@pikku/core 0.7.7 → 0.8.0

package/src/events/queue/queue.types.ts

@@ -0,0 +1,176 @@
+import { APIDocs } from '../../types/core.types.js'
+import { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
+import { QueueConfigMapping } from './validate-worker-config.js'
+
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+export interface PikkuWorkerConfig {
+  /** Optional worker name for identification and monitoring */
+  name?: string
+  /** Number of messages to process in batch / in parallel */
+  batchSize?: number
+  /** Number of messages to prefetch for efficiency */
+  prefetch?: number
+  /** Polling interval for pull-based queues in ms */
+  pollInterval?: number
+  /** Message visibility timeout in seconds */
+  visibilityTimeout?: number
+  /** Duration of job lock in milliseconds */
+  lockDuration?: number
+  /** Number of seconds to wait when queue is empty before polling again */
+  drainDelay?: number
+  /** Keep N completed jobs for inspection */
+  removeOnComplete?: number
+  /** Keep N failed jobs for inspection */
+  removeOnFail?: number
+  /** Maximum number of times a job can be recovered from stalled state */
+  maxStalledCount?: number
+  /** Condition to start processor at instance creation */
+  autorun?: boolean
+}
+
+/**
+ * Configuration for individual jobs - how jobs behave
+ */
+export interface PikkuJobConfig {
+  /** Maximum retry attempts for failed jobs */
+  retryAttempts?: number
+  /** Initial retry delay in milliseconds */
+  retryDelay?: number
+  /** Retry backoff strategy */
+  retryBackoff?: 'linear' | 'exponential' | 'fixed'
+  /** Queue for failed messages after max retries */
+  deadLetterQueue?: string
+  /** How long to retain completed jobs in seconds */
+  messageRetention?: number
+  /** Job priority (higher numbers = higher priority) */
+  priority?: number
+  /** Enable FIFO ordering where supported */
+  fifo?: boolean
+  /** Job timeout in milliseconds */
+  timeout?: number
+  /** Delay before job execution in milliseconds */
+  delay?: number
+}
+
+/**
+ * Configuration validation result with warnings and fallbacks
+ */
+export interface ConfigValidationResult {
+  applied: Partial<PikkuWorkerConfig>
+  ignored: Partial<PikkuWorkerConfig>
+  warnings: string[]
+  fallbacks: { [key: string]: any }
+}
+
+/**
+ * Queue job representation
+ */
+export type QueueJobStatus =
+  | 'waiting'
+  | 'active'
+  | 'completed'
+  | 'failed'
+  | 'delayed'
+
+export type QueueJobMetadata = {
+  progress?: number | string | object | undefined | boolean
+  attemptsMade?: number
+  maxAttempts?: number
+  processedAt?: Date
+  completedAt?: Date
+  failedAt?: Date
+  error?: string
+  createdAt: Date
+}
+
+export interface QueueJob<T = any, R = any> {
+  id: string
+  queueName: string
+  status: () => Promise<QueueJobStatus> | QueueJobStatus
+  data: T
+  result?: R
+  waitForCompletion?: (ttl?: number) => Promise<R>
+  metadata?: () => Promise<QueueJobMetadata> | QueueJobMetadata
+}
+
+/**
+ * Job options for queue operations
+ */
+export interface JobOptions {
+  priority?: number
+  delay?: number
+  attempts?: number
+  backoff?: string | { type: string; delay?: number }
+  removeOnComplete?: number
+  removeOnFail?: number
+  jobId?: string
+}
+
+/**
+ * Queue provider interface for job publishing operations
+ */
+export interface QueueService {
+  /** Whether this queue provider supports job results */
+  readonly supportsResults: boolean
+
+  /** Add a job to the queue with type safety */
+  add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>
+
+  /** Get job status and result */
+  getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>
+}
+
+/**
+ * Queue service interface that queue adapters implement
+ */
+export interface QueueWorkers {
+  /** Service name identifier */
+  name: string
+
+  /** Whether this queue service supports job results */
+  supportsResults: boolean
+
+  /** Configuration mapping for validation */
+  configMappings: QueueConfigMapping
+
+  /** Scan state and register all compatible processors */
+  registerQueues(): Promise<Record<string, ConfigValidationResult[]>>
+
+  /** Close all queues and connections */
+  close(): Promise<void>
+}
+
+/**
+ * Queue processor metadata
+ */
+export type queueWorkersMeta = Record<
+  string,
+  {
+    pikkuFuncName: string
+    schemaName?: string
+    queueName: string
+    session?: undefined
+    docs?: APIDocs
+    tags?: string[]
+    config?: PikkuWorkerConfig
+  }
+>
+
+/**
+ * Core queue processor definition
+ */
+export type CoreQueueWorker<
+  APIFunction = CoreAPIFunctionSessionless<any, any>,
+> = {
+  queueName: string
+  func: APIFunction
+  config?: PikkuWorkerConfig
+  docs?: APIDocs
+  session?: undefined
+  tags?: string[]
+}

package/src/events/queue/register-queue-helper.ts

@@ -0,0 +1,62 @@
+import type { ConfigValidationResult, CoreQueueWorker } from './queue.types.js'
+import { getQueueWorkers } from './queue-runner.js'
+import {
+  QueueConfigMapping,
+  validateWorkerConfig,
+} from './validate-worker-config.js'
+import { Logger } from '../../services/logger.js'
+
+/**
+ * Queue processor registration callback
+ */
+export type QueueRegistrationCallback<T = any> = (
+  queueName: string,
+  processor: CoreQueueWorker
+) => Promise<T>
+
+/**
+ * Helper function to register queue processors with validation
+ * This centralizes the common logic for looping over processors and validating configs
+ *
+ * @param configMappings - Configuration mapping for the queue implementation
+ * @param registerCallback - Callback to register each individual queue processor
+ * @param logger - Optional logger for info/error messages
+ * @returns Record of validation results by queue name
+ */
+export async function registerqueueWorkers<T = any>(
+  configMappings: QueueConfigMapping,
+  logger: Logger,
+  registerCallback: QueueRegistrationCallback<T>
+): Promise<Record<string, ConfigValidationResult[]>> {
+  const configValidation: Record<string, ConfigValidationResult[]> = {}
+  const queueWorkers = getQueueWorkers()
+
+  for (const [queueName, processor] of queueWorkers) {
+    logger?.info(`Registering queue processor: ${queueName}`)
+    // Validate the processor configuration
+    const validationResult = validateWorkerConfig(
+      configMappings,
+      processor.config
+    )
+
+    // Store validation results
+    configValidation[queueName] = configValidation[queueName] || []
+    configValidation[queueName].push(validationResult)
+
+    if (validationResult.warnings.length > 0) {
+      logger?.warn(
+        `Configuration warnings for queue ${queueName}:`,
+        validationResult.warnings
+      )
+    }
+
+    try {
+      await registerCallback(queueName, processor)
+      logger?.info(`Successfully registered queue processor: ${queueName}`)
+    } catch (error) {
+      logger?.error(`Failed to register queue processor ${queueName}:`, error)
+    }
+  }
+
+  return configValidation
+}

package/src/events/queue/validate-worker-config.ts

@@ -0,0 +1,113 @@
+import { PikkuWorkerConfig, ConfigValidationResult } from './queue.types.js'
+
+/**
+ * Configuration mapping structure for queue implementations
+ */
+export interface QueueConfigMapping {
+  /** Configurations that are directly supported */
+  supported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** The property name in the underlying queue system */
+        queueProperty?: string
+        /** Optional transform function for the value */
+        transform?: (value: any) => any
+        /** Description of what this configuration does */
+        description: string
+      }
+    >
+  >
+
+  /** Configurations that are not supported */
+  unsupported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration is not supported */
+        reason: string
+        /** Detailed explanation of what the queue system does instead */
+        explanation: string
+      }
+    >
+  >
+
+  /** Configurations that have partial support or workarounds */
+  fallbacks: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration uses a fallback */
+        reason: string
+        /** Detailed explanation of the fallback behavior */
+        explanation: string
+        /** The fallback value or description */
+        fallbackValue: string
+      }
+    >
+  >
+}
+
+/**
+ * Validates worker configuration using a mapping table
+ * This provides a flexible, maintainable way to validate configurations
+ * across different queue implementations
+ *
+ * @param config - The worker configuration to validate
+ * @param configMapping - The mapping table defining supported/unsupported configurations
+ * @returns Validation result with applied, ignored, warnings, and fallbacks
+ */
+export function validateWorkerConfig(
+  configMapping: QueueConfigMapping,
+  config: PikkuWorkerConfig = {}
+): ConfigValidationResult {
+  const applied: Partial<PikkuWorkerConfig> = {}
+  const ignored: Partial<PikkuWorkerConfig> = {}
+  const warnings: string[] = []
+  const fallbacks: { [key: string]: any } = {}
+
+  // Process each configuration property
+  for (const [key, value] of Object.entries(config)) {
+    if (value === undefined) continue
+
+    const configKey = key as keyof PikkuWorkerConfig
+
+    // Check if it's a supported configuration
+    if (configKey in configMapping.supported) {
+      applied[configKey] = value
+      continue
+    }
+
+    // Check if it's a fallback configuration
+    if (configKey in configMapping.fallbacks) {
+      const fallbackMapping = configMapping.fallbacks[configKey]!
+      applied[configKey] = value
+      fallbacks[key] = fallbackMapping.fallbackValue
+      warnings.push(
+        `${key}: ${fallbackMapping.reason}. ${fallbackMapping.explanation}`
+      )
+      continue
+    }
+
+    // Check if it's an unsupported configuration
+    if (configKey in configMapping.unsupported) {
+      ignored[configKey] = value
+      const mapping = configMapping.unsupported[configKey]!
+      warnings.push(`${key}: ${mapping.reason}. ${mapping.explanation}`)
+      continue
+    }
+
+    // Unknown configuration
+    ignored[configKey] = value
+    warnings.push(
+      `${key}: Unknown configuration option for this queue implementation`
+    )
+  }
+
+  return {
+    applied,
+    ignored,
+    warnings,
+    fallbacks,
+  }
+}

package/src/events/rpc/index.ts

@@ -0,0 +1,2 @@
+export { initialize } from './rpc-runner.js'
+export type { PikkuRPC, RPCMeta } from './rpc-types.js'

package/src/{rpc → events/rpc}/rpc-runner.ts

@@ -1,6 +1,6 @@
-import { CoreServices } from '../types/core.types.js'
-import { runPikkuFunc } from '../function/function-runner.js'
-import { pikkuState } from '../pikku-state.js'
+import { CoreServices } from '../../types/core.types.js'
+import { runPikkuFunc } from '../../function/function-runner.js'
+import { pikkuState } from '../../pikku-state.js'
 import { RPCMeta } from './rpc-types.js'
 
 // Type for the RPC service configuration

package/src/events/scheduler/index.ts

@@ -0,0 +1,9 @@
+export * from './scheduler.types.js'
+
+export {
+  runScheduledTask,
+  addScheduledTask,
+  getScheduledTasks,
+} from './scheduler-runner.js'
+
+export * from './log-schedulers.js'

package/src/{scheduler → events/scheduler}/log-schedulers.ts

@@ -1,5 +1,5 @@
-import { pikkuState } from '../pikku-state.js'
-import { Logger } from '../services/index.js'
+import { pikkuState } from '../../pikku-state.js'
+import { Logger } from '../../services/index.js'
 
 /**
  * Logs all the loaded scheduled tasks.

package/src/{scheduler → events/scheduler}/scheduler-runner.ts

@@ -3,13 +3,13 @@ import type {
   CoreSingletonServices,
   CoreUserSession,
   CreateSessionServices,
-} from '../types/core.types.js'
+} from '../../types/core.types.js'
 import type { CoreScheduledTask } from './scheduler.types.js'
-import type { CoreAPIFunctionSessionless } from '../function/functions.types.js'
-import { getErrorResponse } from '../errors/error-handler.js'
-import { closeSessionServices } from '../utils.js'
-import { pikkuState } from '../pikku-state.js'
-import { addFunction, runPikkuFunc } from '../function/function-runner.js'
+import type { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
+import { getErrorResponse } from '../../errors/error-handler.js'
+import { closeSessionServices } from '../../utils.js'
+import { pikkuState } from '../../pikku-state.js'
+import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
 import { rpcService } from '../rpc/rpc-runner.js'
 
 export type RunScheduledTasksParams = {
@@ -50,10 +50,7 @@ class ScheduledTaskNotFoundError extends Error {
   }
 }
 
-export async function runScheduledTask<
-  SingletonServices extends CoreSingletonServices = CoreSingletonServices,
-  UserSession extends CoreUserSession = CoreUserSession,
->({
+export async function runScheduledTask({
   name,
   session,
   singletonServices,
@@ -108,3 +105,7 @@ export async function runScheduledTask<
     }
   }
 }
+
+export const getScheduledTasks = () => {
+  return pikkuState('scheduler', 'tasks')
+}

package/src/{scheduler → events/scheduler}/scheduler.types.ts

@@ -1,5 +1,5 @@
-import { APIDocs, CoreUserSession } from '../types/core.types.js'
-import { CoreAPIFunctionSessionless } from '../function/functions.types.js'
+import { APIDocs, CoreUserSession } from '../../types/core.types.js'
+import { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
 
 /**
  * Represents metadata for scheduled tasks, including title, schedule, and documentation.
@@ -22,12 +22,10 @@ export type ScheduledTasksMeta<UserSession extends CoreUserSession = any> =
  */
 export type CoreScheduledTask<
   APIFunction = CoreAPIFunctionSessionless<void, void>,
-  UserSession extends CoreUserSession = CoreUserSession,
 > = {
   name: string
   schedule: string
   func: APIFunction
   docs?: APIDocs
-  session?: UserSession
   tags?: string[]
 }

package/src/function/function-runner.ts

@@ -1,9 +1,14 @@
 import { ForbiddenError } from '../errors/errors.js'
+import { runMiddleware } from '../middleware-runner.js'
 import { verifyPermissions } from '../permissions.js'
 import { pikkuState } from '../pikku-state.js'
 import { coerceTopLevelDataFromSchema, validateSchema } from '../schema.js'
 import { Logger } from '../services/logger.js'
-import { CoreServices, CoreUserSession } from '../types/core.types.js'
+import {
+  CoreServices,
+  CoreUserSession,
+  PikkuFunctionMiddleware,
+} from '../types/core.types.js'
 import {
   CorePermissionGroup,
   CorePikkuFunctionConfig,
@@ -145,5 +150,18 @@ export const runPikkuFunc = async <In = any, Out = any>(
     throw new ForbiddenError('Permission denied')
   }
 
+  if (funcConfig.middleware) {
+    return (await runMiddleware<PikkuFunctionMiddleware>(
+      allServices,
+      {
+        http: allServices.http,
+        mcp: allServices.mcp,
+        rpc: allServices.rpc,
+      },
+      funcConfig.middleware,
+      async () => await funcConfig.func(allServices, data, session!)
+    )) as Out
+  }
+
   return (await funcConfig.func(allServices, data, session!)) as Out
 }

package/src/function/functions.types.ts

@@ -1,8 +1,9 @@
-import { PikkuChannel } from '../channel/channel.types.js'
+import { PikkuChannel } from '../events/channel/channel.types.js'
 import type {
   CoreServices,
   CoreSingletonServices,
   CoreUserSession,
+  PikkuFunctionMiddleware,
 } from '../types/core.types.js'
 
 /**
@@ -85,4 +86,5 @@ export type CorePikkuFunctionConfig<
   func: APIFunction
   auth?: boolean
   permissions?: CorePermissionGroup<APIPermission>
+  middleware?: PikkuFunctionMiddleware[]
 }

package/src/handle-error.ts

@@ -1,7 +1,7 @@
 import { getErrorResponse } from './errors/error-handler.js'
 import { NotFoundError } from './errors/errors.js'
 import { Logger } from './services/logger.js'
-import { PikkuHTTP } from './http/http.types.js'
+import { PikkuHTTP } from './events/http/http.types.js'
 
 /**
  * Handle errors that occur during route processing
@@ -14,7 +14,7 @@ import { PikkuHTTP } from './http/http.types.js'
  * @param {boolean} respondWith404 - Whether to respond with 404 for NotFoundError
  * @param {boolean} bubbleError - Whether to throw the error after handling
  */
-export const handleError = (
+export const handleHTTPError = (
   e: any,
   http: PikkuHTTP | undefined,
   trackerId: string | undefined,

package/src/index.ts

@@ -6,15 +6,32 @@ export * from './function/index.js'
 export * from './pikku-request.js'
 export * from './pikku-response.js'
 export * from './services/index.js'
-export * from './http/index.js'
-export * from './channel/index.js'
-export * from './scheduler/index.js'
+export * from './events/http/index.js'
+export * from './events/channel/index.js'
+export * from './events/scheduler/index.js'
+export * from './events/rpc/index.js'
+export * from './events/queue/index.js'
+export * from './events/mcp/index.js'
 export * from './errors/index.js'
 export * from './middleware/index.js'
 export * from './time-utils.js'
 export * from './utils.js'
 export { pikkuState } from './pikku-state.js'
 export { runMiddleware } from './middleware-runner.js'
-export { addHTTPRoute, addMiddleware } from './http/http-runner.js'
-export { addChannel } from './channel/channel-runner.js'
-export { addScheduledTask } from './scheduler/scheduler-runner.js'
+export { addHTTPRoute, addMiddleware } from './events/http/http-runner.js'
+export { addChannel } from './events/channel/channel-runner.js'
+export { addScheduledTask } from './events/scheduler/scheduler-runner.js'
+export {
+  addMCPResource,
+  addMCPTool,
+  addMCPPrompt,
+  runMCPResource,
+  runMCPTool,
+  runMCPPrompt,
+  getMCPTools,
+  getMCPResources,
+  getMCPPrompts,
+  getMCPResourcesMeta,
+  getMCPToolsMeta,
+  getMCPPromptsMeta,
+} from './events/mcp/mcp-runner.js'

package/src/middleware-runner.ts

@@ -1,6 +1,7 @@
 import { UserSessionService } from './services/user-session-service.js'
 import {
   CoreSingletonServices,
+  PikkuFunctionMiddleware,
   PikkuInteraction,
   PikkuMiddleware,
 } from './types/core.types.js'
@@ -22,22 +23,28 @@ import {
  *   async () => { return await runMain(); }
  * );
  */
-export const runMiddleware = async (
+export const runMiddleware = async <
+  Middleware extends
+    | PikkuMiddleware
+    | PikkuFunctionMiddleware = PikkuMiddleware,
+>(
   services: CoreSingletonServices & {
-    userSession: UserSessionService<any>
+    userSession?: UserSessionService<any>
   },
   interaction: PikkuInteraction,
-  middlewares: PikkuMiddleware[],
-  main?: () => Promise<void>
-): Promise<void> => {
+  middlewares: Middleware[],
+  main?: () => Promise<unknown>
+): Promise<unknown> => {
+  let result: any
   const dispatch = async (index: number): Promise<any> => {
     if (middlewares && index < middlewares.length) {
-      await middlewares[index]!(services, interaction, () =>
+      return await middlewares[index]!(services as any, interaction, () =>
         dispatch(index + 1)
       )
     } else if (main) {
-      return await main()
+      result = await main()
     }
   }
   await dispatch(0)
+  return result
 }

package/src/pikku-state.ts

@@ -1,17 +1,29 @@
-import { ChannelsMeta, CoreAPIChannel } from './channel/channel.types.js'
+import { ChannelsMeta, CoreAPIChannel } from './events/channel/channel.types.js'
 import {
   CoreHTTPFunctionRoute,
   HTTPMethod,
   HTTPRoutesMeta,
-} from './http/http.types.js'
+} from './events/http/http.types.js'
 import { FunctionsMeta, PikkuMiddleware } from './types/core.types.js'
 import {
   CoreScheduledTask,
   ScheduledTasksMeta,
-} from './scheduler/scheduler.types.js'
+} from './events/scheduler/scheduler.types.js'
 import { ErrorDetails, PikkuError } from './errors/error-handler.js'
 import { CorePikkuFunctionConfig } from './function/functions.types.js'
-import { RPCMeta } from './rpc/rpc-types.js'
+import { RPCMeta } from './events/rpc/rpc-types.js'
+import {
+  queueWorkersMeta,
+  CoreQueueWorker,
+} from './events/queue/queue.types.js'
+import {
+  CoreMCPResource,
+  CoreMCPTool,
+  CoreMCPPrompt,
+  MCPResourceMeta,
+  MCPToolMeta,
+  MCPPromptMeta,
+} from './events/mcp/mcp.types.js'
 
 interface PikkuState {
   function: {
@@ -41,6 +53,18 @@ interface PikkuState {
     tasks: Map<string, CoreScheduledTask>
     meta: ScheduledTasksMeta
   }
+  queue: {
+    registrations: Map<string, CoreQueueWorker>
+    meta: queueWorkersMeta
+  }
+  mcp: {
+    resources: Map<string, CoreMCPResource>
+    resourcesMeta: MCPResourceMeta
+    tools: Map<string, CoreMCPTool>
+    toolsMeta: MCPToolMeta
+    prompts: Map<string, CoreMCPPrompt>
+    promptsMeta: MCPPromptMeta
+  }
   misc: {
     errors: Map<PikkuError, ErrorDetails>
     schemas: Map<string, any>
@@ -70,6 +94,18 @@ export const resetPikkuState = () => {
       tasks: new Map(),
       meta: [] as unknown as ScheduledTasksMeta,
     },
+    queue: {
+      registrations: new Map(),
+      meta: {},
+    },
+    mcp: {
+      resources: new Map(),
+      resourcesMeta: {} as MCPResourceMeta,
+      tools: new Map(),
+      toolsMeta: {} as MCPToolMeta,
+      prompts: new Map(),
+      promptsMeta: {} as MCPPromptMeta,
+    },
     misc: {
       errors: globalThis.pikkuState?.misc?.errors || new Map(),
       schemas: globalThis.pikkuState?.misc?.schema || new Map(),

package/src/services/user-session-service.ts

@@ -1,4 +1,4 @@
-import { ChannelStore } from '../channel/channel-store.js'
+import { ChannelStore } from '../events/channel/channel-store.js'
 import { CoreUserSession } from '../types/core.types.js'
 
 export interface UserSessionService<UserSession extends CoreUserSession> {