@pikku/core 0.7.8 → 0.8.1

package/src/events/queue/queue.types.ts

@@ -0,0 +1,176 @@
+import { APIDocs } from '../../types/core.types.js'
+import { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
+import { QueueConfigMapping } from './validate-worker-config.js'
+
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+export interface PikkuWorkerConfig {
+  /** Optional worker name for identification and monitoring */
+  name?: string
+  /** Number of messages to process in batch / in parallel */
+  batchSize?: number
+  /** Number of messages to prefetch for efficiency */
+  prefetch?: number
+  /** Polling interval for pull-based queues in ms */
+  pollInterval?: number
+  /** Message visibility timeout in seconds */
+  visibilityTimeout?: number
+  /** Duration of job lock in milliseconds */
+  lockDuration?: number
+  /** Number of seconds to wait when queue is empty before polling again */
+  drainDelay?: number
+  /** Keep N completed jobs for inspection */
+  removeOnComplete?: number
+  /** Keep N failed jobs for inspection */
+  removeOnFail?: number
+  /** Maximum number of times a job can be recovered from stalled state */
+  maxStalledCount?: number
+  /** Condition to start processor at instance creation */
+  autorun?: boolean
+}
+
+/**
+ * Configuration for individual jobs - how jobs behave
+ */
+export interface PikkuJobConfig {
+  /** Maximum retry attempts for failed jobs */
+  retryAttempts?: number
+  /** Initial retry delay in milliseconds */
+  retryDelay?: number
+  /** Retry backoff strategy */
+  retryBackoff?: 'linear' | 'exponential' | 'fixed'
+  /** Queue for failed messages after max retries */
+  deadLetterQueue?: string
+  /** How long to retain completed jobs in seconds */
+  messageRetention?: number
+  /** Job priority (higher numbers = higher priority) */
+  priority?: number
+  /** Enable FIFO ordering where supported */
+  fifo?: boolean
+  /** Job timeout in milliseconds */
+  timeout?: number
+  /** Delay before job execution in milliseconds */
+  delay?: number
+}
+
+/**
+ * Configuration validation result with warnings and fallbacks
+ */
+export interface ConfigValidationResult {
+  applied: Partial<PikkuWorkerConfig>
+  ignored: Partial<PikkuWorkerConfig>
+  warnings: string[]
+  fallbacks: { [key: string]: any }
+}
+
+/**
+ * Queue job representation
+ */
+export type QueueJobStatus =
+  | 'waiting'
+  | 'active'
+  | 'completed'
+  | 'failed'
+  | 'delayed'
+
+export type QueueJobMetadata = {
+  progress?: number | string | object | undefined | boolean
+  attemptsMade?: number
+  maxAttempts?: number
+  processedAt?: Date
+  completedAt?: Date
+  failedAt?: Date
+  error?: string
+  createdAt: Date
+}
+
+export interface QueueJob<T = any, R = any> {
+  id: string
+  queueName: string
+  status: () => Promise<QueueJobStatus> | QueueJobStatus
+  data: T
+  result?: R
+  waitForCompletion?: (ttl?: number) => Promise<R>
+  metadata?: () => Promise<QueueJobMetadata> | QueueJobMetadata
+}
+
+/**
+ * Job options for queue operations
+ */
+export interface JobOptions {
+  priority?: number
+  delay?: number
+  attempts?: number
+  backoff?: string | { type: string; delay?: number }
+  removeOnComplete?: number
+  removeOnFail?: number
+  jobId?: string
+}
+
+/**
+ * Queue provider interface for job publishing operations
+ */
+export interface QueueService {
+  /** Whether this queue provider supports job results */
+  readonly supportsResults: boolean
+
+  /** Add a job to the queue with type safety */
+  add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>
+
+  /** Get job status and result */
+  getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>
+}
+
+/**
+ * Queue service interface that queue adapters implement
+ */
+export interface QueueWorkers {
+  /** Service name identifier */
+  name: string
+
+  /** Whether this queue service supports job results */
+  supportsResults: boolean
+
+  /** Configuration mapping for validation */
+  configMappings: QueueConfigMapping
+
+  /** Scan state and register all compatible processors */
+  registerQueues(): Promise<Record<string, ConfigValidationResult[]>>
+
+  /** Close all queues and connections */
+  close(): Promise<void>
+}
+
+/**
+ * Queue processor metadata
+ */
+export type queueWorkersMeta = Record<
+  string,
+  {
+    pikkuFuncName: string
+    schemaName?: string
+    queueName: string
+    session?: undefined
+    docs?: APIDocs
+    tags?: string[]
+    config?: PikkuWorkerConfig
+  }
+>
+
+/**
+ * Core queue processor definition
+ */
+export type CoreQueueWorker<
+  APIFunction = CoreAPIFunctionSessionless<any, any>,
+> = {
+  queueName: string
+  func: APIFunction
+  config?: PikkuWorkerConfig
+  docs?: APIDocs
+  session?: undefined
+  tags?: string[]
+}

package/src/events/queue/register-queue-helper.ts

@@ -0,0 +1,62 @@
+import type { ConfigValidationResult, CoreQueueWorker } from './queue.types.js'
+import { getQueueWorkers } from './queue-runner.js'
+import {
+  QueueConfigMapping,
+  validateWorkerConfig,
+} from './validate-worker-config.js'
+import { Logger } from '../../services/logger.js'
+
+/**
+ * Queue processor registration callback
+ */
+export type QueueRegistrationCallback<T = any> = (
+  queueName: string,
+  processor: CoreQueueWorker
+) => Promise<T>
+
+/**
+ * Helper function to register queue processors with validation
+ * This centralizes the common logic for looping over processors and validating configs
+ *
+ * @param configMappings - Configuration mapping for the queue implementation
+ * @param registerCallback - Callback to register each individual queue processor
+ * @param logger - Optional logger for info/error messages
+ * @returns Record of validation results by queue name
+ */
+export async function registerQueueWorkers<T = any>(
+  configMappings: QueueConfigMapping,
+  logger: Logger,
+  registerCallback: QueueRegistrationCallback<T>
+): Promise<Record<string, ConfigValidationResult[]>> {
+  const configValidation: Record<string, ConfigValidationResult[]> = {}
+  const queueWorkers = getQueueWorkers()
+
+  for (const [queueName, processor] of queueWorkers) {
+    logger?.info(`Registering queue processor: ${queueName}`)
+    // Validate the processor configuration
+    const validationResult = validateWorkerConfig(
+      configMappings,
+      processor.config
+    )
+
+    // Store validation results
+    configValidation[queueName] = configValidation[queueName] || []
+    configValidation[queueName].push(validationResult)
+
+    if (validationResult.warnings.length > 0) {
+      logger?.warn(
+        `Configuration warnings for queue ${queueName}:`,
+        validationResult.warnings
+      )
+    }
+
+    try {
+      await registerCallback(queueName, processor)
+      logger?.info(`Successfully registered queue processor: ${queueName}`)
+    } catch (error) {
+      logger?.error(`Failed to register queue processor ${queueName}:`, error)
+    }
+  }
+
+  return configValidation
+}

package/src/events/queue/validate-worker-config.ts

@@ -0,0 +1,113 @@
+import { PikkuWorkerConfig, ConfigValidationResult } from './queue.types.js'
+
+/**
+ * Configuration mapping structure for queue implementations
+ */
+export interface QueueConfigMapping {
+  /** Configurations that are directly supported */
+  supported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** The property name in the underlying queue system */
+        queueProperty?: string
+        /** Optional transform function for the value */
+        transform?: (value: any) => any
+        /** Description of what this configuration does */
+        description: string
+      }
+    >
+  >
+
+  /** Configurations that are not supported */
+  unsupported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration is not supported */
+        reason: string
+        /** Detailed explanation of what the queue system does instead */
+        explanation: string
+      }
+    >
+  >
+
+  /** Configurations that have partial support or workarounds */
+  fallbacks: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration uses a fallback */
+        reason: string
+        /** Detailed explanation of the fallback behavior */
+        explanation: string
+        /** The fallback value or description */
+        fallbackValue: string
+      }
+    >
+  >
+}
+
+/**
+ * Validates worker configuration using a mapping table
+ * This provides a flexible, maintainable way to validate configurations
+ * across different queue implementations
+ *
+ * @param config - The worker configuration to validate
+ * @param configMapping - The mapping table defining supported/unsupported configurations
+ * @returns Validation result with applied, ignored, warnings, and fallbacks
+ */
+export function validateWorkerConfig(
+  configMapping: QueueConfigMapping,
+  config: PikkuWorkerConfig = {}
+): ConfigValidationResult {
+  const applied: Partial<PikkuWorkerConfig> = {}
+  const ignored: Partial<PikkuWorkerConfig> = {}
+  const warnings: string[] = []
+  const fallbacks: { [key: string]: any } = {}
+
+  // Process each configuration property
+  for (const [key, value] of Object.entries(config)) {
+    if (value === undefined) continue
+
+    const configKey = key as keyof PikkuWorkerConfig
+
+    // Check if it's a supported configuration
+    if (configKey in configMapping.supported) {
+      applied[configKey] = value
+      continue
+    }
+
+    // Check if it's a fallback configuration
+    if (configKey in configMapping.fallbacks) {
+      const fallbackMapping = configMapping.fallbacks[configKey]!
+      applied[configKey] = value
+      fallbacks[key] = fallbackMapping.fallbackValue
+      warnings.push(
+        `${key}: ${fallbackMapping.reason}. ${fallbackMapping.explanation}`
+      )
+      continue
+    }
+
+    // Check if it's an unsupported configuration
+    if (configKey in configMapping.unsupported) {
+      ignored[configKey] = value
+      const mapping = configMapping.unsupported[configKey]!
+      warnings.push(`${key}: ${mapping.reason}. ${mapping.explanation}`)
+      continue
+    }
+
+    // Unknown configuration
+    ignored[configKey] = value
+    warnings.push(
+      `${key}: Unknown configuration option for this queue implementation`
+    )
+  }
+
+  return {
+    applied,
+    ignored,
+    warnings,
+    fallbacks,
+  }
+}

package/src/events/rpc/index.ts

@@ -0,0 +1,2 @@
+export { initialize } from './rpc-runner.js'
+export type { PikkuRPC, RPCMeta } from './rpc-types.js'

package/src/{rpc → events/rpc}/rpc-runner.ts

@@ -1,6 +1,6 @@
-import { CoreServices } from '../types/core.types.js'
-import { runPikkuFunc } from '../function/function-runner.js'
-import { pikkuState } from '../pikku-state.js'
+import { CoreServices } from '../../types/core.types.js'
+import { runPikkuFunc } from '../../function/function-runner.js'
+import { pikkuState } from '../../pikku-state.js'
 import { RPCMeta } from './rpc-types.js'
 
 // Type for the RPC service configuration

package/src/events/scheduler/index.ts

@@ -0,0 +1,9 @@
+export * from './scheduler.types.js'
+
+export {
+  runScheduledTask,
+  addScheduledTask,
+  getScheduledTasks,
+} from './scheduler-runner.js'
+
+export * from './log-schedulers.js'

package/src/{scheduler → events/scheduler}/log-schedulers.ts

@@ -1,5 +1,5 @@
-import { pikkuState } from '../pikku-state.js'
-import { Logger } from '../services/index.js'
+import { pikkuState } from '../../pikku-state.js'
+import { Logger } from '../../services/index.js'
 
 /**
  * Logs all the loaded scheduled tasks.

package/src/{scheduler → events/scheduler}/scheduler-runner.ts

@@ -3,13 +3,13 @@ import type {
   CoreSingletonServices,
   CoreUserSession,
   CreateSessionServices,
-} from '../types/core.types.js'
+} from '../../types/core.types.js'
 import type { CoreScheduledTask } from './scheduler.types.js'
-import type { CoreAPIFunctionSessionless } from '../function/functions.types.js'
-import { getErrorResponse } from '../errors/error-handler.js'
-import { closeSessionServices } from '../utils.js'
-import { pikkuState } from '../pikku-state.js'
-import { addFunction, runPikkuFunc } from '../function/function-runner.js'
+import type { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
+import { getErrorResponse } from '../../errors/error-handler.js'
+import { closeSessionServices } from '../../utils.js'
+import { pikkuState } from '../../pikku-state.js'
+import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
 import { rpcService } from '../rpc/rpc-runner.js'
 
 export type RunScheduledTasksParams = {
@@ -50,10 +50,7 @@ class ScheduledTaskNotFoundError extends Error {
   }
 }
 
-export async function runScheduledTask<
-  SingletonServices extends CoreSingletonServices = CoreSingletonServices,
-  UserSession extends CoreUserSession = CoreUserSession,
->({
+export async function runScheduledTask({
   name,
   session,
   singletonServices,
@@ -108,3 +105,7 @@ export async function runScheduledTask<
     }
   }
 }
+
+export const getScheduledTasks = () => {
+  return pikkuState('scheduler', 'tasks')
+}

package/src/{scheduler → events/scheduler}/scheduler.types.ts

@@ -1,5 +1,5 @@
-import { APIDocs, CoreUserSession } from '../types/core.types.js'
-import { CoreAPIFunctionSessionless } from '../function/functions.types.js'
+import { APIDocs, CoreUserSession } from '../../types/core.types.js'
+import { CoreAPIFunctionSessionless } from '../../function/functions.types.js'
 
 /**
  * Represents metadata for scheduled tasks, including title, schedule, and documentation.
@@ -22,12 +22,10 @@ export type ScheduledTasksMeta<UserSession extends CoreUserSession = any> =
  */
 export type CoreScheduledTask<
   APIFunction = CoreAPIFunctionSessionless<void, void>,
-  UserSession extends CoreUserSession = CoreUserSession,
 > = {
   name: string
   schedule: string
   func: APIFunction
   docs?: APIDocs
-  session?: UserSession
   tags?: string[]
 }

package/src/function/function-runner.ts

@@ -1,64 +1,18 @@
 import { ForbiddenError } from '../errors/errors.js'
+import { runMiddleware } from '../middleware-runner.js'
 import { verifyPermissions } from '../permissions.js'
 import { pikkuState } from '../pikku-state.js'
 import { coerceTopLevelDataFromSchema, validateSchema } from '../schema.js'
-import { Logger } from '../services/logger.js'
-import { CoreServices, CoreUserSession } from '../types/core.types.js'
+import {
+  CoreServices,
+  CoreUserSession,
+  PikkuFunctionMiddleware,
+} from '../types/core.types.js'
 import {
   CorePermissionGroup,
   CorePikkuFunctionConfig,
 } from './functions.types.js'
 
-// Permission merging function
-function mergePermissions(
-  logger: Logger,
-  funcName: string,
-  funcPermissions?: CorePermissionGroup,
-  transportPermissions?: CorePermissionGroup
-): CorePermissionGroup | undefined {
-  if (!funcPermissions && !transportPermissions) {
-    return undefined
-  }
-
-  if (!funcPermissions) {
-    return transportPermissions
-  }
-
-  if (!transportPermissions) {
-    return funcPermissions
-  }
-
-  // Start with a copy of function permissions
-  const mergedPermissions = { ...funcPermissions }
-
-  // Merge in transport permissions
-  for (const [key, transportValue] of Object.entries(transportPermissions)) {
-    if (key in mergedPermissions) {
-      // For permission arrays, concatenate and deduplicate values
-      if (
-        Array.isArray(mergedPermissions[key]) &&
-        Array.isArray(transportValue)
-      ) {
-        mergedPermissions[key] = [
-          ...new Set([...mergedPermissions[key], ...transportValue]),
-        ]
-      }
-      // For other types, warn about conflict and use the more restrictive (transport-level) value
-      else {
-        logger.warn(
-          `Permission conflict on key "${key}" for function "${funcName}". Using transport-level permission.`
-        )
-        mergedPermissions[key] = transportValue
-      }
-    } else {
-      // If key doesn't exist in function permissions, add it
-      mergedPermissions[key] = transportValue
-    }
-  }
-
-  return mergedPermissions
-}
-
 export const addFunction = (
   funcName: string,
   funcConfig: CorePikkuFunctionConfig<any, any>
@@ -86,12 +40,14 @@ export const runPikkuFunc = async <In = any, Out = any>(
     data,
     session,
     permissions: transportPermissions,
+    middleware: transportMiddleware,
     coerceDataFromSchema,
   }: {
     getAllServices: () => Promise<CoreServices> | CoreServices
     data: In
     session?: CoreUserSession
     permissions?: CorePermissionGroup
+    middleware?: PikkuFunctionMiddleware[]
     coerceDataFromSchema?: boolean
   }
 ): Promise<Out> => {
@@ -127,23 +83,40 @@ export const runPikkuFunc = async <In = any, Out = any>(
     }
   }
 
-  // Execute permission checks
-  const mergedPermissions = mergePermissions(
-    allServices.logger,
-    funcName,
-    funcConfig.permissions,
-    transportPermissions
-  )
-  const permissioned = await verifyPermissions(
-    mergedPermissions,
-    allServices,
-    data,
-    session
-  )
+  let permissioned = true
+  if (funcConfig.permissions) {
+    permissioned = await verifyPermissions(
+      funcConfig.permissions,
+      allServices,
+      data,
+      session
+    )
+  }
+  if (!permissioned && transportPermissions) {
+    permissioned = await verifyPermissions(
+      transportPermissions,
+      allServices,
+      data,
+      session
+    )
+  }
 
   if (permissioned === false) {
     throw new ForbiddenError('Permission denied')
   }
 
+  if (transportMiddleware || funcConfig.middleware) {
+    return (await runMiddleware<PikkuFunctionMiddleware>(
+      allServices,
+      {
+        http: allServices.http,
+        mcp: allServices.mcp,
+        rpc: allServices.rpc,
+      },
+      [...(transportMiddleware || []), ...(funcConfig.middleware || [])],
+      async () => await funcConfig.func(allServices, data, session!)
+    )) as Out
+  }
+
   return (await funcConfig.func(allServices, data, session!)) as Out
 }

package/src/function/functions.types.ts

@@ -1,8 +1,9 @@
-import { PikkuChannel } from '../channel/channel.types.js'
+import { PikkuChannel } from '../events/channel/channel.types.js'
 import type {
   CoreServices,
   CoreSingletonServices,
   CoreUserSession,
+  PikkuFunctionMiddleware,
 } from '../types/core.types.js'
 
 /**
@@ -85,4 +86,5 @@ export type CorePikkuFunctionConfig<
   func: APIFunction
   auth?: boolean
   permissions?: CorePermissionGroup<APIPermission>
+  middleware?: PikkuFunctionMiddleware[]
 }

package/src/handle-error.ts

@@ -1,7 +1,7 @@
 import { getErrorResponse } from './errors/error-handler.js'
 import { NotFoundError } from './errors/errors.js'
 import { Logger } from './services/logger.js'
-import { PikkuHTTP } from './http/http.types.js'
+import { PikkuHTTP } from './events/http/http.types.js'
 
 /**
  * Handle errors that occur during route processing
@@ -14,7 +14,7 @@ import { PikkuHTTP } from './http/http.types.js'
  * @param {boolean} respondWith404 - Whether to respond with 404 for NotFoundError
  * @param {boolean} bubbleError - Whether to throw the error after handling
  */
-export const handleError = (
+export const handleHTTPError = (
   e: any,
   http: PikkuHTTP | undefined,
   trackerId: string | undefined,

package/src/index.ts

@@ -5,16 +5,37 @@ export * from './types/core.types.js'
 export * from './function/index.js'
 export * from './pikku-request.js'
 export * from './pikku-response.js'
-export * from './services/index.js'
-export * from './http/index.js'
-export * from './channel/index.js'
-export * from './scheduler/index.js'
+export * from './services/logger.js'
+export * from './services/schema-service.js'
+export * from './services/jwt-service.js'
+export * from './services/secret-service.js'
+export * from './services/user-session-service.js'
+export * from './events/http/index.js'
+export * from './events/channel/index.js'
+export * from './events/scheduler/index.js'
+export * from './events/rpc/index.js'
+export * from './events/queue/index.js'
+export * from './events/mcp/index.js'
 export * from './errors/index.js'
 export * from './middleware/index.js'
 export * from './time-utils.js'
 export * from './utils.js'
 export { pikkuState } from './pikku-state.js'
 export { runMiddleware } from './middleware-runner.js'
-export { addHTTPRoute, addMiddleware } from './http/http-runner.js'
-export { addChannel } from './channel/channel-runner.js'
-export { addScheduledTask } from './scheduler/scheduler-runner.js'
+export { addHTTPRoute, addMiddleware } from './events/http/http-runner.js'
+export { addChannel } from './events/channel/channel-runner.js'
+export { addScheduledTask } from './events/scheduler/scheduler-runner.js'
+export {
+  addMCPResource,
+  addMCPTool,
+  addMCPPrompt,
+  runMCPResource,
+  runMCPTool,
+  runMCPPrompt,
+  getMCPTools,
+  getMCPResources,
+  getMCPPrompts,
+  getMCPResourcesMeta,
+  getMCPToolsMeta,
+  getMCPPromptsMeta,
+} from './events/mcp/mcp-runner.js'

package/src/middleware/auth-apikey.ts

@@ -33,7 +33,7 @@ export const authAPIKey = <
     }
 )) => {
   const middleware: PikkuMiddleware = async (services, { http }, next) => {
-    const { userSession: userSessionService, jwt: jwtService } = services
+    const { userSession: userSessionService, jwt: jwtService } = services as any
     if (!http?.request || userSessionService.get()) {
       return next()
     }

package/src/middleware/auth-bearer.ts

@@ -28,7 +28,7 @@ export const authBearer = <
   ) => Promise<UserSession> | UserSession
 } = {}): PikkuMiddleware => {
   const middleware: PikkuMiddleware = async (services, { http }, next) => {
-    const { userSession: userSessionService, jwt: jwtService } = services
+    const { userSession: userSessionService, jwt: jwtService } = services as any
     // Skip if session already exists.
     if (!http?.request || userSessionService.get()) {
       return next()