@pikku/core 0.12.4 → 0.12.5

package/src/wirings/queue/queue-runner.ts

@@ -0,0 +1,192 @@
+import type { PikkuWire } from '../../types/core.types.js'
+import type { CoreQueueWorker, QueueJob, PikkuQueue } from './queue.types.js'
+import type {
+  CorePikkuFunctionConfig,
+  CorePikkuFunctionSessionless,
+} from '../../function/functions.types.js'
+import { getErrorResponse, PikkuError } from '../../errors/error-handler.js'
+import { PikkuMissingMetaError } from '../../errors/errors.js'
+import {
+  getSingletonServices,
+  getCreateWireServices,
+  pikkuState,
+} from '../../pikku-state.js'
+import { addFunction, runPikkuFunc } from '../../function/function-runner.js'
+
+/**
+ * Error class for queue processor not found
+ */
+class QueueWorkerNotFoundError extends PikkuError {
+  constructor(name: string) {
+    super(`Queue processor not found: ${name}`)
+  }
+}
+
+/**
+ * Error class for when a queue job is explicitly failed
+ */
+export class QueueJobFailedError extends PikkuError {
+  constructor(jobId: string, reason?: string) {
+    super(`Queue job ${jobId} failed${reason ? `: ${reason}` : ''}`)
+    this.name = 'QueueJobFailedError'
+  }
+}
+
+/**
+ * Error class for when a queue job is explicitly discarded
+ */
+export class QueueJobDiscardedError extends PikkuError {
+  constructor(jobId: string, reason?: string) {
+    super(`Queue job ${jobId} discarded${reason ? `: ${reason}` : ''}`)
+    this.name = 'QueueJobDiscardedError'
+  }
+}
+/**
+ * Add a queue processor to the system
+ */
+export const wireQueueWorker = <
+  InputData = any,
+  OutputData = any,
+  PikkuFunctionConfig extends CorePikkuFunctionConfig<
+    CorePikkuFunctionSessionless<InputData, OutputData>
+  > = CorePikkuFunctionConfig<
+    CorePikkuFunctionSessionless<InputData, OutputData>
+  >,
+>(
+  queueWorker: CoreQueueWorker<PikkuFunctionConfig>
+) => {
+  // Get processor metadata
+  const meta = pikkuState(null, 'queue', 'meta')
+  const processorMeta = meta[queueWorker.name]
+  if (!processorMeta) {
+    throw new PikkuMissingMetaError(
+      `Missing generated metadata for queue worker '${queueWorker.name}'`
+    )
+  }
+
+  // Register the function with pikku
+  addFunction(processorMeta.pikkuFuncId, {
+    func: queueWorker.func.func,
+    auth: queueWorker.func.auth,
+    permissions: queueWorker.func.permissions,
+    middleware: queueWorker.func.middleware as any,
+    tags: queueWorker.func.tags,
+  })
+
+  // Store processor definition in state - runtime adapters will pick this up
+  const registrations = pikkuState(null, 'queue', 'registrations')
+  registrations.set(queueWorker.name, queueWorker)
+}
+
+/**
+ * Get all registered queue processors
+ */
+export function getQueueWorkers(): Map<string, CoreQueueWorker> {
+  return pikkuState(null, 'queue', 'registrations')
+}
+
+/**
+ * Stop and remove a queue processor
+ */
+export async function removeQueueWorker(name: string): Promise<void> {
+  const registrations = pikkuState(null, 'queue', 'registrations')
+  const registration = registrations.get(name)
+
+  if (!registration) {
+    throw new QueueWorkerNotFoundError(name)
+  }
+
+  registrations.delete(name)
+}
+
+/**
+ * Process a single queue job - this function is called by queue consumers
+ */
+export async function runQueueJob({
+  job,
+  updateProgress,
+}: {
+  job: QueueJob
+  updateProgress?: (progress: number | string | object) => Promise<void>
+}): Promise<void> {
+  const singletonServices = getSingletonServices()
+  const createWireServices = getCreateWireServices()
+  const logger = singletonServices.logger
+
+  const meta = pikkuState(null, 'queue', 'meta')
+  const processorMeta = meta[job.queueName]
+  if (!processorMeta) {
+    throw new PikkuMissingMetaError(
+      `Missing generated metadata for queue worker '${job.queueName}'`
+    )
+  }
+
+  // Get the queue worker registration to access middleware
+  const registrations = pikkuState(null, 'queue', 'registrations')
+  const queueWorker = registrations.get(job.queueName)
+  if (!queueWorker) {
+    throw new Error(`Queue worker registration not found for: ${job.queueName}`)
+  }
+
+  // Create the queue wire object
+  const queue: PikkuQueue = {
+    queueName: job.queueName,
+    jobId: job.id,
+    updateProgress:
+      updateProgress ||
+      (async (progress: number | string | object) => {
+        logger.info(`Job ${job.id} progress: ${progress}`)
+        // Default implementation - just log the progress
+      }),
+    fail: async (reason?: string) => {
+      throw new QueueJobFailedError(job.id, reason)
+    },
+    discard: async (reason?: string) => {
+      throw new QueueJobDiscardedError(job.id, reason)
+    },
+  }
+
+  try {
+    logger.info(`Processing job ${job.id} in queue ${job.queueName}`)
+
+    const wire: PikkuWire = {
+      queue,
+    }
+
+    // Execute the pikku function with the job data
+    const result = await runPikkuFunc(
+      'queue',
+      job.queueName,
+      processorMeta.pikkuFuncId,
+      {
+        singletonServices,
+        createWireServices,
+        auth: false,
+        data: () => job.data,
+        inheritedMiddleware: processorMeta.middleware,
+        wireMiddleware: queueWorker.middleware,
+        tags: queueWorker.tags,
+        wire,
+      }
+    )
+
+    logger.debug(
+      `Successfully processed job ${job.id} in queue ${job.queueName}`
+    )
+
+    return result
+  } catch (error: any) {
+    logger.error(
+      `Error processing job ${job.id} in queue ${job.queueName}:`,
+      error
+    )
+
+    const errorResponse = getErrorResponse(error)
+    if (errorResponse != null) {
+      logger.error('Processed error response:', errorResponse)
+    }
+
+    // Re-throw the error so the queue service can handle retries/DLQ
+    throw error
+  }
+}

package/src/wirings/queue/queue.types.ts

@@ -0,0 +1,189 @@
+import type { CommonWireMeta } from '../../types/core.types.js'
+import type { CorePikkuFunctionConfig } from '../../function/functions.types.js'
+import type { QueueConfigMapping } from './validate-worker-config.js'
+
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+/**
+ * Configuration for queue workers - how jobs are processed
+ */
+export interface PikkuWorkerConfig {
+  /** Optional worker name for identification and monitoring */
+  name?: string
+  /** Number of messages to process in batch / in parallel */
+  batchSize?: number
+  /** Number of messages to prefetch for efficiency */
+  prefetch?: number
+  /** Polling interval for pull-based queues in ms */
+  pollInterval?: number
+  /** Message visibility timeout in seconds */
+  visibilityTimeout?: number
+  /** Duration of job lock in milliseconds */
+  lockDuration?: number
+  /** Number of seconds to wait when queue is empty before polling again */
+  drainDelay?: number
+  /** Keep N completed jobs for inspection */
+  removeOnComplete?: number
+  /** Keep N failed jobs for inspection */
+  removeOnFail?: number
+  /** Maximum number of times a job can be recovered from stalled state */
+  maxStalledCount?: number
+  /** Condition to start processor at instance creation */
+  autorun?: boolean
+}
+
+/**
+ * Configuration for individual jobs - how jobs behave
+ */
+export interface PikkuJobConfig {
+  /** Maximum retry attempts for failed jobs */
+  retryAttempts?: number
+  /** Initial retry delay in milliseconds */
+  retryDelay?: number
+  /** Retry backoff strategy */
+  retryBackoff?: 'linear' | 'exponential' | 'fixed'
+  /** Queue for failed messages after max retries */
+  deadLetterQueue?: string
+  /** How long to retain completed jobs in seconds */
+  messageRetention?: number
+  /** Job priority (higher numbers = higher priority) */
+  priority?: number
+  /** Enable FIFO ordering where supported */
+  fifo?: boolean
+  /** Job timeout in milliseconds */
+  timeout?: number
+  /** Delay before job execution in milliseconds */
+  delay?: number
+}
+
+/**
+ * Configuration validation result with warnings and fallbacks
+ */
+export interface ConfigValidationResult {
+  applied: Partial<PikkuWorkerConfig>
+  ignored: Partial<PikkuWorkerConfig>
+  warnings: string[]
+  fallbacks: { [key: string]: any }
+}
+
+/**
+ * Queue job representation
+ */
+export type QueueJobStatus =
+  | 'waiting'
+  | 'active'
+  | 'completed'
+  | 'failed'
+  | 'delayed'
+
+export type QueueJobMetadata = {
+  progress?: number | string | object | undefined | boolean
+  attemptsMade?: number
+  maxAttempts?: number
+  processedAt?: Date
+  completedAt?: Date
+  failedAt?: Date
+  error?: string
+  createdAt: Date
+}
+
+export interface QueueJob<T = any, R = any> {
+  id: string
+  queueName: string
+  status: () => Promise<QueueJobStatus> | QueueJobStatus
+  data: T
+  result?: R
+  waitForCompletion?: (ttl?: number) => Promise<R>
+  metadata?: () => Promise<QueueJobMetadata> | QueueJobMetadata
+}
+
+/**
+ * Job options for queue operations
+ */
+export interface JobOptions {
+  priority?: number
+  delay?: number
+  attempts?: number
+  backoff?: string | { type: string; delay?: number }
+  removeOnComplete?: number
+  removeOnFail?: number
+  jobId?: string
+}
+
+/**
+ * Queue provider interface for job publishing operations
+ */
+export interface QueueService {
+  /** Whether this queue provider supports job results */
+  readonly supportsResults: boolean
+
+  /** Add a job to the queue with type safety */
+  add<T>(queueName: string, data: T, options?: JobOptions): Promise<string>
+
+  /** Get job status and result */
+  getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>
+}
+
+/**
+ * Queue service interface that queue adapters implement
+ */
+export interface QueueWorkers {
+  /** Service name identifier */
+  name: string
+
+  /** Whether this queue service supports job results */
+  supportsResults: boolean
+
+  /** Configuration mapping for validation */
+  configMappings: QueueConfigMapping
+
+  /** Scan state and register all compatible processors */
+  registerQueues(): Promise<Record<string, ConfigValidationResult[]>>
+}
+
+/**
+ * Queue processor metadata
+ */
+export type QueueWorkersMeta = Record<
+  string,
+  CommonWireMeta & {
+    name: string
+    config?: PikkuWorkerConfig
+  }
+>
+
+/**
+ * Core queue processor definition
+ */
+export type CoreQueueWorker<
+  PikkuFunctionConfig extends CorePikkuFunctionConfig<
+    any,
+    any,
+    any
+  > = CorePikkuFunctionConfig<any, any, any>,
+> = {
+  name: string
+  func: PikkuFunctionConfig
+  config?: PikkuWorkerConfig
+  errors?: string[]
+  tags?: string[]
+  middleware?: PikkuFunctionConfig['middleware']
+}
+
+/**
+ * Represents a queue wire object for middleware
+ * Provides information and actions for the current queue job execution
+ */
+export interface PikkuQueue {
+  /** The name of the queue being processed */
+  queueName: string
+  /** The current job ID */
+  jobId: string
+  /** Update job progress (0-100 or custom value) */
+  updateProgress: (progress: number | string | object) => Promise<void>
+  /** Fail the current job with optional reason */
+  fail: (reason?: string) => Promise<void>
+  /** Discard/delete the job without retrying */
+  discard: (reason?: string) => Promise<void>
+}

package/src/wirings/queue/register-queue-helper.ts

@@ -0,0 +1,60 @@
+import type { ConfigValidationResult, CoreQueueWorker } from './queue.types.js'
+import { getQueueWorkers } from './queue-runner.js'
+import type { QueueConfigMapping } from './validate-worker-config.js'
+import { validateWorkerConfig } from './validate-worker-config.js'
+import type { Logger } from '../../services/logger.js'
+
+/**
+ * Queue processor registration callback
+ */
+export type QueueRegistrationCallback<T = any> = (
+  queueName: string,
+  processor: CoreQueueWorker
+) => Promise<T>
+
+/**
+ * Helper function to register queue processors with validation
+ * This centralizes the common logic for looping over processors and validating configs
+ *
+ * @param configMappings - Configuration mapping for the queue implementation
+ * @param registerCallback - Callback to register each individual queue processor
+ * @param logger - Optional logger for info/error messages
+ * @returns Record of validation results by queue name
+ */
+export async function registerQueueWorkers<T = any>(
+  configMappings: QueueConfigMapping,
+  logger: Logger,
+  registerCallback: QueueRegistrationCallback<T>
+): Promise<Record<string, ConfigValidationResult[]>> {
+  const configValidation: Record<string, ConfigValidationResult[]> = {}
+  const queueWorkers = getQueueWorkers()
+
+  for (const [queueName, processor] of queueWorkers) {
+    logger?.info(`Registering queue processor: ${queueName}`)
+    // Validate the processor configuration
+    const validationResult = validateWorkerConfig(
+      configMappings,
+      processor.config
+    )
+
+    // Store validation results
+    configValidation[queueName] = configValidation[queueName] || []
+    configValidation[queueName].push(validationResult)
+
+    if (validationResult.warnings.length > 0) {
+      logger?.warn(
+        `Configuration warnings for queue ${queueName}:`,
+        validationResult.warnings
+      )
+    }
+
+    try {
+      await registerCallback(queueName, processor)
+      logger?.info(`Successfully registered queue processor: ${queueName}`)
+    } catch (error) {
+      logger?.error(`Failed to register queue processor ${queueName}:`, error)
+    }
+  }
+
+  return configValidation
+}

package/src/wirings/queue/validate-worker-config.test.ts

@@ -0,0 +1,108 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert'
+import { validateWorkerConfig } from './validate-worker-config.js'
+import type { QueueConfigMapping } from './validate-worker-config.js'
+
+const createTestMapping = (): QueueConfigMapping => ({
+  supported: {
+    concurrency: {
+      queueProperty: 'concurrency',
+      description: 'Number of concurrent workers',
+    },
+    maxRetries: {
+      queueProperty: 'maxRetries',
+      description: 'Maximum retry attempts',
+    },
+  },
+  unsupported: {
+    rateLimitMax: {
+      reason: 'Not supported',
+      explanation: 'This queue does not support rate limiting',
+    },
+  },
+  fallbacks: {
+    retryDelay: {
+      reason: 'Partial support',
+      explanation: 'Uses fixed delay instead of exponential',
+      fallbackValue: '1000ms fixed',
+    },
+  },
+})
+
+describe('validateWorkerConfig', () => {
+  test('should return empty results for empty config', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, {})
+    assert.deepStrictEqual(result.applied, {})
+    assert.deepStrictEqual(result.ignored, {})
+    assert.deepStrictEqual(result.warnings, [])
+    assert.deepStrictEqual(result.fallbacks, {})
+  })
+
+  test('should return empty results for undefined config', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping)
+    assert.deepStrictEqual(result.applied, {})
+    assert.deepStrictEqual(result.ignored, {})
+  })
+
+  test('should apply supported configurations', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, { concurrency: 5 } as any)
+    assert.deepStrictEqual(result.applied, { concurrency: 5 })
+    assert.deepStrictEqual(result.ignored, {})
+    assert.strictEqual(result.warnings.length, 0)
+  })
+
+  test('should ignore unsupported configurations with warning', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, { rateLimitMax: 100 } as any)
+    assert.deepStrictEqual(result.ignored, { rateLimitMax: 100 })
+    assert.strictEqual(result.warnings.length, 1)
+    assert.ok(result.warnings[0].includes('Not supported'))
+  })
+
+  test('should handle fallback configurations', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, { retryDelay: 2000 } as any)
+    assert.deepStrictEqual(result.applied, { retryDelay: 2000 })
+    assert.deepStrictEqual(result.fallbacks, { retryDelay: '1000ms fixed' })
+    assert.strictEqual(result.warnings.length, 1)
+    assert.ok(result.warnings[0].includes('Partial support'))
+  })
+
+  test('should warn about unknown configurations', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, {
+      unknownOption: 'val',
+    } as any)
+    assert.deepStrictEqual(result.ignored, { unknownOption: 'val' })
+    assert.strictEqual(result.warnings.length, 1)
+    assert.ok(result.warnings[0].includes('Unknown configuration'))
+  })
+
+  test('should skip undefined values', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, {
+      concurrency: undefined,
+    } as any)
+    assert.deepStrictEqual(result.applied, {})
+  })
+
+  test('should handle mixed supported and unsupported configs', () => {
+    const mapping = createTestMapping()
+    const result = validateWorkerConfig(mapping, {
+      concurrency: 3,
+      maxRetries: 5,
+      rateLimitMax: 100,
+      retryDelay: 1000,
+    } as any)
+    assert.deepStrictEqual(result.applied, {
+      concurrency: 3,
+      maxRetries: 5,
+      retryDelay: 1000,
+    })
+    assert.deepStrictEqual(result.ignored, { rateLimitMax: 100 })
+    assert.strictEqual(result.warnings.length, 2)
+  })
+})

package/src/wirings/queue/validate-worker-config.ts

@@ -0,0 +1,116 @@
+import type {
+  PikkuWorkerConfig,
+  ConfigValidationResult,
+} from './queue.types.js'
+
+/**
+ * Configuration mapping structure for queue implementations
+ */
+export interface QueueConfigMapping {
+  /** Configurations that are directly supported */
+  supported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** The property name in the underlying queue system */
+        queueProperty?: string
+        /** Optional transform function for the value */
+        transform?: (value: any) => any
+        /** Description of what this configuration does */
+        description: string
+      }
+    >
+  >
+
+  /** Configurations that are not supported */
+  unsupported: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration is not supported */
+        reason: string
+        /** Detailed explanation of what the queue system does instead */
+        explanation: string
+      }
+    >
+  >
+
+  /** Configurations that have partial support or workarounds */
+  fallbacks: Partial<
+    Record<
+      keyof PikkuWorkerConfig,
+      {
+        /** Why this configuration uses a fallback */
+        reason: string
+        /** Detailed explanation of the fallback behavior */
+        explanation: string
+        /** The fallback value or description */
+        fallbackValue: string
+      }
+    >
+  >
+}
+
+/**
+ * Validates worker configuration using a mapping table
+ * This provides a flexible, maintainable way to validate configurations
+ * across different queue implementations
+ *
+ * @param config - The worker configuration to validate
+ * @param configMapping - The mapping table defining supported/unsupported configurations
+ * @returns Validation result with applied, ignored, warnings, and fallbacks
+ */
+export function validateWorkerConfig(
+  configMapping: QueueConfigMapping,
+  config: PikkuWorkerConfig = {}
+): ConfigValidationResult {
+  const applied: Partial<PikkuWorkerConfig> = {}
+  const ignored: Partial<PikkuWorkerConfig> = {}
+  const warnings: string[] = []
+  const fallbacks: { [key: string]: any } = {}
+
+  // Process each configuration property
+  for (const [key, value] of Object.entries(config)) {
+    if (value === undefined) continue
+
+    const configKey = key as keyof PikkuWorkerConfig
+
+    // Check if it's a supported configuration
+    if (configKey in configMapping.supported) {
+      applied[configKey] = value
+      continue
+    }
+
+    // Check if it's a fallback configuration
+    if (configKey in configMapping.fallbacks) {
+      const fallbackMapping = configMapping.fallbacks[configKey]!
+      applied[configKey] = value
+      fallbacks[key] = fallbackMapping.fallbackValue
+      warnings.push(
+        `${key}: ${fallbackMapping.reason}. ${fallbackMapping.explanation}`
+      )
+      continue
+    }
+
+    // Check if it's an unsupported configuration
+    if (configKey in configMapping.unsupported) {
+      ignored[configKey] = value
+      const mapping = configMapping.unsupported[configKey]!
+      warnings.push(`${key}: ${mapping.reason}. ${mapping.explanation}`)
+      continue
+    }
+
+    // Unknown configuration
+    ignored[configKey] = value
+    warnings.push(
+      `${key}: Unknown configuration option for this queue implementation`
+    )
+  }
+
+  return {
+    applied,
+    ignored,
+    warnings,
+    fallbacks,
+  }
+}

package/src/wirings/rpc/index.ts

@@ -0,0 +1,4 @@
+export { PikkuRPCService, rpcService, RPCNotFoundError } from './rpc-runner.js'
+export type { PikkuRPC, RPCMeta } from './rpc-types.js'
+export { wireAddon } from './wire-addon.js'
+export type { WireAddonConfig } from './wire-addon.js'