@pikku/queue-pg-boss 0.12.10 → 0.12.12

package/src/pg-boss-queue-service.ts

@@ -1,113 +0,0 @@
-import type { PgBoss, Request as PgBossRequest } from 'pg-boss'
-import type { QueueJob, QueueService } from '@pikku/core/queue'
-import type { JobOptions } from '@pikku/core/queue'
-import { mapPgBossJobToQueueJob } from './utils.js'
-
-export const mapPikkuJobToPgBoss = (
-  options?: JobOptions
-): PgBossRequest['options'] => {
-  const pgBossOptions: any = {}
-
-  if (options?.priority !== undefined) {
-    pgBossOptions.priority = options.priority
-  }
-
-  if (options?.delay !== undefined) {
-    pgBossOptions.startAfter = new Date(Date.now() + options.delay)
-  }
-
-  if (options?.jobId !== undefined) {
-    pgBossOptions.singletonKey = options.jobId
-  }
-
-  if (options?.group !== undefined) {
-    pgBossOptions.group = {
-      id: options.group.id,
-      ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
-    }
-  }
-
-  // Map retry options
-  if (options?.attempts !== undefined) {
-    pgBossOptions.retryLimit = options.attempts - 1
-  }
-
-  if (options?.backoff !== undefined) {
-    const type =
-      typeof options.backoff === 'string'
-        ? options.backoff
-        : options.backoff.type
-    const delay =
-      typeof options.backoff === 'object' ? options.backoff.delay : undefined
-    pgBossOptions.retryBackoff = type === 'exponential'
-    if (delay !== undefined) {
-      // pg-boss uses seconds, we use milliseconds — never round a sub-second
-      // delay down to 0, which pg-boss treats as retry-immediately
-      pgBossOptions.retryDelay = Math.max(1, Math.round(delay / 1000))
-    } else if (type === 'exponential') {
-      // pg-boss computes exponential backoff as retry_delay * 2^n, and the
-      // queue-level retry_delay defaults to 0 — without an explicit base every
-      // "exponential" retry fires immediately. 1s base → ~1s, 2s, 4s, 8s…
-      pgBossOptions.retryDelay = 1
-    }
-  }
-
-  if (options?.removeOnComplete !== undefined) {
-    pgBossOptions.onComplete = options.removeOnComplete === 0
-  }
-
-  if (options?.removeOnFail !== undefined) {
-    // pg-boss doesn't have a direct equivalent, but we can track this for consistency
-    // Note: pg-boss keeps failed jobs by default
-  }
-
-  return pgBossOptions as PgBossRequest['options']
-}
-
-/**
- * pg-boss/PostgreSQL queue service implementation
- * Supports job results, retries, and PostgreSQL-based queuing
- */
-export class PgBossQueueService implements QueueService {
-  readonly name = 'pg-boss'
-  readonly supportsResults = true
-  protected pgBoss: PgBoss
-
-  constructor(pgBoss: PgBoss) {
-    this.pgBoss = pgBoss
-  }
-
-  /**
-   * Add a job to the queue
-   */
-  public async add<In>(
-    queueName: string,
-    data: In,
-    options?: JobOptions
-  ): Promise<string> {
-    await this.pgBoss.createQueue(queueName)
-    const jobId = await this.pgBoss.send(
-      queueName,
-      data as any,
-      mapPikkuJobToPgBoss(options)
-    )
-    if (!jobId) {
-      throw new Error('Failed to create job')
-    }
-    return jobId
-  }
-
-  /**
-   * Get job by ID
-   */
-  async getJob<T, R>(
-    queueName: string,
-    jobId: string
-  ): Promise<QueueJob<T, R> | null> {
-    const job = await this.pgBoss.getJobById<T>(queueName, jobId)
-    if (!job) {
-      return null
-    }
-    return mapPgBossJobToQueueJob<T, R>(job, this.pgBoss)
-  }
-}

package/src/pg-boss-queue-worker.test.ts

@@ -1,93 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import {
-  PgBossQueueWorkers,
-  mapPikkuWorkerToPgBoss,
-} from './pg-boss-queue-worker.js'
-
-describe('PgBossQueueWorkers', () => {
-  test('requires a logger (explicit or from singleton services)', async () => {
-    const workers = new PgBossQueueWorkers({} as any)
-
-    await assert.rejects(() => workers.registerQueues(), {
-      message:
-        'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first',
-    })
-  })
-})
-
-describe('mapPikkuWorkerToPgBoss', () => {
-  test('parallelism maps to independent workers (localConcurrency), never batch fetching', () => {
-    // batchSize: 1 keeps each worker single-job so a slow job can never hold its
-    // batch siblings — head-of-line blocking is impossible.
-    assert.deepEqual(mapPikkuWorkerToPgBoss(), {
-      localConcurrency: 10,
-      batchSize: 1,
-    })
-
-    assert.deepEqual(mapPikkuWorkerToPgBoss({ batchSize: 25 }), {
-      localConcurrency: 25,
-      batchSize: 1,
-    })
-  })
-
-  test('pollInterval is converted from ms to seconds', () => {
-    assert.deepEqual(mapPikkuWorkerToPgBoss({ pollInterval: 3000 }), {
-      localConcurrency: 10,
-      batchSize: 1,
-      pollingIntervalSeconds: 3,
-    })
-  })
-
-  test('groupConcurrency caps how much of the worker one group can take', () => {
-    assert.deepEqual(mapPikkuWorkerToPgBoss({ groupConcurrency: 2 }), {
-      localConcurrency: 10,
-      batchSize: 1,
-      localGroupConcurrency: 2,
-    })
-  })
-
-  test('tiered groupConcurrency passes tiers through', () => {
-    assert.deepEqual(
-      mapPikkuWorkerToPgBoss({
-        batchSize: 20,
-        groupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
-      }),
-      {
-        localConcurrency: 20,
-        batchSize: 1,
-        localGroupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
-      }
-    )
-  })
-
-  test('group limits are clamped to the worker concurrency', () => {
-    // pg-boss asserts every group limit is <= localConcurrency and throws at
-    // worker start otherwise — clamp so an over-eager config degrades to "this
-    // group may use the whole worker" instead of failing to start.
-    assert.deepEqual(
-      mapPikkuWorkerToPgBoss({
-        batchSize: 4,
-        groupConcurrency: { default: 10, tiers: { greedy: 99 } },
-      }),
-      {
-        localConcurrency: 4,
-        batchSize: 1,
-        localGroupConcurrency: { default: 4, tiers: { greedy: 4 } },
-      }
-    )
-
-    assert.deepEqual(
-      mapPikkuWorkerToPgBoss({ batchSize: 3, groupConcurrency: 8 }),
-      { localConcurrency: 3, batchSize: 1, localGroupConcurrency: 3 }
-    )
-  })
-
-  test('no group config means no group limits', () => {
-    assert.equal(
-      'localGroupConcurrency' in mapPikkuWorkerToPgBoss({ batchSize: 5 }),
-      false
-    )
-  })
-})

package/src/pg-boss-queue-worker.ts

@@ -1,231 +0,0 @@
-import type { PgBoss, WorkOptions } from 'pg-boss'
-import type {
-  QueueWorkers,
-  PikkuWorkerConfig,
-  QueueConfigMapping,
-  ConfigValidationResult,
-} from '@pikku/core/queue'
-import { runQueueJob } from '@pikku/core/queue'
-import type { Logger } from '@pikku/core/services'
-import {
-  registerQueueWorkers,
-  QueueJobFailedError,
-  QueueJobDiscardedError,
-} from '@pikku/core/queue'
-import { pikkuState } from '@pikku/core/state'
-import { mapPgBossJobToQueueJob } from './utils.js'
-
-export const mapPikkuWorkerToPgBoss = (
-  workerConfig?: PikkuWorkerConfig
-): WorkOptions => {
-  // Map the requested parallelism to pg-boss `localConcurrency` (independent
-  // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
-  // batch to one handler and only fetches again once that handler resolves — so
-  // a single slow job blocks every sibling in its batch AND stalls the next
-  // fetch. On a shared queue that is head-of-line blocking: one long job can
-  // freeze the entire worker and starve everything queued behind it.
-  // `localConcurrency` spawns N pollers that each fetch+process a single job and
-  // refill the instant they finish, so a slow job never holds up the others.
-  const localConcurrency = workerConfig?.batchSize ?? 10
-  const workerOptions: WorkOptions = {
-    localConcurrency,
-    batchSize: 1,
-  }
-
-  if (workerConfig?.pollInterval !== undefined) {
-    workerOptions.pollingIntervalSeconds = Math.round(
-      workerConfig.pollInterval / 1000
-    )
-  }
-
-  // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
-  // already at capacity are excluded from the fetch itself, so an at-capacity
-  // group costs nothing rather than being fetched and restored. pg-boss asserts
-  // every limit is <= localConcurrency, so clamp instead of letting a config
-  // that reads as "let this tier use more" throw at worker start.
-  if (workerConfig?.groupConcurrency !== undefined) {
-    const clamp = (limit: number) => Math.min(limit, localConcurrency)
-    workerOptions.localGroupConcurrency =
-      typeof workerConfig.groupConcurrency === 'number'
-        ? clamp(workerConfig.groupConcurrency)
-        : {
-            default: clamp(workerConfig.groupConcurrency.default),
-            ...(workerConfig.groupConcurrency.tiers
-              ? {
-                  tiers: Object.fromEntries(
-                    Object.entries(workerConfig.groupConcurrency.tiers).map(
-                      ([tier, limit]) => [tier, clamp(limit)]
-                    )
-                  ),
-                }
-              : {}),
-          }
-  }
-
-  return workerOptions
-}
-
-/**
- * pg-boss/PostgreSQL queue worker implementation
- * Supports job results, retries, and PostgreSQL-based queuing
- */
-export class PgBossQueueWorkers implements QueueWorkers {
-  readonly name = 'pg-boss'
-  readonly supportsResults = true
-
-  /**
-   * Configuration mapping rules for pg-boss
-   * This defines how Pikku worker configs map to pg-boss configurations
-   */
-  readonly configMappings: QueueConfigMapping = {
-    // Configurations that are directly supported
-    supported: {
-      batchSize: {
-        queueProperty: 'localConcurrency',
-        description: 'Number of jobs to process concurrently',
-      },
-      groupConcurrency: {
-        queueProperty: 'localGroupConcurrency',
-        description:
-          'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
-      },
-      pollInterval: {
-        queueProperty: 'pollingIntervalSeconds',
-        transform: (value: number) => Math.round(value / 1000),
-        description:
-          'How often to poll for new jobs (converted from ms to seconds)',
-      },
-    },
-
-    // Configurations that are not supported but have reasonable explanations
-    unsupported: {
-      name: {
-        reason: 'Worker names are not supported in pg-boss',
-        explanation:
-          'pg-boss identifies workers by their queue name and process, not custom names',
-      },
-      autorun: {
-        reason: 'Autorun is not configurable in pg-boss (always enabled)',
-        explanation:
-          'pg-boss workers automatically start processing when created',
-      },
-      lockDuration: {
-        reason: 'Lock duration is managed differently in pg-boss',
-        explanation:
-          'pg-boss uses job-level expiration instead of worker-level locks',
-      },
-      drainDelay: {
-        reason: 'Drain delay is not configurable in pg-boss',
-        explanation: 'pg-boss handles graceful shutdown internally',
-      },
-      maxStalledCount: {
-        reason: 'Max stalled count is not configurable in pg-boss',
-        explanation:
-          'pg-boss handles stalled jobs through its built-in retry mechanism',
-      },
-      prefetch: {
-        reason: 'Prefetch is managed internally by pg-boss',
-        explanation:
-          'pg-boss optimizes job fetching automatically based on batch size',
-      },
-      visibilityTimeout: {
-        reason: "Visibility timeout concept doesn't apply to pg-boss",
-        explanation:
-          'pg-boss uses PostgreSQL locks and job expiration instead of visibility timeout',
-      },
-    },
-
-    // Configurations that have partial support or workarounds
-    fallbacks: {
-      removeOnComplete: {
-        reason: 'Job retention is managed through pg-boss archival system',
-        explanation:
-          'pg-boss automatically archives completed jobs based on database settings',
-        fallbackValue: 'Managed by pg-boss database archival',
-      },
-      removeOnFail: {
-        reason:
-          'Failed job retention is managed through pg-boss archival system',
-        explanation:
-          'pg-boss automatically archives failed jobs based on database settings',
-        fallbackValue: 'Managed by pg-boss database archival',
-      },
-    },
-  }
-
-  private pgBoss: PgBoss
-  private activeWorkers = new Map<string, string>()
-
-  constructor(pgBoss: PgBoss) {
-    this.pgBoss = pgBoss
-  }
-
-  /**
-   * Scan state and register all compatible processors
-   */
-  async registerQueues(
-    logger?: Logger
-  ): Promise<Record<string, ConfigValidationResult[]>> {
-    if (!logger) {
-      logger = pikkuState(null, 'package', 'singletonServices')?.logger
-    }
-    if (!logger) {
-      throw new Error(
-        'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first'
-      )
-    }
-
-    return await registerQueueWorkers(
-      this.configMappings,
-      logger,
-      async (queueName, processor) => {
-        await this.pgBoss.createQueue(queueName)
-        const workerId = await this.pgBoss.work(
-          processor.name,
-          {
-            ...mapPikkuWorkerToPgBoss(processor.config),
-            includeMetadata: true,
-          },
-          async (jobs) => {
-            if (!jobs || jobs.length === 0) {
-              logger.warn(`No jobs received for queue ${queueName}`)
-              return
-            }
-            // Process all jobs in parallel
-            await Promise.all(
-              jobs.map(async (job) => {
-                try {
-                  await runQueueJob({
-                    job: mapPgBossJobToQueueJob(job, this.pgBoss),
-                  })
-                } catch (error: unknown) {
-                  if (error instanceof QueueJobFailedError) {
-                    // Mark this specific job as failed
-                    await this.pgBoss.fail(queueName, job.id, {
-                      message: error.message,
-                    })
-                  } else if (error instanceof QueueJobDiscardedError) {
-                    // For pg-boss, complete the job successfully to discard it
-                    logger.info(
-                      `PgBoss job ${job.id} discarded: ${error.message}`
-                    )
-                    await this.pgBoss.complete(queueName, job.id)
-                  } else {
-                    // For other errors, fail the job
-                    await this.pgBoss.fail(queueName, job.id, {
-                      message:
-                        error instanceof Error
-                          ? error.message
-                          : 'Unknown error',
-                    })
-                  }
-                }
-              })
-            )
-          }
-        )
-        this.activeWorkers.set(queueName, workerId)
-      }
-    )
-  }
-}

package/src/pg-boss-scheduler-service.ts

@@ -1,192 +0,0 @@
-import type { PgBoss } from 'pg-boss'
-import { runScheduledTask } from '@pikku/core/scheduler'
-import { SchedulerService } from '@pikku/core/services'
-import type { CoreUserSession } from '@pikku/core/types'
-import type {
-  ScheduledTaskInfo,
-  ScheduledTaskSummary,
-} from '@pikku/core/services'
-import { parseDurationString } from '@pikku/core/utils'
-import { pikkuState } from '@pikku/core/state'
-import { getScheduledTasks } from '@pikku/core/scheduler'
-
-/**
- * Data stored in scheduled job
- */
-interface ScheduledJobData {
-  rpcName: string
-  data?: any
-  session?: CoreUserSession
-}
-
-/** Prefix for pg-boss queues/schedules backing recurring scheduled tasks. */
-const RECURRING_TASK_PREFIX = 'pikku-recurring-scheduled-task_'
-
-/**
- * pg-boss scheduler service implementation
- * Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
- * Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
- */
-export class PgBossSchedulerService extends SchedulerService {
-  private scheduledCronNames: string[] = []
-
-  constructor(private pgBoss: PgBoss) {
-    super()
-  }
-
-  /**
-   * Initialize - no-op as pg-boss is already initialized by factory
-   */
-  async init(): Promise<void> {
-    // No-op - pg-boss is already initialized by factory
-  }
-
-  /**
-   * Schedule a one-off delayed RPC call
-   * Uses pg-boss send() with startAfter for one-time execution
-   */
-  async scheduleRPC(
-    delay: number | string,
-    rpcName: string,
-    data?: any,
-    session?: CoreUserSession
-  ): Promise<string> {
-    // Parse delay if it's a string
-    const delayMs =
-      typeof delay === 'string' ? parseDurationString(delay) : delay
-
-    const jobData: ScheduledJobData = {
-      rpcName,
-      data,
-      session,
-    }
-
-    // Use send() with startAfter for one-off delayed execution
-    const taskId = await this.pgBoss.send(
-      'pikku-remote-internal-rpc',
-      jobData,
-      {
-        startAfter: new Date(Date.now() + delayMs),
-        singletonKey: `${rpcName}-${Date.now()}`, // Ensure uniqueness
-      }
-    )
-
-    if (!taskId) {
-      throw new Error('Failed to schedule RPC')
-    }
-
-    return taskId
-  }
-
-  /**
-   * Unschedule (cancel) a task by ID
-   */
-  async unschedule(taskId: string): Promise<boolean> {
-    await this.pgBoss.cancel('pikku-remote-internal-rpc', taskId)
-    return true
-  }
-
-  /**
-   * Get a scheduled task by ID with full details
-   */
-  async getTask(taskId: string): Promise<ScheduledTaskInfo | null> {
-    const job = await this.pgBoss.getJobById(
-      'pikku-remote-internal-rpc',
-      taskId
-    )
-
-    if (!job) {
-      return null
-    }
-
-    const jobData = job.data as ScheduledJobData
-
-    return {
-      taskId: job.id,
-      rpcName: jobData.rpcName,
-      scheduledFor: job.startAfter || job.createdOn,
-      data: jobData.data,
-      session: jobData.session,
-      status: job.state as any,
-    }
-  }
-
-  /**
-   * Get all scheduled tasks with minimal info
-   */
-  async getAllTasks(): Promise<ScheduledTaskSummary[]> {
-    // pg-boss doesn't have a simple way to list all pending jobs
-    // This would require querying the database directly or maintaining a separate index
-    // For now, return empty array - can be implemented if needed
-    return []
-  }
-
-  /**
-   * Close - no-op as pg-boss lifecycle is managed by factory
-   */
-  async close(): Promise<void> {
-    // No-op - pg-boss lifecycle is managed by factory
-  }
-
-  /**
-   * Start recurring scheduled tasks.
-   * Registers a pg-boss worker to process recurring jobs via runScheduledTask.
-   */
-  async start(): Promise<void> {
-    const logger = pikkuState(null, 'package', 'singletonServices')!.logger
-    const scheduledTasks = getScheduledTasks()
-
-    const expectedCronNames = new Set<string>()
-    for (const [name, task] of scheduledTasks) {
-      const cronName = `${RECURRING_TASK_PREFIX}${name.replace(/[^a-zA-Z0-9_\-.]/g, '_')}`
-      expectedCronNames.add(cronName)
-      await this.pgBoss.createQueue(cronName)
-      await this.pgBoss.schedule(cronName, task.schedule, {
-        rpcName: name,
-      } as ScheduledJobData)
-      this.scheduledCronNames.push(cronName)
-
-      // Register a worker to process the scheduled jobs
-      await this.pgBoss.work<ScheduledJobData>(cronName, async (jobs) => {
-        for (const job of jobs) {
-          const { rpcName } = job.data
-          logger.info(`Running scheduled task: ${rpcName}`)
-          await runScheduledTask({ name: rpcName })
-        }
-      })
-    }
-
-    // Prune orphaned schedules. A task removed from code leaves its pg-boss
-    // schedule behind; with no worker registered it keeps producing jobs that
-    // pile up as 'created' forever (this is how a removed task flooded a queue
-    // with thousands of stuck jobs). Drop any recurring schedule + queue that's
-    // no longer registered so task removals self-heal on the next boot.
-    try {
-      const existing = await this.pgBoss.getSchedules()
-      for (const schedule of existing) {
-        if (
-          schedule.name.startsWith(RECURRING_TASK_PREFIX) &&
-          !expectedCronNames.has(schedule.name)
-        ) {
-          await this.pgBoss.unschedule(schedule.name)
-          await this.pgBoss.deleteQueue(schedule.name)
-          logger.warn(`Pruned orphaned scheduled task queue: ${schedule.name}`)
-        }
-      }
-    } catch (err) {
-      logger.warn(
-        `Failed to prune orphaned scheduled tasks: ${err instanceof Error ? err.message : String(err)}`
-      )
-    }
-  }
-
-  /**
-   * Stop recurring scheduled tasks by unscheduling them from pg-boss.
-   */
-  async stop(): Promise<void> {
-    for (const cronName of this.scheduledCronNames) {
-      await this.pgBoss.unschedule(cronName)
-    }
-    this.scheduledCronNames = []
-  }
-}