@pikku/queue-pg-boss 0.7.0

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@pikku/queue-pg-boss",
+  "version": "0.7.0",
+  "author": "yasser.fadl@gmail.com",
+  "license": "MIT",
+  "module": "dist/index.js",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "tsc": "tsc",
+    "ncu": "npx npm-check-updates",
+    "build": "tsc -b",
+    "release": "npm run build && npm test"
+  },
+  "peerDependencies": {
+    "@pikku/core": "^0.7.0"
+  },
+  "dependencies": {
+    "pg-boss": "^10.3.2"
+  },
+  "devDependencies": {
+    "typescript": "^5.6"
+  }
+}

package/run-tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cd "$(dirname "$0")"
+node --loader tsx --test --experimental-test-coverage src/**/*.test.ts "$@"

package/src/index.ts

@@ -0,0 +1,18 @@
+/**
+ * @module @pikku/queue-pg-boss
+ *
+ * PostgreSQL queue service for Pikku using pg-boss
+ * Provides job results, retries, priorities, and PostgreSQL-based queuing
+ */
+
+export { PgBossQueueService } from './pg-boss-queue-service.js'
+export { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
+
+// Re-export core queue types for convenience
+export type {
+  QueueService,
+  PikkuWorkerConfig,
+  PikkuJobConfig,
+  QueueJob,
+  JobOptions,
+} from '@pikku/core/queue'

package/src/pg-boss-queue-service.ts

@@ -0,0 +1,88 @@
+import PgBoss from 'pg-boss'
+import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue'
+import { mapPgBossJobToQueueJob } from './utils.js'
+
+export const mapPikkuJobToPgBoss = (
+  options?: JobOptions
+): PgBoss.JobOptions => {
+  const pgBossOptions: PgBoss.JobOptions = {}
+
+  if (options?.priority !== undefined) {
+    pgBossOptions.priority = options.priority
+  }
+
+  if (options?.delay !== undefined) {
+    pgBossOptions.startAfter = new Date(Date.now() + options.delay)
+  }
+
+  if (options?.jobId !== undefined) {
+    pgBossOptions.singletonKey = options.jobId
+  }
+
+  return pgBossOptions
+}
+
+/**
+ * pg-boss/PostgreSQL queue service implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export class PgBossQueueService implements QueueService {
+  readonly name = 'pg-boss'
+  readonly supportsResults = true
+  protected pgBoss: PgBoss
+
+  constructor(optionsOrUrl: PgBoss.ConstructorOptions | string) {
+    if (typeof optionsOrUrl === 'string') {
+      // If a string is provided, treat it as the connection URL
+      optionsOrUrl = { connectionString: optionsOrUrl }
+    }
+    this.pgBoss = new PgBoss(optionsOrUrl)
+  }
+
+  /**
+   * Initialize the pg-boss instance
+   */
+  async init(): Promise<void> {
+    await this.pgBoss.start()
+  }
+
+  /**
+   * Add a job to the queue
+   */
+  public async add<In>(
+    queueName: string,
+    data: In,
+    options?: JobOptions
+  ): Promise<string> {
+    const jobId = await this.pgBoss.send(
+      queueName,
+      data as any,
+      mapPikkuJobToPgBoss(options)
+    )
+    if (!jobId) {
+      throw new Error('Failed to create job')
+    }
+    return jobId
+  }
+
+  /**
+   * Get job by ID
+   */
+  async getJob<T, R>(
+    queueName: string,
+    jobId: string
+  ): Promise<QueueJob<T, R> | null> {
+    const job = await this.pgBoss.getJobById<T>(queueName, jobId)
+    if (!job) {
+      return null
+    }
+    return mapPgBossJobToQueueJob<T, R>(job, this.pgBoss)
+  }
+
+  /**
+   * Close the pg-boss connection
+   */
+  async close(): Promise<void> {
+    await this.pgBoss.stop()
+  }
+}

package/src/pg-boss-queue-worker.ts

@@ -0,0 +1,186 @@
+import PgBoss from 'pg-boss'
+import type {
+  QueueWorkers,
+  PikkuWorkerConfig,
+  QueueConfigMapping,
+  ConfigValidationResult,
+} from '@pikku/core/queue'
+import { runQueueJob, registerqueueWorkers } from '@pikku/core/queue'
+import {
+  CoreServices,
+  CoreSingletonServices,
+  CreateSessionServices,
+} from '@pikku/core'
+import { mapPgBossJobToQueueJob } from './utils.js'
+
+export const mapPikkuWorkerToPgBoss = (
+  workerConfig?: PikkuWorkerConfig
+): PgBoss.WorkOptions => {
+  const workerOptions: PgBoss.WorkOptions = {}
+
+  if (workerConfig?.batchSize !== undefined) {
+    workerOptions.batchSize = workerConfig.batchSize
+  }
+
+  if (workerConfig?.pollInterval !== undefined) {
+    workerOptions.pollingIntervalSeconds = Math.round(
+      workerConfig.pollInterval / 1000
+    )
+  }
+
+  return workerOptions
+}
+
+/**
+ * pg-boss/PostgreSQL queue worker implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export class PgBossQueueWorkers implements QueueWorkers {
+  readonly name = 'pg-boss'
+  readonly supportsResults = true
+
+  /**
+   * Configuration mapping rules for pg-boss
+   * This defines how Pikku worker configs map to pg-boss configurations
+   */
+  readonly configMappings: QueueConfigMapping = {
+    // Configurations that are directly supported
+    supported: {
+      batchSize: {
+        queueProperty: 'batchSize',
+        description: 'Number of jobs to process in a single batch',
+      },
+      pollInterval: {
+        queueProperty: 'pollingIntervalSeconds',
+        transform: (value: number) => Math.round(value / 1000),
+        description:
+          'How often to poll for new jobs (converted from ms to seconds)',
+      },
+    },
+
+    // Configurations that are not supported but have reasonable explanations
+    unsupported: {
+      name: {
+        reason: 'Worker names are not supported in pg-boss',
+        explanation:
+          'pg-boss identifies workers by their queue name and process, not custom names',
+      },
+      autorun: {
+        reason: 'Autorun is not configurable in pg-boss (always enabled)',
+        explanation:
+          'pg-boss workers automatically start processing when created',
+      },
+      lockDuration: {
+        reason: 'Lock duration is managed differently in pg-boss',
+        explanation:
+          'pg-boss uses job-level expiration instead of worker-level locks',
+      },
+      drainDelay: {
+        reason: 'Drain delay is not configurable in pg-boss',
+        explanation: 'pg-boss handles graceful shutdown internally',
+      },
+      maxStalledCount: {
+        reason: 'Max stalled count is not configurable in pg-boss',
+        explanation:
+          'pg-boss handles stalled jobs through its built-in retry mechanism',
+      },
+      prefetch: {
+        reason: 'Prefetch is managed internally by pg-boss',
+        explanation:
+          'pg-boss optimizes job fetching automatically based on batch size',
+      },
+      visibilityTimeout: {
+        reason: "Visibility timeout concept doesn't apply to pg-boss",
+        explanation:
+          'pg-boss uses PostgreSQL locks and job expiration instead of visibility timeout',
+      },
+    },
+
+    // Configurations that have partial support or workarounds
+    fallbacks: {
+      removeOnComplete: {
+        reason: 'Job retention is managed through pg-boss archival system',
+        explanation:
+          'pg-boss automatically archives completed jobs based on database settings',
+        fallbackValue: 'Managed by pg-boss database archival',
+      },
+      removeOnFail: {
+        reason:
+          'Failed job retention is managed through pg-boss archival system',
+        explanation:
+          'pg-boss automatically archives failed jobs based on database settings',
+        fallbackValue: 'Managed by pg-boss database archival',
+      },
+    },
+  }
+
+  private pgBoss: PgBoss
+  private activeWorkers = new Map<string, string>()
+
+  constructor(
+    options: PgBoss.ConstructorOptions | string,
+    private singletonServices: CoreSingletonServices,
+    private createSessionServices?: CreateSessionServices<
+      CoreSingletonServices,
+      CoreServices,
+      any
+    >
+  ) {
+    if (typeof options === 'string') {
+      // If a string is provided, treat it as the connection string
+      options = { connectionString: options }
+    }
+    this.pgBoss = new PgBoss(options)
+  }
+
+  /**
+   * Initialize pg-boss
+   */
+  async init(): Promise<void> {
+    await this.pgBoss.start()
+  }
+
+  /**
+   * Scan state and register all compatible processors
+   */
+  async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
+    return await registerqueueWorkers(
+      this.configMappings,
+      this.singletonServices.logger,
+      async (queueName, processor) => {
+        await this.pgBoss.createQueue(queueName)
+        const workerId = await this.pgBoss.work<any>(
+          processor.queueName,
+          {
+            ...mapPikkuWorkerToPgBoss(processor.config),
+            includeMetadata: true,
+          },
+          async ([job]) => {
+            if (!this.singletonServices) {
+              throw new Error('Singleton services not available')
+            }
+            if (!job) {
+              this.singletonServices.logger.warn(
+                `No job received for queue ${queueName}`
+              )
+              return
+            }
+            await runQueueJob({
+              singletonServices: this.singletonServices,
+              createSessionServices: this.createSessionServices,
+              job: mapPgBossJobToQueueJob(job, this.pgBoss),
+            })
+          }
+        )
+        this.activeWorkers.set(queueName, workerId)
+      }
+    )
+  }
+
+  /**
+   * Close all workers and connections
+   */
+  async close(): Promise<void> {
+    await this.pgBoss.stop()
+  }
+}

package/src/utils.ts

@@ -0,0 +1,87 @@
+import { QueueJob, QueueJobStatus } from '@pikku/core/queue'
+import PgBoss from 'pg-boss'
+
+/**
+ * Map pg-boss job state to our queue job status
+ */
+const mapPgBossStateToStatus = (state: string): QueueJobStatus => {
+  switch (state) {
+    case 'created':
+    case 'retry':
+      return 'waiting'
+    case 'active':
+      return 'active'
+    case 'completed':
+      return 'completed'
+    case 'failed':
+      return 'failed'
+    case 'cancelled':
+      return 'failed'
+    default:
+      return 'waiting'
+  }
+}
+
+/**
+ * Map pg-boss job to QueueJob interface
+ */
+export const mapPgBossJobToQueueJob = <In, Out>(
+  pgBossJob: PgBoss.JobWithMetadata<In>,
+  pgBossInstance: PgBoss
+): QueueJob<In, Out> => {
+  return {
+    queueName: pgBossJob.name,
+    id: pgBossJob.id || '',
+    data: pgBossJob.data,
+    status: async () => {
+      const jobWithState = await pgBossInstance.getJobById(
+        pgBossJob.name,
+        pgBossJob.id
+      )
+      return mapPgBossStateToStatus(jobWithState?.state || 'created')
+    },
+    waitForCompletion: async (ttl?: number): Promise<Out> => {
+      // pg-boss doesn't have a built-in waitForCompletion, so we'll poll
+      const startTime = Date.now()
+      const timeout = ttl || 30000 // 30 seconds default
+
+      while (Date.now() - startTime < timeout) {
+        const job = await pgBossInstance.getJobById(
+          pgBossJob.name,
+          pgBossJob.id
+        )
+        if (job?.state === 'completed') {
+          return job.output as Out
+        }
+        if (job?.state === 'failed' || job?.state === 'cancelled') {
+          throw new Error(`Job failed: ${JSON.stringify(job.output)}`)
+        }
+        // Wait 100ms before polling again
+        await new Promise((resolve) => setTimeout(resolve, 100))
+      }
+
+      throw new Error('Job did not complete within timeout')
+    },
+    metadata: () => {
+      return {
+        progress: 0, // pg-boss doesn't have built-in progress tracking
+        attemptsMade: pgBossJob.retryCount || 0,
+        maxAttempts: pgBossJob.retryLimit || 1,
+        createdAt: pgBossJob.createdOn
+          ? new Date(pgBossJob.createdOn)
+          : new Date(),
+        processedAt: pgBossJob.startedOn
+          ? new Date(pgBossJob.startedOn)
+          : undefined,
+        completedAt: pgBossJob.completedOn
+          ? new Date(pgBossJob.completedOn)
+          : undefined,
+        failedAt:
+          pgBossJob.state === 'failed' && pgBossJob.completedOn
+            ? new Date(pgBossJob.completedOn)
+            : undefined,
+        error: (pgBossJob.data as any).error,
+      }
+    },
+  }
+}

package/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "module": "Node16",
+    "outDir": "dist",
+    "target": "esnext",
+    "declaration": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["**/*.test.ts", "node_modules"],
+  "references": [
+    {
+      "path": "../../core/tsconfig.json"
+    }
+  ]
+}