@pikku/queue-pg-boss 0.8.0 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @pikku/services-queue-bullmq
 
+## 0.9.0
+
+### Breaking Changes
+
+- Normalized all transports to use "wirings" instead of events/routes/transports for consistency across the framework
+
+
 ## 0.8.0
 
 - **PG Boss integration**: Postgres queue service implementation

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * @module @pikku/queue-pg-boss
+ *
+ * PostgreSQL queue service for Pikku using pg-boss
+ * Provides job results, retries, priorities, and PostgreSQL-based queuing
+ */
+export { PgBossQueueService } from './pg-boss-queue-service.js';
+export { PgBossQueueWorkers } from './pg-boss-queue-worker.js';
+export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';

package/dist/index.js

@@ -0,0 +1,8 @@
+/**
+ * @module @pikku/queue-pg-boss
+ *
+ * PostgreSQL queue service for Pikku using pg-boss
+ * Provides job results, retries, priorities, and PostgreSQL-based queuing
+ */
+export { PgBossQueueService } from './pg-boss-queue-service.js';
+export { PgBossQueueWorkers } from './pg-boss-queue-worker.js';

package/dist/pg-boss-queue-service.d.ts

@@ -0,0 +1,29 @@
+import PgBoss from 'pg-boss';
+import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue';
+export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBoss.JobOptions;
+/**
+ * pg-boss/PostgreSQL queue service implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export declare class PgBossQueueService implements QueueService {
+    readonly name = "pg-boss";
+    readonly supportsResults = true;
+    protected pgBoss: PgBoss;
+    constructor(optionsOrUrl: PgBoss.ConstructorOptions | string);
+    /**
+     * Initialize the pg-boss instance
+     */
+    init(): Promise<void>;
+    /**
+     * Add a job to the queue
+     */
+    add<In>(queueName: string, data: In, options?: JobOptions): Promise<string>;
+    /**
+     * Get job by ID
+     */
+    getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>;
+    /**
+     * Close the pg-boss connection
+     */
+    close(): Promise<void>;
+}

package/dist/pg-boss-queue-service.js

@@ -0,0 +1,63 @@
+import PgBoss from 'pg-boss';
+import { mapPgBossJobToQueueJob } from './utils.js';
+export const mapPikkuJobToPgBoss = (options) => {
+    const pgBossOptions = {};
+    if (options?.priority !== undefined) {
+        pgBossOptions.priority = options.priority;
+    }
+    if (options?.delay !== undefined) {
+        pgBossOptions.startAfter = new Date(Date.now() + options.delay);
+    }
+    if (options?.jobId !== undefined) {
+        pgBossOptions.singletonKey = options.jobId;
+    }
+    return pgBossOptions;
+};
+/**
+ * pg-boss/PostgreSQL queue service implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export class PgBossQueueService {
+    name = 'pg-boss';
+    supportsResults = true;
+    pgBoss;
+    constructor(optionsOrUrl) {
+        if (typeof optionsOrUrl === 'string') {
+            // If a string is provided, treat it as the connection URL
+            optionsOrUrl = { connectionString: optionsOrUrl };
+        }
+        this.pgBoss = new PgBoss(optionsOrUrl);
+    }
+    /**
+     * Initialize the pg-boss instance
+     */
+    async init() {
+        await this.pgBoss.start();
+    }
+    /**
+     * Add a job to the queue
+     */
+    async add(queueName, data, options) {
+        const jobId = await this.pgBoss.send(queueName, data, mapPikkuJobToPgBoss(options));
+        if (!jobId) {
+            throw new Error('Failed to create job');
+        }
+        return jobId;
+    }
+    /**
+     * Get job by ID
+     */
+    async getJob(queueName, jobId) {
+        const job = await this.pgBoss.getJobById(queueName, jobId);
+        if (!job) {
+            return null;
+        }
+        return mapPgBossJobToQueueJob(job, this.pgBoss);
+    }
+    /**
+     * Close the pg-boss connection
+     */
+    async close() {
+        await this.pgBoss.stop();
+    }
+}

package/dist/pg-boss-queue-worker.d.ts

@@ -0,0 +1,34 @@
+import PgBoss from 'pg-boss';
+import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
+import { CoreServices, CoreSingletonServices, CreateSessionServices } from '@pikku/core';
+export declare const mapPikkuWorkerToPgBoss: (workerConfig?: PikkuWorkerConfig) => PgBoss.WorkOptions;
+/**
+ * pg-boss/PostgreSQL queue worker implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export declare class PgBossQueueWorkers implements QueueWorkers {
+    private singletonServices;
+    private createSessionServices?;
+    readonly name = "pg-boss";
+    readonly supportsResults = true;
+    /**
+     * Configuration mapping rules for pg-boss
+     * This defines how Pikku worker configs map to pg-boss configurations
+     */
+    readonly configMappings: QueueConfigMapping;
+    private pgBoss;
+    private activeWorkers;
+    constructor(options: PgBoss.ConstructorOptions | string, singletonServices: CoreSingletonServices, createSessionServices?: CreateSessionServices<CoreSingletonServices, CoreServices, any>);
+    /**
+     * Initialize pg-boss
+     */
+    init(): Promise<void>;
+    /**
+     * Scan state and register all compatible processors
+     */
+    registerQueues(): Promise<Record<string, ConfigValidationResult[]>>;
+    /**
+     * Close all workers and connections
+     */
+    close(): Promise<void>;
+}

package/dist/pg-boss-queue-worker.js

@@ -0,0 +1,134 @@
+import PgBoss from 'pg-boss';
+import { runQueueJob, registerQueueWorkers } from '@pikku/core/queue';
+import { mapPgBossJobToQueueJob } from './utils.js';
+export const mapPikkuWorkerToPgBoss = (workerConfig) => {
+    const workerOptions = {};
+    if (workerConfig?.batchSize !== undefined) {
+        workerOptions.batchSize = workerConfig.batchSize;
+    }
+    if (workerConfig?.pollInterval !== undefined) {
+        workerOptions.pollingIntervalSeconds = Math.round(workerConfig.pollInterval / 1000);
+    }
+    return workerOptions;
+};
+/**
+ * pg-boss/PostgreSQL queue worker implementation
+ * Supports job results, retries, and PostgreSQL-based queuing
+ */
+export class PgBossQueueWorkers {
+    singletonServices;
+    createSessionServices;
+    name = 'pg-boss';
+    supportsResults = true;
+    /**
+     * Configuration mapping rules for pg-boss
+     * This defines how Pikku worker configs map to pg-boss configurations
+     */
+    configMappings = {
+        // Configurations that are directly supported
+        supported: {
+            batchSize: {
+                queueProperty: 'batchSize',
+                description: 'Number of jobs to process in a single batch',
+            },
+            pollInterval: {
+                queueProperty: 'pollingIntervalSeconds',
+                transform: (value) => Math.round(value / 1000),
+                description: 'How often to poll for new jobs (converted from ms to seconds)',
+            },
+        },
+        // Configurations that are not supported but have reasonable explanations
+        unsupported: {
+            name: {
+                reason: 'Worker names are not supported in pg-boss',
+                explanation: 'pg-boss identifies workers by their queue name and process, not custom names',
+            },
+            autorun: {
+                reason: 'Autorun is not configurable in pg-boss (always enabled)',
+                explanation: 'pg-boss workers automatically start processing when created',
+            },
+            lockDuration: {
+                reason: 'Lock duration is managed differently in pg-boss',
+                explanation: 'pg-boss uses job-level expiration instead of worker-level locks',
+            },
+            drainDelay: {
+                reason: 'Drain delay is not configurable in pg-boss',
+                explanation: 'pg-boss handles graceful shutdown internally',
+            },
+            maxStalledCount: {
+                reason: 'Max stalled count is not configurable in pg-boss',
+                explanation: 'pg-boss handles stalled jobs through its built-in retry mechanism',
+            },
+            prefetch: {
+                reason: 'Prefetch is managed internally by pg-boss',
+                explanation: 'pg-boss optimizes job fetching automatically based on batch size',
+            },
+            visibilityTimeout: {
+                reason: "Visibility timeout concept doesn't apply to pg-boss",
+                explanation: 'pg-boss uses PostgreSQL locks and job expiration instead of visibility timeout',
+            },
+        },
+        // Configurations that have partial support or workarounds
+        fallbacks: {
+            removeOnComplete: {
+                reason: 'Job retention is managed through pg-boss archival system',
+                explanation: 'pg-boss automatically archives completed jobs based on database settings',
+                fallbackValue: 'Managed by pg-boss database archival',
+            },
+            removeOnFail: {
+                reason: 'Failed job retention is managed through pg-boss archival system',
+                explanation: 'pg-boss automatically archives failed jobs based on database settings',
+                fallbackValue: 'Managed by pg-boss database archival',
+            },
+        },
+    };
+    pgBoss;
+    activeWorkers = new Map();
+    constructor(options, singletonServices, createSessionServices) {
+        this.singletonServices = singletonServices;
+        this.createSessionServices = createSessionServices;
+        if (typeof options === 'string') {
+            // If a string is provided, treat it as the connection string
+            options = { connectionString: options };
+        }
+        this.pgBoss = new PgBoss(options);
+    }
+    /**
+     * Initialize pg-boss
+     */
+    async init() {
+        await this.pgBoss.start();
+    }
+    /**
+     * Scan state and register all compatible processors
+     */
+    async registerQueues() {
+        return await registerQueueWorkers(this.configMappings, this.singletonServices.logger, async (queueName, processor) => {
+            await this.pgBoss.createQueue(queueName);
+            const workerId = await this.pgBoss.work(processor.queueName, {
+                ...mapPikkuWorkerToPgBoss(processor.config),
+                includeMetadata: true,
+            }, async ([job]) => {
+                if (!this.singletonServices) {
+                    throw new Error('Singleton services not available');
+                }
+                if (!job) {
+                    this.singletonServices.logger.warn(`No job received for queue ${queueName}`);
+                    return;
+                }
+                await runQueueJob({
+                    singletonServices: this.singletonServices,
+                    createSessionServices: this.createSessionServices,
+                    job: mapPgBossJobToQueueJob(job, this.pgBoss),
+                });
+            });
+            this.activeWorkers.set(queueName, workerId);
+        });
+    }
+    /**
+     * Close all workers and connections
+     */
+    async close() {
+        await this.pgBoss.stop();
+    }
+}

package/dist/utils.d.ts

@@ -0,0 +1,6 @@
+import { QueueJob } from '@pikku/core/queue';
+import PgBoss from 'pg-boss';
+/**
+ * Map pg-boss job to QueueJob interface
+ */
+export declare const mapPgBossJobToQueueJob: <In, Out>(pgBossJob: PgBoss.JobWithMetadata<In>, pgBossInstance: PgBoss) => QueueJob<In, Out>;

package/dist/utils.js

@@ -0,0 +1,71 @@
+/**
+ * Map pg-boss job state to our queue job status
+ */
+const mapPgBossStateToStatus = (state) => {
+    switch (state) {
+        case 'created':
+        case 'retry':
+            return 'waiting';
+        case 'active':
+            return 'active';
+        case 'completed':
+            return 'completed';
+        case 'failed':
+            return 'failed';
+        case 'cancelled':
+            return 'failed';
+        default:
+            return 'waiting';
+    }
+};
+/**
+ * Map pg-boss job to QueueJob interface
+ */
+export const mapPgBossJobToQueueJob = (pgBossJob, pgBossInstance) => {
+    return {
+        queueName: pgBossJob.name,
+        id: pgBossJob.id || '',
+        data: pgBossJob.data,
+        status: async () => {
+            const jobWithState = await pgBossInstance.getJobById(pgBossJob.name, pgBossJob.id);
+            return mapPgBossStateToStatus(jobWithState?.state || 'created');
+        },
+        waitForCompletion: async (ttl) => {
+            // pg-boss doesn't have a built-in waitForCompletion, so we'll poll
+            const startTime = Date.now();
+            const timeout = ttl || 30000; // 30 seconds default
+            while (Date.now() - startTime < timeout) {
+                const job = await pgBossInstance.getJobById(pgBossJob.name, pgBossJob.id);
+                if (job?.state === 'completed') {
+                    return job.output;
+                }
+                if (job?.state === 'failed' || job?.state === 'cancelled') {
+                    throw new Error(`Job failed: ${JSON.stringify(job.output)}`);
+                }
+                // Wait 100ms before polling again
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+            throw new Error('Job did not complete within timeout');
+        },
+        metadata: () => {
+            return {
+                progress: 0, // pg-boss doesn't have built-in progress tracking
+                attemptsMade: pgBossJob.retryCount || 0,
+                maxAttempts: pgBossJob.retryLimit || 1,
+                createdAt: pgBossJob.createdOn
+                    ? new Date(pgBossJob.createdOn)
+                    : new Date(),
+                processedAt: pgBossJob.startedOn
+                    ? new Date(pgBossJob.startedOn)
+                    : undefined,
+                completedAt: pgBossJob.completedOn
+                    ? new Date(pgBossJob.completedOn)
+                    : undefined,
+                failedAt: pgBossJob.state === 'failed' && pgBossJob.completedOn
+                    ? new Date(pgBossJob.completedOn)
+                    : undefined,
+                error: pgBossJob.data.error,
+            };
+        },
+    };
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/queue-pg-boss",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",

package/src/pg-boss-queue-worker.ts

@@ -5,7 +5,7 @@ import type {
   QueueConfigMapping,
   ConfigValidationResult,
 } from '@pikku/core/queue'
-import { runQueueJob, registerqueueWorkers } from '@pikku/core/queue'
+import { runQueueJob, registerQueueWorkers } from '@pikku/core/queue'
 import {
   CoreServices,
   CoreSingletonServices,
@@ -144,7 +144,7 @@ export class PgBossQueueWorkers implements QueueWorkers {
    * Scan state and register all compatible processors
    */
   async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
-    return await registerqueueWorkers(
+    return await registerQueueWorkers(
       this.configMappings,
       this.singletonServices.logger,
       async (queueName, processor) => {