@pikku/queue-pg-boss 0.12.6 → 0.12.8

package/CHANGELOG.md

@@ -1,3 +1,58 @@
+## 0.12.8
+
+### Patch Changes
+
+- 6847338: Stop one slow job from holding an entire queue (head-of-line blocking).
+
+  The worker mapped pikku's parallelism to pg-boss `batchSize`, whose handler
+  receives the whole fetched batch and only fetches again once that handler
+  resolves. Because the handler `Promise.all`s the batch, a single long-running
+  job blocked every sibling in its batch _and_ stalled the next fetch — so on a
+  shared queue one slow job could freeze the worker and starve everything behind
+  it (observed with workflow-orchestrator jobs, where a long step pinned all 10
+  slots for tens of minutes).
+
+  Parallelism now maps to pg-boss `localConcurrency` with `batchSize: 1`: N
+  independent workers each fetch and process a single job and refill the instant
+  they finish, so a slow job never holds up the others. Same effective
+  concurrency, no head-of-line blocking.
+
+- e3d4454: Add job groups, so one shared queue can stay fair without splitting into one
+  queue per producer.
+
+  A job may now carry `group: { id, tier }`, and a worker may cap how many jobs
+  of any one group run at once via `groupConcurrency`. On pg-boss this maps to
+  `localGroupConcurrency`, which excludes at-capacity groups from the fetch query
+  itself, so a capped group costs nothing rather than being fetched and restored.
+  BullMQ declares it unsupported (groups are a BullMQ Pro feature) — being
+  push-based, it can simply use a queue per group at no polling cost.
+
+  Workflow services accept a `queueStrategy`. The default `'per-workflow'` is
+  unchanged: every workflow gets its own `wf-orchestrator-*` / `wf-step-*` queue,
+  which is also what lets serverless providers deploy one unit per workflow. The
+  new `'shared-groups'` routes every workflow through the shared
+  orchestrator/step-worker queues and isolates them by group instead, so a
+  monolith runs one set of pollers rather than one per workflow — on a
+  pull-based backend with dozens of workflows that is the difference between
+  hundreds of poll loops and twenty. It is for single-process runtimes only; a
+  per-unit serverless deploy still needs the per-workflow queues to route to its
+  units.
+
+- Updated dependencies [24252b8]
+- Updated dependencies [e3d4454]
+  - @pikku/core@0.12.69
+
+## 0.12.7
+
+### Patch Changes
+
+- 6c30861: fix workflow step retry backoff firing immediately
+  - `@pikku/queue-pg-boss`: `backoff: 'exponential'` mapped to `retryBackoff: true` without a base `retryDelay`; pg-boss computes exponential backoff as `retry_delay * 2^n` with a queue default of 0, so every retry fired immediately. Exponential backoff now gets a 1s base delay, and sub-second fixed delays round up to 1s instead of flooring to 0 (= immediate).
+  - `@pikku/core`: a duration-string `retryDelay` (e.g. `'15s'`) on a workflow step was silently dropped (only numbers were honored) and fell back to exponential. It now resolves to a fixed backoff via `getDurationInMilliseconds`.
+
+- Updated dependencies [6c30861]
+  - @pikku/core@0.12.56
+
 ## 0.12.6
 
 ### Patch Changes

package/dist/pg-boss-queue-service.js

@@ -10,21 +10,30 @@ export const mapPikkuJobToPgBoss = (options) => {
     if (options?.jobId !== undefined) {
         pgBossOptions.singletonKey = options.jobId;
     }
+    if (options?.group !== undefined) {
+        pgBossOptions.group = {
+            id: options.group.id,
+            ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
+        };
+    }
     // Map retry options
     if (options?.attempts !== undefined) {
         pgBossOptions.retryLimit = options.attempts - 1;
     }
     if (options?.backoff !== undefined) {
-        if (typeof options.backoff === 'string') {
-            // If backoff is a string, assume it's 'exponential' or 'fixed'
-            pgBossOptions.retryBackoff = options.backoff === 'exponential';
+        const type = typeof options.backoff === 'string' ? options.backoff : options.backoff.type;
+        const delay = typeof options.backoff === 'object' ? options.backoff.delay : undefined;
+        pgBossOptions.retryBackoff = type === 'exponential';
+        if (delay !== undefined) {
+            // pg-boss uses seconds, we use milliseconds — never round a sub-second
+            // delay down to 0, which pg-boss treats as retry-immediately
+            pgBossOptions.retryDelay = Math.max(1, Math.round(delay / 1000));
         }
-        else if (typeof options.backoff === 'object') {
-            pgBossOptions.retryBackoff = options.backoff.type === 'exponential';
-            if (options.backoff.delay !== undefined) {
-                // pg-boss uses seconds, we use milliseconds
-                pgBossOptions.retryDelay = Math.floor(options.backoff.delay / 1000);
-            }
+        else if (type === 'exponential') {
+            // pg-boss computes exponential backoff as retry_delay * 2^n, and the
+            // queue-level retry_delay defaults to 0 — without an explicit base every
+            // "exponential" retry fires immediately. 1s base → ~1s, 2s, 4s, 8s…
+            pgBossOptions.retryDelay = 1;
         }
     }
     if (options?.removeOnComplete !== undefined) {

package/dist/pg-boss-queue-worker.js

@@ -2,15 +2,41 @@ import { registerQueueWorkers, runQueueJob, QueueJobFailedError, QueueJobDiscard
 import { pikkuState } from '@pikku/core/internal';
 import { mapPgBossJobToQueueJob } from './utils.js';
 export const mapPikkuWorkerToPgBoss = (workerConfig) => {
+    // Map the requested parallelism to pg-boss `localConcurrency` (independent
+    // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
+    // batch to one handler and only fetches again once that handler resolves — so
+    // a single slow job blocks every sibling in its batch AND stalls the next
+    // fetch. On a shared queue that is head-of-line blocking: one long job can
+    // freeze the entire worker and starve everything queued behind it.
+    // `localConcurrency` spawns N pollers that each fetch+process a single job and
+    // refill the instant they finish, so a slow job never holds up the others.
+    const localConcurrency = workerConfig?.batchSize ?? 10;
     const workerOptions = {
-        batchSize: 10,
+        localConcurrency,
+        batchSize: 1,
     };
-    if (workerConfig?.batchSize !== undefined) {
-        workerOptions.batchSize = workerConfig.batchSize;
-    }
     if (workerConfig?.pollInterval !== undefined) {
         workerOptions.pollingIntervalSeconds = Math.round(workerConfig.pollInterval / 1000);
     }
+    // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
+    // already at capacity are excluded from the fetch itself, so an at-capacity
+    // group costs nothing rather than being fetched and restored. pg-boss asserts
+    // every limit is <= localConcurrency, so clamp instead of letting a config
+    // that reads as "let this tier use more" throw at worker start.
+    if (workerConfig?.groupConcurrency !== undefined) {
+        const clamp = (limit) => Math.min(limit, localConcurrency);
+        workerOptions.localGroupConcurrency =
+            typeof workerConfig.groupConcurrency === 'number'
+                ? clamp(workerConfig.groupConcurrency)
+                : {
+                    default: clamp(workerConfig.groupConcurrency.default),
+                    ...(workerConfig.groupConcurrency.tiers
+                        ? {
+                            tiers: Object.fromEntries(Object.entries(workerConfig.groupConcurrency.tiers).map(([tier, limit]) => [tier, clamp(limit)])),
+                        }
+                        : {}),
+                };
+    }
     return workerOptions;
 };
 /**
@@ -28,8 +54,12 @@ export class PgBossQueueWorkers {
         // Configurations that are directly supported
         supported: {
             batchSize: {
-                queueProperty: 'batchSize',
-                description: 'Number of jobs to fetch and process concurrently in parallel',
+                queueProperty: 'localConcurrency',
+                description: 'Number of jobs to process concurrently',
+            },
+            groupConcurrency: {
+                queueProperty: 'localGroupConcurrency',
+                description: 'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
             },
             pollInterval: {
                 queueProperty: 'pollingIntervalSeconds',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/queue-pg-boss",
-  "version": "0.12.6",
+  "version": "0.12.8",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -15,7 +15,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.44"
+    "@pikku/core": "^0.12.69"
   },
   "dependencies": {
     "pg-boss": "^12.14.0"

package/run-tests.sh

@@ -1,5 +1,56 @@
 #!/bin/bash
-cd "$(dirname "$0")"
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$script_dir"
+
+# Enable nullglob to handle cases where no files match the pattern
+shopt -s nullglob
+
+# Initialize variables for options
+watch_mode=false
+coverage_mode=false
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --watch)
+      watch_mode=true
+      shift
+      ;;
+    --coverage)
+      coverage_mode=true
+      shift
+      ;;
+    *)
+      echo "Unknown option: $1"
+      exit 1
+      ;;
+  esac
+done
+
+# Define the pattern to match your test files
+pattern="src/*.test.ts"
+
+# Expand the pattern into an array of files
 files=($(find src -type f -name "*.test.ts"))
-if [ ${#files[@]} -eq 0 ]; then echo "No test files found"; exit 0; fi
-node --import tsx --test "$@" "${files[@]}"
+
+# Check if any files matched the pattern
+if [ ${#files[@]} -eq 0 ]; then
+  echo "No test files found matching pattern: $pattern"
+  exit 0
+fi
+
+# Construct the node command
+node_cmd=(node --import tsx --test)
+
+# Append options based on flags
+if [ "$watch_mode" = true ]; then
+  node_cmd+=(--watch)
+fi
+
+if [ "$coverage_mode" = true ]; then
+  node_cmd+=(--test-coverage-include="src/**/*.{ts,js}" --test-coverage-exclude="**/dist/**" --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info)
+fi
+
+# Execute the node command with the expanded list of files
+"${node_cmd[@]}" "${files[@]}"

package/src/pg-boss-queue-service.test.ts

@@ -16,12 +16,20 @@ describe('mapPikkuJobToPgBoss — retry mapping', () => {
     assert.equal(mapPikkuJobToPgBoss({ attempts: 1 }).retryLimit, 0)
   })
 
-  test("backoff 'exponential' → retryBackoff true", () => {
-    assert.equal(
-      mapPikkuJobToPgBoss({ attempts: 6, backoff: 'exponential' })
-        .retryBackoff,
-      true
-    )
+  test("backoff 'exponential' → retryBackoff true + 1s base delay", () => {
+    const opts = mapPikkuJobToPgBoss({ attempts: 6, backoff: 'exponential' })
+    assert.equal(opts.retryBackoff, true)
+    // pg-boss multiplies retry_delay by 2^n and the queue default is 0 —
+    // without a base, "exponential" retries all fire immediately.
+    assert.equal(opts.retryDelay, 1)
+  })
+
+  test('sub-second fixed delay rounds up to 1s, not down to immediate', () => {
+    const opts = mapPikkuJobToPgBoss({
+      attempts: 4,
+      backoff: { type: 'fixed', delay: 300 },
+    })
+    assert.equal(opts.retryDelay, 1)
   })
 
   test('fixed backoff → retryBackoff false + retryDelay in seconds', () => {
@@ -49,3 +57,22 @@ describe('mapPikkuJobToPgBoss — retry mapping', () => {
     assert.equal(opts.retryBackoff, undefined)
   })
 })
+
+describe('mapPikkuJobToPgBoss — group mapping', () => {
+  test('group id is forwarded so the worker can enforce a per-group limit', () => {
+    assert.deepEqual(mapPikkuJobToPgBoss({ group: { id: 'deployWorkflow' } })
+      .group, { id: 'deployWorkflow' })
+  })
+
+  test('tier is only set when provided', () => {
+    const opts = mapPikkuJobToPgBoss({
+      group: { id: 'deployWorkflow', tier: 'slow' },
+    })
+    assert.deepEqual(opts.group, { id: 'deployWorkflow', tier: 'slow' })
+    assert.equal('tier' in mapPikkuJobToPgBoss({ group: { id: 'a' } }).group!, false)
+  })
+
+  test('no group → no group field', () => {
+    assert.equal(mapPikkuJobToPgBoss({ attempts: 2 }).group, undefined)
+  })
+})

package/src/pg-boss-queue-service.ts

@@ -19,21 +19,33 @@ export const mapPikkuJobToPgBoss = (
     pgBossOptions.singletonKey = options.jobId
   }
 
+  if (options?.group !== undefined) {
+    pgBossOptions.group = {
+      id: options.group.id,
+      ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
+    }
+  }
+
   // Map retry options
   if (options?.attempts !== undefined) {
     pgBossOptions.retryLimit = options.attempts - 1
   }
 
   if (options?.backoff !== undefined) {
-    if (typeof options.backoff === 'string') {
-      // If backoff is a string, assume it's 'exponential' or 'fixed'
-      pgBossOptions.retryBackoff = options.backoff === 'exponential'
-    } else if (typeof options.backoff === 'object') {
-      pgBossOptions.retryBackoff = options.backoff.type === 'exponential'
-      if (options.backoff.delay !== undefined) {
-        // pg-boss uses seconds, we use milliseconds
-        pgBossOptions.retryDelay = Math.floor(options.backoff.delay / 1000)
-      }
+    const type =
+      typeof options.backoff === 'string' ? options.backoff : options.backoff.type
+    const delay =
+      typeof options.backoff === 'object' ? options.backoff.delay : undefined
+    pgBossOptions.retryBackoff = type === 'exponential'
+    if (delay !== undefined) {
+      // pg-boss uses seconds, we use milliseconds — never round a sub-second
+      // delay down to 0, which pg-boss treats as retry-immediately
+      pgBossOptions.retryDelay = Math.max(1, Math.round(delay / 1000))
+    } else if (type === 'exponential') {
+      // pg-boss computes exponential backoff as retry_delay * 2^n, and the
+      // queue-level retry_delay defaults to 0 — without an explicit base every
+      // "exponential" retry fires immediately. 1s base → ~1s, 2s, 4s, 8s…
+      pgBossOptions.retryDelay = 1
     }
   }
 

package/src/pg-boss-queue-worker.test.ts

@@ -1,7 +1,10 @@
 import { describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 
-import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
+import {
+  PgBossQueueWorkers,
+  mapPikkuWorkerToPgBoss,
+} from './pg-boss-queue-worker.js'
 
 describe('PgBossQueueWorkers', () => {
   test('requires a logger (explicit or from singleton services)', async () => {
@@ -13,3 +16,78 @@ describe('PgBossQueueWorkers', () => {
     })
   })
 })
+
+describe('mapPikkuWorkerToPgBoss', () => {
+  test('parallelism maps to independent workers (localConcurrency), never batch fetching', () => {
+    // batchSize: 1 keeps each worker single-job so a slow job can never hold its
+    // batch siblings — head-of-line blocking is impossible.
+    assert.deepEqual(mapPikkuWorkerToPgBoss(), {
+      localConcurrency: 10,
+      batchSize: 1,
+    })
+
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ batchSize: 25 }), {
+      localConcurrency: 25,
+      batchSize: 1,
+    })
+  })
+
+  test('pollInterval is converted from ms to seconds', () => {
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ pollInterval: 3000 }), {
+      localConcurrency: 10,
+      batchSize: 1,
+      pollingIntervalSeconds: 3,
+    })
+  })
+
+  test('groupConcurrency caps how much of the worker one group can take', () => {
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ groupConcurrency: 2 }), {
+      localConcurrency: 10,
+      batchSize: 1,
+      localGroupConcurrency: 2,
+    })
+  })
+
+  test('tiered groupConcurrency passes tiers through', () => {
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({
+        batchSize: 20,
+        groupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
+      }),
+      {
+        localConcurrency: 20,
+        batchSize: 1,
+        localGroupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
+      }
+    )
+  })
+
+  test('group limits are clamped to the worker concurrency', () => {
+    // pg-boss asserts every group limit is <= localConcurrency and throws at
+    // worker start otherwise — clamp so an over-eager config degrades to "this
+    // group may use the whole worker" instead of failing to start.
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({
+        batchSize: 4,
+        groupConcurrency: { default: 10, tiers: { greedy: 99 } },
+      }),
+      {
+        localConcurrency: 4,
+        batchSize: 1,
+        localGroupConcurrency: { default: 4, tiers: { greedy: 4 } },
+      }
+    )
+
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({ batchSize: 3, groupConcurrency: 8 }),
+      { localConcurrency: 3, batchSize: 1, localGroupConcurrency: 3 }
+    )
+  })
+
+  test('no group config means no group limits', () => {
+    assert.equal(
+      'localGroupConcurrency' in mapPikkuWorkerToPgBoss({ batchSize: 5 }),
+      false
+    )
+  })
+})

package/src/pg-boss-queue-worker.ts

@@ -18,12 +18,18 @@ import { mapPgBossJobToQueueJob } from './utils.js'
 export const mapPikkuWorkerToPgBoss = (
   workerConfig?: PikkuWorkerConfig
 ): WorkOptions => {
+  // Map the requested parallelism to pg-boss `localConcurrency` (independent
+  // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
+  // batch to one handler and only fetches again once that handler resolves — so
+  // a single slow job blocks every sibling in its batch AND stalls the next
+  // fetch. On a shared queue that is head-of-line blocking: one long job can
+  // freeze the entire worker and starve everything queued behind it.
+  // `localConcurrency` spawns N pollers that each fetch+process a single job and
+  // refill the instant they finish, so a slow job never holds up the others.
+  const localConcurrency = workerConfig?.batchSize ?? 10
   const workerOptions: WorkOptions = {
-    batchSize: 10,
-  }
-
-  if (workerConfig?.batchSize !== undefined) {
-    workerOptions.batchSize = workerConfig.batchSize
+    localConcurrency,
+    batchSize: 1,
   }
 
   if (workerConfig?.pollInterval !== undefined) {
@@ -32,6 +38,30 @@ export const mapPikkuWorkerToPgBoss = (
     )
   }
 
+  // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
+  // already at capacity are excluded from the fetch itself, so an at-capacity
+  // group costs nothing rather than being fetched and restored. pg-boss asserts
+  // every limit is <= localConcurrency, so clamp instead of letting a config
+  // that reads as "let this tier use more" throw at worker start.
+  if (workerConfig?.groupConcurrency !== undefined) {
+    const clamp = (limit: number) => Math.min(limit, localConcurrency)
+    workerOptions.localGroupConcurrency =
+      typeof workerConfig.groupConcurrency === 'number'
+        ? clamp(workerConfig.groupConcurrency)
+        : {
+            default: clamp(workerConfig.groupConcurrency.default),
+            ...(workerConfig.groupConcurrency.tiers
+              ? {
+                  tiers: Object.fromEntries(
+                    Object.entries(workerConfig.groupConcurrency.tiers).map(
+                      ([tier, limit]) => [tier, clamp(limit)]
+                    )
+                  ),
+                }
+              : {}),
+          }
+  }
+
   return workerOptions
 }
 
@@ -51,9 +81,13 @@ export class PgBossQueueWorkers implements QueueWorkers {
     // Configurations that are directly supported
     supported: {
       batchSize: {
-        queueProperty: 'batchSize',
+        queueProperty: 'localConcurrency',
+        description: 'Number of jobs to process concurrently',
+      },
+      groupConcurrency: {
+        queueProperty: 'localGroupConcurrency',
         description:
-          'Number of jobs to fetch and process concurrently in parallel',
+          'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
       },
       pollInterval: {
         queueProperty: 'pollingIntervalSeconds',