@pikku/queue-pg-boss 0.12.7 → 0.12.9

package/CHANGELOG.md

@@ -1,3 +1,61 @@
+## 0.12.9
+
+### Patch Changes
+
+- 6794681: Publish the ecosystem surface as per-area sub-barrels under `@pikku/core/ecosystem/*`, and point generated code, the CLI, the inspector and the runtime adapters at them.
+
+  348 names that only generated code, the toolchain or a runtime adapter ever imports now have a second home on `@pikku/core/ecosystem/<area>` — one sub-barrel per area, matching how core already publishes its entrypoints, so no single barrel grows without bound and a consumer only pulls in the area it uses.
+
+  This step is additive: every name is still exported from the entrypoint it was published from before, so nothing downstream breaks. Removing them from the app-facing barrels is a later change, and needs a release carrying `./ecosystem/*` first.
+
+- Updated dependencies [7406bfe]
+- Updated dependencies [6794681]
+  - @pikku/core@0.12.84
+
+## 0.12.8
+
+### Patch Changes
+
+- 6847338: Stop one slow job from holding an entire queue (head-of-line blocking).
+
+  The worker mapped pikku's parallelism to pg-boss `batchSize`, whose handler
+  receives the whole fetched batch and only fetches again once that handler
+  resolves. Because the handler `Promise.all`s the batch, a single long-running
+  job blocked every sibling in its batch _and_ stalled the next fetch — so on a
+  shared queue one slow job could freeze the worker and starve everything behind
+  it (observed with workflow-orchestrator jobs, where a long step pinned all 10
+  slots for tens of minutes).
+
+  Parallelism now maps to pg-boss `localConcurrency` with `batchSize: 1`: N
+  independent workers each fetch and process a single job and refill the instant
+  they finish, so a slow job never holds up the others. Same effective
+  concurrency, no head-of-line blocking.
+
+- e3d4454: Add job groups, so one shared queue can stay fair without splitting into one
+  queue per producer.
+
+  A job may now carry `group: { id, tier }`, and a worker may cap how many jobs
+  of any one group run at once via `groupConcurrency`. On pg-boss this maps to
+  `localGroupConcurrency`, which excludes at-capacity groups from the fetch query
+  itself, so a capped group costs nothing rather than being fetched and restored.
+  BullMQ declares it unsupported (groups are a BullMQ Pro feature) — being
+  push-based, it can simply use a queue per group at no polling cost.
+
+  Workflow services accept a `queueStrategy`. The default `'per-workflow'` is
+  unchanged: every workflow gets its own `wf-orchestrator-*` / `wf-step-*` queue,
+  which is also what lets serverless providers deploy one unit per workflow. The
+  new `'shared-groups'` routes every workflow through the shared
+  orchestrator/step-worker queues and isolates them by group instead, so a
+  monolith runs one set of pollers rather than one per workflow — on a
+  pull-based backend with dozens of workflows that is the difference between
+  hundreds of poll loops and twenty. It is for single-process runtimes only; a
+  per-unit serverless deploy still needs the per-workflow queues to route to its
+  units.
+
+- Updated dependencies [24252b8]
+- Updated dependencies [e3d4454]
+  - @pikku/core@0.12.69
+
 ## 0.12.7
 
 ### Patch Changes

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 - present Yasser Fadl and Pikku contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,29 @@
+# @pikku/queue-pg-boss
+
+pg-boss queue, worker and scheduler services for Pikku. Backed by Postgres, so
+it needs no extra infrastructure if you already run one.
+
+## Install
+
+```bash
+npm install @pikku/queue-pg-boss
+```
+
+## Usage
+
+```typescript
+import PgBoss from 'pg-boss'
+import { PgBossQueueService } from '@pikku/queue-pg-boss'
+
+const pgBoss = new PgBoss(connectionString)
+await pgBoss.start()
+
+const queue = new PgBossQueueService(pgBoss)
+```
+
+Use `PgBossQueueWorkers` to process jobs and `PgBossSchedulerService` for cron
+wirings. For a Redis-backed alternative see `@pikku/queue-bullmq`.
+
+## Docs
+
+https://pikku.dev/docs

package/dist/index.d.ts

@@ -8,4 +8,5 @@ export { PgBossServiceFactory } from './pg-boss-service-factory.js';
 export { PgBossQueueService } from './pg-boss-queue-service.js';
 export { PgBossQueueWorkers } from './pg-boss-queue-worker.js';
 export { PgBossSchedulerService } from './pg-boss-scheduler-service.js';
-export type { QueueService, PikkuWorkerConfig, PikkuJobConfig, QueueJob, JobOptions, } from '@pikku/core/queue';
+export type { QueueService, QueueJob } from '@pikku/core/queue';
+export type { PikkuWorkerConfig, PikkuJobConfig, JobOptions, } from '@pikku/core/ecosystem/queue';

package/dist/pg-boss-queue-service.d.ts

@@ -1,5 +1,6 @@
 import type { PgBoss, Request as PgBossRequest } from 'pg-boss';
-import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue';
+import type { QueueService, QueueJob } from '@pikku/core/queue';
+import type { JobOptions } from '@pikku/core/ecosystem/queue';
 export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBossRequest["options"];
 /**
  * pg-boss/PostgreSQL queue service implementation

package/dist/pg-boss-queue-service.js

@@ -10,12 +10,20 @@ export const mapPikkuJobToPgBoss = (options) => {
     if (options?.jobId !== undefined) {
         pgBossOptions.singletonKey = options.jobId;
     }
+    if (options?.group !== undefined) {
+        pgBossOptions.group = {
+            id: options.group.id,
+            ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
+        };
+    }
     // Map retry options
     if (options?.attempts !== undefined) {
         pgBossOptions.retryLimit = options.attempts - 1;
     }
     if (options?.backoff !== undefined) {
-        const type = typeof options.backoff === 'string' ? options.backoff : options.backoff.type;
+        const type = typeof options.backoff === 'string'
+            ? options.backoff
+            : options.backoff.type;
         const delay = typeof options.backoff === 'object' ? options.backoff.delay : undefined;
         pgBossOptions.retryBackoff = type === 'exponential';
         if (delay !== undefined) {

package/dist/pg-boss-queue-worker.d.ts

@@ -1,5 +1,5 @@
 import type { PgBoss, WorkOptions } from 'pg-boss';
-import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
+import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/ecosystem/queue';
 import type { Logger } from '@pikku/core/services';
 export declare const mapPikkuWorkerToPgBoss: (workerConfig?: PikkuWorkerConfig) => WorkOptions;
 /**

package/dist/pg-boss-queue-worker.js

@@ -1,16 +1,43 @@
-import { registerQueueWorkers, runQueueJob, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
-import { pikkuState } from '@pikku/core/internal';
+import { runQueueJob } from '@pikku/core/queue';
+import { registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/ecosystem/queue';
+import { pikkuState } from '@pikku/core/ecosystem';
 import { mapPgBossJobToQueueJob } from './utils.js';
 export const mapPikkuWorkerToPgBoss = (workerConfig) => {
+    // Map the requested parallelism to pg-boss `localConcurrency` (independent
+    // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
+    // batch to one handler and only fetches again once that handler resolves — so
+    // a single slow job blocks every sibling in its batch AND stalls the next
+    // fetch. On a shared queue that is head-of-line blocking: one long job can
+    // freeze the entire worker and starve everything queued behind it.
+    // `localConcurrency` spawns N pollers that each fetch+process a single job and
+    // refill the instant they finish, so a slow job never holds up the others.
+    const localConcurrency = workerConfig?.batchSize ?? 10;
     const workerOptions = {
-        batchSize: 10,
+        localConcurrency,
+        batchSize: 1,
     };
-    if (workerConfig?.batchSize !== undefined) {
-        workerOptions.batchSize = workerConfig.batchSize;
-    }
     if (workerConfig?.pollInterval !== undefined) {
         workerOptions.pollingIntervalSeconds = Math.round(workerConfig.pollInterval / 1000);
     }
+    // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
+    // already at capacity are excluded from the fetch itself, so an at-capacity
+    // group costs nothing rather than being fetched and restored. pg-boss asserts
+    // every limit is <= localConcurrency, so clamp instead of letting a config
+    // that reads as "let this tier use more" throw at worker start.
+    if (workerConfig?.groupConcurrency !== undefined) {
+        const clamp = (limit) => Math.min(limit, localConcurrency);
+        workerOptions.localGroupConcurrency =
+            typeof workerConfig.groupConcurrency === 'number'
+                ? clamp(workerConfig.groupConcurrency)
+                : {
+                    default: clamp(workerConfig.groupConcurrency.default),
+                    ...(workerConfig.groupConcurrency.tiers
+                        ? {
+                            tiers: Object.fromEntries(Object.entries(workerConfig.groupConcurrency.tiers).map(([tier, limit]) => [tier, clamp(limit)])),
+                        }
+                        : {}),
+                };
+    }
     return workerOptions;
 };
 /**
@@ -28,8 +55,12 @@ export class PgBossQueueWorkers {
         // Configurations that are directly supported
         supported: {
             batchSize: {
-                queueProperty: 'batchSize',
-                description: 'Number of jobs to fetch and process concurrently in parallel',
+                queueProperty: 'localConcurrency',
+                description: 'Number of jobs to process concurrently',
+            },
+            groupConcurrency: {
+                queueProperty: 'localGroupConcurrency',
+                description: 'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
             },
             pollInterval: {
                 queueProperty: 'pollingIntervalSeconds',

package/dist/pg-boss-scheduler-service.d.ts

@@ -1,5 +1,6 @@
 import type { PgBoss } from 'pg-boss';
-import type { ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
+import type { CoreUserSession } from '@pikku/core';
+import type { ScheduledTaskInfo, ScheduledTaskSummary } from '@pikku/core/ecosystem/services';
 import { SchedulerService } from '@pikku/core';
 /**
  * pg-boss scheduler service implementation

package/dist/pg-boss-scheduler-service.js

@@ -1,6 +1,8 @@
-import { SchedulerService, parseDurationString } from '@pikku/core';
-import { pikkuState } from '@pikku/core/internal';
-import { runScheduledTask, getScheduledTasks } from '@pikku/core/scheduler';
+import { SchedulerService } from '@pikku/core';
+import { parseDurationString } from '@pikku/core/ecosystem/types';
+import { pikkuState } from '@pikku/core/ecosystem';
+import { runScheduledTask } from '@pikku/core/scheduler';
+import { getScheduledTasks } from '@pikku/core/ecosystem/scheduler';
 /** Prefix for pg-boss queues/schedules backing recurring scheduled tasks. */
 const RECURRING_TASK_PREFIX = 'pikku-recurring-scheduled-task_';
 /**

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@pikku/queue-pg-boss",
-  "version": "0.12.7",
+  "version": "0.12.9",
+  "description": "pg-boss queue, worker and scheduler services for Pikku, backed by Postgres",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -15,7 +16,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.56"
+    "@pikku/core": "^0.12.84"
   },
   "dependencies": {
     "pg-boss": "^12.14.0"

package/src/index.ts

@@ -11,10 +11,9 @@ export { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
 export { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
 
 // Re-export core queue types for convenience
+export type { QueueService, QueueJob } from '@pikku/core/queue'
 export type {
-  QueueService,
   PikkuWorkerConfig,
   PikkuJobConfig,
-  QueueJob,
   JobOptions,
-} from '@pikku/core/queue'
+} from '@pikku/core/ecosystem/queue'

package/src/pg-boss-queue-service.test.ts

@@ -57,3 +57,22 @@ describe('mapPikkuJobToPgBoss — retry mapping', () => {
     assert.equal(opts.retryBackoff, undefined)
   })
 })
+
+describe('mapPikkuJobToPgBoss — group mapping', () => {
+  test('group id is forwarded so the worker can enforce a per-group limit', () => {
+    assert.deepEqual(mapPikkuJobToPgBoss({ group: { id: 'deployWorkflow' } })
+      .group, { id: 'deployWorkflow' })
+  })
+
+  test('tier is only set when provided', () => {
+    const opts = mapPikkuJobToPgBoss({
+      group: { id: 'deployWorkflow', tier: 'slow' },
+    })
+    assert.deepEqual(opts.group, { id: 'deployWorkflow', tier: 'slow' })
+    assert.equal('tier' in mapPikkuJobToPgBoss({ group: { id: 'a' } }).group!, false)
+  })
+
+  test('no group → no group field', () => {
+    assert.equal(mapPikkuJobToPgBoss({ attempts: 2 }).group, undefined)
+  })
+})

package/src/pg-boss-queue-service.ts

@@ -1,5 +1,6 @@
 import type { PgBoss, Request as PgBossRequest } from 'pg-boss'
-import type { QueueService, QueueJob, JobOptions } from '@pikku/core/queue'
+import type { QueueService, QueueJob } from '@pikku/core/queue'
+import type { JobOptions } from '@pikku/core/ecosystem/queue'
 import { mapPgBossJobToQueueJob } from './utils.js'
 
 export const mapPikkuJobToPgBoss = (
@@ -19,6 +20,13 @@ export const mapPikkuJobToPgBoss = (
     pgBossOptions.singletonKey = options.jobId
   }
 
+  if (options?.group !== undefined) {
+    pgBossOptions.group = {
+      id: options.group.id,
+      ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
+    }
+  }
+
   // Map retry options
   if (options?.attempts !== undefined) {
     pgBossOptions.retryLimit = options.attempts - 1
@@ -26,7 +34,9 @@ export const mapPikkuJobToPgBoss = (
 
   if (options?.backoff !== undefined) {
     const type =
-      typeof options.backoff === 'string' ? options.backoff : options.backoff.type
+      typeof options.backoff === 'string'
+        ? options.backoff
+        : options.backoff.type
     const delay =
       typeof options.backoff === 'object' ? options.backoff.delay : undefined
     pgBossOptions.retryBackoff = type === 'exponential'

package/src/pg-boss-queue-worker.test.ts

@@ -1,7 +1,10 @@
 import { describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 
-import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
+import {
+  PgBossQueueWorkers,
+  mapPikkuWorkerToPgBoss,
+} from './pg-boss-queue-worker.js'
 
 describe('PgBossQueueWorkers', () => {
   test('requires a logger (explicit or from singleton services)', async () => {
@@ -13,3 +16,78 @@ describe('PgBossQueueWorkers', () => {
     })
   })
 })
+
+describe('mapPikkuWorkerToPgBoss', () => {
+  test('parallelism maps to independent workers (localConcurrency), never batch fetching', () => {
+    // batchSize: 1 keeps each worker single-job so a slow job can never hold its
+    // batch siblings — head-of-line blocking is impossible.
+    assert.deepEqual(mapPikkuWorkerToPgBoss(), {
+      localConcurrency: 10,
+      batchSize: 1,
+    })
+
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ batchSize: 25 }), {
+      localConcurrency: 25,
+      batchSize: 1,
+    })
+  })
+
+  test('pollInterval is converted from ms to seconds', () => {
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ pollInterval: 3000 }), {
+      localConcurrency: 10,
+      batchSize: 1,
+      pollingIntervalSeconds: 3,
+    })
+  })
+
+  test('groupConcurrency caps how much of the worker one group can take', () => {
+    assert.deepEqual(mapPikkuWorkerToPgBoss({ groupConcurrency: 2 }), {
+      localConcurrency: 10,
+      batchSize: 1,
+      localGroupConcurrency: 2,
+    })
+  })
+
+  test('tiered groupConcurrency passes tiers through', () => {
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({
+        batchSize: 20,
+        groupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
+      }),
+      {
+        localConcurrency: 20,
+        batchSize: 1,
+        localGroupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
+      }
+    )
+  })
+
+  test('group limits are clamped to the worker concurrency', () => {
+    // pg-boss asserts every group limit is <= localConcurrency and throws at
+    // worker start otherwise — clamp so an over-eager config degrades to "this
+    // group may use the whole worker" instead of failing to start.
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({
+        batchSize: 4,
+        groupConcurrency: { default: 10, tiers: { greedy: 99 } },
+      }),
+      {
+        localConcurrency: 4,
+        batchSize: 1,
+        localGroupConcurrency: { default: 4, tiers: { greedy: 4 } },
+      }
+    )
+
+    assert.deepEqual(
+      mapPikkuWorkerToPgBoss({ batchSize: 3, groupConcurrency: 8 }),
+      { localConcurrency: 3, batchSize: 1, localGroupConcurrency: 3 }
+    )
+  })
+
+  test('no group config means no group limits', () => {
+    assert.equal(
+      'localGroupConcurrency' in mapPikkuWorkerToPgBoss({ batchSize: 5 }),
+      false
+    )
+  })
+})

package/src/pg-boss-queue-worker.ts

@@ -4,26 +4,32 @@ import type {
   PikkuWorkerConfig,
   QueueConfigMapping,
   ConfigValidationResult,
-} from '@pikku/core/queue'
+} from '@pikku/core/ecosystem/queue'
+import { runQueueJob } from '@pikku/core/queue'
 import {
   registerQueueWorkers,
-  runQueueJob,
   QueueJobFailedError,
   QueueJobDiscardedError,
-} from '@pikku/core/queue'
-import { pikkuState } from '@pikku/core/internal'
+} from '@pikku/core/ecosystem/queue'
+import { pikkuState } from '@pikku/core/ecosystem'
 import type { Logger } from '@pikku/core/services'
 import { mapPgBossJobToQueueJob } from './utils.js'
 
 export const mapPikkuWorkerToPgBoss = (
   workerConfig?: PikkuWorkerConfig
 ): WorkOptions => {
+  // Map the requested parallelism to pg-boss `localConcurrency` (independent
+  // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
+  // batch to one handler and only fetches again once that handler resolves — so
+  // a single slow job blocks every sibling in its batch AND stalls the next
+  // fetch. On a shared queue that is head-of-line blocking: one long job can
+  // freeze the entire worker and starve everything queued behind it.
+  // `localConcurrency` spawns N pollers that each fetch+process a single job and
+  // refill the instant they finish, so a slow job never holds up the others.
+  const localConcurrency = workerConfig?.batchSize ?? 10
   const workerOptions: WorkOptions = {
-    batchSize: 10,
-  }
-
-  if (workerConfig?.batchSize !== undefined) {
-    workerOptions.batchSize = workerConfig.batchSize
+    localConcurrency,
+    batchSize: 1,
   }
 
   if (workerConfig?.pollInterval !== undefined) {
@@ -32,6 +38,30 @@ export const mapPikkuWorkerToPgBoss = (
     )
   }
 
+  // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
+  // already at capacity are excluded from the fetch itself, so an at-capacity
+  // group costs nothing rather than being fetched and restored. pg-boss asserts
+  // every limit is <= localConcurrency, so clamp instead of letting a config
+  // that reads as "let this tier use more" throw at worker start.
+  if (workerConfig?.groupConcurrency !== undefined) {
+    const clamp = (limit: number) => Math.min(limit, localConcurrency)
+    workerOptions.localGroupConcurrency =
+      typeof workerConfig.groupConcurrency === 'number'
+        ? clamp(workerConfig.groupConcurrency)
+        : {
+            default: clamp(workerConfig.groupConcurrency.default),
+            ...(workerConfig.groupConcurrency.tiers
+              ? {
+                  tiers: Object.fromEntries(
+                    Object.entries(workerConfig.groupConcurrency.tiers).map(
+                      ([tier, limit]) => [tier, clamp(limit)]
+                    )
+                  ),
+                }
+              : {}),
+          }
+  }
+
   return workerOptions
 }
 
@@ -51,9 +81,13 @@ export class PgBossQueueWorkers implements QueueWorkers {
     // Configurations that are directly supported
     supported: {
       batchSize: {
-        queueProperty: 'batchSize',
+        queueProperty: 'localConcurrency',
+        description: 'Number of jobs to process concurrently',
+      },
+      groupConcurrency: {
+        queueProperty: 'localGroupConcurrency',
         description:
-          'Number of jobs to fetch and process concurrently in parallel',
+          'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
       },
       pollInterval: {
         queueProperty: 'pollingIntervalSeconds',

package/src/pg-boss-scheduler-service.ts

@@ -1,12 +1,14 @@
 import type { PgBoss } from 'pg-boss'
+import type { CoreUserSession } from '@pikku/core'
 import type {
   ScheduledTaskInfo,
   ScheduledTaskSummary,
-  CoreUserSession,
-} from '@pikku/core'
-import { SchedulerService, parseDurationString } from '@pikku/core'
-import { pikkuState } from '@pikku/core/internal'
-import { runScheduledTask, getScheduledTasks } from '@pikku/core/scheduler'
+} from '@pikku/core/ecosystem/services'
+import { SchedulerService } from '@pikku/core'
+import { parseDurationString } from '@pikku/core/ecosystem/types'
+import { pikkuState } from '@pikku/core/ecosystem'
+import { runScheduledTask } from '@pikku/core/scheduler'
+import { getScheduledTasks } from '@pikku/core/ecosystem/scheduler'
 
 /**
  * Data stored in scheduled job