@pikku/queue-bullmq 0.12.2 → 0.12.3

package/CHANGELOG.md

@@ -1,3 +1,66 @@
+## 0.12.3
+
+### Patch Changes
+
+- 92cd5b1: feat(workflow): workflow-owned step retries + stable invocationId
+
+  The workflow — not the queue — now owns step retry policy, and each step
+  invocation gets a stable idempotency key.
+  - **Default `retries: 5` with exponential backoff.** A step with no `retries`
+    previously inherited the queue's bare default (e.g. pg-boss `retry_limit 2`,
+    no backoff) so retries fired instantly and couldn't outlast a transient
+    outage. Retries now default to 5 with backoff, resolved at the workflow layer.
+  - **`retries: 0` is honored.** Dispatch previously passed `undefined` options
+    for `retries: 0`, letting the queue re-run a non-idempotent step up to its own
+    default. The resolved policy now always sets `attempts` (`retries: 0` →
+    `attempts: 1`), so the queue never second-guesses the workflow. The persisted
+    step retries and the dispatched `attempts` are resolved together so
+    "retries exhausted" and "no more redeliveries" are the same event.
+  - **`workflowStep.invocationId`** — a deterministic, dependency-free
+    `uuidv5(runId:stepName)` handed to every step. Unlike `stepId` (minted per
+    attempt), it is identical across retries, so a step can dedupe on it
+    (`ON CONFLICT (invocationId)`, Stripe idempotency keys, etc.).
+  - **queue-bullmq**: `mapPikkuJobToBull` now maps `backoff` (previously dropped,
+    so a step's backoff silently never applied on Redis), and `registerQueues`
+    throws a clear error when no logger is available (matching queue-pg-boss).
+  - **Dispatch failures are recoverable, not fatal.** A step is now marked
+    `scheduled` only _after_ it is successfully handed to its transport (queue or
+    scheduler) — a failed hand-off leaves it `pending` so a replay re-dispatches
+    it, instead of stranding it in `scheduled` (replay would pause forever on a
+    job that was never enqueued). A transport outage (e.g. pg-boss momentarily
+    down) is surfaced as a new `WorkflowDispatchException`, which the orchestrator
+    treats as transient: the run is left running and the orchestrator job is
+    rethrown for redelivery (it replays idempotently from the snapshot) rather
+    than the whole run being marked `failed`. The orchestrator job now also
+    carries its own retry policy, so this holds even when the orchestrator queue
+    is configured `retry_limit 0`. A genuine step error still fails the run.
+  - **Same step name can be invoked multiple times in one run.** Step rows are now
+    keyed per _invocation_: the Nth reach of a step name in a replay resolves to a
+    physical key (`name` for the first, `name#N` for repeats), so a literal
+    duplicate name no longer clobbers the earlier step's state. The first reach
+    keeps the bare name, so existing rows, graph-node matching and `invocationId`s
+    are unchanged. Ordinals are derived deterministically from DSL execution order
+    and reset each replay.
+  - **Step provenance (`fromStepName`) + graph cycles.** Every step now records
+    the predecessor it was scheduled from (`fromStepName`; entry steps have none),
+    persisted on the step row across all stores (in-memory, kysely, redis,
+    mongodb, cloudflare DO) and carried in the queued payload. The DSL wire
+    exposes the derived `fromInvocationId` (`uuidv5(runId:fromStepName)`) so
+    consumers get the stable predecessor key without a second persisted id —
+    `fromStepName` is the source of truth (it is replay-deterministic; `stepId`,
+    minted per row, is not). This makes the walked path reconstructable even when
+    a node is reached more than once: in `a → b → a → c` the second `a` is a
+    distinct ordinal instance (`a#1`) whose `fromStepName` is `b`.
+    The graph runner now supports **cycles**: a forward edge into an
+    already-started node still collapses to a single run (joins/diamonds are
+    unchanged), but a _back-edge_ — one whose target can reach its source — fires
+    a fresh ordinal instance, so a node can loop back to itself. Termination is
+    the graph's responsibility (branch routing must converge); the engine enforces
+    no visit cap.
+
+- Updated dependencies [92cd5b1]
+  - @pikku/core@0.12.38
+
 ## 0.12.0
 
 ## 0.12.2

package/dist/bull-queue-service.js

@@ -11,6 +11,14 @@ export const mapPikkuJobToBull = (options) => {
     if (options?.attempts !== undefined) {
         bullOptions.attempts = options.attempts;
     }
+    // Map retry backoff. BullMQ accepts `{ type, delay }`; our `delay` is in ms,
+    // which is what BullMQ expects too (unlike pg-boss, which uses seconds).
+    if (options?.backoff !== undefined) {
+        bullOptions.backoff =
+            typeof options.backoff === 'string'
+                ? { type: options.backoff }
+                : { type: options.backoff.type, delay: options.backoff.delay };
+    }
     if (options?.removeOnComplete !== undefined) {
         bullOptions.removeOnComplete = options.removeOnComplete;
     }

package/dist/bull-queue-worker.js

@@ -107,7 +107,10 @@ export class BullQueueWorkers {
      * Scan state and register all compatible processors
      */
     async registerQueues() {
-        const logger = pikkuState(null, 'package', 'singletonServices').logger;
+        const logger = pikkuState(null, 'package', 'singletonServices')?.logger;
+        if (!logger) {
+            throw new Error('Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first');
+        }
         return await registerQueueWorkers(this.configMappings, logger, async (queueName, processor) => {
             const worker = new Worker(processor.name, async (job) => {
                 try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/queue-bullmq",
-  "version": "0.12.2",
+  "version": "0.12.3",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/index.js",
@@ -8,12 +8,14 @@
   "type": "module",
   "scripts": {
     "tsc": "tsc",
+    "test": "bash run-tests.sh",
     "ncu": "npx npm-check-updates",
     "build": "tsc -b",
-    "release": "npm run build && npm test"
+    "release": "npm run build && npm test",
+    "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.2"
+    "@pikku/core": "^0.12.38"
   },
   "dependencies": {
     "bullmq": "^5.70.1",
@@ -21,5 +23,8 @@
   },
   "devDependencies": {
     "typescript": "^5.9"
+  },
+  "exports": {
+    ".": "./dist/index.js"
   }
 }

package/run-tests.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 cd "$(dirname "$0")"
-exec node --test --experimental-test-coverage --loader=tsx/esm dist/**/*.test.js
+files=($(find src -type f -name "*.test.ts"))
+if [ ${#files[@]} -eq 0 ]; then echo "No test files found"; exit 0; fi
+node --import tsx --test "$@" "${files[@]}"

package/src/bull-queue-service.test.ts

@@ -0,0 +1,36 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { mapPikkuJobToBull } from './bull-queue-service.js'
+
+// Same contract as the pg-boss adapter: the workflow's resolved retry policy
+// (attempts + backoff) must reach BullMQ. backoff mapping in particular was
+// previously dropped, so a step's backoff silently never applied on Redis.
+describe('mapPikkuJobToBull — retry mapping', () => {
+  test('attempts passes through unchanged (BullMQ uses attempts directly)', () => {
+    assert.equal(mapPikkuJobToBull({ attempts: 6 }).attempts, 6)
+    assert.equal(mapPikkuJobToBull({ attempts: 1 }).attempts, 1)
+  })
+
+  test("string backoff 'exponential' → { type: 'exponential' }", () => {
+    assert.deepEqual(mapPikkuJobToBull({ attempts: 6, backoff: 'exponential' }).backoff, {
+      type: 'exponential',
+    })
+  })
+
+  test('object backoff → { type, delay } (delay stays in ms)', () => {
+    assert.deepEqual(
+      mapPikkuJobToBull({ attempts: 4, backoff: { type: 'fixed', delay: 250 } })
+        .backoff,
+      { type: 'fixed', delay: 250 }
+    )
+  })
+
+  test('no backoff → backoff left unset', () => {
+    assert.equal(mapPikkuJobToBull({ attempts: 6 }).backoff, undefined)
+  })
+
+  test('no options → empty job options', () => {
+    assert.deepEqual(mapPikkuJobToBull(undefined), {})
+  })
+})

package/src/bull-queue-service.ts

@@ -19,6 +19,15 @@ export const mapPikkuJobToBull = (options?: JobOptions): JobsOptions => {
     bullOptions.attempts = options.attempts
   }
 
+  // Map retry backoff. BullMQ accepts `{ type, delay }`; our `delay` is in ms,
+  // which is what BullMQ expects too (unlike pg-boss, which uses seconds).
+  if (options?.backoff !== undefined) {
+    bullOptions.backoff =
+      typeof options.backoff === 'string'
+        ? { type: options.backoff }
+        : { type: options.backoff.type, delay: options.backoff.delay }
+  }
+
   if (options?.removeOnComplete !== undefined) {
     bullOptions.removeOnComplete = options.removeOnComplete
   }

package/src/bull-queue-worker.test.ts

@@ -4,12 +4,12 @@ import assert from 'node:assert/strict'
 import { BullQueueWorkers } from './bull-queue-worker.js'
 
 describe('BullQueueWorkers', () => {
-  test('requires setJobRunner before registerQueues', async () => {
+  test('requires a logger (explicit or from singleton services)', async () => {
     const workers = new BullQueueWorkers({})
 
     await assert.rejects(() => workers.registerQueues(), {
       message:
-        'BullQueueWorkers requires setJobRunner() before registerQueues()',
+        'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first',
     })
   })
 })

package/src/bull-queue-worker.ts

@@ -141,7 +141,12 @@ export class BullQueueWorkers implements QueueWorkers {
    * Scan state and register all compatible processors
    */
   async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
-    const logger = pikkuState(null, 'package', 'singletonServices')!.logger
+    const logger = pikkuState(null, 'package', 'singletonServices')?.logger
+    if (!logger) {
+      throw new Error(
+        'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first'
+      )
+    }
 
     return await registerQueueWorkers(
       this.configMappings,