@pikku/mongodb 0.12.6 → 0.12.7

package/CHANGELOG.md

@@ -1,5 +1,68 @@
 # @pikku/mongodb
 
+## 0.12.7
+
+### Patch Changes
+
+- 92cd5b1: feat(workflow): workflow-owned step retries + stable invocationId
+
+  The workflow — not the queue — now owns step retry policy, and each step
+  invocation gets a stable idempotency key.
+  - **Default `retries: 5` with exponential backoff.** A step with no `retries`
+    previously inherited the queue's bare default (e.g. pg-boss `retry_limit 2`,
+    no backoff) so retries fired instantly and couldn't outlast a transient
+    outage. Retries now default to 5 with backoff, resolved at the workflow layer.
+  - **`retries: 0` is honored.** Dispatch previously passed `undefined` options
+    for `retries: 0`, letting the queue re-run a non-idempotent step up to its own
+    default. The resolved policy now always sets `attempts` (`retries: 0` →
+    `attempts: 1`), so the queue never second-guesses the workflow. The persisted
+    step retries and the dispatched `attempts` are resolved together so
+    "retries exhausted" and "no more redeliveries" are the same event.
+  - **`workflowStep.invocationId`** — a deterministic, dependency-free
+    `uuidv5(runId:stepName)` handed to every step. Unlike `stepId` (minted per
+    attempt), it is identical across retries, so a step can dedupe on it
+    (`ON CONFLICT (invocationId)`, Stripe idempotency keys, etc.).
+  - **queue-bullmq**: `mapPikkuJobToBull` now maps `backoff` (previously dropped,
+    so a step's backoff silently never applied on Redis), and `registerQueues`
+    throws a clear error when no logger is available (matching queue-pg-boss).
+  - **Dispatch failures are recoverable, not fatal.** A step is now marked
+    `scheduled` only _after_ it is successfully handed to its transport (queue or
+    scheduler) — a failed hand-off leaves it `pending` so a replay re-dispatches
+    it, instead of stranding it in `scheduled` (replay would pause forever on a
+    job that was never enqueued). A transport outage (e.g. pg-boss momentarily
+    down) is surfaced as a new `WorkflowDispatchException`, which the orchestrator
+    treats as transient: the run is left running and the orchestrator job is
+    rethrown for redelivery (it replays idempotently from the snapshot) rather
+    than the whole run being marked `failed`. The orchestrator job now also
+    carries its own retry policy, so this holds even when the orchestrator queue
+    is configured `retry_limit 0`. A genuine step error still fails the run.
+  - **Same step name can be invoked multiple times in one run.** Step rows are now
+    keyed per _invocation_: the Nth reach of a step name in a replay resolves to a
+    physical key (`name` for the first, `name#N` for repeats), so a literal
+    duplicate name no longer clobbers the earlier step's state. The first reach
+    keeps the bare name, so existing rows, graph-node matching and `invocationId`s
+    are unchanged. Ordinals are derived deterministically from DSL execution order
+    and reset each replay.
+  - **Step provenance (`fromStepName`) + graph cycles.** Every step now records
+    the predecessor it was scheduled from (`fromStepName`; entry steps have none),
+    persisted on the step row across all stores (in-memory, kysely, redis,
+    mongodb, cloudflare DO) and carried in the queued payload. The DSL wire
+    exposes the derived `fromInvocationId` (`uuidv5(runId:fromStepName)`) so
+    consumers get the stable predecessor key without a second persisted id —
+    `fromStepName` is the source of truth (it is replay-deterministic; `stepId`,
+    minted per row, is not). This makes the walked path reconstructable even when
+    a node is reached more than once: in `a → b → a → c` the second `a` is a
+    distinct ordinal instance (`a#1`) whose `fromStepName` is `b`.
+    The graph runner now supports **cycles**: a forward edge into an
+    already-started node still collapses to a single run (joins/diamonds are
+    unchanged), but a _back-edge_ — one whose target can reach its source — fires
+    a fresh ordinal instance, so a node can loop back to itself. Termination is
+    the graph's responsibility (branch routing must converge); the engine enforces
+    no visit cap.
+
+- Updated dependencies [92cd5b1]
+  - @pikku/core@0.12.38
+
 ## 0.12.6
 
 ### Patch Changes

package/dist/src/mongodb-secret-service.d.ts

@@ -25,6 +25,6 @@ export declare class MongoDBSecretService implements SecretService {
     hasSecret(key: string): Promise<boolean>;
     setSecret(key: string, value: unknown): Promise<void>;
     deleteSecret(key: string): Promise<void>;
-    getSecrets(keys: string[]): Promise<Record<string, unknown>>;
+    getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(keys: (keyof T & string)[]): Promise<T>;
     rotateKEK(): Promise<number>;
 }

package/dist/src/mongodb-workflow-service.d.ts

@@ -1,5 +1,5 @@
 import type { SerializedError } from '@pikku/core';
-import { PikkuWorkflowService, type WorkflowPlannedStep, type WorkflowRun, type WorkflowRunWire, type StepState, type WorkflowStatus, type WorkflowVersionStatus } from '@pikku/core/workflow';
+import { PikkuWorkflowService, type WorkflowPlannedStep, type WorkflowRun, type WorkflowRunWire, type StepState, type StepStatus, type WorkflowStatus, type WorkflowVersionStatus } from '@pikku/core/workflow';
 import type { Db } from 'mongodb';
 export declare class MongoDBWorkflowService extends PikkuWorkflowService {
     private initialized;
@@ -19,7 +19,7 @@ export declare class MongoDBWorkflowService extends PikkuWorkflowService {
     protected insertStepStateImpl(runId: string, stepName: string, rpcName: string | null, data: any, stepOptions?: {
         retries?: number;
         retryDelay?: string | number;
-    }): Promise<StepState>;
+    }, fromStepName?: string): Promise<StepState>;
     getStepState(runId: string, stepName: string): Promise<StepState>;
     getRunHistory(runId: string): Promise<Array<StepState & {
         stepName: string;
@@ -40,6 +40,11 @@ export declare class MongoDBWorkflowService extends PikkuWorkflowService {
         branchKeys: Record<string, string>;
     }>;
     getNodesWithoutSteps(runId: string, nodeIds: string[]): Promise<string[]>;
+    getStepInstances(runId: string): Promise<Array<{
+        stepName: string;
+        status: StepStatus;
+        fromStepName?: string;
+    }>>;
     getNodeResults(runId: string, nodeIds: string[]): Promise<Record<string, any>>;
     protected setBranchTakenImpl(stepId: string, branchKey: string): Promise<void>;
     protected updateRunStateImpl(runId: string, name: string, value: unknown): Promise<void>;

package/dist/src/mongodb-workflow-service.js

@@ -62,7 +62,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
             },
         });
     }
-    async insertStepStateImpl(runId, stepName, rpcName, data, stepOptions) {
+    async insertStepStateImpl(runId, stepName, rpcName, data, stepOptions, fromStepName) {
         const stepId = crypto.randomUUID();
         const now = new Date();
         await this.steps.insertOne({
@@ -77,6 +77,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
             branchTaken: null,
             retries: stepOptions?.retries ?? null,
             retryDelay: stepOptions?.retryDelay?.toString() ?? null,
+            fromStepName: fromStepName ?? null,
             createdAt: now,
             updatedAt: now,
         });
@@ -89,6 +90,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
             attemptCount: 1,
             retries: stepOptions?.retries,
             retryDelay: stepOptions?.retryDelay?.toString(),
+            fromStepName,
             createdAt: now,
             updatedAt: now,
         };
@@ -112,6 +114,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
             attemptCount,
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
+            fromStepName: row.fromStepName ?? undefined,
             createdAt: new Date(row.createdAt),
             updatedAt: new Date(row.updatedAt),
         };
@@ -240,6 +243,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
             attemptCount,
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
+            fromStepName: row.fromStepName ?? undefined,
             createdAt: new Date(row.createdAt),
             updatedAt: new Date(row.updatedAt),
         };
@@ -293,6 +297,17 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
         const existingStepNames = new Set(result.map((r) => r.stepName));
         return nodeIds.filter((id) => !existingStepNames.has(id));
     }
+    async getStepInstances(runId) {
+        const rows = await this.steps
+            .find({ workflowRunId: runId })
+            .project({ stepName: 1, status: 1, fromStepName: 1 })
+            .toArray();
+        return rows.map((r) => ({
+            stepName: r.stepName,
+            status: r.status,
+            fromStepName: r.fromStepName ?? undefined,
+        }));
+    }
     async getNodeResults(runId, nodeIds) {
         if (nodeIds.length === 0)
             return {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/mongodb",
-  "version": "0.12.6",
+  "version": "0.12.7",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/src/index.js",
@@ -17,7 +17,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.27"
+    "@pikku/core": "^0.12.38"
   },
   "dependencies": {
     "mongodb": "^6.12.0"

package/src/mongodb-secret-service.ts

@@ -131,7 +131,9 @@ export class MongoDBSecretService implements SecretService {
     await this.logAudit(key, 'delete')
   }
 
-  async getSecrets(keys: string[]): Promise<Record<string, unknown>> {
+  async getSecrets<
+    T extends Record<string, unknown> = Record<string, unknown>,
+  >(keys: (keyof T & string)[]): Promise<T> {
     const rows = await this.secrets
       .find({ _id: { $in: keys } } as any)
       .toArray()
@@ -148,7 +150,7 @@ export class MongoDBSecretService implements SecretService {
         // skip secrets that fail to decrypt
       }
     }
-    return out
+    return out as T
   }
 
   async rotateKEK(): Promise<number> {

package/src/mongodb-workflow-service.ts

@@ -5,6 +5,7 @@ import {
   type WorkflowRun,
   type WorkflowRunWire,
   type StepState,
+  type StepStatus,
   type WorkflowStatus,
   type WorkflowVersionStatus,
 } from '@pikku/core/workflow'
@@ -41,6 +42,7 @@ interface WorkflowStepDoc {
   branchTaken: string | null
   retries: number | null
   retryDelay: string | null
+  fromStepName?: string | null
   createdAt: Date
   updatedAt: Date
 }
@@ -172,7 +174,8 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
     stepName: string,
     rpcName: string | null,
     data: any,
-    stepOptions?: { retries?: number; retryDelay?: string | number }
+    stepOptions?: { retries?: number; retryDelay?: string | number },
+    fromStepName?: string
   ): Promise<StepState> {
     const stepId = crypto.randomUUID()
     const now = new Date()
@@ -189,6 +192,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
       branchTaken: null,
       retries: stepOptions?.retries ?? null,
       retryDelay: stepOptions?.retryDelay?.toString() ?? null,
+      fromStepName: fromStepName ?? null,
       createdAt: now,
       updatedAt: now,
     })
@@ -203,6 +207,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
       attemptCount: 1,
       retries: stepOptions?.retries,
       retryDelay: stepOptions?.retryDelay?.toString(),
+      fromStepName,
       createdAt: now,
       updatedAt: now,
     }
@@ -232,6 +237,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
       attemptCount,
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
+      fromStepName: row.fromStepName ?? undefined,
       createdAt: new Date(row.createdAt),
       updatedAt: new Date(row.updatedAt),
     }
@@ -427,6 +433,7 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
       attemptCount,
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
+      fromStepName: row.fromStepName ?? undefined,
       createdAt: new Date(row.createdAt),
       updatedAt: new Date(row.updatedAt),
     }
@@ -500,6 +507,20 @@ export class MongoDBWorkflowService extends PikkuWorkflowService {
     return nodeIds.filter((id) => !existingStepNames.has(id))
   }
 
+  async getStepInstances(runId: string): Promise<
+    Array<{ stepName: string; status: StepStatus; fromStepName?: string }>
+  > {
+    const rows = await this.steps
+      .find({ workflowRunId: runId })
+      .project({ stepName: 1, status: 1, fromStepName: 1 })
+      .toArray()
+    return rows.map((r: any) => ({
+      stepName: r.stepName,
+      status: r.status as StepStatus,
+      fromStepName: r.fromStepName ?? undefined,
+    }))
+  }
+
   async getNodeResults(
     runId: string,
     nodeIds: string[]