@pikku/kysely 0.12.16 → 0.12.17

package/CHANGELOG.md

@@ -1,3 +1,66 @@
+## 0.12.17
+
+### Patch Changes
+
+- 92cd5b1: feat(workflow): workflow-owned step retries + stable invocationId
+
+  The workflow — not the queue — now owns step retry policy, and each step
+  invocation gets a stable idempotency key.
+  - **Default `retries: 5` with exponential backoff.** A step with no `retries`
+    previously inherited the queue's bare default (e.g. pg-boss `retry_limit 2`,
+    no backoff) so retries fired instantly and couldn't outlast a transient
+    outage. Retries now default to 5 with backoff, resolved at the workflow layer.
+  - **`retries: 0` is honored.** Dispatch previously passed `undefined` options
+    for `retries: 0`, letting the queue re-run a non-idempotent step up to its own
+    default. The resolved policy now always sets `attempts` (`retries: 0` →
+    `attempts: 1`), so the queue never second-guesses the workflow. The persisted
+    step retries and the dispatched `attempts` are resolved together so
+    "retries exhausted" and "no more redeliveries" are the same event.
+  - **`workflowStep.invocationId`** — a deterministic, dependency-free
+    `uuidv5(runId:stepName)` handed to every step. Unlike `stepId` (minted per
+    attempt), it is identical across retries, so a step can dedupe on it
+    (`ON CONFLICT (invocationId)`, Stripe idempotency keys, etc.).
+  - **queue-bullmq**: `mapPikkuJobToBull` now maps `backoff` (previously dropped,
+    so a step's backoff silently never applied on Redis), and `registerQueues`
+    throws a clear error when no logger is available (matching queue-pg-boss).
+  - **Dispatch failures are recoverable, not fatal.** A step is now marked
+    `scheduled` only _after_ it is successfully handed to its transport (queue or
+    scheduler) — a failed hand-off leaves it `pending` so a replay re-dispatches
+    it, instead of stranding it in `scheduled` (replay would pause forever on a
+    job that was never enqueued). A transport outage (e.g. pg-boss momentarily
+    down) is surfaced as a new `WorkflowDispatchException`, which the orchestrator
+    treats as transient: the run is left running and the orchestrator job is
+    rethrown for redelivery (it replays idempotently from the snapshot) rather
+    than the whole run being marked `failed`. The orchestrator job now also
+    carries its own retry policy, so this holds even when the orchestrator queue
+    is configured `retry_limit 0`. A genuine step error still fails the run.
+  - **Same step name can be invoked multiple times in one run.** Step rows are now
+    keyed per _invocation_: the Nth reach of a step name in a replay resolves to a
+    physical key (`name` for the first, `name#N` for repeats), so a literal
+    duplicate name no longer clobbers the earlier step's state. The first reach
+    keeps the bare name, so existing rows, graph-node matching and `invocationId`s
+    are unchanged. Ordinals are derived deterministically from DSL execution order
+    and reset each replay.
+  - **Step provenance (`fromStepName`) + graph cycles.** Every step now records
+    the predecessor it was scheduled from (`fromStepName`; entry steps have none),
+    persisted on the step row across all stores (in-memory, kysely, redis,
+    mongodb, cloudflare DO) and carried in the queued payload. The DSL wire
+    exposes the derived `fromInvocationId` (`uuidv5(runId:fromStepName)`) so
+    consumers get the stable predecessor key without a second persisted id —
+    `fromStepName` is the source of truth (it is replay-deterministic; `stepId`,
+    minted per row, is not). This makes the walked path reconstructable even when
+    a node is reached more than once: in `a → b → a → c` the second `a` is a
+    distinct ordinal instance (`a#1`) whose `fromStepName` is `b`.
+    The graph runner now supports **cycles**: a forward edge into an
+    already-started node still collapses to a single run (joins/diamonds are
+    unchanged), but a _back-edge_ — one whose target can reach its source — fires
+    a fresh ordinal instance, so a node can loop back to itself. Termination is
+    the graph's responsibility (branch routing must converge); the engine enforces
+    no visit cap.
+
+- Updated dependencies [92cd5b1]
+  - @pikku/core@0.12.38
+
 ## 0.12.16
 
 ### Patch Changes

package/dist/src/kysely-tables.d.ts

@@ -43,6 +43,7 @@ export interface WorkflowStepTable {
     branchTaken: string | null;
     retries: number | null;
     retryDelay: string | null;
+    fromStepName: string | null;
     createdAt: Generated<Date>;
     updatedAt: Generated<Date>;
 }

package/dist/src/kysely-workflow-service.d.ts

@@ -1,5 +1,5 @@
 import type { SerializedError } from '@pikku/core';
-import { PikkuWorkflowService, type WorkflowPlannedStep, type WorkflowRun, type WorkflowRunWire, type StepState, type WorkflowStatus, type WorkflowVersionStatus } from '@pikku/core/workflow';
+import { PikkuWorkflowService, type WorkflowPlannedStep, type WorkflowRun, type WorkflowRunWire, type StepState, type StepStatus, type WorkflowStatus, type WorkflowVersionStatus } from '@pikku/core/workflow';
 import type { Kysely } from 'kysely';
 import type { KyselyPikkuDB } from './kysely-tables.js';
 export declare class KyselyWorkflowService extends PikkuWorkflowService {
@@ -17,7 +17,7 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
     protected insertStepStateImpl(runId: string, stepName: string, rpcName: string | null, data: any, stepOptions?: {
         retries?: number;
         retryDelay?: string | number;
-    }): Promise<StepState>;
+    }, fromStepName?: string): Promise<StepState>;
     getStepState(runId: string, stepName: string): Promise<StepState>;
     getRunHistory(runId: string): Promise<Array<StepState & {
         stepName: string;
@@ -38,6 +38,11 @@ export declare class KyselyWorkflowService extends PikkuWorkflowService {
         branchKeys: Record<string, string>;
     }>;
     getNodesWithoutSteps(runId: string, nodeIds: string[]): Promise<string[]>;
+    getStepInstances(runId: string): Promise<Array<{
+        stepName: string;
+        status: StepStatus;
+        fromStepName?: string;
+    }>>;
     getNodeResults(runId: string, nodeIds: string[]): Promise<Record<string, any>>;
     protected setBranchTakenImpl(stepId: string, branchKey: string): Promise<void>;
     protected updateRunStateImpl(runId: string, name: string, value: unknown): Promise<void>;

package/dist/src/kysely-workflow-service.js

@@ -55,6 +55,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             .addColumn('branch_taken', 'text')
             .addColumn('retries', 'integer')
             .addColumn('retry_delay', 'text')
+            .addColumn('from_step_name', 'text')
             .addColumn('created_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addColumn('updated_at', 'timestamp', (col) => col.defaultTo(sql `CURRENT_TIMESTAMP`).notNull())
             .addUniqueConstraint('workflow_step_run_name_unique', [
@@ -62,6 +63,14 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             'step_name',
         ])
             .execute();
+        // Additive: backfill the column on tables created before from_step_name.
+        await this.db.schema
+            .alterTable('workflow_step')
+            .addColumn('from_step_name', 'text')
+            .execute()
+            .catch(() => {
+            // Column already exists (fresh tables include it above) — nothing to do.
+        });
         await this.db.schema
             .createTable('workflow_step_history')
             .ifNotExists()
@@ -132,7 +141,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             .where('workflowRunId', '=', id)
             .execute();
     }
-    async insertStepStateImpl(runId, stepName, rpcName, data, stepOptions) {
+    async insertStepStateImpl(runId, stepName, rpcName, data, stepOptions, fromStepName) {
         const stepId = crypto.randomUUID();
         const now = new Date();
         await this.db
@@ -146,6 +155,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             status: 'pending',
             retries: stepOptions?.retries ?? null,
             retryDelay: stepOptions?.retryDelay?.toString() ?? null,
+            fromStepName: fromStepName ?? null,
             createdAt: now,
             updatedAt: now,
         })
@@ -159,6 +169,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             attemptCount: 1,
             retries: stepOptions?.retries,
             retryDelay: stepOptions?.retryDelay?.toString(),
+            fromStepName,
             createdAt: now,
             updatedAt: now,
         };
@@ -173,6 +184,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             's.error',
             's.retries',
             's.retryDelay',
+            's.fromStepName',
             's.createdAt',
             's.updatedAt',
         ])
@@ -195,6 +207,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             attemptCount: Number(row.attemptCount),
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
+            fromStepName: row.fromStepName ?? undefined,
             createdAt: new Date(row.createdAt),
             updatedAt: new Date(row.updatedAt),
         };
@@ -348,6 +361,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             's.error',
             's.retries',
             's.retryDelay',
+            's.fromStepName',
             's.createdAt',
             's.updatedAt',
         ])
@@ -366,6 +380,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
             attemptCount: Number(row.attemptCount),
             retries: row.retries != null ? Number(row.retries) : undefined,
             retryDelay: row.retryDelay ?? undefined,
+            fromStepName: row.fromStepName ?? undefined,
             createdAt: new Date(row.createdAt),
             updatedAt: new Date(row.updatedAt),
         };
@@ -420,6 +435,18 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         const existingStepNames = new Set(result.map((r) => r.stepName));
         return nodeIds.filter((id) => !existingStepNames.has(id));
     }
+    async getStepInstances(runId) {
+        const rows = await this.db
+            .selectFrom('workflowStep')
+            .select(['stepName', 'status', 'fromStepName'])
+            .where('workflowRunId', '=', runId)
+            .execute();
+        return rows.map((r) => ({
+            stepName: r.stepName,
+            status: r.status,
+            fromStepName: r.fromStepName ?? undefined,
+        }));
+    }
     async getNodeResults(runId, nodeIds) {
         if (nodeIds.length === 0)
             return {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/kysely",
-  "version": "0.12.16",
+  "version": "0.12.17",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",
   "module": "dist/src/index.js",
@@ -20,7 +20,7 @@
     "prepublishOnly": "yarn build"
   },
   "peerDependencies": {
-    "@pikku/core": "^0.12.28"
+    "@pikku/core": "^0.12.38"
   },
   "dependencies": {
     "kysely": "^0.29.0"

package/src/kysely-tables.ts

@@ -51,6 +51,7 @@ export interface WorkflowStepTable {
   branchTaken: string | null
   retries: number | null
   retryDelay: string | null
+  fromStepName: string | null
   createdAt: Generated<Date>
   updatedAt: Generated<Date>
 }

package/src/kysely-workflow-service.ts

@@ -5,6 +5,7 @@ import {
   type WorkflowRun,
   type WorkflowRunWire,
   type StepState,
+  type StepStatus,
   type WorkflowStatus,
   type WorkflowVersionStatus,
 } from '@pikku/core/workflow'
@@ -79,6 +80,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       .addColumn('branch_taken', 'text')
       .addColumn('retries', 'integer')
       .addColumn('retry_delay', 'text')
+      .addColumn('from_step_name', 'text')
       .addColumn('created_at', 'timestamp', (col) =>
         col.defaultTo(sql`CURRENT_TIMESTAMP`).notNull()
       )
@@ -91,6 +93,15 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       ])
       .execute()
 
+    // Additive: backfill the column on tables created before from_step_name.
+    await this.db.schema
+      .alterTable('workflow_step')
+      .addColumn('from_step_name', 'text')
+      .execute()
+      .catch(() => {
+        // Column already exists (fresh tables include it above) — nothing to do.
+      })
+
     await this.db.schema
       .createTable('workflow_step_history')
       .ifNotExists()
@@ -196,7 +207,8 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     stepName: string,
     rpcName: string | null,
     data: any,
-    stepOptions?: { retries?: number; retryDelay?: string | number }
+    stepOptions?: { retries?: number; retryDelay?: string | number },
+    fromStepName?: string
   ): Promise<StepState> {
     const stepId = crypto.randomUUID()
     const now = new Date()
@@ -212,6 +224,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         status: 'pending',
         retries: stepOptions?.retries ?? null,
         retryDelay: stepOptions?.retryDelay?.toString() ?? null,
+        fromStepName: fromStepName ?? null,
         createdAt: now,
         updatedAt: now,
       })
@@ -227,6 +240,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       attemptCount: 1,
       retries: stepOptions?.retries,
       retryDelay: stepOptions?.retryDelay?.toString(),
+      fromStepName,
       createdAt: now,
       updatedAt: now,
     }
@@ -242,6 +256,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         's.error',
         's.retries',
         's.retryDelay',
+        's.fromStepName',
         's.createdAt',
         's.updatedAt',
       ])
@@ -274,6 +289,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       attemptCount: Number(row.attemptCount),
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
+      fromStepName: row.fromStepName ?? undefined,
       createdAt: new Date(row.createdAt),
       updatedAt: new Date(row.updatedAt),
     }
@@ -467,6 +483,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
         's.error',
         's.retries',
         's.retryDelay',
+        's.fromStepName',
         's.createdAt',
         's.updatedAt',
       ])
@@ -492,6 +509,7 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
       attemptCount: Number(row.attemptCount),
       retries: row.retries != null ? Number(row.retries) : undefined,
       retryDelay: row.retryDelay ?? undefined,
+      fromStepName: row.fromStepName ?? undefined,
       createdAt: new Date(row.createdAt),
       updatedAt: new Date(row.updatedAt),
     }
@@ -568,6 +586,21 @@ export class KyselyWorkflowService extends PikkuWorkflowService {
     return nodeIds.filter((id) => !existingStepNames.has(id))
   }
 
+  async getStepInstances(runId: string): Promise<
+    Array<{ stepName: string; status: StepStatus; fromStepName?: string }>
+  > {
+    const rows = await this.db
+      .selectFrom('workflowStep')
+      .select(['stepName', 'status', 'fromStepName'])
+      .where('workflowRunId', '=', runId)
+      .execute()
+    return rows.map((r) => ({
+      stepName: r.stepName,
+      status: r.status as StepStatus,
+      fromStepName: r.fromStepName ?? undefined,
+    }))
+  }
+
   async getNodeResults(
     runId: string,
     nodeIds: string[]