@pikku/core 0.12.36 → 0.12.38

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -69,6 +69,17 @@ import type { WorkflowService } from '../../services/workflow-service.js'
 import { PikkuError, addError } from '../../errors/error-handler.js'
 import { RPCNotFoundError } from '../rpc/rpc-runner.js'
 import { ChildWorkflowStartedException } from './graph/graph-runner.js'
+import { deriveInvocationId } from './workflow-invocation-id.js'
+import type { JobOptions } from '../queue/queue.types.js'
+
+/**
+ * Default number of retries for a workflow step when none is specified. The
+ * workflow — not the queue — owns retry policy; a step inherits this unless it
+ * sets its own `retries` (including `retries: 0` to opt out entirely). Picked >0
+ * so a transient failure (a DB blip, a downstream restart, a deploy) is ridden
+ * out by default; safe because every step gets a stable `invocationId` to dedupe on.
+ */
+export const DEFAULT_STEP_RETRIES = 5
 
 /**
  * Exception thrown when workflow needs to pause for async step
@@ -109,6 +120,25 @@ export class WorkflowSuspendedException extends Error {
   }
 }
 
+/**
+ * Thrown when a step (or the orchestrator) could not be enqueued — the queue
+ * itself failed (e.g. pg-boss is momentarily down), NOT the step's own logic.
+ * This is transient infrastructure failure: the run is left untouched (the step
+ * stays `pending`, the run stays running) and the orchestrator job is rethrown
+ * so the queue redelivers it and the workflow replays from its snapshot. Treat
+ * it as non-terminal — never mark the run `failed` for it.
+ */
+export class WorkflowDispatchException extends Error {
+  constructor(
+    public readonly runId: string,
+    public readonly stepName: string,
+    options?: { cause?: unknown }
+  ) {
+    super(`Failed to dispatch workflow step '${stepName}' (run ${runId})`, options)
+    this.name = 'WorkflowDispatchException'
+  }
+}
+
 /**
  * Error class for workflow not found
  */
@@ -471,14 +501,16 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepName: string,
     rpcName: string | null,
     data: any,
-    stepOptions?: WorkflowStepOptions
+    stepOptions?: WorkflowStepOptions,
+    fromStepName?: string
   ): Promise<StepState> {
     const step = await this.insertStepStateImpl(
       runId,
       stepName,
       rpcName,
       data,
-      stepOptions
+      stepOptions,
+      fromStepName
     )
     await this.safeMirror(() =>
       this.mirror!.insertStepState(runId, { ...step, stepName, rpcName, data })
@@ -491,7 +523,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepName: string,
     rpcName: string | null,
     data: any,
-    stepOptions?: WorkflowStepOptions
+    stepOptions?: WorkflowStepOptions,
+    fromStepName?: string
   ): Promise<StepState>
 
   /**
@@ -663,6 +696,20 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     nodeIds: string[]
   ): Promise<string[]>
 
+  /**
+   * List every step instance of a run (any status) with its predecessor.
+   * Drives bounded graph revisits: the runner counts instances per logical node
+   * and treats each `fromStepName → node` as a once-fired transition.
+   * @param runId - Run ID
+   */
+  abstract getStepInstances(runId: string): Promise<
+    Array<{
+      stepName: string
+      status: StepStatus
+      fromStepName?: string
+    }>
+  >
+
   /**
    * Get results for specific nodes
    * @param runId - Run ID
@@ -784,9 +831,42 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       const run = await this.getRun(runId)
       workflowName = run?.workflow
     }
-    await queueService.add(this.getOrchestratorQueueName(workflowName), {
-      runId,
-    })
+    // Carry an explicit retry policy on the orchestrator job too. Orchestrator
+    // runs are idempotent (they replay from the snapshot, returning cached step
+    // results), so redelivery is always safe — and it's what lets a transient
+    // dispatch/infra failure recover: the job is rethrown and retried instead of
+    // the run hanging. Passing `attempts` per-job overrides the queue default, so
+    // this holds even when the orchestrator queue is configured `retry_limit 0`.
+    await queueService.add(
+      this.getOrchestratorQueueName(workflowName),
+      { runId },
+      this.resolveStepJobOptions()
+    )
+  }
+
+  /**
+   * Resolve a step's retry policy into queue job options. The workflow is the
+   * sole source of truth for retries: an explicitly-set `retries` (including 0)
+   * is always honored, an unset one defaults to {@link DEFAULT_STEP_RETRIES},
+   * and we ALWAYS pass `attempts` so the queue can never fall back to its own
+   * default — which would re-run a step the workflow said not to retry. Backoff
+   * defaults to exponential whenever there's at least one retry, so retries ride
+   * out a transient outage instead of firing instantly.
+   */
+  protected resolveStepJobOptions(
+    stepOptions?: WorkflowStepOptions
+  ): JobOptions {
+    const retries = stepOptions?.retries ?? DEFAULT_STEP_RETRIES
+    const retryDelay = stepOptions?.retryDelay
+    const backoff =
+      typeof retryDelay === 'number'
+        ? { type: 'fixed', delay: retryDelay }
+        : retryDelay === 'exponential'
+          ? 'exponential'
+          : retries > 0
+            ? 'exponential'
+            : undefined
+    return { attempts: retries + 1, ...(backoff ? { backoff } : {}) }
   }
 
   public async queueStepWorker(
@@ -794,32 +874,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepName: string,
     rpcName: string,
     data: any,
-    stepOptions?: WorkflowStepOptions
+    stepOptions?: WorkflowStepOptions,
+    fromStepName?: string
   ): Promise<void> {
     const queueService = this.verifyQueueService()
-    const retries = stepOptions?.retries ?? 0
-    const retryDelay = stepOptions?.retryDelay
     await queueService.add(
       this.getStepWorkerQueueName(rpcName),
-      JSON.parse(
-        JSON.stringify({
-          runId,
-          stepName,
-          rpcName,
-          data,
-        })
-      ),
-      retries > 0 || retryDelay
-        ? {
-            attempts: retries + 1,
-            backoff:
-              typeof retryDelay === 'number'
-                ? { type: 'fixed', delay: retryDelay }
-                : retryDelay === 'exponential'
-                  ? 'exponential'
-                  : undefined,
-          }
-        : undefined
+      JSON.parse(JSON.stringify({ runId, stepName, rpcName, data, fromStepName })),
+      this.resolveStepJobOptions(stepOptions)
     )
   }
 
@@ -877,7 +939,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepName: string,
     rpcName: string,
     data: unknown,
-    stepOptions?: WorkflowStepOptions
+    stepOptions?: WorkflowStepOptions,
+    fromStepName?: string
   ): Promise<boolean> {
     // Step execution is decided purely by the function's `inline` flag (default
     // true). Only a function explicitly marked `inline: false` dispatches via
@@ -903,28 +966,21 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       )
       return false
     }
-    const retries = stepOptions?.retries ?? 0
-    const retryDelay = stepOptions?.retryDelay
-    await getSingletonServices()!.queueService!.add(
-      this.getStepWorkerQueueName(rpcName),
-      JSON.parse(
-        JSON.stringify({
-          runId,
-          stepName,
-          rpcName,
-          data,
-        })
-      ),
-      {
-        attempts: retries + 1,
-        backoff:
-          typeof retryDelay === 'number'
-            ? { type: 'fixed', delay: retryDelay }
-            : retryDelay === 'exponential'
-              ? 'exponential'
-              : undefined,
-      }
-    )
+    try {
+      await getSingletonServices()!.queueService!.add(
+        this.getStepWorkerQueueName(rpcName),
+        JSON.parse(
+          JSON.stringify({ runId, stepName, rpcName, data, fromStepName })
+        ),
+        this.resolveStepJobOptions(stepOptions)
+      )
+    } catch (cause) {
+      // The queue is down/unreachable — NOT a step failure. Surface it as a
+      // transient dispatch error so the caller leaves the step `pending` and the
+      // orchestrator job is retried (replayed from snapshot) rather than the run
+      // being marked failed. `add` already throws on failure in every adapter.
+      throw new WorkflowDispatchException(runId, stepName, { cause })
+    }
     return true
   }
 
@@ -1037,7 +1093,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         if (
           error.name !== 'WorkflowAsyncException' &&
           error.name !== 'WorkflowCancelledException' &&
-          error.name !== 'WorkflowSuspendedException'
+          error.name !== 'WorkflowSuspendedException' &&
+          // Transient queue failure — leave the run resumable, don't fail it.
+          error.name !== 'WorkflowDispatchException'
         ) {
           await this.updateRunStatus(runId, 'failed', undefined, {
             name: error.name,
@@ -1050,6 +1108,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           )
           throw error
         }
+        if (error.name === 'WorkflowDispatchException') {
+          // Rethrow so the caller (poll loop / starter) sees the transient
+          // failure; the run stays running and can be resumed.
+          throw error
+        }
       } finally {
         this.inlineRuns.delete(runId)
       }
@@ -1092,7 +1155,56 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
+  // Per-run, per-replay ordinal counters (runId → stepName → count).
+  private stepOrdinals = new Map<string, Map<string, number>>()
+  // Previous step key reached in the current DSL walk (runId → stepName), so a
+  // new step records where it came from. Rebuilt each replay alongside ordinals.
+  private stepLineage = new Map<string, string>()
+
+  private resetStepOrdinals(runId: string): void {
+    this.stepOrdinals.set(runId, new Map())
+    this.stepLineage.delete(runId)
+  }
+
+  /** The step the DSL walk last reached (the predecessor for the next step). */
+  private lastStepName(runId: string): string | undefined {
+    return this.stepLineage.get(runId)
+  }
+
+  /**
+   * Physical, replay-stable key for the Nth reach of `logicalStepName` in a run:
+   * bare name for the first reach (ordinal 0, unchanged behavior), `name#N` for
+   * repeats — so the same literal step name can be invoked multiple times without
+   * the rows clobbering. Deterministic given a deterministic DSL body.
+   */
+  private nextStepKey(runId: string, logicalStepName: string): string {
+    let perRun = this.stepOrdinals.get(runId)
+    if (!perRun) {
+      perRun = new Map()
+      this.stepOrdinals.set(runId, perRun)
+    }
+    const ordinal = perRun.get(logicalStepName) ?? 0
+    perRun.set(logicalStepName, ordinal + 1)
+    const stepName =
+      ordinal === 0 ? logicalStepName : `${logicalStepName}#${ordinal}`
+    this.stepLineage.set(runId, stepName)
+    return stepName
+  }
+
   public async runWorkflowJob(runId: string, rpcService: any): Promise<void> {
+    // Fresh ordinal counters per replay so step keys are deterministic.
+    this.resetStepOrdinals(runId)
+    try {
+      await this.runWorkflowJobInner(runId, rpcService)
+    } finally {
+      this.stepOrdinals.delete(runId)
+    }
+  }
+
+  private async runWorkflowJobInner(
+    runId: string,
+    rpcService: any
+  ): Promise<void> {
     const run = await this.getRun(runId)
     if (!run) {
       throw new WorkflowRunNotFoundError(runId)
@@ -1335,17 +1447,25 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         const isGraphWorkflow =
           workflowMeta?.source === 'graph' ||
           workflowMeta?.source === 'dynamic-workflow'
-        if (
-          isGraphWorkflow &&
-          workflowMeta?.nodes &&
-          stepName in workflowMeta.nodes
-        ) {
+        // Map the physical step key back to its logical node: a revisit instance
+        // is `node#N` (ordinal), which isn't a literal key in `nodes`.
+        let graphNodeId: string | undefined
+        if (isGraphWorkflow && workflowMeta?.nodes) {
+          if (stepName in workflowMeta.nodes) {
+            graphNodeId = stepName
+          } else {
+            const hash = stepName.lastIndexOf('#')
+            const base = hash > 0 ? stepName.slice(0, hash) : undefined
+            if (base && base in workflowMeta.nodes) graphNodeId = base
+          }
+        }
+        if (graphNodeId) {
           result = await executeGraphStep(
             this,
             rpcService,
             runId,
             stepState.stepId,
-            stepName,
+            graphNodeId,
             rpcName,
             data,
             run.workflow
@@ -1393,7 +1513,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
               workflowStep: {
                 runId,
                 stepId: stepState.stepId,
+                invocationId: deriveInvocationId(runId, stepName),
                 attemptCount: stepState.attemptCount,
+                fromInvocationId: stepState.fromStepName
+                  ? deriveInvocationId(runId, stepState.fromStepName)
+                  : undefined,
               },
             })
           }
@@ -1424,7 +1548,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         // Store error and mark failed
         await this.setStepError(stepState.stepId, error)
 
-        const maxAttempts = (stepState.retries ?? 0) + 1
+        const maxAttempts = (stepState.retries ?? DEFAULT_STEP_RETRIES) + 1
         const retriesExhausted = stepState.attemptCount >= maxAttempts
 
         if (retriesExhausted) {
@@ -1458,6 +1582,17 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         return
       }
 
+      if (error.name === 'WorkflowDispatchException') {
+        // Transient: the queue was unreachable, not a workflow failure. Leave the
+        // run running and rethrow so the orchestrator job is redelivered and the
+        // workflow replays from its snapshot. Do NOT mark the run failed.
+        getSingletonServices()!.logger.warn(
+          `Workflow run ${runId} could not dispatch a step (queue unavailable); leaving run for orchestrator retry`,
+          error
+        )
+        throw error
+      }
+
       await this.updateRunStatus(runId, 'failed', undefined, {
         message: error.message,
         stack: error.stack,
@@ -1480,12 +1615,23 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
   private async rpcStep(
     runId: string,
-    stepName: string,
+    logicalStepName: string,
     rpcName: string,
     data: any,
     rpcService: any,
     stepOptions?: WorkflowStepOptions
   ): Promise<any> {
+    // Capture the predecessor before nextStepKey advances the lineage to us.
+    const fromStepName = this.lastStepName(runId)
+    const stepName = this.nextStepKey(runId, logicalStepName)
+    // Resolve the retry policy ONCE here so the value persisted on the step
+    // (which drives `retriesExhausted`) is the same one the queue dispatch turns
+    // into `attempts`. Without this the queue could retry N times while the
+    // engine thinks retries are already exhausted (or vice-versa).
+    const resolvedStepOptions: WorkflowStepOptions = {
+      retries: stepOptions?.retries ?? DEFAULT_STEP_RETRIES,
+      retryDelay: stepOptions?.retryDelay,
+    }
     // Check if step already exists
     let stepState: StepState
     try {
@@ -1497,7 +1643,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         stepName,
         rpcName,
         data,
-        stepOptions
+        resolvedStepOptions,
+        fromStepName
       )
     }
 
@@ -1524,26 +1671,30 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       throw new WorkflowAsyncException(runId, stepName)
     }
 
-    // Step is pending - schedule it
-    await this.setStepScheduled(stepState.stepId)
-
     // Hand off to subclass-overridable transport. Default behavior enqueues
     // via the queue service; DO-style subclasses RPC to a step worker.
+    // Dispatch BEFORE marking the step `scheduled`: if the queue is down,
+    // dispatchStep throws WorkflowDispatchException and the step stays `pending`,
+    // so the orchestrator's next replay re-dispatches it. Marking `scheduled`
+    // first would strand the step (replay sees `scheduled`, pauses, never
+    // re-enqueues the job that was never created).
     const dispatched = await this.dispatchStep(
       runId,
       stepName,
       rpcName,
       data,
-      stepOptions
+      resolvedStepOptions,
+      fromStepName
     )
     if (dispatched) {
+      await this.setStepScheduled(stepState.stepId)
       throw new WorkflowAsyncException(runId, stepName)
     }
 
     {
       // Inline (no transport available) - execute locally with retry loop
-      const retries = stepOptions?.retries ?? 0
-      const retryDelay = stepOptions?.retryDelay
+      const retries = resolvedStepOptions.retries ?? this.getConfig().retries
+      const retryDelay = resolvedStepOptions.retryDelay
       let currentStepState = stepState
 
       while (true) {
@@ -1592,7 +1743,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
               workflowStep: {
                 runId,
                 stepId: currentStepState.stepId,
+                invocationId: deriveInvocationId(runId, stepName),
                 attemptCount: currentStepState.attemptCount,
+                fromInvocationId: currentStepState.fromStepName
+                  ? deriveInvocationId(runId, currentStepState.fromStepName)
+                  : undefined,
               },
             })
           }
@@ -1636,10 +1791,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
   private async inlineStep(
     runId: string,
-    stepName: string,
+    logicalStepName: string,
     fn: Function,
     stepOptions?: WorkflowStepOptions
   ): Promise<any> {
+    const fromStepName = this.lastStepName(runId)
+    const stepName = this.nextStepKey(runId, logicalStepName)
     // Check if step already exists
     let stepState: StepState
     try {
@@ -1651,7 +1808,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         stepName,
         null,
         null,
-        stepOptions
+        stepOptions,
+        fromStepName
       )
     }
 
@@ -1728,16 +1886,27 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
-  private async sleepStep(runId: string, stepName: string, duration: number) {
+  private async sleepStep(
+    runId: string,
+    logicalStepName: string,
+    duration: number
+  ) {
+    const fromStepName = this.lastStepName(runId)
+    const stepName = this.nextStepKey(runId, logicalStepName)
     // Check if step already exists
     let stepState: StepState
     try {
       stepState = await this.getStepState(runId, stepName)
     } catch {
       // Step doesn't exist - create it (sleep step, no RPC)
-      stepState = await this.insertStepState(runId, stepName, null, {
-        duration,
-      })
+      stepState = await this.insertStepState(
+        runId,
+        stepName,
+        null,
+        { duration },
+        undefined,
+        fromStepName
+      )
     }
 
     if (stepState.status === 'succeeded') {
@@ -1750,18 +1919,19 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       throw new WorkflowAsyncException(runId, stepName)
     }
 
-    // Step is pending - schedule it
-    await this.setStepScheduled(stepState.stepId)
-
     // Hand off to subclass-overridable transport. Default behavior schedules
     // a delayed sleeper RPC via the scheduler service; DO-style subclasses
-    // override to use native timer primitives (e.g. setAlarm).
-    const scheduled = await this.scheduleSleep(
-      runId,
-      stepState.stepId,
-      duration
-    )
+    // override to use native timer primitives (e.g. setAlarm). Schedule BEFORE
+    // marking `scheduled` so a scheduler outage leaves the step `pending` for
+    // re-scheduling on replay instead of stranding it (see rpcStep).
+    let scheduled: boolean
+    try {
+      scheduled = await this.scheduleSleep(runId, stepState.stepId, duration)
+    } catch (cause) {
+      throw new WorkflowDispatchException(runId, stepName, { cause })
+    }
     if (scheduled) {
+      await this.setStepScheduled(stepState.stepId)
       throw new WorkflowAsyncException(runId, stepName)
     }
 
@@ -1788,7 +1958,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   }
 
   private async suspendStep(runId: string, reason: string): Promise<void> {
-    const suspendStepName = this.getSuspendStepName(reason)
+    const fromStepName = this.lastStepName(runId)
+    const suspendStepName = this.nextStepKey(
+      runId,
+      this.getSuspendStepName(reason)
+    )
     await this.withStepLock(runId, suspendStepName, async () => {
       let stepState: StepState
       try {
@@ -1800,7 +1974,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           'pikkuWorkflowSuspend',
           {
             reason,
-          }
+          },
+          undefined,
+          fromStepName
         )
       }
       if (!stepState.stepId) {
@@ -1810,7 +1986,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           'pikkuWorkflowSuspend',
           {
             reason,
-          }
+          },
+          undefined,
+          fromStepName
         )
       }
 
@@ -1900,7 +2078,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     const singletonServices = getSingletonServices()
     const workflow = singletonServices.config?.workflow
     return {
-      retries: workflow?.retries ?? 0,
+      retries: workflow?.retries ?? DEFAULT_STEP_RETRIES,
       retryDelay: workflow?.retryDelay ?? 0,
       orchestratorQueueName:
         workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',