@pikku/core 0.12.70 → 0.12.72

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -337,15 +337,80 @@ export interface WorkflowRunExtension {
   ): Promise<void>
 }
 
+/**
+ * States a run never leaves. `suspended` is deliberately absent: a suspended
+ * run stops a poll loop but can still be resumed, so anything the process holds
+ * for it has to survive.
+ */
+const WORKFLOW_TERMINAL_STATES: ReadonlySet<string> = new Set([
+  'completed',
+  'failed',
+  'cancelled',
+])
+
+/** First wait when polling a run, before the backoff starts widening it. */
+const WORKFLOW_POLL_MIN_MS = 10
+
+/** How much each successive wait grows, up to the caller's ceiling. */
+const WORKFLOW_POLL_FACTOR = 1.6
+
+/**
+ * Ceiling for the wait on an inline sub-workflow. Lower than a top-level run's
+ * default, because the parent step is blocked on it and every wait here is
+ * added latency in the middle of a workflow rather than at its edge.
+ */
+const WORKFLOW_CHILD_POLL_MAX_MS = 500
+
 /**
  * Abstract workflow state service
  * Implementations provide pluggable storage backends (SQLite, PostgreSQL, etc.)
  * Combines orchestration and step execution
  */
+/**
+ * Everything the engine holds in memory for a run that is executing in this
+ * process. One entry, one lifetime: created when the run starts executing here
+ * and dropped when nothing is holding it open any more.
+ *
+ * The `replay` half is rebuilt from scratch on every orchestrator tick; the
+ * rest outlives individual ticks and belongs to whoever started the run.
+ */
+type RunContext = {
+  /** Executing straight through in-process, without a queue. */
+  inline: boolean
+  replay?: {
+    /** How many times this walk has reached each logical step name. */
+    ordinals: Map<string, number>
+    /** The step key the walk last reached — the next step's predecessor. */
+    lastStep?: string
+    /** Every step of the run as this replay found it, keyed by step name. */
+    steps?: Map<string, StepState>
+    /** The run as this replay found it. Only its immutable half is reused. */
+    run?: WorkflowRun
+  }
+}
+
 export abstract class PikkuWorkflowService implements WorkflowService {
-  private inlineRuns = new Set<string>()
   private runExtension?: WorkflowRunExtension
 
+  private runContexts = new Map<string, RunContext>()
+
+  private contextFor(runId: string): RunContext {
+    let context = this.runContexts.get(runId)
+    if (!context) {
+      context = { inline: false }
+      this.runContexts.set(runId, context)
+    }
+    return context
+  }
+
+  /** Drop a run's context once nothing is holding it open. */
+  private releaseContext(runId: string): void {
+    const context = this.runContexts.get(runId)
+    if (!context) return
+    if (context.inline || context.replay) return
+    this.runContexts.delete(runId)
+  }
+
   protected get logger() {
     return getSingletonServices()?.logger
   }
@@ -372,19 +437,36 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
-  private async safeMirror(fn: () => Promise<void>): Promise<void> {
-    if (!this.mirror) return
-    try {
-      await fn()
-    } catch (err: any) {
+  /**
+   * Perform a state write, then shadow it to the mirror.
+   *
+   * The mirror is an observability sink, never a second source of truth, and
+   * both halves of that follow from this one shape: it is only ever told about
+   * a write that already landed, and a mirror that is down or throwing cannot
+   * fail — or even be seen by — the workflow it is watching.
+   *
+   * @param write - the authoritative write; its result is what the caller gets
+   * @param mirror - shadows the write, given the live mirror and what was written
+   */
+  private async mirrored<T>(
+    write: () => Promise<T>,
+    mirror: (mirror: WorkflowRunMirror, written: T) => Promise<void>
+  ): Promise<T> {
+    const written = await write()
+    if (this.mirror) {
       try {
-        this.logger?.warn?.(
-          `[pikku] WorkflowRunMirror write failed: ${err?.message ?? err}`
-        )
-      } catch {
-        // logger unavailable (e.g. singleton services not initialized) — swallow
+        await mirror(this.mirror, written)
+      } catch (err: any) {
+        try {
+          this.logger?.warn?.(
+            `[pikku] WorkflowRunMirror write failed: ${err?.message ?? err}`
+          )
+        } catch {
+          // logger unavailable (e.g. singleton services not initialized) — swallow
+        }
       }
     }
+    return written
   }
 
   /**
@@ -520,21 +602,24 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * Check if a run is executing inline (without queues)
    */
   protected isInline(runId: string): boolean {
-    return this.inlineRuns.has(runId)
+    return this.runContexts.get(runId)?.inline === true
   }
 
   /**
    * Register a run as inline (for graph-runner to use)
    */
   public registerInlineRun(runId: string): void {
-    this.inlineRuns.add(runId)
+    this.contextFor(runId).inline = true
   }
 
   /**
    * Unregister a run from inline tracking
    */
   public unregisterInlineRun(runId: string): void {
-    this.inlineRuns.delete(runId)
+    const context = this.runContexts.get(runId)
+    if (!context) return
+    context.inline = false
+    this.releaseContext(runId)
   }
 
   public async registerWorkflowVersions(): Promise<void> {
@@ -556,26 +641,27 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       plannedSteps?: WorkflowPlannedStep[]
     }
   ): Promise<string> {
-    const runId = await this.createRunImpl(
-      workflowName,
-      input,
-      inline,
-      graphHash,
-      wire,
-      options
-    )
-    await this.safeMirror(() =>
-      this.mirror!.createRun(
-        runId,
-        workflowName,
-        input,
-        inline,
-        graphHash,
-        wire,
-        options
-      )
+    return this.mirrored(
+      () =>
+        this.createRunImpl(
+          workflowName,
+          input,
+          inline,
+          graphHash,
+          wire,
+          options
+        ),
+      (mirror, runId) =>
+        mirror.createRun(
+          runId,
+          workflowName,
+          input,
+          inline,
+          graphHash,
+          wire,
+          options
+        )
     )
-    return runId
   }
 
   protected abstract createRunImpl(
@@ -702,10 +788,17 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     output?: any,
     error?: SerializedError
   ): Promise<void> {
-    await this.updateRunStatusImpl(id, status, output, error)
-    await this.safeMirror(() =>
-      this.mirror!.updateRunStatus(id, status, output, error)
+    await this.mirrored(
+      () => this.updateRunStatusImpl(id, status, output, error),
+      (mirror) => mirror.updateRunStatus(id, status, output, error)
     )
+    if (WORKFLOW_TERMINAL_STATES.has(status)) {
+      // The run is over: release whatever this process opened for it. Queued
+      // runs never pass through the inline path that does this, so their
+      // context was held for the life of the process.
+      this.runExtension?.detachRunContext(id)
+      this.releaseContext(id)
+    }
   }
 
   protected abstract updateRunStatusImpl(
@@ -736,18 +829,19 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepOptions?: WorkflowStepOptions,
     fromStepName?: string
   ): Promise<StepState> {
-    const step = await this.insertStepStateImpl(
-      runId,
-      stepName,
-      rpcName,
-      data,
-      stepOptions,
-      fromStepName
-    )
-    await this.safeMirror(() =>
-      this.mirror!.insertStepState(runId, { ...step, stepName, rpcName, data })
+    return this.mirrored(
+      () =>
+        this.insertStepStateImpl(
+          runId,
+          stepName,
+          rpcName,
+          data,
+          stepOptions,
+          fromStepName
+        ),
+      (mirror, step) =>
+        mirror.insertStepState(runId, { ...step, stepName, rpcName, data })
     )
-    return step
   }
 
   protected abstract insertStepStateImpl(
@@ -773,8 +867,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * @param stepId - Step ID
    */
   public async setStepRunning(stepId: string): Promise<void> {
-    await this.setStepRunningImpl(stepId)
-    await this.safeMirror(() => this.mirror!.setStepRunning(stepId))
+    await this.mirrored(
+      () => this.setStepRunningImpl(stepId),
+      (mirror) => mirror.setStepRunning(stepId)
+    )
   }
 
   protected abstract setStepRunningImpl(stepId: string): Promise<void>
@@ -785,8 +881,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * @param stepId - Step ID
    */
   public async setStepScheduled(stepId: string): Promise<void> {
-    await this.setStepScheduledImpl(stepId)
-    await this.safeMirror(() => this.mirror!.setStepScheduled(stepId))
+    await this.mirrored(
+      () => this.setStepScheduledImpl(stepId),
+      (mirror) => mirror.setStepScheduled(stepId)
+    )
   }
 
   protected abstract setStepScheduledImpl(stepId: string): Promise<void>
@@ -798,8 +896,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * @param result - Step result
    */
   public async setStepResult(stepId: string, result: any): Promise<void> {
-    await this.setStepResultImpl(stepId, result)
-    await this.safeMirror(() => this.mirror!.setStepResult(stepId, result))
+    await this.mirrored(
+      () => this.setStepResultImpl(stepId, result),
+      (mirror) => mirror.setStepResult(stepId, result)
+    )
   }
 
   protected abstract setStepResultImpl(
@@ -816,9 +916,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepId: string,
     childRunId: string
   ): Promise<void> {
-    await this.setStepChildRunIdImpl(stepId, childRunId)
-    await this.safeMirror(() =>
-      this.mirror!.setStepChildRunId(stepId, childRunId)
+    await this.mirrored(
+      () => this.setStepChildRunIdImpl(stepId, childRunId),
+      (mirror) => mirror.setStepChildRunId(stepId, childRunId)
     )
   }
 
@@ -834,14 +934,18 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * @param error - Error object
    */
   public async setStepError(stepId: string, error: Error): Promise<void> {
-    await this.setStepErrorImpl(stepId, error)
-    const serialized: SerializedError = {
-      message: error.message,
-      stack: error.stack,
-      code: (error as any).code,
-      expected: isExpectedError(error),
-    }
-    await this.safeMirror(() => this.mirror!.setStepError(stepId, serialized))
+    await this.mirrored(
+      () => this.setStepErrorImpl(stepId, error),
+      (mirror) => {
+        const serialized: SerializedError = {
+          message: error.message,
+          stack: error.stack,
+          code: (error as any).code,
+          expected: isExpectedError(error),
+        }
+        return mirror.setStepError(stepId, serialized)
+      }
+    )
   }
 
   protected abstract setStepErrorImpl(
@@ -861,15 +965,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     failedStepId: string,
     status: 'pending' | 'running'
   ): Promise<StepState> {
-    const newStep = await this.createRetryAttemptImpl(failedStepId, status)
-    const stepName = (newStep as any).stepName ?? ''
-    await this.safeMirror(() =>
-      this.mirror!.createRetryAttempt(failedStepId, {
-        ...newStep,
-        stepName,
-      })
+    return this.mirrored(
+      () => this.createRetryAttemptImpl(failedStepId, status),
+      (mirror, newStep) =>
+        mirror.createRetryAttempt(failedStepId, {
+          ...newStep,
+          stepName: (newStep as any).stepName ?? '',
+        })
     )
-    return newStep
   }
 
   protected abstract createRetryAttemptImpl(
@@ -963,8 +1066,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepId: string,
     branchKey: string
   ): Promise<void> {
-    await this.setBranchTakenImpl(stepId, branchKey)
-    await this.safeMirror(() => this.mirror!.setBranchTaken(stepId, branchKey))
+    await this.mirrored(
+      () => this.setBranchTakenImpl(stepId, branchKey),
+      (mirror) => mirror.setBranchTaken(stepId, branchKey)
+    )
   }
 
   protected abstract setBranchTakenImpl(
@@ -983,8 +1088,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     name: string,
     value: unknown
   ): Promise<void> {
-    await this.updateRunStateImpl(runId, name, value)
-    await this.safeMirror(() => this.mirror!.updateRunState(runId, name, value))
+    await this.mirrored(
+      () => this.updateRunStateImpl(runId, name, value),
+      (mirror) => mirror.updateRunState(runId, name, value)
+    )
   }
 
   protected abstract updateRunStateImpl(
@@ -1007,9 +1114,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     source: string,
     status?: WorkflowVersionStatus
   ): Promise<void> {
-    await this.upsertWorkflowVersionImpl(name, graphHash, graph, source, status)
-    await this.safeMirror(() =>
-      this.mirror!.upsertWorkflowVersion(name, graphHash, graph, source, status)
+    await this.mirrored(
+      () =>
+        this.upsertWorkflowVersionImpl(name, graphHash, graph, source, status),
+      (mirror) =>
+        mirror.upsertWorkflowVersion(name, graphHash, graph, source, status)
     )
   }
 
@@ -1026,9 +1135,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     graphHash: string,
     status: WorkflowVersionStatus
   ): Promise<void> {
-    await this.updateWorkflowVersionStatusImpl(name, graphHash, status)
-    await this.safeMirror(() =>
-      this.mirror!.updateWorkflowVersionStatus(name, graphHash, status)
+    await this.mirrored(
+      () => this.updateWorkflowVersionStatusImpl(name, graphHash, status),
+      (mirror) => mirror.updateWorkflowVersionStatus(name, graphHash, status)
     )
   }
 
@@ -1043,10 +1152,6 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     graphHash: string
   ): Promise<{ graph: any; source: string } | null>
 
-  abstract getAIGeneratedWorkflows(
-    agentName?: string
-  ): Promise<Array<{ workflowName: string; graphHash: string; graph: any }>>
-
   // ============================================================================
   // Workflow Lifecycle Methods
   // ============================================================================
@@ -1116,9 +1221,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     const queueService = this.verifyQueueService()
     await queueService.add(
       this.getStepWorkerQueueName(rpcName),
-      JSON.parse(
-        JSON.stringify({ runId, stepName, rpcName, data, fromStepName })
-      ),
+      { runId, stepName, rpcName, data, fromStepName },
       {
         ...this.resolveStepJobOptions(stepOptions),
         // Group by step function, mirroring how per-step queues split them —
@@ -1210,9 +1313,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     try {
       await getSingletonServices()!.queueService!.add(
         this.getStepWorkerQueueName(rpcName),
-        JSON.parse(
-          JSON.stringify({ runId, stepName, rpcName, data, fromStepName })
-        ),
+        { runId, stepName, rpcName, data, fromStepName },
         {
           ...this.resolveStepJobOptions(stepOptions),
           group: this.getJobGroup(rpcName),
@@ -1299,27 +1400,16 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       onRunCreated?: (runId: string) => void
     }
   ): Promise<{ runId: string }> {
-    // Resolve workflow from static meta (root or addon namespace), then dynamic DB
+    // Resolve workflow from static meta (root or addon namespace)
     const resolved = resolveWorkflowMeta(name)
-    let workflowMeta = resolved?.meta
+    const workflowMeta = resolved?.meta
     const packageName = resolved?.packageName ?? null
 
-    if (!workflowMeta) {
-      const dynamicWorkflows = await this.getAIGeneratedWorkflows()
-      const match = dynamicWorkflows.find((w) => w.workflowName === name)
-      if (match?.graph) {
-        workflowMeta = match.graph
-      }
-    }
-
     if (!workflowMeta) {
       throw new WorkflowNotFoundError(name)
     }
 
-    if (
-      workflowMeta.source === 'graph' ||
-      workflowMeta.source === 'dynamic-workflow'
-    ) {
+    if (workflowMeta.source === 'graph') {
       const shouldInline =
         options?.inline || !getSingletonServices()?.queueService
       return runWorkflowGraph(
@@ -1366,7 +1456,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     await this.runExtension?.attachRunContext(runId, workflowMeta, options)
 
     if (shouldInline) {
-      this.inlineRuns.add(runId)
+      this.registerInlineRun(runId)
       try {
         await this.runWorkflowJob(runId, rpcService)
       } catch (error: any) {
@@ -1399,7 +1489,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           throw error
         }
       } finally {
-        this.inlineRuns.delete(runId)
+        this.unregisterInlineRun(runId)
         this.runExtension?.detachRunContext(runId)
       }
     } else {
@@ -1415,7 +1505,6 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     rpcService: any,
     options?: { pollIntervalMs?: number; wire?: WorkflowRunWire }
   ): Promise<any> {
-    const pollInterval = options?.pollIntervalMs ?? 1000
     const { runId } = await this.startWorkflow(
       name,
       input,
@@ -1423,38 +1512,155 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       rpcService,
       { inline: true }
     )
+    const run = await this.awaitRunEnd(runId, options?.pollIntervalMs ?? 1000)
+    if (run.status === 'failed') {
+      throw new WorkflowRunFailedError(run.error?.message)
+    }
+    if (run.status === 'cancelled') {
+      throw new WorkflowRunCancelledError()
+    }
+    return run.output
+  }
+
+  /**
+   * Read a run until it reaches an end state, backing off as it drags on.
+   *
+   * A fixed interval is wrong at both ends: it makes a workflow that finished
+   * in milliseconds wait out the whole interval anyway, and it keeps reading a
+   * long-running one at full rate for as long as it lasts. Starting short and
+   * growing to `maxIntervalMs` returns quick runs promptly while a slow run's
+   * read cost grows logarithmically rather than linearly with its duration.
+   */
+  protected async awaitRunEnd(
+    runId: string,
+    maxIntervalMs: number
+  ): Promise<WorkflowRun> {
+    let interval = Math.min(WORKFLOW_POLL_MIN_MS, maxIntervalMs)
     while (true) {
       const run = await this.getRun(runId)
       if (!run) {
         throw new WorkflowRunNotFoundError(runId)
       }
       if (WORKFLOW_END_STATES.has(run.status)) {
-        if (run.status === 'failed') {
-          throw new WorkflowRunFailedError(run.error?.message)
-        }
-        if (run.status === 'cancelled') {
-          throw new WorkflowRunCancelledError()
-        }
-        return run.output
+        return run
       }
-      await new Promise((resolve) => setTimeout(resolve, pollInterval))
+      await this.waitBeforeNextRead(interval)
+      interval = Math.min(interval * WORKFLOW_POLL_FACTOR, maxIntervalMs)
     }
   }
 
-  // Per-run, per-replay ordinal counters (runId → stepName → count).
-  private stepOrdinals = new Map<string, Map<string, number>>()
-  // Previous step key reached in the current DSL walk (runId → stepName), so a
-  // new step records where it came from. Rebuilt each replay alongside ordinals.
-  private stepLineage = new Map<string, string>()
+  /**
+   * Wait between two reads of a run.
+   *
+   * Its own method so the backoff schedule can be asserted on directly. Timing
+   * a poll loop by the clock measures the host's scheduler as much as the
+   * policy — `setTimeout(40)` routinely returns late on a loaded runner — which
+   * makes the obvious test both slow and flaky.
+   */
+  protected async waitBeforeNextRead(ms: number): Promise<void> {
+    await new Promise((resolve) => setTimeout(resolve, ms))
+  }
+
+  /**
+   * Every step of a run in one read, or `null` if this backend has no bulk read.
+   *
+   * A replay walks the DSL body from the top, and each step it passes asks for
+   * its own row — so a run of N steps costs N reads per replay and O(N^2) over
+   * its lifetime. Backends that can answer this in a single query collapse that
+   * to one read per replay.
+   */
+  protected async listStepStates(
+    _runId: string
+  ): Promise<Array<StepState & { stepName: string }> | null> {
+    return null
+  }
+
+  /**
+   * Begin a replay pass: fresh ordinal counters, and one read of the steps the
+   * run has already taken so the walk back to where it left off is served from
+   * memory. Safe because a pass reaches each step key at most once, and the
+   * steps it replays past are `succeeded` and therefore immutable.
+   */
+  private async beginReplay(runId: string): Promise<void> {
+    const context = this.contextFor(runId)
+    context.replay = { ordinals: new Map() }
+    const steps = await this.listStepStates(runId)
+    if (steps) {
+      context.replay.steps = new Map(steps.map((step) => [step.stepName, step]))
+    }
+  }
+
+  private endReplay(runId: string): void {
+    const context = this.runContexts.get(runId)
+    if (!context) return
+    context.replay = undefined
+    this.releaseContext(runId)
+  }
+
+  /**
+   * The step row for `stepName`, creating it if the run has not reached it
+   * before. Served from the replay snapshot when one is loaded.
+   */
+  private async loadOrCreateStep(
+    runId: string,
+    stepName: string,
+    create: () => Promise<StepState>
+  ): Promise<StepState> {
+    const snapshot = this.runContexts.get(runId)?.replay?.steps
+    if (snapshot) {
+      const cached = snapshot.get(stepName)
+      if (cached) {
+        return cached
+      }
+    } else {
+      try {
+        return await this.getStepState(runId, stepName)
+      } catch {
+        // No row yet — fall through and create it.
+      }
+    }
 
-  private resetStepOrdinals(runId: string): void {
-    this.stepOrdinals.set(runId, new Map())
-    this.stepLineage.delete(runId)
+    let step: StepState
+    try {
+      step = await create()
+    } catch (error) {
+      // A concurrent replay of this run created the row after the snapshot was
+      // taken. Its state is the truth; if it isn't really there, the insert
+      // failed for its own reasons and that error is the one worth seeing.
+      try {
+        step = await this.getStepState(runId, stepName)
+      } catch {
+        throw error
+      }
+    }
+    snapshot?.set(stepName, step)
+    return step
+  }
+
+  /**
+   * The run's immutable half — which workflow it is, the wire it was started
+   * on, its input. `getRun` is otherwise called several times per step for
+   * answers that were all fixed at creation, so a replay reads it once and
+   * hands the same object to everyone who only needs that half.
+   *
+   * Anyone who needs `status`, `output`, `error` or `state` must call `getRun`:
+   * those move while the run executes, and a cached copy would be a lie.
+   */
+  private async getRunIdentity(runId: string): Promise<WorkflowRun | null> {
+    const replay = this.runContexts.get(runId)?.replay
+    if (replay?.run) {
+      return replay.run
+    }
+    const run = await this.getRun(runId)
+    if (run && replay) {
+      replay.run = run
+    }
+    return run
   }
 
   /** The step the DSL walk last reached (the predecessor for the next step). */
   private lastStepName(runId: string): string | undefined {
-    return this.stepLineage.get(runId)
+    return this.runContexts.get(runId)?.replay?.lastStep
   }
 
   /**
@@ -1464,26 +1670,26 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    * the rows clobbering. Deterministic given a deterministic DSL body.
    */
   private nextStepKey(runId: string, logicalStepName: string): string {
-    let perRun = this.stepOrdinals.get(runId)
-    if (!perRun) {
-      perRun = new Map()
-      this.stepOrdinals.set(runId, perRun)
-    }
-    const ordinal = perRun.get(logicalStepName) ?? 0
-    perRun.set(logicalStepName, ordinal + 1)
+    const context = this.contextFor(runId)
+    const replay: NonNullable<RunContext['replay']> = (context.replay ??= {
+      ordinals: new Map(),
+    })
+    const ordinal = replay.ordinals.get(logicalStepName) ?? 0
+    replay.ordinals.set(logicalStepName, ordinal + 1)
     const stepName =
       ordinal === 0 ? logicalStepName : `${logicalStepName}#${ordinal}`
-    this.stepLineage.set(runId, stepName)
+    replay.lastStep = stepName
     return stepName
   }
 
   public async runWorkflowJob(runId: string, rpcService: any): Promise<void> {
-    // Fresh ordinal counters per replay so step keys are deterministic.
-    this.resetStepOrdinals(runId)
+    // Fresh ordinal counters per replay so step keys are deterministic, and one
+    // read of the steps the run has already taken.
+    await this.beginReplay(runId)
     try {
       await this.runWorkflowJobInner(runId, rpcService)
     } finally {
-      this.stepOrdinals.delete(runId)
+      this.endReplay(runId)
     }
   }
 
@@ -1491,7 +1697,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     runId: string,
     rpcService: any
   ): Promise<void> {
-    const run = await this.getRun(runId)
+    // Caches the run for the rest of this replay, so the steps it walks don't
+    // each re-read the workflow name and wire it already has.
+    const run = await this.getRunIdentity(runId)
     if (!run) {
       throw new WorkflowRunNotFoundError(runId)
     }
@@ -1509,10 +1717,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       return
     }
 
-    if (
-      workflowMeta?.source === 'graph' ||
-      workflowMeta?.source === 'dynamic-workflow'
-    ) {
+    if (workflowMeta?.source === 'graph') {
       await continueGraph(this, runId, run.workflow)
       const updatedRun = await this.getRun(runId)
       if (updatedRun?.status === 'completed') {
@@ -1529,29 +1734,6 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       return
     }
 
-    if (!workflowMeta) {
-      const dynamicWorkflows = await this.getAIGeneratedWorkflows()
-      const match = dynamicWorkflows.find(
-        (w) => w.workflowName === run.workflow
-      )
-      if (match?.graph) {
-        await continueGraph(this, runId, run.workflow, match.graph)
-        const updatedRun = await this.getRun(runId)
-        if (updatedRun?.status === 'completed') {
-          await this.onChildWorkflowCompleted(updatedRun, updatedRun.output)
-        } else if (
-          updatedRun?.status === 'failed' ||
-          updatedRun?.status === 'cancelled'
-        ) {
-          await this.onChildWorkflowFailed(
-            updatedRun,
-            new Error(updatedRun.error?.message || 'Child workflow failed')
-          )
-        }
-        return
-      }
-    }
-
     const registrations = pikkuState(pkgName, 'workflows', 'registrations')
     const workflow = registrations.get(resolved?.resolvedName ?? run.workflow)
     if (!workflow) {
@@ -1764,9 +1946,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       const meta = pikkuState(null, 'workflows', 'meta')
       const workflowMeta = meta[run.workflow]
 
-      const isGraphWorkflow =
-        workflowMeta?.source === 'graph' ||
-        workflowMeta?.source === 'dynamic-workflow'
+      const isGraphWorkflow = workflowMeta?.source === 'graph'
       // Map the physical step key back to its logical node: a revisit instance
       // is `node#N` (ordinal), which isn't a literal key in `nodes`.
       let graphNodeId: string | undefined
@@ -1833,7 +2013,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
             stepState,
             rpcName,
             data,
-            rpcService
+            rpcService,
+            run
           )
         }
       }
@@ -1938,11 +2119,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     stepState: StepState,
     rpcName: string,
     data: any,
-    rpcService: any
+    rpcService: any,
+    knownRun?: WorkflowRun | null
   ): Promise<any> {
     // Carry the run's pikkuUserId onto the step wire so authed steps rehydrate their
     // session on the queued path too (the bare job wire lacks it; inline already has it).
-    const run = await this.getRun(runId)
+    const run = knownRun ?? (await this.getRunIdentity(runId))
     return rpcService.rpcWithWire(rpcName, data, {
       ...(run?.wire?.pikkuUserId ? { pikkuUserId: run.wire.pikkuUserId } : {}),
       workflowStep: {
@@ -2052,13 +2234,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       actor: stepOptions?.actor,
       onError: stepOptions?.onError,
     }
-    // Check if step already exists
-    let stepState: StepState
-    try {
-      stepState = await this.getStepState(runId, stepName)
-    } catch {
-      // Step doesn't exist - create it
-      stepState = await this.insertStepState(
+    // Reuse the step if the run already reached it, otherwise create it.
+    const stepState = await this.loadOrCreateStep(runId, stepName, () =>
+      this.insertStepState(
         runId,
         stepName,
         rpcName,
@@ -2066,7 +2244,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         resolvedStepOptions,
         fromStepName
       )
-    }
+    )
 
     if (stepState.status === 'succeeded') {
       // Return cached result
@@ -2160,24 +2338,17 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           )
           await this.setStepChildRunId(currentStepState.stepId, childRunId)
           // Poll until child workflow completes
-          while (true) {
-            const childRun = await this.getRun(childRunId)
-            if (!childRun) {
-              throw new WorkflowRunNotFoundError(childRunId)
-            }
-            if (WORKFLOW_END_STATES.has(childRun.status)) {
-              if (childRun.status === 'failed') {
-                throw new Error(
-                  childRun.error?.message || 'Sub-workflow failed'
-                )
-              }
-              if (childRun.status === 'cancelled') {
-                throw new Error('Sub-workflow was cancelled')
-              }
-              return childRun.output
-            }
-            await new Promise((resolve) => setTimeout(resolve, 500))
+          const childRun = await this.awaitRunEnd(
+            childRunId,
+            WORKFLOW_CHILD_POLL_MAX_MS
+          )
+          if (childRun.status === 'failed') {
+            throw new Error(childRun.error?.message || 'Sub-workflow failed')
+          }
+          if (childRun.status === 'cancelled') {
+            throw new Error('Sub-workflow was cancelled')
           }
+          return childRun.output
         }
         return this.invokeStepRpc(
           runId,
@@ -2222,13 +2393,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   ): Promise<any> {
     const fromStepName = this.lastStepName(runId)
     const stepName = this.nextStepKey(runId, logicalStepName)
-    // Check if step already exists
-    let stepState: StepState
-    try {
-      stepState = await this.getStepState(runId, stepName)
-    } catch {
-      // Step doesn't exist - create it (inline, so never dispatched)
-      stepState = await this.insertStepState(
+    // Reuse the step if the run already reached it, otherwise create it
+    // (inline, so never dispatched).
+    const stepState = await this.loadOrCreateStep(runId, stepName, () =>
+      this.insertStepState(
         runId,
         stepName,
         rpcName,
@@ -2236,7 +2404,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         stepOptions,
         fromStepName
       )
-    }
+    )
 
     if (stepState.status === 'succeeded') {
       // Return cached result
@@ -2288,13 +2456,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   ) {
     const fromStepName = this.lastStepName(runId)
     const stepName = this.nextStepKey(runId, logicalStepName)
-    // Check if step already exists
-    let stepState: StepState
-    try {
-      stepState = await this.getStepState(runId, stepName)
-    } catch {
-      // Step doesn't exist - create it (sleep step, no RPC)
-      stepState = await this.insertStepState(
+    // Reuse the step if the run already reached it, otherwise create it
+    // (sleep step, no RPC).
+    const stepState = await this.loadOrCreateStep(runId, stepName, () =>
+      this.insertStepState(
         runId,
         stepName,
         null,
@@ -2302,7 +2467,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         undefined,
         fromStepName
       )
-    }
+    )
 
     if (stepState.status === 'succeeded') {
       // Sleep already completed, return immediately