@pikku/core 0.12.38 → 0.12.39

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -66,10 +66,20 @@ import {
   runFromMeta,
 } from './graph/graph-runner.js'
 import type { WorkflowService } from '../../services/workflow-service.js'
-import { PikkuError, addError } from '../../errors/error-handler.js'
+import {
+  PikkuError,
+  addError,
+  isExpectedError,
+} from '../../errors/error-handler.js'
 import { RPCNotFoundError } from '../rpc/rpc-runner.js'
 import { ChildWorkflowStartedException } from './graph/graph-runner.js'
 import { deriveInvocationId } from './workflow-invocation-id.js'
+import {
+  buildRunTimeline,
+  reconstructStateAt,
+  type RunTimeline,
+  type ReconstructedRunState,
+} from './run-timeline.js'
 import type { JobOptions } from '../queue/queue.types.js'
 
 /**
@@ -416,7 +426,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     // Build step summaries from history (latest attempt per step)
     const stepMap = new Map<
       string,
-      { status: StepStatus; startedAt?: Date; completedAt?: Date }
+      { status: StepStatus; startedAt?: Date; completedAt?: Date; attempts: number }
     >()
     for (const step of history) {
       const existing = stepMap.get(step.stepName)
@@ -425,6 +435,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           status: step.status,
           startedAt: step.runningAt ?? step.createdAt,
           completedAt: step.succeededAt ?? step.failedAt,
+          attempts: step.attemptCount,
         })
       }
     }
@@ -436,6 +447,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         s.startedAt && s.completedAt
           ? s.completedAt.getTime() - s.startedAt.getTime()
           : undefined,
+      attempts: s.attempts,
     }))
 
     return {
@@ -453,6 +465,33 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
+  /**
+   * Build the run's time-travel event stream from durable history.
+   * @param id - Run ID
+   * @returns Ordered timeline, or null if the run doesn't exist
+   */
+  public async getRunTimeline(id: string): Promise<RunTimeline | null> {
+    const run = await this.getRun(id)
+    if (!run) return null
+    return buildRunTimeline(await this.getRunHistory(id))
+  }
+
+  /**
+   * Reconstruct the run's state at a point in its timeline.
+   * @param id - Run ID
+   * @param at - A seq index (inclusive) or a Date (inclusive); omit for the
+   *             final state.
+   * @returns Reconstructed state, or null if the run doesn't exist
+   */
+  public async reconstructRunStateAt(
+    id: string,
+    at?: number | Date
+  ): Promise<ReconstructedRunState | null> {
+    const timeline = await this.getRunTimeline(id)
+    if (!timeline) return null
+    return reconstructStateAt(timeline, at ?? timeline.length - 1)
+  }
+
   /**
    * Get workflow run history (all step attempts in chronological order)
    * @param runId - Run ID
@@ -607,6 +646,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       message: error.message,
       stack: error.stack,
       code: (error as any).code,
+      expected: isExpectedError(error),
     }
     await this.safeMirror(() => this.mirror!.setStepError(stepId, serialized))
   }
@@ -1102,9 +1142,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
             message: error.message,
             stack: error.stack,
           })
+          // An expected failure (a PikkuError, e.g. a build gate tripping) —
+          // its message is the whole story, so don't dump the stack. The
+          // `expected` flag survives the step-boundary rehydration that strips
+          // the class. Anything else is an uncaught/unexpected error: log it in
+          // full so the trace is there to debug.
           getSingletonServices()!.logger.error(
             `Workflow ${name} (run ${runId}) failed:`,
-            error
+            isExpectedError(error) ? error.message : error
           )
           throw error
         }
@@ -1509,17 +1554,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
               )
             }
           } else {
-            result = await rpcService.rpcWithWire(rpcName, data, {
-              workflowStep: {
-                runId,
-                stepId: stepState.stepId,
-                invocationId: deriveInvocationId(runId, stepName),
-                attemptCount: stepState.attemptCount,
-                fromInvocationId: stepState.fromStepName
-                  ? deriveInvocationId(runId, stepState.fromStepName)
-                  : undefined,
-              },
-            })
+            result = await this.invokeStepRpc(
+              runId,
+              stepName,
+              stepState,
+              rpcName,
+              data,
+              rpcService
+            )
           }
         }
 
@@ -1613,6 +1655,82 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     return getSingletonServices()!.queueService!
   }
 
+  /**
+   * Invoke a step's RPC with the workflow-step wire (step identity + provenance).
+   * Identical for the queue executor and the inline executor — the only thing
+   * that differs between transports is who calls it, not the call itself.
+   */
+  private async invokeStepRpc(
+    runId: string,
+    stepName: string,
+    stepState: StepState,
+    rpcName: string,
+    data: any,
+    rpcService: any
+  ): Promise<any> {
+    return rpcService.rpcWithWire(rpcName, data, {
+      workflowStep: {
+        runId,
+        stepId: stepState.stepId,
+        invocationId: deriveInvocationId(runId, stepName),
+        attemptCount: stepState.attemptCount,
+        fromInvocationId: stepState.fromStepName
+          ? deriveInvocationId(runId, stepState.fromStepName)
+          : undefined,
+      },
+    })
+  }
+
+  /**
+   * Inline (straight-through) step execution with an in-process retry loop —
+   * shared by inline RPC steps and inline function steps. Same scaffolding
+   * (running → result, or fail → retry-attempt → backoff → retry) wrapped
+   * around a step-specific `doWork` body. Stays O(K): no suspend/replay.
+   *
+   * `onError` is an optional hook for terminal errors that must NOT retry
+   * (e.g. RPC-not-found → suspend the run for redeploy). If it throws, the
+   * loop exits immediately without recording a step error or retrying.
+   */
+  private async runInlineRetryLoop(
+    stepState: StepState,
+    retries: number,
+    retryDelay: WorkflowStepOptions['retryDelay'],
+    doWork: (currentStepState: StepState) => Promise<any>,
+    onError?: (error: any) => Promise<void>
+  ): Promise<any> {
+    let currentStepState = stepState
+    while (true) {
+      try {
+        await this.setStepRunning(currentStepState.stepId)
+        const result = await doWork(currentStepState)
+        await this.setStepResult(currentStepState.stepId, result)
+        return result
+      } catch (error: any) {
+        if (onError) await onError(error)
+
+        // Record the error (marks step as failed)
+        await this.setStepError(currentStepState.stepId, error)
+
+        if (currentStepState.attemptCount < retries) {
+          // Create a new pending retry attempt, then back off if configured.
+          currentStepState = await this.createRetryAttempt(
+            currentStepState.stepId,
+            'pending'
+          )
+          if (retryDelay) {
+            await new Promise((resolve) =>
+              setTimeout(resolve, getDurationInMilliseconds(retryDelay))
+            )
+          }
+          // Continue loop to retry
+        } else {
+          // No more retries, fail the workflow
+          throw error
+        }
+      }
+    }
+  }
+
   private async rpcStep(
     runId: string,
     logicalStepName: string,
@@ -1691,102 +1809,70 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       throw new WorkflowAsyncException(runId, stepName)
     }
 
-    {
-      // Inline (no transport available) - execute locally with retry loop
-      const retries = resolvedStepOptions.retries ?? this.getConfig().retries
-      const retryDelay = resolvedStepOptions.retryDelay
-      let currentStepState = stepState
-
-      while (true) {
-        try {
-          await this.setStepRunning(currentStepState.stepId)
-          // Check if the name refers to a workflow
-          const workflowMeta = pikkuState(null, 'workflows', 'meta')[rpcName]
-          let result: any
-          if (workflowMeta) {
-            const childWire = {
-              type: 'workflow',
-              id: rpcName,
-              parentRunId: runId,
-              pikkuUserId: rpcService.wire?.pikkuUserId,
+    // Inline (no transport available) - execute locally with the shared retry
+    // loop. The body resolves to a sub-workflow result or a plain RPC result.
+    const retries = resolvedStepOptions.retries ?? this.getConfig().retries
+    const retryDelay = resolvedStepOptions.retryDelay
+
+    return this.runInlineRetryLoop(
+      stepState,
+      retries,
+      retryDelay,
+      async (currentStepState) => {
+        // Check if the name refers to a workflow
+        const workflowMeta = pikkuState(null, 'workflows', 'meta')[rpcName]
+        if (workflowMeta) {
+          const childWire = {
+            type: 'workflow',
+            id: rpcName,
+            parentRunId: runId,
+            pikkuUserId: rpcService.wire?.pikkuUserId,
+          }
+          const { runId: childRunId } = await this.startWorkflow(
+            rpcName,
+            data,
+            childWire,
+            rpcService,
+            { inline: true }
+          )
+          await this.setStepChildRunId(currentStepState.stepId, childRunId)
+          // Poll until child workflow completes
+          while (true) {
+            const childRun = await this.getRun(childRunId)
+            if (!childRun) {
+              throw new WorkflowRunNotFoundError(childRunId)
             }
-            const { runId: childRunId } = await this.startWorkflow(
-              rpcName,
-              data,
-              childWire,
-              rpcService,
-              { inline: true }
-            )
-            await this.setStepChildRunId(currentStepState.stepId, childRunId)
-            // Poll until child workflow completes
-            while (true) {
-              const childRun = await this.getRun(childRunId)
-              if (!childRun) {
-                throw new WorkflowRunNotFoundError(childRunId)
+            if (WORKFLOW_END_STATES.has(childRun.status)) {
+              if (childRun.status === 'failed') {
+                throw new Error(childRun.error?.message || 'Sub-workflow failed')
               }
-              if (WORKFLOW_END_STATES.has(childRun.status)) {
-                if (childRun.status === 'failed') {
-                  throw new Error(
-                    childRun.error?.message || 'Sub-workflow failed'
-                  )
-                }
-                if (childRun.status === 'cancelled') {
-                  throw new Error('Sub-workflow was cancelled')
-                }
-                result = childRun.output
-                break
+              if (childRun.status === 'cancelled') {
+                throw new Error('Sub-workflow was cancelled')
               }
-              await new Promise((resolve) => setTimeout(resolve, 500))
-            }
-          } else {
-            result = await rpcService.rpcWithWire(rpcName, data, {
-              workflowStep: {
-                runId,
-                stepId: currentStepState.stepId,
-                invocationId: deriveInvocationId(runId, stepName),
-                attemptCount: currentStepState.attemptCount,
-                fromInvocationId: currentStepState.fromStepName
-                  ? deriveInvocationId(runId, currentStepState.fromStepName)
-                  : undefined,
-              },
-            })
-          }
-          await this.setStepResult(currentStepState.stepId, result)
-          return result
-        } catch (error: any) {
-          if (error instanceof RPCNotFoundError) {
-            await this.updateRunStatus(runId, 'suspended', undefined, {
-              message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
-              code: 'RPC_NOT_FOUND',
-            })
-            throw error
-          }
-
-          // Record the error (marks step as failed)
-          await this.setStepError(currentStepState.stepId, error)
-
-          // Check if we should retry
-          if (currentStepState.attemptCount < retries) {
-            // Create a new pending retry attempt
-            currentStepState = await this.createRetryAttempt(
-              currentStepState.stepId,
-              'pending'
-            )
-
-            // Wait for retry delay if specified
-            if (retryDelay) {
-              await new Promise((resolve) =>
-                setTimeout(resolve, getDurationInMilliseconds(retryDelay))
-              )
+              return childRun.output
             }
-            // Continue loop to retry
-          } else {
-            // No more retries, fail the workflow
-            throw error
+            await new Promise((resolve) => setTimeout(resolve, 500))
           }
         }
+        return this.invokeStepRpc(
+          runId,
+          stepName,
+          currentStepState,
+          rpcName,
+          data,
+          rpcService
+        )
+      },
+      async (error) => {
+        if (error instanceof RPCNotFoundError) {
+          await this.updateRunStatus(runId, 'suspended', undefined, {
+            message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+            code: 'RPC_NOT_FOUND',
+          })
+          throw error
+        }
       }
-    }
+    )
   }
 
   private async inlineStep(
@@ -1821,44 +1907,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     // Execute inline function
     const retries = stepOptions?.retries ?? this.getConfig().retries
     const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay
-    let currentStepState = stepState
 
     // Check if we're running inline (in-memory) or remote (queue-based)
     if (this.isInline(runId)) {
-      // Inline mode - execute with retry loop
-      while (true) {
-        try {
-          await this.setStepRunning(currentStepState.stepId)
-          const result = await fn()
-          await this.setStepResult(currentStepState.stepId, result)
-          return result
-        } catch (error: any) {
-          // Record the error (marks step as failed)
-          await this.setStepError(currentStepState.stepId, error)
-
-          // Check if we should retry
-          if (currentStepState.attemptCount < retries) {
-            // Create a new pending retry attempt
-            currentStepState = await this.createRetryAttempt(
-              currentStepState.stepId,
-              'pending'
-            )
-
-            // Wait for retry delay if specified
-            if (retryDelay) {
-              await new Promise((resolve) =>
-                setTimeout(resolve, getDurationInMilliseconds(retryDelay))
-              )
-            }
-            // Continue loop to retry
-          } else {
-            // No more retries, fail the workflow
-            throw error
-          }
-        }
-      }
+      // Inline mode - execute with the shared in-process retry loop.
+      return this.runInlineRetryLoop(stepState, retries, retryDelay, () => fn())
     } else {
-      // Remote mode - use queue-based retry
+      // Remote mode - single attempt, then suspend for orchestrator-driven retry.
+      let currentStepState = stepState
       try {
         await this.setStepRunning(currentStepState.stepId)
         const result = await fn()

package/src/wirings/workflow/run-timeline.test.ts

@@ -0,0 +1,211 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import {
+  buildRunTimeline,
+  reconstructStateAt,
+  reconstructFinalState,
+} from './run-timeline.js'
+import type { StepState } from './workflow.types.js'
+
+type HistoryEntry = StepState & { stepName: string }
+
+const T = (ms: number) => new Date(1_700_000_000_000 + ms)
+
+/** A succeeded step attempt with the standard lifecycle timestamps. */
+const ok = (
+  stepName: string,
+  base: number,
+  opts: { from?: string; result?: unknown; attempt?: number } = {}
+): HistoryEntry =>
+  ({
+    stepId: `${stepName}-${opts.attempt ?? 1}`,
+    stepName,
+    status: 'succeeded',
+    attemptCount: opts.attempt ?? 1,
+    fromStepName: opts.from,
+    result: opts.result,
+    createdAt: T(base),
+    runningAt: T(base + 1),
+    succeededAt: T(base + 2),
+    updatedAt: T(base + 2),
+  }) as HistoryEntry
+
+const failed = (
+  stepName: string,
+  base: number,
+  opts: { from?: string; attempt?: number; message?: string } = {}
+): HistoryEntry =>
+  ({
+    stepId: `${stepName}-${opts.attempt ?? 1}`,
+    stepName,
+    status: 'failed',
+    attemptCount: opts.attempt ?? 1,
+    fromStepName: opts.from,
+    error: { message: opts.message ?? 'boom' },
+    createdAt: T(base),
+    runningAt: T(base + 1),
+    failedAt: T(base + 2),
+    updatedAt: T(base + 2),
+  }) as HistoryEntry
+
+describe('buildRunTimeline', () => {
+  test('explodes each attempt into ordered lifecycle events', () => {
+    const tl = buildRunTimeline([ok('begin', 0, { result: { n: 1 } })])
+    assert.deepEqual(
+      tl.map((e) => e.type),
+      ['pending', 'running', 'succeeded']
+    )
+    // seq is monotonic from 0
+    assert.deepEqual(
+      tl.map((e) => e.seq),
+      [0, 1, 2]
+    )
+    // result rides only on the succeeded event
+    assert.deepEqual(tl[2]!.result, { n: 1 })
+    assert.equal(tl[0]!.result, undefined)
+  })
+
+  test('orders across steps by timestamp, lifecycle, then history index', () => {
+    // two steps; b created at the same instant begin succeeds
+    const tl = buildRunTimeline([
+      ok('begin', 0, { result: 1 }),
+      ok('next', 2, { from: 'begin', result: 2 }),
+    ])
+    // begin.pending(0) begin.running(1) begin.succeeded(2)==next.pending(2)
+    // → succeeded sorts before pending at the same instant (lifecycle 3 vs 0)?
+    // No: pending=0 < succeeded=3, so next.pending precedes begin.succeeded.
+    const at2 = tl.filter((e) => e.at.getTime() === T(2).getTime())
+    assert.deepEqual(
+      at2.map((e) => `${e.stepName}:${e.type}`),
+      ['next:pending', 'begin:succeeded']
+    )
+  })
+
+  test('carries provenance on the created event only', () => {
+    const tl = buildRunTimeline([ok('next', 5, { from: 'begin' })])
+    const created = tl.find((e) => e.type === 'pending')!
+    assert.equal(created.fromStepName, 'begin')
+    assert.equal(
+      tl.find((e) => e.type === 'succeeded')!.fromStepName,
+      undefined
+    )
+  })
+
+  test('terminal event is driven by status, not lifecycle timestamps', () => {
+    // Kysely leaves succeededAt/runningAt null but the row's status + result
+    // are authoritative — the terminal event must still fire (off updatedAt).
+    const noStamps = {
+      stepId: 's-1',
+      stepName: 'begin',
+      status: 'succeeded',
+      attemptCount: 1,
+      result: { ok: 1 },
+      createdAt: T(0),
+      updatedAt: T(5),
+    } as HistoryEntry
+    const tl = buildRunTimeline([noStamps])
+    assert.deepEqual(
+      tl.map((e) => e.type),
+      ['pending', 'succeeded']
+    )
+    const succeeded = tl.find((e) => e.type === 'succeeded')!
+    assert.equal(succeeded.at.getTime(), T(5).getTime(), 'falls back to updatedAt')
+    assert.deepEqual(succeeded.result, { ok: 1 })
+    assert.deepEqual(reconstructFinalState(tl).results, { begin: { ok: 1 } })
+  })
+})
+
+describe('reconstructStateAt', () => {
+  // begin → next → finish, each succeeding in order
+  const history: HistoryEntry[] = [
+    ok('begin', 0, { result: { step: 'begin' } }),
+    ok('next', 10, { from: 'begin', result: { step: 'next' } }),
+    ok('finish', 20, { from: 'next', result: { step: 'finish' } }),
+  ]
+  const tl = buildRunTimeline(history)
+
+  test('a point before the first event is the empty initial state', () => {
+    const s = reconstructStateAt(tl, -1)
+    assert.equal(s.seq, -1)
+    assert.deepEqual(s.steps, [])
+    assert.deepEqual(s.results, {})
+    assert.equal(s.phase, 'pending')
+  })
+
+  test('mid-run by seq: only steps up to that event exist', () => {
+    // fold through begin.succeeded (seq 2)
+    const s = reconstructStateAt(tl, 2)
+    assert.deepEqual(s.path, ['begin'])
+    assert.equal(s.steps[0]!.status, 'succeeded')
+    assert.deepEqual(s.results, { begin: { step: 'begin' } })
+    // next hasn't been created yet at this point
+    assert.equal(s.phase, 'idle')
+  })
+
+  test('mid-run captures an in-flight step as running', () => {
+    // begin done (0,1,2), next created+running (3,4) but not yet succeeded
+    const s = reconstructStateAt(tl, 4)
+    assert.deepEqual(s.path, ['begin', 'next'])
+    assert.equal(s.steps[1]!.stepName, 'next')
+    assert.equal(s.steps[1]!.status, 'running')
+    assert.equal(s.steps[1]!.fromStepName, 'begin')
+    assert.equal(s.phase, 'running')
+    // next's result not yet available in the replay cache
+    assert.deepEqual(s.results, { begin: { step: 'begin' } })
+  })
+
+  test('time-travel by Date folds every event at or before the instant', () => {
+    const s = reconstructStateAt(tl, T(12))
+    // begin fully done (<=2), next created(10)+running(11) but succeeded at 12
+    assert.deepEqual(s.path, ['begin', 'next'])
+    assert.equal(s.steps[1]!.status, 'succeeded')
+    assert.deepEqual(s.results, {
+      begin: { step: 'begin' },
+      next: { step: 'next' },
+    })
+  })
+
+  test('final state has the full walked path and all results', () => {
+    const s = reconstructFinalState(tl)
+    assert.deepEqual(s.path, ['begin', 'next', 'finish'])
+    assert.deepEqual(s.results, {
+      begin: { step: 'begin' },
+      next: { step: 'next' },
+      finish: { step: 'finish' },
+    })
+    assert.equal(s.phase, 'idle')
+  })
+})
+
+describe('reconstructStateAt — retries', () => {
+  // attempt 1 fails, attempt 2 (created later) succeeds
+  const history: HistoryEntry[] = [
+    failed('flaky', 0, { message: 'first try' }),
+    ok('flaky', 10, { result: { ok: true }, attempt: 2 }),
+  ]
+  const tl = buildRunTimeline(history)
+
+  test('between attempts the step is failed', () => {
+    const s = reconstructStateAt(tl, T(2)) // through failedAt of attempt 1
+    assert.equal(s.steps[0]!.status, 'failed')
+    assert.equal(s.steps[0]!.error?.message, 'first try')
+    assert.equal(s.phase, 'failed')
+    assert.deepEqual(s.results, {})
+  })
+
+  test("the retry's created event reopens the step and clears the error", () => {
+    const s = reconstructStateAt(tl, T(10)) // attempt-2 pending
+    assert.equal(s.steps[0]!.status, 'pending')
+    assert.equal(s.steps[0]!.error, undefined)
+    assert.equal(s.steps[0]!.attemptCount, 2)
+    assert.equal(s.phase, 'running')
+  })
+
+  test('after the retry succeeds the result is available', () => {
+    const s = reconstructFinalState(tl)
+    assert.equal(s.steps[0]!.status, 'succeeded')
+    assert.deepEqual(s.results, { flaky: { ok: true } })
+    assert.equal(s.path.length, 1, 'a retried step is still one path entry')
+  })
+})