@pikku/core 0.12.69 → 0.12.71

package/src/wirings/workflow/workflow-dispatch-payload.test.ts

@@ -0,0 +1,59 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/**
+ * Captures what the workflow service hands the queue, so a test can tell
+ * whether core serialised the payload on the way past.
+ */
+function serviceWithCapturingQueue() {
+  const added: Array<{ queueName: string; data: any }> = []
+  pikkuState(null, 'package', 'singletonServices', {
+    queueService: {
+      add: async (queueName: string, data: any) => {
+        added.push({ queueName, data })
+        return 'job-1'
+      },
+    },
+    logger: silentLogger,
+  } as any)
+  return { ws: new InMemoryWorkflowService() as any, added }
+}
+
+describe('dispatching a step does not serialise the payload twice', () => {
+  test('the queue receives the payload core was given', async () => {
+    const { ws, added } = serviceWithCapturingQueue()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    const payload = { when: new Date('2020-01-01T00:00:00.000Z'), n: 1 }
+
+    await ws.queueStepWorker(runId, 'step-1', 'rpc.fn', payload)
+
+    assert.equal(added.length, 1)
+    assert.ok(
+      added[0]!.data.data.when instanceof Date,
+      'core round-tripped the payload through JSON before handing it to a queue that serialises it anyway'
+    )
+    assert.equal(added[0]!.data.data.n, 1)
+    assert.equal(added[0]!.data.runId, runId)
+    assert.equal(added[0]!.data.stepName, 'step-1')
+  })
+
+  test('a payload the queue cannot serialise is still the queue’s problem, not a silent drop', async () => {
+    const { ws, added } = serviceWithCapturingQueue()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    const circular: any = { name: 'loop' }
+    circular.self = circular
+
+    await ws.queueStepWorker(runId, 'step-1', 'rpc.fn', circular)
+
+    assert.equal(added[0]!.data.data.self, added[0]!.data.data)
+  })
+})

package/src/wirings/workflow/workflow-mirror.test.ts

@@ -0,0 +1,178 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+import type { WorkflowRunMirror } from './workflow.types.js'
+
+/**
+ * Every method on `WorkflowRunMirror`. A mirror method that is added to the
+ * interface but never wired into the service writes nothing at runtime, and
+ * nothing else would catch that — so the list is spelled out here.
+ */
+const MIRROR_METHODS = [
+  'createRun',
+  'updateRunStatus',
+  'insertStepState',
+  'setStepRunning',
+  'setStepScheduled',
+  'setStepResult',
+  'setStepChildRunId',
+  'setStepError',
+  'createRetryAttempt',
+  'setBranchTaken',
+  'updateRunState',
+  'upsertWorkflowVersion',
+  'updateWorkflowVersionStatus',
+] as const
+
+type Call = { method: string; args: any[] }
+
+/** A mirror that records every call, and optionally throws on all of them. */
+const recordingMirror = (
+  onCall?: (method: string) => void
+): { mirror: WorkflowRunMirror; calls: Call[] } => {
+  const calls: Call[] = []
+  const mirror = {} as any
+  for (const method of MIRROR_METHODS) {
+    mirror[method] = async (...args: any[]) => {
+      calls.push({ method, args })
+      onCall?.(method)
+    }
+  }
+  return { mirror, calls }
+}
+
+const service = (mirror?: WorkflowRunMirror) => {
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: { error() {}, info() {}, warn() {}, debug() {} },
+  } as any)
+  const ws = new InMemoryWorkflowService() as any
+  ws.mirror = mirror
+  return ws
+}
+
+/** Drive every write the service mirrors, in dependency order. */
+const driveEveryWrite = async (ws: any) => {
+  const runId = await ws.createRun('flow', { a: 1 }, false, 'hash', {
+    type: 'test',
+  })
+  const step = await ws.insertStepState(runId, 'step-1', 'rpc.fn', { x: 1 })
+  await ws.setStepRunning(step.stepId)
+  await ws.setStepScheduled(step.stepId)
+  await ws.setStepResult(step.stepId, 'ok')
+  await ws.setStepChildRunId(step.stepId, 'child-1')
+  await ws.setStepError(step.stepId, new Error('boom'))
+  await ws.createRetryAttempt(step.stepId, 'pending')
+  await ws.setBranchTaken(step.stepId, 'left')
+  await ws.updateRunState(runId, 'total', 42)
+  await ws.upsertWorkflowVersion('flow', 'hash', { nodes: {} }, 'code')
+  await ws.updateWorkflowVersionStatus('flow', 'hash', 'archived')
+  await ws.updateRunStatus(runId, 'completed', 'output')
+  return { runId, step }
+}
+
+describe('mirroring workflow state', () => {
+  test('every mirror method is wired to the write it shadows', async () => {
+    const { mirror, calls } = recordingMirror()
+    await driveEveryWrite(service(mirror))
+
+    assert.deepEqual(
+      [...new Set(calls.map((c) => c.method))].sort(),
+      [...MIRROR_METHODS].sort()
+    )
+  })
+
+  test('a mirror write carries the same identifiers as the write it shadows', async () => {
+    const { mirror, calls } = recordingMirror()
+    const ws = service(mirror)
+    const { runId, step } = await driveEveryWrite(ws)
+
+    const call = (method: string) => calls.find((c) => c.method === method)!
+    assert.equal(call('createRun').args[0], runId)
+    assert.equal(call('createRun').args[1], 'flow')
+    assert.equal(call('insertStepState').args[0], runId)
+    assert.equal(call('insertStepState').args[1].stepId, step.stepId)
+    assert.equal(call('insertStepState').args[1].stepName, 'step-1')
+    assert.equal(call('insertStepState').args[1].rpcName, 'rpc.fn')
+    assert.equal(call('setStepResult').args[1], 'ok')
+    assert.equal(call('setStepChildRunId').args[1], 'child-1')
+    assert.equal(call('setBranchTaken').args[1], 'left')
+    assert.equal(call('updateRunState').args[2], 42)
+    assert.equal(call('updateRunStatus').args[1], 'completed')
+  })
+
+  test('a step error reaches the mirror serialised, not as a live Error', async () => {
+    const { mirror, calls } = recordingMirror()
+    await driveEveryWrite(service(mirror))
+
+    const serialised = calls.find((c) => c.method === 'setStepError')!.args[1]
+    assert.equal(serialised instanceof Error, false)
+    assert.equal(serialised.message, 'boom')
+    assert.equal(typeof serialised.stack, 'string')
+    assert.equal(typeof serialised.expected, 'boolean')
+  })
+
+  test('the mirror only ever sees a write that already landed', async () => {
+    const seen: Array<string | undefined> = []
+    const { mirror } = recordingMirror()
+    const ws = service(mirror)
+    ;(mirror as any).updateRunStatus = async () => {
+      seen.push((await ws.getRun(runIdRef.id))?.status)
+    }
+    const runIdRef = { id: '' }
+
+    runIdRef.id = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.updateRunStatus(runIdRef.id, 'completed', 'output')
+
+    assert.deepEqual(
+      seen,
+      ['completed'],
+      'the mirror ran before the write it was mirroring'
+    )
+  })
+
+  test('a write that fails is not mirrored', async () => {
+    const { mirror, calls } = recordingMirror()
+    const ws = service(mirror)
+    ws.updateRunStatusImpl = async () => {
+      throw new Error('database down')
+    }
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+
+    await assert.rejects(
+      ws.updateRunStatus(runId, 'completed'),
+      /database down/
+    )
+    assert.equal(
+      calls.some((c) => c.method === 'updateRunStatus'),
+      false,
+      'a status the store rejected was reported to the mirror as if it had happened'
+    )
+  })
+})
+
+describe('a mirror that is broken', () => {
+  test('no write fails because the mirror did', async () => {
+    const { mirror } = recordingMirror((method) => {
+      throw new Error(`mirror down on ${method}`)
+    })
+    const ws = service(mirror)
+
+    const { runId } = await driveEveryWrite(ws)
+
+    assert.equal((await ws.getRun(runId))?.status, 'completed')
+    assert.equal((await ws.getRun(runId))?.output, 'output')
+    assert.deepEqual(await ws.getRunState(runId), { total: 42 })
+  })
+
+  test('no mirror configured is not an error path', async () => {
+    const ws = service(undefined)
+    const { runId } = await driveEveryWrite(ws)
+    assert.equal((await ws.getRun(runId))?.status, 'completed')
+  })
+})

package/src/wirings/workflow/workflow-replay-snapshot.test.ts

@@ -0,0 +1,139 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+function inlineService() {
+  pikkuState(null, 'package', 'singletonServices', {
+    queueService: { add: async () => {} },
+    logger: silentLogger,
+  } as any)
+  return new InMemoryWorkflowService()
+}
+
+/**
+ * Every per-step read the service issues. In a real backend each one is its own
+ * round-trip, so a replay that walks N completed steps costs N of them — and
+ * over the life of a run that is O(N^2).
+ */
+function countStepReads(ws: any) {
+  const counter = { reads: 0 }
+  const original = ws.getStepState.bind(ws)
+  ws.getStepState = async (...args: any[]) => {
+    counter.reads++
+    return original(...args)
+  }
+  return counter
+}
+
+async function seedRun(ws: any, steps: number) {
+  const runId = await ws.createRun('flow', {}, true, 'hash', { type: 'test' })
+  ws.registerInlineRun(runId)
+  await ws.beginReplay(runId)
+  for (let i = 0; i < steps; i++) {
+    await ws.inlineStep(runId, `step-${i}`, async () => i)
+  }
+  ws.endReplay(runId)
+  return runId
+}
+
+describe('replay reads the run steps once, not once per step', () => {
+  test('a replay over completed steps issues no per-step reads', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 8)
+
+    const counter = countStepReads(ws)
+    await ws.beginReplay(runId)
+    for (let i = 0; i < 8; i++) {
+      await ws.inlineStep(runId, `step-${i}`, async () => {
+        throw new Error(`step-${i} re-ran instead of replaying its cached result`)
+      })
+    }
+    ws.endReplay(runId)
+
+    assert.equal(
+      counter.reads,
+      0,
+      `the replay still read each step individually (${counter.reads} reads for 8 steps)`
+    )
+  })
+
+  test('the snapshot still returns each step its own cached result', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 4)
+
+    await ws.beginReplay(runId)
+    const replayed: unknown[] = []
+    for (let i = 0; i < 4; i++) {
+      replayed.push(
+        await ws.inlineStep(runId, `step-${i}`, async () => 'must not run')
+      )
+    }
+    ws.endReplay(runId)
+
+    assert.deepEqual(replayed, [0, 1, 2, 3])
+  })
+
+  test('a step reached for the first time mid-replay is created, not mistaken for a miss', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 2)
+
+    await ws.beginReplay(runId)
+    await ws.inlineStep(runId, 'step-0', async () => 'cached')
+    await ws.inlineStep(runId, 'step-1', async () => 'cached')
+    const fresh = await ws.inlineStep(runId, 'step-new', async () => 'ran')
+    ws.endReplay(runId)
+
+    assert.equal(fresh, 'ran')
+    assert.equal((await ws.getStepState(runId, 'step-new')).result, 'ran')
+  })
+
+  test('a second replay sees what the previous one wrote', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 1)
+
+    await ws.beginReplay(runId)
+    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
+    await ws.inlineStep(runId, 'late', async () => 'first')
+    ws.endReplay(runId)
+
+    await ws.beginReplay(runId)
+    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
+    const late = await ws.inlineStep(runId, 'late', async () => 'must not run')
+    ws.endReplay(runId)
+
+    assert.equal(late, 'first', 'the second replay re-ran a step the first one completed')
+  })
+
+  test('outside a replay, steps are still read one at a time', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 3)
+
+    const counter = countStepReads(ws)
+    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
+
+    assert.equal(
+      counter.reads,
+      1,
+      'a call outside a replay must read the step directly rather than trust a stale snapshot'
+    )
+  })
+
+  test('a backend with no bulk read falls back to per-step reads', async () => {
+    const ws: any = inlineService()
+    const runId = await seedRun(ws, 3)
+    ws.listStepStates = async () => null
+
+    const counter = countStepReads(ws)
+    await ws.beginReplay(runId)
+    for (let i = 0; i < 3; i++) {
+      await ws.inlineStep(runId, `step-${i}`, async () => 'must not run')
+    }
+    ws.endReplay(runId)
+
+    assert.equal(counter.reads, 3)
+  })
+})

package/src/wirings/workflow/workflow-run-context.test.ts

@@ -0,0 +1,177 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+function service() {
+  pikkuState(null, 'package', 'singletonServices', {
+    queueService: { add: async () => {} },
+    logger: silentLogger,
+  } as any)
+  return new InMemoryWorkflowService() as any
+}
+
+const newRun = (ws: any, inline = false) =>
+  ws.createRun('flow', {}, inline, 'hash', { type: 'test' })
+
+const trackDetaches = (ws: any): string[] => {
+  const detached: string[] = []
+  ws.setRunExtension(() => ({
+    attachRunContext: async () => {},
+    detachRunContext: (runId: string) => detached.push(runId),
+    decorateRunWire: () => {},
+    decorateWorkflowWire: () => {},
+    onBeforeRunFunc: async () => {},
+    onAfterRunFunc: async () => {},
+  }))
+  return detached
+}
+
+/**
+ * A long-lived process orchestrates many runs. Anything it keeps per run has to
+ * be released when that run stops executing here, or the engine grows for the
+ * life of the deployment.
+ */
+describe('per-run state is released when the run stops executing here', () => {
+  test('a finished replay leaves nothing behind', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    await ws.beginReplay(runId)
+    await ws.inlineStep(runId, 'a', async () => 1)
+    ws.endReplay(runId)
+
+    assert.equal(
+      ws.runContexts.size,
+      0,
+      'the replay held on to its ordinals, lineage or step snapshot'
+    )
+  })
+
+  test('the step the walk last reached does not survive its replay', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    await ws.beginReplay(runId)
+    await ws.inlineStep(runId, 'a', async () => 1)
+    ws.endReplay(runId)
+
+    assert.equal(ws.lastStepName(runId), undefined)
+  })
+
+  test('unregistering an inline run releases it', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    ws.registerInlineRun(runId)
+    assert.equal(ws.isInline(runId), true)
+
+    ws.unregisterInlineRun(runId)
+    assert.equal(ws.isInline(runId), false)
+    assert.equal(ws.runContexts.size, 0)
+  })
+
+  test('a replay inside an inline run does not release the inline flag', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    ws.registerInlineRun(runId)
+    await ws.beginReplay(runId)
+    ws.endReplay(runId)
+
+    assert.equal(
+      ws.isInline(runId),
+      true,
+      'ending a replay dropped state the run still needs'
+    )
+  })
+
+  test("a queued run's extension context is released when the run reaches a terminal state", async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+    const detached = trackDetaches(ws)
+
+    await ws.updateRunStatus(runId, 'completed', null)
+
+    assert.deepEqual(
+      detached,
+      [runId],
+      'an extension holds live per-run state; a completed run must not keep it'
+    )
+    assert.equal(ws.runContexts.size, 0)
+  })
+
+  test('a suspended run keeps its extension context, because it can still be resumed', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+    const detached = trackDetaches(ws)
+
+    await ws.updateRunStatus(runId, 'suspended')
+
+    assert.deepEqual(detached, [])
+  })
+})
+
+describe('a replay reads the run once', () => {
+  test('the steps of a replay share one read of the run', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    let reads = 0
+    const getRun = ws.getRun.bind(ws)
+    ws.getRun = async (id: string) => {
+      reads++
+      return getRun(id)
+    }
+
+    await ws.beginReplay(runId)
+    await ws.getRunIdentity(runId)
+    await ws.getRunIdentity(runId)
+    await ws.getRunIdentity(runId)
+    ws.endReplay(runId)
+
+    assert.equal(reads, 1, `the run was read ${reads} times in one replay`)
+  })
+
+  test('the next replay re-reads the run rather than trusting the last one', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    let reads = 0
+    const getRun = ws.getRun.bind(ws)
+    ws.getRun = async (id: string) => {
+      reads++
+      return getRun(id)
+    }
+
+    await ws.beginReplay(runId)
+    await ws.getRunIdentity(runId)
+    ws.endReplay(runId)
+    await ws.beginReplay(runId)
+    await ws.getRunIdentity(runId)
+    ws.endReplay(runId)
+
+    assert.equal(reads, 2)
+  })
+
+  test('outside a replay the run is read every time', async () => {
+    const ws = service()
+    const runId = await newRun(ws)
+
+    let reads = 0
+    const getRun = ws.getRun.bind(ws)
+    ws.getRun = async (id: string) => {
+      reads++
+      return getRun(id)
+    }
+
+    await ws.getRunIdentity(runId)
+    await ws.getRunIdentity(runId)
+
+    assert.equal(reads, 2, 'a cached run outside a replay would go stale')
+    assert.equal(ws.runContexts.size, 0, 'reading a run must not create state')
+  })
+})

package/src/wirings/workflow/workflow-run-polling.test.ts

@@ -0,0 +1,132 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+import type { WorkflowRun } from './workflow.types.js'
+
+/**
+ * A workflow service whose run reads are counted and timestamped, and whose
+ * run finishes only once it has been read `endAfter` times.
+ */
+class PollProbe extends InMemoryWorkflowService {
+  readonly reads: number[] = []
+  /** Every wait the loop asked for, in order — the schedule, not the clock. */
+  readonly waits: number[] = []
+  endAfter = 1
+
+  override async waitBeforeNextRead(ms: number): Promise<void> {
+    this.waits.push(ms)
+  }
+
+  override async startWorkflow(): Promise<{ runId: string }> {
+    return { runId: 'run-1' }
+  }
+
+  override async getRun(_runId: string): Promise<WorkflowRun | null> {
+    this.reads.push(Date.now())
+    const done = this.reads.length >= this.endAfter
+    return {
+      id: 'run-1',
+      status: done ? 'completed' : 'running',
+      output: 'result',
+    } as WorkflowRun
+  }
+}
+
+const probe = () => {
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: { error() {}, info() {}, warn() {}, debug() {} },
+  } as any)
+  return new PollProbe()
+}
+
+describe('waiting for a run to finish', () => {
+  test('a run that finishes quickly is not held for a whole poll interval', async () => {
+    const ws = probe()
+    ws.endAfter = 2
+
+    const output = await ws.runToCompletion('flow', {}, {} as any)
+
+    assert.equal(output, 'result')
+    assert.equal(ws.waits.length, 1)
+    assert.ok(
+      ws.waits[0]! < 1000,
+      `a run done after one poll waited the full ${ws.waits[0]}ms default — a fixed interval makes every fast workflow pay for it`
+    )
+  })
+
+  test('reads spread out as a run keeps running', async () => {
+    const ws = probe()
+    ws.endAfter = 8
+
+    await ws.runToCompletion('flow', {}, {} as any)
+
+    assert.ok(ws.waits.length >= 6)
+    assert.ok(
+      ws.waits.at(-1)! > ws.waits[0]! * 2,
+      `waits stayed flat (${ws.waits.join('ms, ')}ms) — a run that drags on keeps costing the same read rate forever`
+    )
+    for (let i = 1; i < ws.waits.length; i++) {
+      assert.ok(
+        ws.waits[i]! >= ws.waits[i - 1]!,
+        `wait ${i} shrank (${ws.waits.join('ms, ')}ms)`
+      )
+    }
+  })
+
+  test('the wait never grows past the configured interval', async () => {
+    const ws = probe()
+    ws.endAfter = 12
+
+    await ws.runToCompletion('flow', {}, {} as any, { pollIntervalMs: 40 })
+
+    for (const wait of ws.waits) {
+      assert.ok(wait <= 40, `a wait of ${wait}ms overshot the 40ms ceiling`)
+    }
+    assert.equal(
+      ws.waits.at(-1),
+      40,
+      `the ceiling was never reached (${ws.waits.join('ms, ')}ms)`
+    )
+  })
+
+  test('a run that is already finished is returned without any wait', async () => {
+    const ws = probe()
+    ws.endAfter = 1
+
+    await ws.runToCompletion('flow', {}, {} as any)
+
+    assert.equal(ws.reads.length, 1)
+    assert.deepEqual(ws.waits, [])
+  })
+})
+
+describe('a run that ends badly', () => {
+  test('a failed run surfaces its error message', async () => {
+    const ws = probe()
+    ws.getRun = async () =>
+      ({
+        id: 'run-1',
+        status: 'failed',
+        error: { message: 'step blew up' },
+      }) as WorkflowRun
+
+    await assert.rejects(ws.runToCompletion('flow', {}, {} as any), /blew up/)
+  })
+
+  test('a cancelled run rejects', async () => {
+    const ws = probe()
+    ws.getRun = async () =>
+      ({ id: 'run-1', status: 'cancelled' }) as WorkflowRun
+
+    await assert.rejects(ws.runToCompletion('flow', {}, {} as any))
+  })
+
+  test('a run that disappears rejects rather than polling forever', async () => {
+    const ws = probe()
+    ws.getRun = async () => null
+
+    await assert.rejects(ws.runToCompletion('flow', {}, {} as any), /run-1/)
+  })
+})