@pikku/core 0.12.107 → 0.12.112

package/src/wirings/workflow/workflow-queue-workers.test.ts

@@ -1,131 +0,0 @@
-import { beforeEach, describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import { resetPikkuState, pikkuState } from '../../pikku-state.js'
-import {
-  pikkuWorkflowWorkerFunc,
-  pikkuWorkflowOrchestratorFunc,
-  pikkuWorkflowSleeperFunc,
-} from './workflow-queue-workers.js'
-
-beforeEach(() => {
-  resetPikkuState()
-})
-
-describe('workflow-queue-workers', () => {
-  test('worker calls executeWorkflowStep on the workflow service', async () => {
-    const calls: unknown[][] = []
-    const rpc = { rpcWithWire: async () => ({ ok: true }) }
-
-    pikkuState(null, 'package', 'singletonServices', {
-      workflowService: {
-        executeWorkflowStep: async (...args: unknown[]) => {
-          calls.push(args)
-        },
-      },
-    } as never)
-
-    await pikkuWorkflowWorkerFunc(
-      {},
-      {
-        runId: 'run-1',
-        stepName: 'step-a',
-        rpcName: 'rpc.doThing',
-        data: { ok: true },
-      },
-      { rpc: rpc as never }
-    )
-
-    assert.deepEqual(calls, [
-      ['run-1', 'step-a', 'rpc.doThing', { ok: true }, rpc],
-    ])
-  })
-
-  test('worker throws when workflow service is missing', async () => {
-    pikkuState(null, 'package', 'singletonServices', {} as never)
-
-    await assert.rejects(
-      () =>
-        pikkuWorkflowWorkerFunc(
-          {},
-          {
-            runId: 'run-1',
-            stepName: 'step-a',
-            rpcName: 'rpc.doThing',
-            data: {},
-          },
-          { rpc: {} as never }
-        ),
-      {
-        message:
-          'Workflow service not initialized: cannot execute workflow step for runId run-1, stepName step-a',
-      }
-    )
-  })
-
-  test('orchestrator calls orchestrateWorkflow on the workflow service', async () => {
-    const calls: unknown[][] = []
-    const rpc = { rpcWithWire: async () => ({ ok: true }) }
-
-    pikkuState(null, 'package', 'singletonServices', {
-      workflowService: {
-        orchestrateWorkflow: async (...args: unknown[]) => {
-          calls.push(args)
-        },
-      },
-    } as never)
-
-    await pikkuWorkflowOrchestratorFunc(
-      {},
-      { runId: 'run-2' },
-      { rpc: rpc as never }
-    )
-
-    assert.deepEqual(calls, [['run-2', rpc]])
-  })
-
-  test('orchestrator throws when workflow service is missing', async () => {
-    pikkuState(null, 'package', 'singletonServices', {} as never)
-
-    await assert.rejects(
-      () =>
-        pikkuWorkflowOrchestratorFunc(
-          {},
-          { runId: 'run-2' },
-          { rpc: {} as never }
-        ),
-      {
-        message:
-          'Workflow service not initialized: cannot orchestrate workflow for runId run-2',
-      }
-    )
-  })
-
-  test('sleeper calls executeWorkflowSleepCompleted on the workflow service', async () => {
-    const calls: unknown[][] = []
-
-    pikkuState(null, 'package', 'singletonServices', {
-      workflowService: {
-        executeWorkflowSleepCompleted: async (...args: unknown[]) => {
-          calls.push(args)
-        },
-      },
-    } as never)
-
-    await pikkuWorkflowSleeperFunc({}, { runId: 'run-3', stepId: 'step-3' })
-
-    assert.deepEqual(calls, [['run-3', 'step-3']])
-  })
-
-  test('sleeper throws when workflow service is missing', async () => {
-    pikkuState(null, 'package', 'singletonServices', {} as never)
-
-    await assert.rejects(
-      () => pikkuWorkflowSleeperFunc({}, { runId: 'run-3', stepId: 'step-3' }),
-      {
-        message:
-          'Workflow service not initialized: cannot execute workflow sleep completed for runId run-3, stepId step-3',
-      }
-    )
-  })
-})

package/src/wirings/workflow/workflow-queue-workers.ts

@@ -1,66 +0,0 @@
-import { getSingletonServices } from '../../pikku-state.js'
-import type { PikkuRPC } from '../rpc/rpc-types.js'
-
-export interface WorkflowStepInput {
-  runId: string
-  stepName: string
-  rpcName: string
-  data: unknown
-  fromStepName?: string
-}
-
-export interface PikkuWorkflowOrchestratorInput {
-  runId: string
-}
-
-export interface PikkuWorkflowSleeperInput {
-  runId: string
-  stepId: string
-}
-
-export async function pikkuWorkflowWorkerFunc(
-  _services: Record<string, unknown>,
-  { runId, stepName, rpcName, data }: WorkflowStepInput,
-  { rpc }: { rpc: PikkuRPC }
-): Promise<void> {
-  const services = getSingletonServices()
-  if (!services?.workflowService) {
-    throw new Error(
-      `Workflow service not initialized: cannot execute workflow step for runId ${runId}, stepName ${stepName}`
-    )
-  }
-  await services.workflowService.executeWorkflowStep(
-    runId,
-    stepName,
-    rpcName,
-    data,
-    rpc
-  )
-}
-
-export async function pikkuWorkflowOrchestratorFunc(
-  _services: Record<string, unknown>,
-  { runId }: PikkuWorkflowOrchestratorInput,
-  { rpc }: { rpc: PikkuRPC }
-): Promise<void> {
-  const services = getSingletonServices()
-  if (!services?.workflowService) {
-    throw new Error(
-      `Workflow service not initialized: cannot orchestrate workflow for runId ${runId}`
-    )
-  }
-  await services.workflowService.orchestrateWorkflow(runId, rpc)
-}
-
-export async function pikkuWorkflowSleeperFunc(
-  _services: Record<string, unknown>,
-  { runId, stepId }: PikkuWorkflowSleeperInput
-): Promise<void> {
-  const services = getSingletonServices()
-  if (!services?.workflowService) {
-    throw new Error(
-      `Workflow service not initialized: cannot execute workflow sleep completed for runId ${runId}, stepId ${stepId}`
-    )
-  }
-  await services.workflowService.executeWorkflowSleepCompleted(runId, stepId)
-}

package/src/wirings/workflow/workflow-recovery.ts

@@ -1,192 +0,0 @@
-import type { Logger } from '../../services/logger.js'
-import {
-  DEFAULT_STALLED_RUN_LIMIT,
-  DEFAULT_STALLED_RUN_MS,
-  DEFAULT_UNDISPATCHED_STEP_LIMIT,
-  DEFAULT_UNDISPATCHED_STEP_MS,
-  REDISPATCH_BACKOFF_MAX_ENTRIES,
-  REDISPATCH_BACKOFF_MAX_MS,
-  REDISPATCH_BACKOFF_MS,
-} from './workflow-constants.js'
-
-/**
- * Per-process, advisory record of when a run may next be re-dispatched.
- *
- * Keyed by run rather than by step because the run is the unit of re-drive:
- * `resumeWorkflow` replays the whole run and re-dispatches every step still
- * owed a job, so holding off a single step while resuming its run would
- * suppress nothing.
- *
- * For the same reason one instance is shared by every sweep rather than kept
- * per sweep. The record is of the action, not of the signal that prompted it:
- * a stalled run and an undispatched step are different observations, but both
- * are answered by the one orchestrator message, so a run the relay re-drove a
- * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
- * is what makes the guarantee a message-per-run-per-window instead of one per
- * sweep, and no recovery is lost by it: whichever sweep gets there first
- * performs the identical re-drive, and the cap keeps the delay at 10m.
- *
- * Losing this on restart costs extra dispatches, never correctness.
- */
-export class RedispatchBackoff {
-  private readonly eligibleAt = new Map<string, number>()
-  private readonly delays = new Map<string, number>()
-
-  public isEligible(runId: string, now: number): boolean {
-    const at = this.eligibleAt.get(runId)
-    return at === undefined || at <= now
-  }
-
-  public note(runId: string, now: number): void {
-    const previous = this.delays.get(runId)
-    const delay = Math.min(
-      previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2,
-      REDISPATCH_BACKOFF_MAX_MS
-    )
-    // A run that settles is never returned again, so entries are only evicted
-    // by this bound — oldest first, which is also least recently re-dispatched.
-    if (this.eligibleAt.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
-      const oldest = this.eligibleAt.keys().next()
-      if (!oldest.done) {
-        this.eligibleAt.delete(oldest.value)
-        this.delays.delete(oldest.value)
-      }
-    }
-    this.delays.set(runId, delay)
-    this.eligibleAt.set(runId, now + delay)
-  }
-}
-
-type SweepDeps = {
-  resume: (runId: string) => Promise<void>
-  logger?: Logger
-}
-
-const resumeEach = async (
-  runIds: Iterable<string>,
-  { resume, logger }: SweepDeps,
-  failure: (runId: string, detail: string) => string
-): Promise<string[]> => {
-  const succeeded: string[] = []
-  for (const runId of runIds) {
-    try {
-      await resume(runId)
-      succeeded.push(runId)
-    } catch (err) {
-      // One unresumable run must not stop the sweep from recovering the rest.
-      logger?.error(
-        failure(runId, err instanceof Error ? err.message : String(err))
-      )
-    }
-  }
-  return succeeded
-}
-
-/**
- * Re-drive runs whose next move was lost, and report which were resumed.
- *
- * Arming a step is two writes to two systems — the step row, then the queue or
- * scheduler job — so a process that dies between them leaves a run that is
- * `running` with nothing in flight. Nothing notices: the run parks on a step
- * that will never complete and never error, so it neither finishes nor fails.
- * (Seen on a `workflow.sleep()`: a deploy restart landed between the sleep
- * step's insert and its timer, parking the run permanently.)
- *
- * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted step
- * state, and every settled step is memoized, so resuming a run that was not
- * actually stuck costs an orchestration pass and changes nothing. That
- * idempotence is what makes an idle-time heuristic safe here; a run that is
- * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
- *
- * Idempotent is not free, though: a run stays stalled until something clears
- * the reason it stalled, so an unconditional sweep re-queues the same runs on
- * every tick forever. That is how a handful of wedged runs became a queue of
- * thousands that could not drain — each pass added work the previous pass had
- * not finished. The same per-run backoff the relay uses bounds it: a run is
- * re-driven, then held off for a doubling delay, so a sweep costs at most one
- * message per run per window rather than one per run per tick.
- */
-export const sweepStalledRuns = async (
-  findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
-  backoff: RedispatchBackoff,
-  options: { stalledAfterMs?: number; limit?: number } | undefined,
-  deps: SweepDeps
-): Promise<{ resumed: string[] }> => {
-  const before = new Date(
-    Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
-  )
-  const found = await findStalledRunIds(
-    before,
-    options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
-  )
-
-  const now = Date.now()
-  const runIds: string[] = []
-  for (const runId of found) {
-    if (!backoff.isEligible(runId, now)) continue
-    // Noted before the resume, not after: a run whose resume throws is exactly
-    // the run most likely to still be here next tick, and re-driving it every
-    // tick is the amplification this backoff exists to stop.
-    backoff.note(runId, now)
-    runIds.push(runId)
-  }
-
-  return {
-    resumed: await resumeEach(
-      runIds,
-      deps,
-      (runId, detail) =>
-        `Failed to resume stalled workflow run ${runId}: ${detail}`
-    ),
-  }
-}
-
-/**
- * Re-drive steps whose dispatch was lost, and report which runs were nudged.
- *
- * The step row is the outbox record and this is the relay. Age is the only
- * signal available — a step `pending` because its dispatch was lost is
- * indistinguishable from one whose job is merely still queued — so a step past
- * `undispatchedAfterMs` is re-dispatched regardless, and correctness rests on
- * the claim in `executeWorkflowStepInner` rather than on the guess being right.
- * A redundant dispatch costs one queue message: the loser reads `running` and
- * returns without invoking anything.
- *
- * Re-dispatches back off per run (doubling from 30s, capped at 10m) so a
- * genuine queue backlog is not amplified by a tick that keeps firing at the
- * steps the backlog is already delaying.
- */
-export const sweepUndispatchedSteps = async (
-  findUndispatchedSteps: (
-    before: Date,
-    limit: number
-  ) => Promise<Array<{ runId: string; stepId: string }>>,
-  backoff: RedispatchBackoff,
-  options: { undispatchedAfterMs?: number; limit?: number } | undefined,
-  deps: SweepDeps
-): Promise<{ redispatched: string[] }> => {
-  const before = new Date(
-    Date.now() - (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS)
-  )
-  const steps = await findUndispatchedSteps(
-    before,
-    options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT
-  )
-
-  const now = Date.now()
-  const runIds = new Set<string>()
-  for (const { runId } of steps) {
-    if (!backoff.isEligible(runId, now)) continue
-    backoff.note(runId, now)
-    runIds.add(runId)
-  }
-
-  return {
-    redispatched: await resumeEach(
-      runIds,
-      deps,
-      (runId, detail) =>
-        `Failed to re-dispatch workflow run ${runId}: ${detail}`
-    ),
-  }
-}

package/src/wirings/workflow/workflow-replay-snapshot.test.ts

@@ -1,140 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
-import { pikkuState } from '../../pikku-state.js'
-
-const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
-
-function inlineService() {
-  pikkuState(null, 'package', 'singletonServices', {
-    queueService: { add: async () => {} },
-    logger: silentLogger,
-  } as any)
-  return new InMemoryWorkflowService()
-}
-
-function countStepReads(ws: any) {
-  const counter = { reads: 0 }
-  const original = ws.getStepState.bind(ws)
-  ws.getStepState = async (...args: any[]) => {
-    counter.reads++
-    return original(...args)
-  }
-  return counter
-}
-
-async function seedRun(ws: any, steps: number) {
-  const runId = await ws.createRun('flow', {}, true, 'hash', { type: 'test' })
-  ws.registerInlineRun(runId)
-  await ws.beginReplay(runId)
-  for (let i = 0; i < steps; i++) {
-    await ws.inlineStep(runId, `step-${i}`, async () => i)
-  }
-  ws.endReplay(runId)
-  return runId
-}
-
-describe('replay reads the run steps once, not once per step', () => {
-  test('a replay over completed steps issues no per-step reads', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 8)
-
-    const counter = countStepReads(ws)
-    await ws.beginReplay(runId)
-    for (let i = 0; i < 8; i++) {
-      await ws.inlineStep(runId, `step-${i}`, async () => {
-        throw new Error(
-          `step-${i} re-ran instead of replaying its cached result`
-        )
-      })
-    }
-    ws.endReplay(runId)
-
-    assert.equal(
-      counter.reads,
-      0,
-      `the replay still read each step individually (${counter.reads} reads for 8 steps)`
-    )
-  })
-
-  test('the snapshot still returns each step its own cached result', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 4)
-
-    await ws.beginReplay(runId)
-    const replayed: unknown[] = []
-    for (let i = 0; i < 4; i++) {
-      replayed.push(
-        await ws.inlineStep(runId, `step-${i}`, async () => 'must not run')
-      )
-    }
-    ws.endReplay(runId)
-
-    assert.deepEqual(replayed, [0, 1, 2, 3])
-  })
-
-  test('a step reached for the first time mid-replay is created, not mistaken for a miss', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 2)
-
-    await ws.beginReplay(runId)
-    await ws.inlineStep(runId, 'step-0', async () => 'cached')
-    await ws.inlineStep(runId, 'step-1', async () => 'cached')
-    const fresh = await ws.inlineStep(runId, 'step-new', async () => 'ran')
-    ws.endReplay(runId)
-
-    assert.equal(fresh, 'ran')
-    assert.equal((await ws.getStepState(runId, 'step-new')).result, 'ran')
-  })
-
-  test('a second replay sees what the previous one wrote', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 1)
-
-    await ws.beginReplay(runId)
-    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
-    await ws.inlineStep(runId, 'late', async () => 'first')
-    ws.endReplay(runId)
-
-    await ws.beginReplay(runId)
-    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
-    const late = await ws.inlineStep(runId, 'late', async () => 'must not run')
-    ws.endReplay(runId)
-
-    assert.equal(
-      late,
-      'first',
-      'the second replay re-ran a step the first one completed'
-    )
-  })
-
-  test('outside a replay, steps are still read one at a time', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 3)
-
-    const counter = countStepReads(ws)
-    await ws.inlineStep(runId, 'step-0', async () => 'must not run')
-
-    assert.equal(
-      counter.reads,
-      1,
-      'a call outside a replay must read the step directly rather than trust a stale snapshot'
-    )
-  })
-
-  test('a backend with no bulk read falls back to per-step reads', async () => {
-    const ws: any = inlineService()
-    const runId = await seedRun(ws, 3)
-    ws.listStepStates = async () => null
-
-    const counter = countStepReads(ws)
-    await ws.beginReplay(runId)
-    for (let i = 0; i < 3; i++) {
-      await ws.inlineStep(runId, `step-${i}`, async () => 'must not run')
-    }
-    ws.endReplay(runId)
-
-    assert.equal(counter.reads, 3)
-  })
-})

package/src/wirings/workflow/workflow-retry-policy.test.ts

@@ -1,113 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
-import { pikkuState } from '../../pikku-state.js'
-import { DEFAULT_STEP_RETRIES } from './workflow-constants.js'
-import type { PikkuWorkflowService } from './pikku-workflow-service.js'
-import { deriveInvocationId } from './workflow-invocation-id.js'
-import type { WorkflowStepOptions } from './workflow.types.js'
-
-const UUID_RE =
-  /^[0-9a-f]{8}-[0-9a-f]{4}-5[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/
-
-class TestWorkflowService extends InMemoryWorkflowService {
-  public resolve(options?: WorkflowStepOptions) {
-    return (this as PikkuWorkflowService as any).resolveStepJobOptions(options)
-  }
-}
-
-describe('resolveStepJobOptions — workflow owns retry policy', () => {
-  const ws = new TestWorkflowService()
-
-  test('unset retries → default + exponential backoff (rides out outages)', () => {
-    assert.deepEqual(ws.resolve(undefined), {
-      attempts: DEFAULT_STEP_RETRIES + 1,
-      backoff: 'exponential',
-    })
-  })
-
-  test('retries: 0 → attempts: 1 and NO backoff (the non-idempotent opt-out)', () => {
-    assert.deepEqual(ws.resolve({ retries: 0 }), { attempts: 1 })
-  })
-
-  test('explicit retries always honored', () => {
-    assert.deepEqual(ws.resolve({ retries: 3 }), {
-      attempts: 4,
-      backoff: 'exponential',
-    })
-  })
-
-  test('numeric retryDelay → fixed backoff', () => {
-    assert.deepEqual(ws.resolve({ retries: 3, retryDelay: 250 }), {
-      attempts: 4,
-      backoff: { type: 'fixed', delay: 250 },
-    })
-  })
-
-  test("duration-string retryDelay ('15s') → fixed backoff in ms", () => {
-    assert.deepEqual(ws.resolve({ retries: 3, retryDelay: '15s' }), {
-      attempts: 4,
-      backoff: { type: 'fixed', delay: 15000 },
-    })
-  })
-
-  test("retryDelay: 'exponential' → exponential backoff", () => {
-    assert.deepEqual(ws.resolve({ retries: 2, retryDelay: 'exponential' }), {
-      attempts: 3,
-      backoff: 'exponential',
-    })
-  })
-})
-
-describe('executeWorkflowStep surfaces a stable invocationId', () => {
-  test('invocationId is stable across a retry; stepId is per-attempt', async () => {
-    const ws = new InMemoryWorkflowService()
-    pikkuState(null, 'package', 'singletonServices', {
-      queueService: { add: async () => {} },
-      logger: { error() {}, info() {}, warn() {}, debug() {} },
-    } as any)
-
-    const captured: Array<{
-      invocationId: string
-      stepId: string
-      attemptCount: number
-    }> = []
-    const rpc = {
-      rpcWithWire: async (_n: string, _d: any, opts: any) => {
-        captured.push(opts.workflowStep)
-        if (captured.length === 1) throw new Error('transient boom')
-        return { ok: true }
-      },
-    }
-
-    const runId = await ws.createRun('invflow', {}, false, 'hash', {
-      type: 'test',
-    })
-    await ws.insertStepState(runId, 'updateUser', 'doUpdateUser', {})
-
-    await assert.rejects(
-      ws.executeWorkflowStep(runId, 'updateUser', 'doUpdateUser', {}, rpc)
-    )
-    await ws.executeWorkflowStep(runId, 'updateUser', 'doUpdateUser', {}, rpc)
-
-    assert.equal(captured.length, 2, 'step ran twice')
-    assert.equal(
-      captured[0]!.invocationId,
-      captured[1]!.invocationId,
-      'invocationId is identical across retries — the dedupe key'
-    )
-    assert.equal(
-      captured[0]!.invocationId,
-      deriveInvocationId(runId, 'updateUser')
-    )
-    assert.match(captured[0]!.invocationId, UUID_RE)
-    assert.equal(captured[0]!.attemptCount, 1)
-    assert.equal(captured[1]!.attemptCount, 2)
-    assert.notEqual(
-      captured[0]!.stepId,
-      captured[1]!.stepId,
-      'stepId is minted per attempt — must NOT be used as the dedupe key'
-    )
-  })
-})