@pikku/core 0.12.107 → 0.12.111

package/src/wirings/workflow/workflow-stalled-recovery.test.ts

@@ -1,152 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
-import { pikkuState } from '../../pikku-state.js'
-
-const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
-
-/**
- * `recoverStalledRuns` re-drives a run by putting it back on the orchestrator
- * queue, so the queue is the observation point.
- */
-function trackResumes(): { runIds: string[] } {
-  const seen: string[] = []
-  pikkuState(null, 'package', 'singletonServices', {
-    queueService: {
-      add: async (_queue: string, data: { runId: string }) => {
-        seen.push(data.runId)
-      },
-    },
-    logger: silentLogger,
-  } as any)
-  return { runIds: seen }
-}
-
-/** Backdates every persisted timestamp so the run reads as idle. */
-async function backdate(
-  ws: InMemoryWorkflowService,
-  runId: string,
-  ms: number
-): Promise<void> {
-  const past = new Date(Date.now() - ms)
-  const run = await ws.getRun(runId)
-  ;(run as any).updatedAt = past
-  for (const step of await ws.getRunHistory(runId)) {
-    ;(step as any).updatedAt = past
-  }
-}
-
-describe('stalled run recovery', () => {
-  test('resumes a running run left with a pending step and nothing in flight', async () => {
-    const resumes = trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    // A sleep whose timer was never armed: the step row exists, nothing will
-    // ever complete it.
-    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
-    await backdate(ws, runId, 10 * 60_000)
-
-    const { resumed } = await ws.recoverStalledRuns()
-
-    assert.deepEqual(resumed, [runId], 'the orphaned run is resumed')
-    assert.deepEqual(resumes.runIds, [runId], 'it is put back on the queue')
-  })
-
-  test('leaves a run alone while a step is still scheduled', async () => {
-    trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    const step = await ws.insertStepState(runId, 'Wait 1h', null, {
-      duration: 3_600_000,
-    })
-    await ws.setStepScheduled(step.stepId)
-    await backdate(ws, runId, 10 * 60_000)
-
-    const { resumed } = await ws.recoverStalledRuns()
-
-    assert.deepEqual(
-      resumed,
-      [],
-      'a legitimately sleeping run is not a stalled run'
-    )
-  })
-
-  test('leaves a run alone until it has actually gone idle', async () => {
-    trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
-
-    const { resumed } = await ws.recoverStalledRuns()
-
-    assert.deepEqual(resumed, [], 'a run that just moved is not swept')
-  })
-
-  test('backs off rather than re-resuming the same run every tick', async () => {
-    const resumes = trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
-    await backdate(ws, runId, 10 * 60_000)
-
-    const first = await ws.recoverStalledRuns()
-    // Still stalled: a resume does not clear whatever wedged the run, so an
-    // unconditional sweep would re-queue it on this tick and on every tick
-    // after it.
-    const second = await ws.recoverStalledRuns()
-
-    assert.deepEqual(first.resumed, [runId], 'the first sweep resumes it')
-    assert.deepEqual(
-      second.resumed,
-      [],
-      'the next sweep is held off by the backoff'
-    )
-    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
-  })
-
-  test('does not re-drive a run the relay has just re-dispatched', async () => {
-    const resumes = trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    // Old enough to read as both an undispatched step and a stalled run.
-    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
-    await backdate(ws, runId, 10 * 60_000)
-
-    const relayed = await ws.relayUndispatchedSteps()
-    const swept = await ws.recoverStalledRuns()
-
-    assert.deepEqual(relayed.redispatched, [runId], 'the relay re-drives it')
-    assert.deepEqual(
-      swept.resumed,
-      [],
-      'the sweep adds nothing the relay has not already queued'
-    )
-    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
-  })
-
-  test('leaves a finished run alone', async () => {
-    trackResumes()
-    const ws = new InMemoryWorkflowService()
-    const runId = await ws.createRun('flow', {}, false, 'hash', {
-      type: 'test',
-    })
-    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
-    await ws.updateRunStatus(runId, 'completed')
-    await backdate(ws, runId, 10 * 60_000)
-
-    const { resumed } = await ws.recoverStalledRuns()
-
-    assert.deepEqual(resumed, [], 'only running runs are swept')
-  })
-})

package/src/wirings/workflow/workflow-status-stream.test.ts

@@ -1,354 +0,0 @@
-import { test, describe } from 'node:test'
-import assert from 'node:assert/strict'
-import { streamWorkflowRunStatus } from './workflow-status-stream.js'
-import type { WorkflowRunService } from './workflow.types.js'
-
-const run = (over: Record<string, unknown> = {}) =>
-  ({
-    id: 'run-1',
-    workflow: 'checkout',
-    status: 'running',
-    input: {},
-    wire: {},
-    createdAt: new Date(),
-    updatedAt: new Date(),
-    ...over,
-  }) as any
-
-const step = (stepName: string, status: string, over: any = {}) =>
-  ({ stepId: stepName, stepName, status, attemptCount: 1, ...over }) as any
-
-/**
- * Each poll is driven by a scripted sequence rather than a clock, so a test
- * asserts what the stream sends without waiting for one.
- */
-const harness = (
-  polls: Array<{ run: any; steps: any[] }>,
-  { detailed = false, session = undefined as any } = {}
-) => {
-  const sent: any[] = []
-  let closed = false
-  let index = 0
-  const workflowRunService = {
-    getRun: async () => {
-      const frame = polls[Math.min(index, polls.length - 1)]
-      return frame!.run
-    },
-    getRunSteps: async () => {
-      const frame = polls[Math.min(index, polls.length - 1)]
-      index += 1
-      return frame!.steps
-    },
-  } as unknown as WorkflowRunService
-
-  return {
-    sent,
-    closed: () => closed,
-    stream: () =>
-      streamWorkflowRunStatus({
-        workflowRunService,
-        runId: 'run-1',
-        channel: {
-          send: async (data: any) => {
-            sent.push(data)
-          },
-          close: async () => {
-            closed = true
-          },
-        },
-        session,
-        detailed,
-        pollIntervalMs: 1,
-      }),
-  }
-}
-
-describe('streamWorkflowRunStatus', () => {
-  test('a run that is already finished never starts a timer', async () => {
-    const h = harness([
-      { run: run({ status: 'completed' }), steps: [step('a', 'succeeded')] },
-    ])
-    await h.stream()
-    assert.deepEqual(
-      h.sent.map((frame) => frame.type),
-      ['update', 'done']
-    )
-    assert.equal(h.closed(), true)
-  })
-
-  test('a run nobody can find closes the stream rather than hanging', async () => {
-    const h = harness([{ run: null, steps: [] }])
-    await h.stream()
-    assert.equal(h.sent.length, 0)
-    assert.equal(h.closed(), true)
-  })
-
-  // Checked on every poll, not just the first: a stream that outlives a session
-  // should stop rather than keep reporting.
-  test('a run belonging to someone else is refused', async () => {
-    const h = harness(
-      [{ run: run({ wire: { pikkuUserId: 'someone-else' } }), steps: [] }],
-      { session: { userId: 'me' } }
-    )
-    await assert.rejects(
-      h.stream(),
-      /Not authorized to access this workflow run/
-    )
-  })
-
-  // A deterministic run knows its whole shape up front, so the client can draw
-  // every step — including the ones not started — before anything runs.
-  test('a deterministic run sends its planned shape first', async () => {
-    const h = harness([
-      {
-        run: run({
-          status: 'completed',
-          deterministic: true,
-          plannedSteps: [{ stepName: 'a' }, { stepName: 'b' }],
-        }),
-        steps: [step('a', 'succeeded')],
-      },
-    ])
-    await h.stream()
-    const init = h.sent[0]
-    assert.equal(init.type, 'init')
-    assert.equal(init.deterministic, true)
-    assert.deepEqual(init.steps, [
-      { stepName: 'a', status: 'succeeded' },
-      // Planned but not started, which is the whole reason to send this frame.
-      { stepName: 'b', status: 'pending' },
-    ])
-  })
-
-  test('a dynamic run has no shape to announce, so it announces none', async () => {
-    const h = harness([
-      { run: run({ status: 'completed' }), steps: [step('a', 'succeeded')] },
-    ])
-    await h.stream()
-    assert.equal(
-      h.sent.some((frame) => frame.type === 'init'),
-      false
-    )
-  })
-
-  // A run sitting on a slow step should cost one message, not one per poll.
-  test('nothing is sent while nothing has changed', async () => {
-    const h = harness([
-      { run: run(), steps: [step('a', 'running')] },
-      { run: run(), steps: [step('a', 'running')] },
-      { run: run(), steps: [step('a', 'running')] },
-      {
-        run: run({ status: 'completed' }),
-        steps: [step('a', 'succeeded')],
-      },
-    ])
-    await h.stream()
-    assert.deepEqual(
-      h.sent.map((frame) => frame.type),
-      ['update', 'update', 'done']
-    )
-  })
-
-  test('every terminal status ends the stream', async () => {
-    for (const status of ['completed', 'failed', 'cancelled']) {
-      const h = harness([{ run: run({ status }), steps: [] }])
-      await h.stream()
-      assert.equal(h.sent.at(-1)!.type, 'done', status)
-      assert.equal(h.closed(), true, status)
-    }
-  })
-
-  // The whole difference between the two scaffolded routes. A workflow's output
-  // and its error messages are internal detail, and a step that spawned a child
-  // run says so only to tooling that can follow it.
-  test('the user-facing stream reports progress and nothing else', async () => {
-    const h = harness([
-      {
-        run: run({
-          status: 'failed',
-          output: { card: '4242' },
-          error: { message: 'declined at acquirer', name: 'Error' },
-        }),
-        steps: [step('a', 'failed', { childRunId: 'child-1' })],
-      },
-    ])
-    await h.stream()
-    const update = h.sent[0]
-    assert.equal(update.status, 'failed')
-    assert.equal('output' in update, false)
-    assert.equal('error' in update, false)
-    assert.deepEqual(update.steps, [{ stepName: 'a', status: 'failed' }])
-  })
-
-  test('the detailed stream carries what the run produced', async () => {
-    const h = harness(
-      [
-        {
-          run: run({
-            status: 'failed',
-            output: { card: '4242' },
-            error: { message: 'declined at acquirer', name: 'Error' },
-          }),
-          steps: [step('a', 'failed', { childRunId: 'child-1' })],
-        },
-      ],
-      { detailed: true }
-    )
-    await h.stream()
-    const update = h.sent[0]
-    assert.deepEqual(update.output, { card: '4242' })
-    assert.equal(update.error.message, 'declined at acquirer')
-    assert.deepEqual(update.steps, [
-      { stepName: 'a', status: 'failed', childRunId: 'child-1' },
-    ])
-  })
-
-  // The detailed stream compares output too, so a run whose steps are unchanged
-  // but whose output has moved on still reports it.
-  test('a change only the detailed stream can see still reaches it', async () => {
-    const frames = [
-      { run: run({ output: { progress: 1 } }), steps: [step('a', 'running')] },
-      { run: run({ output: { progress: 2 } }), steps: [step('a', 'running')] },
-      {
-        run: run({ status: 'completed', output: { progress: 2 } }),
-        steps: [step('a', 'succeeded')],
-      },
-    ]
-    const quiet = harness(frames)
-    await quiet.stream()
-    assert.equal(
-      quiet.sent.filter((frame) => frame.type === 'update').length,
-      2
-    )
-
-    const loud = harness(frames, { detailed: true })
-    await loud.stream()
-    assert.equal(loud.sent.filter((frame) => frame.type === 'update').length, 3)
-  })
-
-  // Without this the throw is an unhandled rejection from a timer callback,
-  // which takes the process with it rather than failing the request.
-  test('a poll that throws after the first stops the stream, loudly', async () => {
-    let calls = 0
-    await assert.rejects(
-      streamWorkflowRunStatus({
-        workflowRunService: {
-          getRun: async () => {
-            calls += 1
-            if (calls > 1) {
-              throw new Error('the database went away')
-            }
-            return run()
-          },
-          getRunSteps: async () => [],
-        } as unknown as WorkflowRunService,
-        runId: 'run-1',
-        channel: { send: async () => {}, close: async () => {} },
-        session: undefined,
-        pollIntervalMs: 1,
-      }),
-      /the database went away/
-    )
-  })
-})
-
-/**
- * A stream that throws still has a channel open on the other end. The owner
- * check runs on every poll precisely so a session that loses access stops the
- * stream — which is only true if stopping also closes it.
- */
-describe('streamWorkflowRunStatus closes the channel when a poll throws', () => {
-  const failingStream = (failOn: number) => {
-    let closed = false
-    let polls = 0
-    const workflowRunService = {
-      getRun: async () => {
-        polls += 1
-        if (polls >= failOn) {
-          throw new Error('run store unavailable')
-        }
-        return run()
-      },
-      getRunSteps: async () => [step('charge', 'running')],
-    } as unknown as WorkflowRunService
-
-    return {
-      closed: () => closed,
-      stream: () =>
-        streamWorkflowRunStatus({
-          workflowRunService,
-          runId: 'run-1',
-          channel: {
-            send: async () => {},
-            close: async () => {
-              closed = true
-            },
-          },
-          session: undefined as any,
-          pollIntervalMs: 1,
-        }),
-    }
-  }
-
-  test('the first poll throwing closes the channel and rethrows', async () => {
-    const h = failingStream(1)
-    await assert.rejects(h.stream(), /run store unavailable/)
-    assert.equal(h.closed(), true, 'a throw must not leave the channel open')
-  })
-
-  test('a later poll throwing closes the channel and rethrows', async () => {
-    const h = failingStream(3)
-    await assert.rejects(h.stream(), /run store unavailable/)
-    assert.equal(h.closed(), true, 'a throw must not leave the channel open')
-  })
-})
-
-/**
- * The poll used to be on a fixed interval, which fires whether or not the
- * previous one has come back. Two in flight at once both see `initSent` unset
- * and send the init frame twice.
- */
-describe('streamWorkflowRunStatus never runs two polls at once', () => {
-  test('a poll slower than the interval does not overlap the next', async () => {
-    const sent: any[] = []
-    let inFlight = 0
-    let overlapped = false
-    let polls = 0
-
-    const workflowRunService = {
-      getRun: async () => {
-        inFlight += 1
-        if (inFlight > 1) overlapped = true
-        await new Promise((r) => setTimeout(r, 15))
-        polls += 1
-        inFlight -= 1
-        return run({
-          status: polls >= 3 ? 'completed' : 'running',
-          deterministic: true,
-          plannedSteps: [{ stepName: 'charge' }],
-        })
-      },
-      getRunSteps: async () => [step('charge', 'running')],
-    } as unknown as WorkflowRunService
-
-    await streamWorkflowRunStatus({
-      workflowRunService,
-      runId: 'run-1',
-      channel: {
-        send: async (d: any) => {
-          sent.push(d)
-        },
-        close: async () => {},
-      },
-      session: undefined as any,
-      pollIntervalMs: 1,
-    })
-
-    assert.equal(overlapped, false, 'two polls must never be in flight together')
-    assert.equal(
-      sent.filter((f) => f.type === 'init').length,
-      1,
-      'the init frame is sent exactly once'
-    )
-  })
-})

package/src/wirings/workflow/workflow-status-stream.ts

@@ -1,144 +0,0 @@
-import type { CoreUserSession } from '../../types/core.types.js'
-import type { PikkuChannel } from '../channel/channel.types.js'
-import { assertWorkflowRunOwner } from './workflow-run-ownership.js'
-import type { WorkflowRunService, WorkflowStatus } from './workflow.types.js'
-
-/**
- * The status stream behind the scaffolded workflow SSE routes.
- *
- * Two routes share it, and they differ by one thing: whether the caller is
- * trusted with what the run produced. A user-facing frontend gets step names
- * and statuses; an admin console also gets the output, the error and the child
- * run ids. That is a parameter, not a second copy of the loop.
- */
-
-const TERMINAL: ReadonlySet<string> = new Set<WorkflowStatus>([
-  'completed',
-  'failed',
-  'cancelled',
-])
-
-const DEFAULT_POLL_INTERVAL_MS = 500
-
-export interface WorkflowStatusStreamParams {
-  workflowRunService: WorkflowRunService
-  runId: string
-  channel: Pick<PikkuChannel<unknown, any>, 'send' | 'close'>
-  session: CoreUserSession | undefined
-  /**
-   * Whether to include what the run produced. Off for the user-facing route:
-   * a workflow's output and its error messages are internal detail, and a step
-   * that spawned a child run says so only to tooling that can follow it.
-   */
-  detailed?: boolean
-  pollIntervalMs?: number
-}
-
-/**
- * Streams one run's progress until it reaches a terminal state.
- *
- * Polled rather than subscribed because a run's steps are written by whichever
- * worker picked them up, in whichever process — there is no in-memory event to
- * listen for that every deployment shape would deliver.
- *
- * Each poll sends only when something changed, compared by a hash of exactly
- * what this stream reports. A run that sits on a slow step for a minute costs
- * one message, not a hundred and twenty.
- */
-export const streamWorkflowRunStatus = async ({
-  workflowRunService,
-  runId,
-  channel,
-  session,
-  detailed = false,
-  pollIntervalMs = DEFAULT_POLL_INTERVAL_MS,
-}: WorkflowStatusStreamParams): Promise<void> => {
-  let lastHash = ''
-  let initSent = false
-
-  const poll = async (): Promise<boolean> => {
-    const run = await workflowRunService.getRun(runId)
-    if (!run) {
-      await channel.close()
-      return false
-    }
-    // Checked on every poll, not just the first: ownership is read from the run
-    // itself, and a stream that outlives a session should stop rather than keep
-    // reporting.
-    assertWorkflowRunOwner(run.wire, session)
-
-    const steps = await workflowRunService.getRunSteps(runId)
-
-    // A deterministic run knows its whole shape up front, so the client can
-    // draw every step — including the ones not started — before anything runs.
-    // A dynamic run has nothing to send here, and gets no init frame.
-    if (!initSent && run.deterministic) {
-      const statusByStep = new Map(
-        steps.map((step) => [step.stepName, step.status])
-      )
-      await channel.send({
-        type: 'init',
-        deterministic: true,
-        steps: (run.plannedSteps ?? []).map((step) => ({
-          stepName: step.stepName,
-          status: statusByStep.get(step.stepName) ?? 'pending',
-        })),
-      })
-      initSent = true
-    }
-
-    const hash = JSON.stringify({
-      s: run.status,
-      ...(detailed ? { o: run.output } : {}),
-      steps: steps.map((step) => [step.stepName, step.status]),
-    })
-
-    if (hash !== lastHash) {
-      lastHash = hash
-      await channel.send({
-        type: 'update',
-        status: run.status,
-        ...(detailed ? { output: run.output, error: run.error } : {}),
-        steps: steps.map((step) => ({
-          stepName: step.stepName,
-          status: step.status,
-          ...(detailed && step.childRunId
-            ? { childRunId: step.childRunId }
-            : {}),
-        })),
-      })
-    }
-
-    if (TERMINAL.has(run.status)) {
-      await channel.send({ type: 'done' })
-      await channel.close()
-      return false
-    }
-    return true
-  }
-
-  // Every exit from here closes the channel, including the ones a throw takes:
-  // `assertWorkflowRunOwner` rejecting a session that lost access is exactly
-  // the case where the stream should end rather than be left hanging open.
-  try {
-    // A run that is already finished is answered without ever starting a timer.
-    if (!(await poll())) {
-      return
-    }
-
-    // The next poll is scheduled when the previous one resolves rather than on
-    // a fixed interval. A timer that fires regardless would let two polls
-    // overlap on a slow store — both seeing `initSent` unset and sending the
-    // init frame twice, and racing `lastHash` into out-of-order updates.
-    while (
-      await new Promise<boolean>((resolve, reject) => {
-        setTimeout(() => void poll().then(resolve, reject), pollIntervalMs)
-      })
-    ) {
-      // The condition is the whole loop: poll until it says to stop.
-    }
-  } catch (error) {
-    await channel.close()
-    throw error
-  }
-}

package/src/wirings/workflow/workflow-step-claim.ts

@@ -1,46 +0,0 @@
-import { WorkflowStepFunctionMismatchError } from './workflow-errors.js'
-import type { StepState } from './workflow.types.js'
-
-/** What claiming a step needs from the workflow service. */
-export type StepClaimStore = {
-  getStepState(runId: string, stepName: string): Promise<StepState>
-  setStepRunning(stepId: string): Promise<void>
-  createRetryAttempt(
-    failedStepId: string,
-    status: 'pending' | 'running'
-  ): Promise<StepState>
-}
-
-/**
- * Decide whether this dispatch owns the step, by reading its state and then
- * writing it — which only excludes a concurrent dispatch when the caller holds
- * a lock that genuinely excludes one.
- *
- * A store that can express the whole decision as a single conditional write
- * should do that instead of calling this.
- */
-export const claimStepByReadThenWrite = async (
-  store: StepClaimStore,
-  runId: string,
-  stepName: string,
-  rpcName: string
-): Promise<StepState | null> => {
-  const stepState = await store.getStepState(runId, stepName)
-  // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
-  if (
-    stepState.rpcName !== undefined &&
-    stepState.rpcName !== (rpcName ?? null)
-  ) {
-    throw new WorkflowStepFunctionMismatchError(runId, stepName)
-  }
-  if (stepState.status === 'succeeded' || stepState.status === 'running') {
-    return null
-  }
-  if (stepState.status === 'failed') {
-    return store.createRetryAttempt(stepState.stepId, 'running')
-  }
-  if (stepState.status === 'pending' || stepState.status === 'scheduled') {
-    await store.setStepRunning(stepState.stepId)
-  }
-  return stepState
-}