@pikku/core 0.12.77 → 0.12.79

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -275,6 +275,31 @@ const WORKFLOW_TERMINAL_STATES: ReadonlySet<string> = new Set([
   'cancelled',
 ])
 
+/** Idle window before a `running` run with nothing in flight is treated as stalled. */
+const DEFAULT_STALLED_RUN_MS = 5 * 60_000
+
+/** Runs re-driven per `recoverStalledRuns` call, so one sweep is bounded. */
+const DEFAULT_STALLED_RUN_LIMIT = 100
+
+/**
+ * How long a step may sit `pending` before the relay assumes its dispatch was
+ * lost. This is a bet that no healthy queue takes this long to move a job from
+ * `pending` to `running`; set it above the observed p99 of that latency.
+ */
+const DEFAULT_UNDISPATCHED_STEP_MS = 30_000
+
+/** Steps re-driven per `relayUndispatchedSteps` call, so one tick is bounded. */
+const DEFAULT_UNDISPATCHED_STEP_LIMIT = 100
+
+/** First wait before a step already re-dispatched once is re-dispatched again. */
+const REDISPATCH_BACKOFF_MS = 30_000
+
+/** Ceiling on the doubling backoff, so a permanently stuck step still gets swept. */
+const REDISPATCH_BACKOFF_MAX_MS = 10 * 60_000
+
+/** Bound on the in-process backoff map, so a long-lived process cannot grow it without bound. */
+const REDISPATCH_BACKOFF_MAX_ENTRIES = 10_000
+
 const WORKFLOW_POLL_MIN_MS = 10
 
 const WORKFLOW_POLL_FACTOR = 1.6
@@ -908,6 +933,179 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     )
   }
 
+  /**
+   * Ids of runs that are stalled: still `running`, with no step in a state that
+   * something is expected to complete (`running`, `scheduled`, `suspended`),
+   * and no step activity since `before`.
+   *
+   * Returns nothing by default so a store that cannot express the query keeps
+   * working unchanged; a store that overrides it gains crash recovery through
+   * `recoverStalledRuns`.
+   */
+  protected async findStalledRunIds(
+    _before: Date,
+    _limit: number
+  ): Promise<string[]> {
+    return []
+  }
+
+  /**
+   * Re-drive runs whose next move was lost, and report which were resumed.
+   *
+   * Arming a step is two writes to two systems — the step row, then the queue
+   * or scheduler job — so a process that dies between them leaves a run that is
+   * `running` with nothing in flight. Nothing notices: the run parks on a step
+   * that will never complete and never error, so it neither finishes nor fails.
+   * (Seen on a `workflow.sleep()`: a deploy restart landed between the sleep
+   * step's insert and its timer, parking the run permanently.)
+   *
+   * Replay is the recovery — `resumeWorkflow` re-orchestrates from persisted
+   * step state, and every settled step is memoized, so resuming a run that was
+   * not actually stuck costs an orchestration pass and changes nothing. That
+   * idempotence is what makes an idle-time heuristic safe here; a run that is
+   * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+   *
+   * This is not self-starting. Call it from a scheduled task at whatever
+   * interval suits the workload.
+   */
+  public async recoverStalledRuns(options?: {
+    stalledAfterMs?: number
+    limit?: number
+  }): Promise<{ resumed: string[] }> {
+    const before = new Date(
+      Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
+    )
+    const runIds = await this.findStalledRunIds(
+      before,
+      options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
+    )
+    const resumed: string[] = []
+    for (const runId of runIds) {
+      try {
+        await this.resumeWorkflow(runId)
+        resumed.push(runId)
+      } catch (err) {
+        // One unresumable run must not stop the sweep from recovering the rest.
+        getSingletonServices()?.logger?.error(
+          `Failed to resume stalled workflow run ${runId}: ${err instanceof Error ? err.message : String(err)}`
+        )
+      }
+    }
+    return { resumed }
+  }
+
+  /**
+   * Runs holding a step that has sat `pending` since before `before`, paired
+   * with the step that flagged them.
+   *
+   * Returns nothing by default so a store that cannot express the query keeps
+   * working unchanged — and, because it does not opt in, gains no re-dispatches
+   * either. A store must have an atomic `withStepLock` before overriding this,
+   * or no concurrency for one to exclude: the relay makes duplicate dispatch
+   * routine, and the claim in `executeWorkflowStepInner` is what keeps a
+   * duplicate from becoming a second execution. `kysely-postgres` and
+   * `kysely-mysql` qualify on the lock, `in-memory` on being inline and
+   * single-process; `mongodb` and `kysely-sqlite` qualify on neither.
+   */
+  protected async findUndispatchedSteps(
+    _before: Date,
+    _limit: number
+  ): Promise<Array<{ runId: string; stepId: string }>> {
+    return []
+  }
+
+  /**
+   * Re-drive steps whose dispatch was lost, and report which runs were nudged.
+   *
+   * Arming a step is two writes to two systems: the step row lands `pending`,
+   * then a queue or scheduler job is published. Nothing spans both, so a crash
+   * in between leaves a durable row that nothing will ever pick up — the run
+   * neither finishes nor fails. (Seen on a `workflow.sleep()`: a deploy restart
+   * landed between the sleep step's insert and its timer.)
+   *
+   * The row is the outbox record and this is the relay. Age is the only signal
+   * available — a step `pending` because its dispatch was lost is
+   * indistinguishable from one whose job is merely still queued — so a step
+   * past `undispatchedAfterMs` is re-dispatched regardless, and correctness
+   * rests on the claim rather than on the guess being right. A redundant
+   * dispatch costs one queue message: the loser reads `running` and returns
+   * without invoking anything.
+   *
+   * Re-dispatches back off per step (doubling from 30s, capped at 10m) so a
+   * genuine queue backlog is not amplified by a tick that keeps firing at the
+   * steps the backlog is already delaying. The backoff is per process and
+   * advisory — losing it on restart costs extra dispatches, never correctness.
+   *
+   * This is not self-starting. Call it from a scheduled task; ~30s suits a
+   * queue whose `pending`→`running` latency is well under that.
+   */
+  public async relayUndispatchedSteps(options?: {
+    undispatchedAfterMs?: number
+    limit?: number
+  }): Promise<{ redispatched: string[] }> {
+    const before = new Date(
+      Date.now() -
+        (options?.undispatchedAfterMs ?? DEFAULT_UNDISPATCHED_STEP_MS)
+    )
+    const steps = await this.findUndispatchedSteps(
+      before,
+      options?.limit ?? DEFAULT_UNDISPATCHED_STEP_LIMIT
+    )
+
+    // Backoff is keyed by run, not step, because the run is the unit of
+    // re-drive: `resumeWorkflow` replays the whole run and re-dispatches every
+    // step still owed a job. Holding off a single step while resuming its run
+    // would not suppress anything.
+    const now = Date.now()
+    const runIds = new Set<string>()
+    for (const { runId } of steps) {
+      const eligibleAt = this.redispatchBackoff.get(runId)
+      if (eligibleAt !== undefined && eligibleAt > now) {
+        continue
+      }
+      this.noteRedispatch(runId, now)
+      runIds.add(runId)
+    }
+
+    const redispatched: string[] = []
+    for (const runId of runIds) {
+      try {
+        await this.resumeWorkflow(runId)
+        redispatched.push(runId)
+      } catch (err) {
+        // One unresumable run must not stop the tick from relaying the rest.
+        getSingletonServices()?.logger?.error(
+          `Failed to re-dispatch workflow run ${runId}: ${err instanceof Error ? err.message : String(err)}`
+        )
+      }
+    }
+    return { redispatched }
+  }
+
+  /** Advisory, per-process record of when a run may next be re-dispatched. */
+  private readonly redispatchBackoff = new Map<string, number>()
+
+  private readonly redispatchDelays = new Map<string, number>()
+
+  private noteRedispatch(runId: string, now: number): void {
+    const previous = this.redispatchDelays.get(runId)
+    const delay = Math.min(
+      previous === undefined ? REDISPATCH_BACKOFF_MS : previous * 2,
+      REDISPATCH_BACKOFF_MAX_MS
+    )
+    // A run that settles is never returned again, so entries are only evicted by
+    // this bound — oldest first, which is also least recently re-dispatched.
+    if (this.redispatchBackoff.size >= REDISPATCH_BACKOFF_MAX_ENTRIES) {
+      const oldest = this.redispatchBackoff.keys().next()
+      if (!oldest.done) {
+        this.redispatchBackoff.delete(oldest.value)
+        this.redispatchDelays.delete(oldest.value)
+      }
+    }
+    this.redispatchDelays.set(runId, delay)
+    this.redispatchBackoff.set(runId, now + delay)
+  }
+
   protected resolveStepJobOptions(
     stepOptions?: WorkflowStepOptions
   ): JobOptions {

package/src/wirings/workflow/workflow-dispatch-relay.test.ts

@@ -0,0 +1,128 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/**
+ * `relayUndispatchedSteps` re-drives a run by putting it back on the
+ * orchestrator queue, so the queue is the observation point.
+ */
+function trackResumes(): { runIds: string[] } {
+  const seen: string[] = []
+  pikkuState(null, 'package', 'singletonServices', {
+    queueService: {
+      add: async (_queue: string, data: { runId: string }) => {
+        seen.push(data.runId)
+      },
+    },
+    logger: silentLogger,
+  } as any)
+  return { runIds: seen }
+}
+
+/** Backdates a step's timestamp so it reads as undispatched. */
+async function backdateSteps(
+  ws: InMemoryWorkflowService,
+  runId: string,
+  ms: number
+): Promise<void> {
+  const past = new Date(Date.now() - ms)
+  for (const step of await ws.getRunHistory(runId)) {
+    ;(step as any).updatedAt = past
+  }
+}
+
+describe('undispatched step relay', () => {
+  test('re-dispatches a step whose arming was lost', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // The row committed; the process died before the timer was armed.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdateSteps(ws, runId, 60_000)
+
+    const { redispatched } = await ws.relayUndispatchedSteps()
+
+    assert.deepEqual(redispatched, [runId], 'the run is re-dispatched')
+    assert.deepEqual(resumes.runIds, [runId], 'it is put back on the queue')
+  })
+
+  test('leaves a freshly written step alone', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // Written milliseconds ago — its dispatch is almost certainly in flight.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+
+    const { redispatched } = await ws.relayUndispatchedSteps()
+
+    assert.deepEqual(redispatched, [], 'nothing is re-dispatched')
+    assert.deepEqual(resumes.runIds, [], 'nothing is queued')
+  })
+
+  test('backs off rather than re-dispatching the same step every tick', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Charge card', 'charge', {})
+    await backdateSteps(ws, runId, 60_000)
+
+    const first = await ws.relayUndispatchedSteps()
+    // The step is still pending — a real queue backlog looks exactly like this.
+    const second = await ws.relayUndispatchedSteps()
+
+    assert.deepEqual(first.redispatched, [runId], 'the first tick relays it')
+    assert.deepEqual(
+      second.redispatched,
+      [],
+      'the next tick is held off by the backoff'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
+  test('ignores steps belonging to a settled run', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdateSteps(ws, runId, 60_000)
+    await ws.updateRunStatus(runId, 'completed', {})
+
+    const { redispatched } = await ws.relayUndispatchedSteps()
+
+    assert.deepEqual(redispatched, [], 'a finished run is not re-dispatched')
+    assert.deepEqual(resumes.runIds, [], 'nothing is queued')
+  })
+
+  test('re-dispatches each stuck run once per tick', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runIds: string[] = []
+    for (const n of [1, 2]) {
+      const runId = await ws.createRun('flow', {}, false, 'hash', {
+        type: 'test',
+      })
+      // Two orphaned steps in one run must still produce a single resume.
+      await ws.insertStepState(runId, `Wait ${n}a`, null, { duration: 1000 })
+      await ws.insertStepState(runId, `Wait ${n}b`, null, { duration: 1000 })
+      await backdateSteps(ws, runId, 60_000)
+      runIds.push(runId)
+    }
+
+    const { redispatched } = await ws.relayUndispatchedSteps()
+
+    assert.deepEqual(redispatched.sort(), runIds.sort(), 'both runs relayed')
+    assert.equal(resumes.runIds.length, 2, 'one queue message per run')
+  })
+})

package/src/wirings/workflow/workflow-stalled-recovery.test.ts

@@ -0,0 +1,106 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/**
+ * `recoverStalledRuns` re-drives a run by putting it back on the orchestrator
+ * queue, so the queue is the observation point.
+ */
+function trackResumes(): { runIds: string[] } {
+  const seen: string[] = []
+  pikkuState(null, 'package', 'singletonServices', {
+    queueService: {
+      add: async (_queue: string, data: { runId: string }) => {
+        seen.push(data.runId)
+      },
+    },
+    logger: silentLogger,
+  } as any)
+  return { runIds: seen }
+}
+
+/** Backdates every persisted timestamp so the run reads as idle. */
+async function backdate(
+  ws: InMemoryWorkflowService,
+  runId: string,
+  ms: number
+): Promise<void> {
+  const past = new Date(Date.now() - ms)
+  const run = await ws.getRun(runId)
+  ;(run as any).updatedAt = past
+  for (const step of await ws.getRunHistory(runId)) {
+    ;(step as any).updatedAt = past
+  }
+}
+
+describe('stalled run recovery', () => {
+  test('resumes a running run left with a pending step and nothing in flight', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // A sleep whose timer was never armed: the step row exists, nothing will
+    // ever complete it.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const { resumed } = await ws.recoverStalledRuns()
+
+    assert.deepEqual(resumed, [runId], 'the orphaned run is resumed')
+    assert.deepEqual(resumes.runIds, [runId], 'it is put back on the queue')
+  })
+
+  test('leaves a run alone while a step is still scheduled', async () => {
+    trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    const step = await ws.insertStepState(runId, 'Wait 1h', null, {
+      duration: 3_600_000,
+    })
+    await ws.setStepScheduled(step.stepId)
+    await backdate(ws, runId, 10 * 60_000)
+
+    const { resumed } = await ws.recoverStalledRuns()
+
+    assert.deepEqual(
+      resumed,
+      [],
+      'a legitimately sleeping run is not a stalled run'
+    )
+  })
+
+  test('leaves a run alone until it has actually gone idle', async () => {
+    trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+
+    const { resumed } = await ws.recoverStalledRuns()
+
+    assert.deepEqual(resumed, [], 'a run that just moved is not swept')
+  })
+
+  test('leaves a finished run alone', async () => {
+    trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await ws.updateRunStatus(runId, 'completed')
+    await backdate(ws, runId, 10 * 60_000)
+
+    const { resumed } = await ws.recoverStalledRuns()
+
+    assert.deepEqual(resumed, [], 'only running runs are swept')
+  })
+})