@pikku/core 0.12.88 → 0.12.89

package/CHANGELOG.md

@@ -1,3 +1,39 @@
+## 0.12.89
+
+### Patch Changes
+
+- 32616af: Carry the trace id across a remote RPC hop
+
+  `ContextAwareRPCService` sent the wire's trace id as `x-trace-id`, but the HTTP
+  runner on the receiving end reads `x-request-id` — the header every other sender
+  uses, including `buildRemoteHeaders`, which every deployment service goes
+  through. The receiving side therefore ignored the incoming id and generated a
+  fresh one, so a trace broke at each remote RPC boundary instead of spanning it.
+  Remote RPC now sends `x-request-id` too.
+
+- 6848cd9: fix(workflow): back off the stalled-run sweep, and skip runs that cannot move
+
+  `sweepUndispatchedSteps` has always consulted a per-run backoff so a genuine
+  queue backlog is not amplified by a tick that keeps firing at the steps the
+  backlog is already delaying. `sweepStalledRuns` — its sibling, doing the same
+  re-drive through the same orchestrator queue — had none, and re-resumed every
+  stalled run on every tick. A resume does not clear whatever wedged a run, so
+  the same runs came back on the next tick and the next: in production seven
+  permanently stuck runs refilled a purged orchestrator queue at seven messages a
+  minute, and a backlog of six thousand could never drain because each pass added
+  work the previous pass had not finished. It now takes the same backoff, and
+  both sweeps share one instance — the record belongs to the re-drive, not to the
+  signal that asked for it, so a run the relay nudged a moment ago is not nudged
+  again by the sweep.
+
+  `runWorkflowJob` also now returns immediately for a run in a terminal state
+  instead of taking the run lock and replaying the workflow body. The orchestrator
+  queue is at-least-once and the relay re-dispatches on purpose, so a message for
+  a run that already settled is routine — and replaying one could park the body on
+  a wait that nothing would ever satisfy, holding the run lock, and the pooled
+  connection under it, until something external gave up. `suspended` is
+  deliberately not included: it ends a pass, not the run.
+
 ## 0.12.88
 
 ### Patch Changes

package/dist/types/core.types.d.ts

@@ -182,7 +182,7 @@ export type PikkuWire<In = unknown, Out = unknown, HasInitialSession extends boo
      * sets it; services that log fall back to the singleton logger.
      */
     logger: Logger;
-    /** Trace ID for distributed tracing — propagated across remote RPC calls via x-trace-id header */
+    /** Trace ID for distributed tracing — propagated across remote RPC calls via the x-request-id header */
     traceId: string;
     functionId: string;
     addonNamespace: string;

package/dist/wirings/rpc/rpc-runner.js

@@ -245,7 +245,7 @@ export class ContextAwareRPCService {
             headers.authorization = `Bearer ${token}`;
         }
         if (this.wire.traceId) {
-            headers['x-trace-id'] = this.wire.traceId;
+            headers['x-request-id'] = this.wire.traceId;
         }
         const base = serverUrl.replace(/\/+$/, '');
         const res = await fetch(`${base}/remote/rpc/${encodeURIComponent(remoteFn)}`, {

package/dist/wirings/workflow/pikku-workflow-service.js

@@ -8,7 +8,7 @@ import { RPCNotFoundError } from '../rpc/rpc-runner.js';
 import { deriveInvocationId } from './workflow-invocation-id.js';
 import { approvalDeciderFrom } from './workflow-approval-policy.js';
 import { buildRunTimeline, reconstructStateAt, } from './run-timeline.js';
-import { DEFAULT_STEP_RETRIES, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, } from './workflow-constants.js';
+import { DEFAULT_STEP_RETRIES, WORKFLOW_CHILD_POLL_MAX_MS, WORKFLOW_END_STATES, WORKFLOW_POLL_FACTOR, WORKFLOW_POLL_MIN_MS, WORKFLOW_TERMINAL_STATES, isRunSettled, } from './workflow-constants.js';
 import { WorkflowAsyncException, WorkflowCancelledException, WorkflowDispatchException, WorkflowNotFoundError, WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotFoundError, WorkflowStepNameNotString, WorkflowSuspendedException, } from './workflow-errors.js';
 import { resolveWorkflowMeta } from './workflow-meta-resolver.js';
 import { jobGroupFor, orchestratorQueueName, resolveWorkflowConfig, stepJobOptions, stepWorkerQueueName, } from './workflow-queue-routing.js';
@@ -284,7 +284,7 @@ export class PikkuWorkflowService {
      * scheduled task at whatever interval suits the workload.
      */
     async recoverStalledRuns(options) {
-        return sweepStalledRuns((before, limit) => this.findStalledRunIds(before, limit), options, this.sweepDeps);
+        return sweepStalledRuns((before, limit) => this.findStalledRunIds(before, limit), this.redispatchBackoff, options, this.sweepDeps);
     }
     /**
      * Re-drive steps whose dispatch was lost. Not self-starting — call it from a
@@ -546,6 +546,8 @@ export class PikkuWorkflowService {
         if (!run) {
             throw new WorkflowRunNotFoundError(runId);
         }
+        if (isRunSettled(run.status))
+            return;
         const resolved = resolveWorkflowMeta(run.workflow);
         const workflowMeta = resolved?.meta;
         const pkgName = resolved?.packageName ?? null;

package/dist/wirings/workflow/workflow-constants.d.ts

@@ -3,6 +3,23 @@ export declare const DEFAULT_STEP_RETRIES = 5;
 export declare const WORKFLOW_END_STATES: ReadonlySet<string>;
 /** Statuses a run cannot leave at all. */
 export declare const WORKFLOW_TERMINAL_STATES: ReadonlySet<string>;
+/**
+ * True for a run that will never move again, whatever arrives for it.
+ *
+ * Worth checking before doing anything with an orchestrator message, because
+ * such a message is routine rather than exceptional: the queue is
+ * at-least-once, the relay re-dispatches on purpose, and a run can settle
+ * while a message for it is still in flight. Replaying one is not free —
+ * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * body re-entered after its run failed can park on a wait that nothing will
+ * ever satisfy, holding the lock and the connection under it until something
+ * external gives up. Every leaked advisory lock seen in production traced back
+ * to that: a granted lock, an idle session, and a run already `failed`.
+ *
+ * Note `suspended` is deliberately absent. It ends a run's *current* pass but
+ * not the run, which resumes when its approval or signal arrives.
+ */
+export declare const isRunSettled: (status: string) => boolean;
 export declare const WORKFLOW_POLL_MIN_MS = 10;
 export declare const WORKFLOW_POLL_FACTOR = 1.6;
 export declare const WORKFLOW_CHILD_POLL_MAX_MS = 500;

package/dist/wirings/workflow/workflow-constants.js

@@ -12,6 +12,23 @@ export const WORKFLOW_TERMINAL_STATES = new Set([
     'failed',
     'cancelled',
 ]);
+/**
+ * True for a run that will never move again, whatever arrives for it.
+ *
+ * Worth checking before doing anything with an orchestrator message, because
+ * such a message is routine rather than exceptional: the queue is
+ * at-least-once, the relay re-dispatches on purpose, and a run can settle
+ * while a message for it is still in flight. Replaying one is not free —
+ * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * body re-entered after its run failed can park on a wait that nothing will
+ * ever satisfy, holding the lock and the connection under it until something
+ * external gives up. Every leaked advisory lock seen in production traced back
+ * to that: a granted lock, an idle session, and a run already `failed`.
+ *
+ * Note `suspended` is deliberately absent. It ends a run's *current* pass but
+ * not the run, which resumes when its approval or signal arrives.
+ */
+export const isRunSettled = (status) => WORKFLOW_TERMINAL_STATES.has(status);
 export const WORKFLOW_POLL_MIN_MS = 10;
 export const WORKFLOW_POLL_FACTOR = 1.6;
 export const WORKFLOW_CHILD_POLL_MAX_MS = 500;

package/dist/wirings/workflow/workflow-recovery.d.ts

@@ -7,6 +7,15 @@ import type { Logger } from '../../services/logger.js';
  * owed a job, so holding off a single step while resuming its run would
  * suppress nothing.
  *
+ * For the same reason one instance is shared by every sweep rather than kept
+ * per sweep. The record is of the action, not of the signal that prompted it:
+ * a stalled run and an undispatched step are different observations, but both
+ * are answered by the one orchestrator message, so a run the relay re-drove a
+ * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
+ * is what makes the guarantee a message-per-run-per-window instead of one per
+ * sweep, and no recovery is lost by it: whichever sweep gets there first
+ * performs the identical re-drive, and the cap keeps the delay at 10m.
+ *
  * Losing this on restart costs extra dispatches, never correctness.
  */
 export declare class RedispatchBackoff {
@@ -34,8 +43,16 @@ type SweepDeps = {
  * actually stuck costs an orchestration pass and changes nothing. That
  * idempotence is what makes an idle-time heuristic safe here; a run that is
  * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ *
+ * Idempotent is not free, though: a run stays stalled until something clears
+ * the reason it stalled, so an unconditional sweep re-queues the same runs on
+ * every tick forever. That is how a handful of wedged runs became a queue of
+ * thousands that could not drain — each pass added work the previous pass had
+ * not finished. The same per-run backoff the relay uses bounds it: a run is
+ * re-driven, then held off for a doubling delay, so a sweep costs at most one
+ * message per run per window rather than one per run per tick.
  */
-export declare const sweepStalledRuns: (findStalledRunIds: (before: Date, limit: number) => Promise<string[]>, options: {
+export declare const sweepStalledRuns: (findStalledRunIds: (before: Date, limit: number) => Promise<string[]>, backoff: RedispatchBackoff, options: {
     stalledAfterMs?: number;
     limit?: number;
 } | undefined, deps: SweepDeps) => Promise<{

package/dist/wirings/workflow/workflow-recovery.js

@@ -7,6 +7,15 @@ import { DEFAULT_STALLED_RUN_LIMIT, DEFAULT_STALLED_RUN_MS, DEFAULT_UNDISPATCHED
  * owed a job, so holding off a single step while resuming its run would
  * suppress nothing.
  *
+ * For the same reason one instance is shared by every sweep rather than kept
+ * per sweep. The record is of the action, not of the signal that prompted it:
+ * a stalled run and an undispatched step are different observations, but both
+ * are answered by the one orchestrator message, so a run the relay re-drove a
+ * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
+ * is what makes the guarantee a message-per-run-per-window instead of one per
+ * sweep, and no recovery is lost by it: whichever sweep gets there first
+ * performs the identical re-drive, and the cap keeps the delay at 10m.
+ *
  * Losing this on restart costs extra dispatches, never correctness.
  */
 export class RedispatchBackoff {
@@ -61,10 +70,29 @@ const resumeEach = async (runIds, { resume, logger }, failure) => {
  * actually stuck costs an orchestration pass and changes nothing. That
  * idempotence is what makes an idle-time heuristic safe here; a run that is
  * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ *
+ * Idempotent is not free, though: a run stays stalled until something clears
+ * the reason it stalled, so an unconditional sweep re-queues the same runs on
+ * every tick forever. That is how a handful of wedged runs became a queue of
+ * thousands that could not drain — each pass added work the previous pass had
+ * not finished. The same per-run backoff the relay uses bounds it: a run is
+ * re-driven, then held off for a doubling delay, so a sweep costs at most one
+ * message per run per window rather than one per run per tick.
  */
-export const sweepStalledRuns = async (findStalledRunIds, options, deps) => {
+export const sweepStalledRuns = async (findStalledRunIds, backoff, options, deps) => {
     const before = new Date(Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS));
-    const runIds = await findStalledRunIds(before, options?.limit ?? DEFAULT_STALLED_RUN_LIMIT);
+    const found = await findStalledRunIds(before, options?.limit ?? DEFAULT_STALLED_RUN_LIMIT);
+    const now = Date.now();
+    const runIds = [];
+    for (const runId of found) {
+        if (!backoff.isEligible(runId, now))
+            continue;
+        // Noted before the resume, not after: a run whose resume throws is exactly
+        // the run most likely to still be here next tick, and re-driving it every
+        // tick is the amplification this backoff exists to stop.
+        backoff.note(runId, now);
+        runIds.push(runId);
+    }
     return {
         resumed: await resumeEach(runIds, deps, (runId, detail) => `Failed to resume stalled workflow run ${runId}: ${detail}`),
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pikku/core",
-  "version": "0.12.88",
+  "version": "0.12.89",
   "description": "The Pikku runtime — functions, wirings, services, middleware and types",
   "author": "yasser.fadl@gmail.com",
   "license": "MIT",

package/src/types/core.types.ts

@@ -240,7 +240,7 @@ export type PikkuWire<
    * sets it; services that log fall back to the singleton logger.
    */
   logger: Logger
-  /** Trace ID for distributed tracing — propagated across remote RPC calls via x-trace-id header */
+  /** Trace ID for distributed tracing — propagated across remote RPC calls via the x-request-id header */
   traceId: string
   functionId: string
   addonNamespace: string

package/src/wirings/rpc/rpc-runner.test.ts

@@ -900,7 +900,12 @@ describe('wireRemoteAddon dispatch', () => {
         calls[0]!.init.headers.authorization,
         'Bearer secret-value-for-REGISTRY_TOKEN'
       )
-      assert.equal(calls[0]!.init.headers['x-trace-id'], 'trace-r')
+      assert.equal(
+        calls[0]!.init.headers['x-request-id'],
+        'trace-r',
+        'a remote RPC must send the trace id under the header the receiving ' +
+          'runner reads, or the trace chain breaks at the hop'
+      )
     } finally {
       restoreFetch()
     }

package/src/wirings/rpc/rpc-runner.ts

@@ -356,7 +356,7 @@ export class ContextAwareRPCService {
       headers.authorization = `Bearer ${token}`
     }
     if (this.wire.traceId) {
-      headers['x-trace-id'] = this.wire.traceId
+      headers['x-request-id'] = this.wire.traceId
     }
 
     const base = serverUrl.replace(/\/+$/, '')

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -59,6 +59,7 @@ import {
   WORKFLOW_POLL_FACTOR,
   WORKFLOW_POLL_MIN_MS,
   WORKFLOW_TERMINAL_STATES,
+  isRunSettled,
 } from './workflow-constants.js'
 import {
   WorkflowAsyncException,
@@ -672,6 +673,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   }): Promise<{ resumed: string[] }> {
     return sweepStalledRuns(
       (before, limit) => this.findStalledRunIds(before, limit),
+      this.redispatchBackoff,
       options,
       this.sweepDeps
     )
@@ -1084,6 +1086,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     if (!run) {
       throw new WorkflowRunNotFoundError(runId)
     }
+    if (isRunSettled(run.status)) return
 
     const resolved = resolveWorkflowMeta(run.workflow)
     const workflowMeta = resolved?.meta

package/src/wirings/workflow/workflow-constants.ts

@@ -15,6 +15,25 @@ export const WORKFLOW_TERMINAL_STATES: ReadonlySet<string> = new Set([
   'cancelled',
 ])
 
+/**
+ * True for a run that will never move again, whatever arrives for it.
+ *
+ * Worth checking before doing anything with an orchestrator message, because
+ * such a message is routine rather than exceptional: the queue is
+ * at-least-once, the relay re-dispatches on purpose, and a run can settle
+ * while a message for it is still in flight. Replaying one is not free —
+ * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * body re-entered after its run failed can park on a wait that nothing will
+ * ever satisfy, holding the lock and the connection under it until something
+ * external gives up. Every leaked advisory lock seen in production traced back
+ * to that: a granted lock, an idle session, and a run already `failed`.
+ *
+ * Note `suspended` is deliberately absent. It ends a run's *current* pass but
+ * not the run, which resumes when its approval or signal arrives.
+ */
+export const isRunSettled = (status: string): boolean =>
+  WORKFLOW_TERMINAL_STATES.has(status)
+
 export const WORKFLOW_POLL_MIN_MS = 10
 
 export const WORKFLOW_POLL_FACTOR = 1.6

package/src/wirings/workflow/workflow-recovery.ts

@@ -17,6 +17,15 @@ import {
  * owed a job, so holding off a single step while resuming its run would
  * suppress nothing.
  *
+ * For the same reason one instance is shared by every sweep rather than kept
+ * per sweep. The record is of the action, not of the signal that prompted it:
+ * a stalled run and an undispatched step are different observations, but both
+ * are answered by the one orchestrator message, so a run the relay re-drove a
+ * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
+ * is what makes the guarantee a message-per-run-per-window instead of one per
+ * sweep, and no recovery is lost by it: whichever sweep gets there first
+ * performs the identical re-drive, and the cap keeps the delay at 10m.
+ *
  * Losing this on restart costs extra dispatches, never correctness.
  */
 export class RedispatchBackoff {
@@ -88,19 +97,40 @@ const resumeEach = async (
  * actually stuck costs an orchestration pass and changes nothing. That
  * idempotence is what makes an idle-time heuristic safe here; a run that is
  * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ *
+ * Idempotent is not free, though: a run stays stalled until something clears
+ * the reason it stalled, so an unconditional sweep re-queues the same runs on
+ * every tick forever. That is how a handful of wedged runs became a queue of
+ * thousands that could not drain — each pass added work the previous pass had
+ * not finished. The same per-run backoff the relay uses bounds it: a run is
+ * re-driven, then held off for a doubling delay, so a sweep costs at most one
+ * message per run per window rather than one per run per tick.
  */
 export const sweepStalledRuns = async (
   findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
+  backoff: RedispatchBackoff,
   options: { stalledAfterMs?: number; limit?: number } | undefined,
   deps: SweepDeps
 ): Promise<{ resumed: string[] }> => {
   const before = new Date(
     Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
   )
-  const runIds = await findStalledRunIds(
+  const found = await findStalledRunIds(
     before,
     options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
   )
+
+  const now = Date.now()
+  const runIds: string[] = []
+  for (const runId of found) {
+    if (!backoff.isEligible(runId, now)) continue
+    // Noted before the resume, not after: a run whose resume throws is exactly
+    // the run most likely to still be here next tick, and re-driving it every
+    // tick is the amplification this backoff exists to stop.
+    backoff.note(runId, now)
+    runIds.push(runId)
+  }
+
   return {
     resumed: await resumeEach(
       runIds,

package/src/wirings/workflow/workflow-stalled-recovery.test.ts

@@ -89,6 +89,52 @@ describe('stalled run recovery', () => {
     assert.deepEqual(resumed, [], 'a run that just moved is not swept')
   })
 
+  test('backs off rather than re-resuming the same run every tick', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const first = await ws.recoverStalledRuns()
+    // Still stalled: a resume does not clear whatever wedged the run, so an
+    // unconditional sweep would re-queue it on this tick and on every tick
+    // after it.
+    const second = await ws.recoverStalledRuns()
+
+    assert.deepEqual(first.resumed, [runId], 'the first sweep resumes it')
+    assert.deepEqual(
+      second.resumed,
+      [],
+      'the next sweep is held off by the backoff'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
+  test('does not re-drive a run the relay has just re-dispatched', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // Old enough to read as both an undispatched step and a stalled run.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const relayed = await ws.relayUndispatchedSteps()
+    const swept = await ws.recoverStalledRuns()
+
+    assert.deepEqual(relayed.redispatched, [runId], 'the relay re-drives it')
+    assert.deepEqual(
+      swept.resumed,
+      [],
+      'the sweep adds nothing the relay has not already queued'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
   test('leaves a finished run alone', async () => {
     trackResumes()
     const ws = new InMemoryWorkflowService()

package/src/wirings/workflow/workflow-terminal-run-guard.test.ts

@@ -0,0 +1,105 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/** Records every run lock taken, which is the whole point of the guard. */
+class LockSpyWorkflowService extends InMemoryWorkflowService {
+  public readonly locked: string[] = []
+
+  public override async withRunLock<T>(
+    id: string,
+    fn: () => Promise<T>
+  ): Promise<T> {
+    this.locked.push(id)
+    return super.withRunLock(id, fn)
+  }
+}
+
+/** A registered, fully described workflow, so nothing else can short-circuit. */
+const startRun = async (): Promise<{
+  service: LockSpyWorkflowService
+  runId: string
+  entered: () => boolean
+}> => {
+  resetPikkuState()
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+    queueService: { add: async () => 'job-1' },
+  } as any)
+
+  let bodyEntered = false
+  const func = async () => {
+    bodyEntered = true
+  }
+  pikkuState(null, 'workflows', 'meta', {
+    flow: { name: 'flow', pikkuFuncId: 'flow', source: 'dsl' },
+  } as any)
+  pikkuState(null, 'workflows', 'registrations').set('flow', {
+    name: 'flow',
+    func,
+  } as any)
+  pikkuState(null, 'function', 'meta', {
+    flow: {
+      pikkuFuncId: 'flow',
+      inputSchemaName: null,
+      outputSchemaName: null,
+      sessionless: true,
+    },
+  } as any)
+  pikkuState(null, 'function', 'functions').set('flow', { func } as any)
+
+  const service = new LockSpyWorkflowService()
+  const runId = await service.createRun('flow', {}, false, '', { type: 'test' })
+  return { service, runId, entered: () => bodyEntered }
+}
+
+/**
+ * The leak this guards, read straight off production: every granted advisory
+ * lock held by an idle session mapped to a run that was already `failed`. The
+ * orchestrator queue is at-least-once, so a message for a settled run is
+ * routine — and answering it by taking the run lock and replaying the body is
+ * how a run that can never move again ends up holding a lock and a pooled
+ * connection while it waits on something that will never arrive.
+ */
+describe('an orchestrator message for a run that already settled', () => {
+  for (const status of ['failed', 'completed', 'cancelled'] as const) {
+    test(`a ${status} run is answered without taking its lock`, async () => {
+      const { service, runId, entered } = await startRun()
+      await service.updateRunStatus(runId, status)
+
+      await service.runWorkflowJob(runId, {} as any)
+
+      assert.deepEqual(
+        service.locked,
+        [],
+        'the run lock was taken for a run that can never move again'
+      )
+      assert.equal(
+        entered(),
+        false,
+        'the workflow body was replayed after the run had settled'
+      )
+    })
+  }
+
+  /**
+   * Guards the tests above: they would pass just as well for a service that
+   * refused to orchestrate anything at all.
+   */
+  test('a suspended run is still orchestrated', async () => {
+    const { service, runId } = await startRun()
+    await service.updateRunStatus(runId, 'suspended')
+
+    await service.runWorkflowJob(runId, {} as any)
+
+    assert.deepEqual(
+      service.locked,
+      [runId],
+      'suspended ends a pass, not the run — it resumes when its signal arrives'
+    )
+  })
+})