@pikku/core 0.12.86 → 0.12.89

package/src/wirings/agent/agent.types.ts

@@ -6,6 +6,7 @@ import type {
   CorePikkuMiddleware,
   MiddlewareMetadata,
 } from '../../middleware/middleware.types.js'
+import type { CoreSingletonServices } from '../../types/core.types.js'
 import type { PermissionMetadata } from '../../function/function-meta.types.js'
 import type { AIProviderOptions } from '../../services/agent-runner-service.js'
 import type { PikkuChannel } from '../channel/channel.types.js'
@@ -180,10 +181,10 @@ export interface AgentToolDef extends Partial<ApprovalPolicy> {
 
 export interface PikkuAgentMiddlewareHooks<
   State extends Record<string, unknown> = Record<string, unknown>,
-  Services = any,
+  SingletonServices extends CoreSingletonServices = CoreSingletonServices,
 > {
   modifyInput?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       messages: AgentMessage[]
       instructions: string
@@ -210,7 +211,7 @@ export interface PikkuAgentMiddlewareHooks<
     | { messages: AgentMessage[]; instructions: string }
 
   modifyOutputStream?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       event: AgentStreamEvent
       allEvents: readonly AgentStreamEvent[]
@@ -262,7 +263,7 @@ export interface PikkuAgentMiddlewareHooks<
    * leaves them untouched.
    */
   modifyOutput?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       text: string
       messages: AgentMessage[]
@@ -282,7 +283,7 @@ export interface PikkuAgentMiddlewareHooks<
       }
 
   beforeToolCall?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       toolName: string
       toolCallId: string
@@ -294,7 +295,7 @@ export interface PikkuAgentMiddlewareHooks<
     | void
 
   afterToolCall?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       toolName: string
       toolCallId: string
@@ -305,7 +306,7 @@ export interface PikkuAgentMiddlewareHooks<
   ) => Promise<{ result: unknown } | void> | { result: unknown } | void
 
   afterStep?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       stepNumber: number
       text: string
@@ -323,7 +324,7 @@ export interface PikkuAgentMiddlewareHooks<
   ) => Promise<void> | void
 
   onError?: (
-    services: Services,
+    services: SingletonServices,
     ctx: {
       error: Error
       stepNumber: number

package/src/wirings/rpc/rpc-runner.test.ts

@@ -900,7 +900,12 @@ describe('wireRemoteAddon dispatch', () => {
         calls[0]!.init.headers.authorization,
         'Bearer secret-value-for-REGISTRY_TOKEN'
       )
-      assert.equal(calls[0]!.init.headers['x-trace-id'], 'trace-r')
+      assert.equal(
+        calls[0]!.init.headers['x-request-id'],
+        'trace-r',
+        'a remote RPC must send the trace id under the header the receiving ' +
+          'runner reads, or the trace chain breaks at the hop'
+      )
     } finally {
       restoreFetch()
     }

package/src/wirings/rpc/rpc-runner.ts

@@ -356,7 +356,7 @@ export class ContextAwareRPCService {
       headers.authorization = `Bearer ${token}`
     }
     if (this.wire.traceId) {
-      headers['x-trace-id'] = this.wire.traceId
+      headers['x-request-id'] = this.wire.traceId
     }
 
     const base = serverUrl.replace(/\/+$/, '')

package/src/wirings/variable/validate-variable-definitions.test.ts

@@ -89,26 +89,27 @@ describe('validateAndBuildVariableDefinitionsMeta', () => {
     assert.deepStrictEqual(result, {})
   })
 
-  test('carries required into the meta so only the opted-in ones block a deploy', () => {
+  test('carries optional into the meta so only the opted-out ones skip the gate', () => {
     const result = validateAndBuildVariableDefinitionsMeta(
       [
-        {
-          name: 'consoleUrl',
-          displayName: 'Console URL',
-          variableId: 'CONSOLE_URL',
-          required: true,
-          sourceFile: 'a.ts',
-        },
         {
           name: 'corsOrigins',
           displayName: 'Allowed Browser Origins',
           variableId: 'CORS_ORIGINS',
+          optional: true,
+          sourceFile: 'a.ts',
+        },
+        {
+          name: 'consoleUrl',
+          displayName: 'Console URL',
+          variableId: 'CONSOLE_URL',
           sourceFile: 'a.ts',
         },
       ] as any,
       new Map()
     )
-    assert.strictEqual(result['consoleUrl']!.required, true)
-    assert.strictEqual(result['corsOrigins']!.required, undefined)
+    assert.strictEqual(result['corsOrigins']!.optional, true)
+    // Undeclared means required, which is what makes the gate ask by default.
+    assert.strictEqual(result['consoleUrl']!.optional, undefined)
   })
 })

package/src/wirings/variable/validate-variable-definitions.ts

@@ -43,7 +43,7 @@ export function validateAndBuildVariableDefinitionsMeta(
           description: def.description,
           variableId: def.variableId,
           schema: def.schema,
-          required: def.required,
+          optional: def.optional,
           docsUrl: def.docsUrl,
           sourceFile: def.sourceFile,
         }
@@ -60,7 +60,7 @@ export function validateAndBuildVariableDefinitionsMeta(
         description: def.description,
         variableId: def.variableId,
         schema: def.schema,
-        required: def.required,
+        optional: def.optional,
         docsUrl: def.docsUrl,
         sourceFile: def.sourceFile,
       }

package/src/wirings/variable/variable.types.ts

@@ -5,13 +5,19 @@ export type CoreVariable<T = unknown> = {
   variableId: string
   schema: T
   /**
-   * A variable is OPTIONAL by default because `variables.get` returns
-   * `T | undefined` and never throws — every caller already handles absence, so
-   * a deploy gate that blocks on one contradicts the API. Mark a variable
-   * `required` for the few whose absence genuinely breaks the app; only those
-   * block a deploy.
+   * A variable is REQUIRED by default, and marking it `optional` is how a
+   * declaration says its absence is a supported state. Same flag, same
+   * polarity and same meaning as `CoreSecret.optional` — one word to learn
+   * rather than two with opposite senses.
+   *
+   * Defaulting to required rather than following `variables.get`'s
+   * `T | undefined` return is deliberate. That signature describes what a
+   * caller must HANDLE, not whether a deployment is correct without the value:
+   * an undefined feature flag is fine, an undefined API base URL is an outage
+   * that the type system cannot tell apart. Declaring the difference is the
+   * point of the flag, and the safe default for an undeclared one is to ask.
    */
-  required?: boolean
+  optional?: boolean
   docsUrl?: string
 }
 
@@ -21,7 +27,7 @@ export type VariableDefinitionMeta = {
   description?: string
   variableId: string
   schema?: Record<string, unknown> | string
-  required?: boolean
+  optional?: boolean
   docsUrl?: string
   sourceFile?: string
 }

package/src/wirings/workflow/pikku-scenario-service.ts

@@ -59,8 +59,34 @@ const assertionStep = <T extends WorkflowStepOptions>(
   ({ ...options, retries: options?.retries ?? 0 }) as T & { retries: number }
 
 export { addFeature, resolveFeatureScenarios } from './feature.js'
-export type * from './scenario.types.js'
-export type * from './scenario-run.types.js'
+export type {
+  CoreFeature,
+  CoreFeatureScenario,
+  FeatureMeta,
+  FeaturesMeta,
+  PikkuBrowserWire,
+  PikkuScenarioWire,
+  ScenarioBrowserFailure,
+  ScenarioBrowserProvider,
+  ScenarioEnvironment,
+  ScenarioStepKind,
+  ScenarioStepMeta,
+  ScenarioStepOptions,
+  ScenarioStepPhase,
+  ScenarioSurface,
+  TestIdSelector,
+} from './scenario.types.js'
+export type {
+  ScenarioArtifact,
+  ScenarioFailureDetail,
+  ScenarioResult,
+  ScenarioRunRecord,
+  ScenarioRunReport,
+  ScenarioRunStatus,
+  ScenarioRunStore,
+  ScenarioRunSummary,
+  ScenarioStepRow,
+} from './scenario-run.types.js'
 export { SCENARIO_SURFACES } from './scenario-step.types.js'
 
 // Which of a step's bindings run: one for an action, every witness for a `then`
@@ -979,10 +1005,7 @@ export class PikkuScenarioService implements WorkflowRunExtension {
    * Whether a step is driven by a persona, and so must be given one. Stamped by
    * the definer from a `browser` binding or an explicit `actor: true`.
    */
-  private requiresActor(
-    packageName: string | null,
-    stepFunc: string
-  ): boolean {
+  private requiresActor(packageName: string | null, stepFunc: string): boolean {
     return (
       this.scenarioStepConfig(packageName, stepFunc)?.requiresActor === true
     )

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -59,6 +59,7 @@ import {
   WORKFLOW_POLL_FACTOR,
   WORKFLOW_POLL_MIN_MS,
   WORKFLOW_TERMINAL_STATES,
+  isRunSettled,
 } from './workflow-constants.js'
 import {
   WorkflowAsyncException,
@@ -68,7 +69,6 @@ import {
   WorkflowRunCancelledError,
   WorkflowRunFailedError,
   WorkflowRunNotFoundError,
-  WorkflowStepFunctionMismatchError,
   WorkflowStepNameNotString,
   WorkflowSuspendedException,
 } from './workflow-errors.js'
@@ -95,6 +95,7 @@ import {
 } from './workflow-approval.js'
 import { auditApprovalDecision } from './workflow-approval-audit.js'
 import { recordSuspension, suspendStepNameFor } from './workflow-suspend.js'
+import { claimStepByReadThenWrite } from './workflow-step-claim.js'
 import {
   RedispatchBackoff,
   sweepStalledRuns,
@@ -639,12 +640,12 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    *
    * Returns nothing by default so a store that cannot express the query keeps
    * working unchanged — and, because it does not opt in, gains no re-dispatches
-   * either. A store must have an atomic `withStepLock` before overriding this,
-   * or no concurrency for one to exclude: the relay makes duplicate dispatch
-   * routine, and the claim in `executeWorkflowStepInner` is what keeps a
-   * duplicate from becoming a second execution. `kysely-postgres` and
-   * `kysely-mysql` qualify on the lock, `in-memory` on being inline and
-   * single-process; `mongodb` and `kysely-sqlite` qualify on neither.
+   * either. A store must have an atomic `claimStepForExecution` before
+   * overriding this, or no concurrency for one to exclude: the relay makes
+   * duplicate dispatch routine, and the claim is what keeps a duplicate from
+   * becoming a second execution. Every `@pikku/kysely` dialect qualifies on its
+   * status-guarded claim, `in-memory` on being inline and single-process;
+   * `mongodb` still qualifies on neither.
    */
   protected async findUndispatchedSteps(
     _before: Date,
@@ -672,6 +673,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   }): Promise<{ resumed: string[] }> {
     return sweepStalledRuns(
       (before, limit) => this.findStalledRunIds(before, limit),
+      this.redispatchBackoff,
       options,
       this.sweepDeps
     )
@@ -1084,6 +1086,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     if (!run) {
       throw new WorkflowRunNotFoundError(runId)
     }
+    if (isRunSettled(run.status)) return
 
     const resolved = resolveWorkflowMeta(run.workflow)
     const workflowMeta = resolved?.meta
@@ -1300,6 +1303,29 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
+  /**
+   * Take sole ownership of a step before it runs, returning the state to run
+   * under — or `null` when another dispatch already owns it.
+   *
+   * Dispatch is at-least-once by design: the relay re-dispatches steps it
+   * believes were dropped, and a queue can redeliver a job it already handed
+   * out. This is the one place that keeps a duplicate dispatch from becoming a
+   * second execution of a side-effecting step, so it is only as strong as the
+   * exclusion it is built on — and `withStepLock` excludes nothing unless the
+   * store backs it with a real primitive. A store able to express the decision
+   * as one conditional write should override this rather than reach for a lock,
+   * which is what `@pikku/kysely` does with a status-guarded `UPDATE`.
+   */
+  protected async claimStepForExecution(
+    runId: string,
+    stepName: string,
+    rpcName: string
+  ): Promise<StepState | null> {
+    return this.withStepLock(runId, stepName, () =>
+      claimStepByReadThenWrite(this, runId, stepName, rpcName)
+    )
+  }
+
   private async executeWorkflowStepInner(
     runId: string,
     stepName: string,
@@ -1307,26 +1333,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     data: any,
     rpcService: PikkuRPC
   ): Promise<void> {
-    const claimed = await this.withStepLock(runId, stepName, async () => {
-      const stepState = await this.getStepState(runId, stepName)
-      // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
-      if (
-        stepState.rpcName !== undefined &&
-        stepState.rpcName !== (rpcName ?? null)
-      ) {
-        throw new WorkflowStepFunctionMismatchError(runId, stepName)
-      }
-      if (stepState.status === 'succeeded' || stepState.status === 'running') {
-        return null
-      }
-      if (stepState.status === 'failed') {
-        return this.createRetryAttempt(stepState.stepId, 'running')
-      }
-      if (stepState.status === 'pending' || stepState.status === 'scheduled') {
-        await this.setStepRunning(stepState.stepId)
-      }
-      return stepState
-    })
+    const claimed = await this.claimStepForExecution(runId, stepName, rpcName)
 
     if (!claimed) {
       return

package/src/wirings/workflow/workflow-constants.ts

@@ -15,6 +15,25 @@ export const WORKFLOW_TERMINAL_STATES: ReadonlySet<string> = new Set([
   'cancelled',
 ])
 
+/**
+ * True for a run that will never move again, whatever arrives for it.
+ *
+ * Worth checking before doing anything with an orchestrator message, because
+ * such a message is routine rather than exceptional: the queue is
+ * at-least-once, the relay re-dispatches on purpose, and a run can settle
+ * while a message for it is still in flight. Replaying one is not free —
+ * `runWorkflowJob` takes the run lock and re-enters the workflow body, and a
+ * body re-entered after its run failed can park on a wait that nothing will
+ * ever satisfy, holding the lock and the connection under it until something
+ * external gives up. Every leaked advisory lock seen in production traced back
+ * to that: a granted lock, an idle session, and a run already `failed`.
+ *
+ * Note `suspended` is deliberately absent. It ends a run's *current* pass but
+ * not the run, which resumes when its approval or signal arrives.
+ */
+export const isRunSettled = (status: string): boolean =>
+  WORKFLOW_TERMINAL_STATES.has(status)
+
 export const WORKFLOW_POLL_MIN_MS = 10
 
 export const WORKFLOW_POLL_FACTOR = 1.6

package/src/wirings/workflow/workflow-recovery.ts

@@ -17,6 +17,15 @@ import {
  * owed a job, so holding off a single step while resuming its run would
  * suppress nothing.
  *
+ * For the same reason one instance is shared by every sweep rather than kept
+ * per sweep. The record is of the action, not of the signal that prompted it:
+ * a stalled run and an undispatched step are different observations, but both
+ * are answered by the one orchestrator message, so a run the relay re-drove a
+ * moment ago gains nothing from the stalled sweep re-driving it again. Sharing
+ * is what makes the guarantee a message-per-run-per-window instead of one per
+ * sweep, and no recovery is lost by it: whichever sweep gets there first
+ * performs the identical re-drive, and the cap keeps the delay at 10m.
+ *
  * Losing this on restart costs extra dispatches, never correctness.
  */
 export class RedispatchBackoff {
@@ -88,19 +97,40 @@ const resumeEach = async (
  * actually stuck costs an orchestration pass and changes nothing. That
  * idempotence is what makes an idle-time heuristic safe here; a run that is
  * legitimately mid-sleep is excluded anyway, since its step is `scheduled`.
+ *
+ * Idempotent is not free, though: a run stays stalled until something clears
+ * the reason it stalled, so an unconditional sweep re-queues the same runs on
+ * every tick forever. That is how a handful of wedged runs became a queue of
+ * thousands that could not drain — each pass added work the previous pass had
+ * not finished. The same per-run backoff the relay uses bounds it: a run is
+ * re-driven, then held off for a doubling delay, so a sweep costs at most one
+ * message per run per window rather than one per run per tick.
  */
 export const sweepStalledRuns = async (
   findStalledRunIds: (before: Date, limit: number) => Promise<string[]>,
+  backoff: RedispatchBackoff,
   options: { stalledAfterMs?: number; limit?: number } | undefined,
   deps: SweepDeps
 ): Promise<{ resumed: string[] }> => {
   const before = new Date(
     Date.now() - (options?.stalledAfterMs ?? DEFAULT_STALLED_RUN_MS)
   )
-  const runIds = await findStalledRunIds(
+  const found = await findStalledRunIds(
     before,
     options?.limit ?? DEFAULT_STALLED_RUN_LIMIT
   )
+
+  const now = Date.now()
+  const runIds: string[] = []
+  for (const runId of found) {
+    if (!backoff.isEligible(runId, now)) continue
+    // Noted before the resume, not after: a run whose resume throws is exactly
+    // the run most likely to still be here next tick, and re-driving it every
+    // tick is the amplification this backoff exists to stop.
+    backoff.note(runId, now)
+    runIds.push(runId)
+  }
+
   return {
     resumed: await resumeEach(
       runIds,

package/src/wirings/workflow/workflow-stalled-recovery.test.ts

@@ -89,6 +89,52 @@ describe('stalled run recovery', () => {
     assert.deepEqual(resumed, [], 'a run that just moved is not swept')
   })
 
+  test('backs off rather than re-resuming the same run every tick', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const first = await ws.recoverStalledRuns()
+    // Still stalled: a resume does not clear whatever wedged the run, so an
+    // unconditional sweep would re-queue it on this tick and on every tick
+    // after it.
+    const second = await ws.recoverStalledRuns()
+
+    assert.deepEqual(first.resumed, [runId], 'the first sweep resumes it')
+    assert.deepEqual(
+      second.resumed,
+      [],
+      'the next sweep is held off by the backoff'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
+  test('does not re-drive a run the relay has just re-dispatched', async () => {
+    const resumes = trackResumes()
+    const ws = new InMemoryWorkflowService()
+    const runId = await ws.createRun('flow', {}, false, 'hash', {
+      type: 'test',
+    })
+    // Old enough to read as both an undispatched step and a stalled run.
+    await ws.insertStepState(runId, 'Wait 15s', null, { duration: 15000 })
+    await backdate(ws, runId, 10 * 60_000)
+
+    const relayed = await ws.relayUndispatchedSteps()
+    const swept = await ws.recoverStalledRuns()
+
+    assert.deepEqual(relayed.redispatched, [runId], 'the relay re-drives it')
+    assert.deepEqual(
+      swept.resumed,
+      [],
+      'the sweep adds nothing the relay has not already queued'
+    )
+    assert.deepEqual(resumes.runIds, [runId], 'only one queue message')
+  })
+
   test('leaves a finished run alone', async () => {
     trackResumes()
     const ws = new InMemoryWorkflowService()

package/src/wirings/workflow/workflow-step-claim.ts

@@ -0,0 +1,46 @@
+import { WorkflowStepFunctionMismatchError } from './workflow-errors.js'
+import type { StepState } from './workflow.types.js'
+
+/** What claiming a step needs from the workflow service. */
+export type StepClaimStore = {
+  getStepState(runId: string, stepName: string): Promise<StepState>
+  setStepRunning(stepId: string): Promise<void>
+  createRetryAttempt(
+    failedStepId: string,
+    status: 'pending' | 'running'
+  ): Promise<StepState>
+}
+
+/**
+ * Decide whether this dispatch owns the step, by reading its state and then
+ * writing it — which only excludes a concurrent dispatch when the caller holds
+ * a lock that genuinely excludes one.
+ *
+ * A store that can express the whole decision as a single conditional write
+ * should do that instead of calling this.
+ */
+export const claimStepByReadThenWrite = async (
+  store: StepClaimStore,
+  runId: string,
+  stepName: string,
+  rpcName: string
+): Promise<StepState | null> => {
+  const stepState = await store.getStepState(runId, stepName)
+  // knowledge: decisions/security/a-step-runs-the-function-the-workflow-dispatched-it-with.md
+  if (
+    stepState.rpcName !== undefined &&
+    stepState.rpcName !== (rpcName ?? null)
+  ) {
+    throw new WorkflowStepFunctionMismatchError(runId, stepName)
+  }
+  if (stepState.status === 'succeeded' || stepState.status === 'running') {
+    return null
+  }
+  if (stepState.status === 'failed') {
+    return store.createRetryAttempt(stepState.stepId, 'running')
+  }
+  if (stepState.status === 'pending' || stepState.status === 'scheduled') {
+    await store.setStepRunning(stepState.stepId)
+  }
+  return stepState
+}

package/src/wirings/workflow/workflow-terminal-run-guard.test.ts

@@ -0,0 +1,105 @@
+import { describe, test } from 'node:test'
+import assert from 'node:assert/strict'
+
+import { InMemoryWorkflowService } from '../../services/in-memory-workflow-service.js'
+import { pikkuState, resetPikkuState } from '../../pikku-state.js'
+
+const silentLogger = { error() {}, info() {}, warn() {}, debug() {} }
+
+/** Records every run lock taken, which is the whole point of the guard. */
+class LockSpyWorkflowService extends InMemoryWorkflowService {
+  public readonly locked: string[] = []
+
+  public override async withRunLock<T>(
+    id: string,
+    fn: () => Promise<T>
+  ): Promise<T> {
+    this.locked.push(id)
+    return super.withRunLock(id, fn)
+  }
+}
+
+/** A registered, fully described workflow, so nothing else can short-circuit. */
+const startRun = async (): Promise<{
+  service: LockSpyWorkflowService
+  runId: string
+  entered: () => boolean
+}> => {
+  resetPikkuState()
+  pikkuState(null, 'package', 'singletonServices', {
+    logger: silentLogger,
+    queueService: { add: async () => 'job-1' },
+  } as any)
+
+  let bodyEntered = false
+  const func = async () => {
+    bodyEntered = true
+  }
+  pikkuState(null, 'workflows', 'meta', {
+    flow: { name: 'flow', pikkuFuncId: 'flow', source: 'dsl' },
+  } as any)
+  pikkuState(null, 'workflows', 'registrations').set('flow', {
+    name: 'flow',
+    func,
+  } as any)
+  pikkuState(null, 'function', 'meta', {
+    flow: {
+      pikkuFuncId: 'flow',
+      inputSchemaName: null,
+      outputSchemaName: null,
+      sessionless: true,
+    },
+  } as any)
+  pikkuState(null, 'function', 'functions').set('flow', { func } as any)
+
+  const service = new LockSpyWorkflowService()
+  const runId = await service.createRun('flow', {}, false, '', { type: 'test' })
+  return { service, runId, entered: () => bodyEntered }
+}
+
+/**
+ * The leak this guards, read straight off production: every granted advisory
+ * lock held by an idle session mapped to a run that was already `failed`. The
+ * orchestrator queue is at-least-once, so a message for a settled run is
+ * routine — and answering it by taking the run lock and replaying the body is
+ * how a run that can never move again ends up holding a lock and a pooled
+ * connection while it waits on something that will never arrive.
+ */
+describe('an orchestrator message for a run that already settled', () => {
+  for (const status of ['failed', 'completed', 'cancelled'] as const) {
+    test(`a ${status} run is answered without taking its lock`, async () => {
+      const { service, runId, entered } = await startRun()
+      await service.updateRunStatus(runId, status)
+
+      await service.runWorkflowJob(runId, {} as any)
+
+      assert.deepEqual(
+        service.locked,
+        [],
+        'the run lock was taken for a run that can never move again'
+      )
+      assert.equal(
+        entered(),
+        false,
+        'the workflow body was replayed after the run had settled'
+      )
+    })
+  }
+
+  /**
+   * Guards the tests above: they would pass just as well for a service that
+   * refused to orchestrate anything at all.
+   */
+  test('a suspended run is still orchestrated', async () => {
+    const { service, runId } = await startRun()
+    await service.updateRunStatus(runId, 'suspended')
+
+    await service.runWorkflowJob(runId, {} as any)
+
+    assert.deepEqual(
+      service.locked,
+      [runId],
+      'suspended ends a pass, not the run — it resumes when its signal arrives'
+    )
+  })
+})