@stigmer/runner 3.15.1 → 3.15.2-dev.20260913223433

package/src/activities/execute-deep-agent/__tests__/hermetic/hitl-sequential-gates.test.ts

@@ -0,0 +1,170 @@
+/**
+ * Hermetic goldens: two SEQUENTIAL approval gates — three invocations on the
+ * durable (sqlite) checkpointer.
+ *
+ * Turn 1 opens exactly as every gated arm does (the shared
+ * `gate.turn1.status.json`): gated `execute` A, WAITING_FOR_APPROVAL.
+ * APPROVE. Turn 2 resumes inside gate A, runs A, and the model proposes
+ * gated `execute` B: the graph pauses again and the activity persists
+ * WAITING_FOR_APPROVAL with BOTH rows — A COMPLETED (its committed identity
+ * and result intact), B WAITING. APPROVE. Turn 3 resumes inside gate B, runs
+ * B, the model closes, COMPLETED with both rows COMPLETED and each decision
+ * recorded on its own row.
+ *
+ * Invariant carried from the retired `sequential-gate-resume.test.ts`: the
+ * status persisted at gate B must hold gate A's already-committed tool-call
+ * row — a status rebuilt from scratch that dropped it would trip the server's
+ * append-only-at-identity guard and strand the execution in
+ * WAITING_FOR_APPROVAL with zero pending approvals. That test modelled the
+ * memory checkpointer's replay through a mocked graph. On the real graph the
+ * memory posture (a test-only opt-in; `config.ts` resolves `sqlite` locally
+ * and `http` in cloud) re-interrupts the same call on replay and duplicates
+ * the row (S3 M0 finding F-M0-3), so the invariant is recorded on the durable
+ * posture, where the reinvocation resumes instead of replaying.
+ *
+ * What every reinvocation golden here shows since S3 M2a (the runtime over
+ * the native adapter; Q-S3-16 and Q-M2a-2): `startedAt` is THIS turn's own,
+ * not turn 1's carried by the orchestrator's whole-status clone; the
+ * `fileReviewEventStream` carries only the events THIS turn authored (the
+ * runtime seeds messages, sub-agents, artifacts, write-backs, todos, usage,
+ * the memory report and the structured output — never the server-owned
+ * ledgers, which the server appends to by event id in `update-status.ts`
+ * `appendRunnerEvents`; the hermetic record replaces wholesale, so a
+ * golden reads as what the runner SENT); and `streamingUsage` carries the
+ * runtime accountant's fields (`model`, `estimatedCostUsd`, the requested
+ * tier and thinking mode).
+ *
+ * Regenerate ONLY after a deliberate behavior change:
+ *   npx vitest run src/activities/execute-deep-agent/__tests__/hermetic -u
+ */
+
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { toJson } from "@bufbuild/protobuf";
+import {
+  AgentExecutionStatusSchema,
+  type AgentExecutionStatus,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import {
+  ApprovalAction,
+  ExecutionPhase,
+  ToolCallStatus,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import {
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import {
+  CLOSING_TURN,
+  EXECUTE_CALL_A,
+  EXECUTE_CALL_B,
+  GATED_OPENING_TURN,
+  GATE_TURN1_GOLDEN,
+  executeApprovalMessage,
+} from "../../__test-utils__/hitl-script.js";
+
+const USER_MESSAGE = "Run the command for me.";
+const DECIDED_A_AT = "2026-01-01T00:00:30.000Z";
+const DECIDED_B_AT = "2026-01-01T00:01:00.000Z";
+
+function statusJson(status: AgentExecutionStatus): string {
+  return JSON.stringify(toJson(AgentExecutionStatusSchema, status), null, 2) + "\n";
+}
+
+describe("ExecuteDeepAgent hermetic — two sequential gates (sqlite)", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it("keeps gate A's committed row through gate B's pause and completes after both approvals", async () => {
+    // ── Arrange ──────────────────────────────────────────────────────────────
+    const record = deepAgentExecutionRecord({ message: USER_MESSAGE });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      checkpointer: "sqlite",
+      script: () => ({
+        turns: [
+          GATED_OPENING_TURN,
+          { text: "Now the second command.", toolCalls: [EXECUTE_CALL_B], usage: { inputTokens: 1_600, outputTokens: 40 } },
+          CLOSING_TURN,
+        ],
+      }),
+    });
+
+    // ── Turn 1: gate A ───────────────────────────────────────────────────────
+    const turn1 = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+    expect(turn1.outcome.kind).toBe("returned");
+    expect(record.waitingToolCalls().map((tc) => tc.id)).toEqual([EXECUTE_CALL_A.id]);
+    const atGateA = record.lastFullStatus!;
+    await expect(statusJson(atGateA), "the same opening as every gated arm").toMatchFileSnapshot(GATE_TURN1_GOLDEN);
+    expect(record.decideWaitingToolCalls(ApprovalAction.APPROVE, DECIDED_A_AT)).toBe(1);
+
+    // ── Turn 2: A runs, gate B pauses ────────────────────────────────────────
+    const turn2 = await runDeepAgentTurn(scenario, { turnSeq: 1 });
+    expect(turn2.outcome.kind).toBe("returned");
+    expect((turn2.outcome as { value: Record<string, unknown> }).value.phase).toBe("EXECUTION_WAITING_FOR_APPROVAL");
+    const atGateB = record.lastFullStatus!;
+    const rowsAtB = atGateB.messages.flatMap((m) => m.toolCalls);
+    expect(rowsAtB.map((tc) => [tc.id, tc.status]), "A committed and kept, B waiting").toEqual([
+      [EXECUTE_CALL_A.id, ToolCallStatus.TOOL_CALL_COMPLETED],
+      [EXECUTE_CALL_B.id, ToolCallStatus.TOOL_CALL_WAITING_APPROVAL],
+    ]);
+    expect(rowsAtB[0].result).toContain("hermetic-a");
+    expect(rowsAtB[0].approvalAction).toBe(ApprovalAction.APPROVE);
+    expect(rowsAtB[1].approvalMessage).toBe(executeApprovalMessage(EXECUTE_CALL_B));
+    expect(record.waitingToolCalls().map((tc) => tc.id), "exactly one pending approval").toEqual([EXECUTE_CALL_B.id]);
+    // The gate-B transcript is a superset of the gate-A one: every id, every message, in order.
+    const gateAIds = atGateA.messages.flatMap((m) => m.toolCalls).map((tc) => tc.id);
+    expect(rowsAtB.map((tc) => tc.id).slice(0, gateAIds.length)).toEqual(gateAIds);
+    expect(atGateB.messages.length).toBeGreaterThanOrEqual(atGateA.messages.length);
+    await expect(statusJson(atGateB)).toMatchFileSnapshot("./goldens/sequential-gates.turn2.status.json");
+    expect(record.decideWaitingToolCalls(ApprovalAction.APPROVE, DECIDED_B_AT)).toBe(1);
+
+    // ── Turn 3: B runs, the run completes ────────────────────────────────────
+    const turn3 = await runDeepAgentTurn(scenario, { turnSeq: 2 });
+    expect(turn3.outcome.kind).toBe("returned");
+    expect((turn3.outcome as { value: Record<string, unknown> }).value.phase).toBe("EXECUTION_COMPLETED");
+    expect(record.persistedPhases.filter((p) => p === ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL)).toHaveLength(2);
+    const final = record.lastFullStatus!;
+    const rows = final.messages.flatMap((m) => m.toolCalls);
+    expect(rows.map((tc) => [tc.id, tc.status, tc.approvalAction, tc.approvalDecidedAt])).toEqual([
+      [EXECUTE_CALL_A.id, ToolCallStatus.TOOL_CALL_COMPLETED, ApprovalAction.APPROVE, DECIDED_A_AT],
+      [EXECUTE_CALL_B.id, ToolCallStatus.TOOL_CALL_COMPLETED, ApprovalAction.APPROVE, DECIDED_B_AT],
+    ]);
+    expect(rows[1].result).toContain("hermetic-b");
+
+    // ── Assert: hermeticity ──────────────────────────────────────────────────
+    expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+
+    // ── Assert: the golden ───────────────────────────────────────────────────
+    await expect(statusJson(final)).toMatchFileSnapshot("./goldens/sequential-gates.turn3.status.json");
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hermetic/inline-artifact-across-gate.test.ts

@@ -0,0 +1,175 @@
+/**
+ * Hermetic goldens: an INLINE-PUBLISHED ARTIFACT survives a HITL gate — the
+ * arbiter for Q-S3-16 (the seed's breadth), two invocations on the durable
+ * (sqlite) checkpointer.
+ *
+ * Turn 1: the model writes a report (`write_file`; under capture mode in a
+ * non-git session workspace the write FLOWS into the CAS ledger), and
+ * `streaming-side-effects.ts` hands the path to `InlinePublisher`, which
+ * uploads the bytes to artifact storage and appends an `ExecutionArtifact`
+ * (`storageKey = artifacts/<executionId>/<name>`, a content hash, the
+ * scripted clock) to the status — a mid-turn publish the Cursor harness never
+ * performs. The model then proposes a gated `execute`; WAITING_FOR_APPROVAL.
+ *
+ * Turn 2: after APPROVE, the reinvocation seeds its status from the persisted
+ * one. On this harness `seedStatusFromExecution` clones the WHOLE status, so
+ * the artifact list arrives intact and the run's final status still lists the
+ * report. The server replaces `artifacts` wholesale whenever the request's
+ * list is non-empty (`update-status.ts`), so a runtime that seeded messages
+ * only and then published nothing new would carry an empty list — and the
+ * server would keep the old one, but a runtime that published one NEW
+ * artifact from an empty list would ERASE the prior one. Q-S3-16's
+ * `seedFromPersistedStatus` carries `artifacts` for exactly this reason;
+ * this pair of goldens (`inline-artifact.turn1`, `inline-artifact.turn2`) is
+ * what proves it did.
+ *
+ * Also recorded, F-M0-8 (a shape fact for the owner): the write's tool row
+ * shows `requiresApproval: true` with `approvalPolicySource` BUILTIN_CATEGORY
+ * — the category policy's verdict — while the row is COMPLETED because
+ * capture mode let it flow; the artifact carries no URL (the local store's
+ * serve URL never enters the record).
+ *
+ * What every reinvocation golden here shows since S3 M2a (the runtime over
+ * the native adapter; Q-S3-16 and Q-M2a-2): `startedAt` is THIS turn's own,
+ * not turn 1's carried by the orchestrator's whole-status clone; the
+ * `fileReviewEventStream` carries only the events THIS turn authored (the
+ * runtime seeds messages, sub-agents, artifacts, write-backs, todos, usage,
+ * the memory report and the structured output — never the server-owned
+ * ledgers, which the server appends to by event id in `update-status.ts`
+ * `appendRunnerEvents`; the hermetic record replaces wholesale, so a
+ * golden reads as what the runner SENT); and `streamingUsage` carries the
+ * runtime accountant's fields (`model`, `estimatedCostUsd`, the requested
+ * tier and thinking mode).
+ *
+ * Golden hunk since S3 M4 (Q-M4-7, ruled Q-M4-12 A): turn 1's
+ * `fileChangeProgress` is the TRUE snapshot at the WAITING write
+ * (`filesChanged: 1`, `report.md` ADD +3, `capturedAt` t=2) where it was a
+ * stale t=0 snapshot taken by the stream loop before the engine wrote the
+ * file — no later mid-stream persist crossed the floor, so the pause used
+ * to persist "0 files" beside a one-file change set. The runtime now
+ * refreshes the strip on every write, the terminal one included.
+ *
+ * Regenerate ONLY after a deliberate behavior change:
+ *   npx vitest run src/activities/execute-deep-agent/__tests__/hermetic -u
+ */
+
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { toJson } from "@bufbuild/protobuf";
+import {
+  AgentExecutionStatusSchema,
+  type AgentExecutionStatus,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import {
+  ApprovalAction,
+  ExecutionPhase,
+  ToolCallStatus,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import {
+  FIXTURE,
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import { CLOSING_TURN, EXECUTE_CALL_A } from "../../__test-utils__/hitl-script.js";
+
+const REPORT = "report.md";
+const REPORT_BODY = "# Report\n\nAll fixtures nominal.\n";
+const WRITE_CALL_ID = "call-hermetic-write-0001";
+const DECIDED_AT = "2026-01-01T00:00:30.000Z";
+
+function statusJson(status: AgentExecutionStatus): string {
+  return JSON.stringify(toJson(AgentExecutionStatusSchema, status), null, 2) + "\n";
+}
+
+describe("ExecuteDeepAgent hermetic — inline artifact across a gate (sqlite)", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it("publishes the report inline in turn 1 and still lists it after the approved turn 2", async () => {
+    // ── Arrange ──────────────────────────────────────────────────────────────
+    const record = deepAgentExecutionRecord({ message: "Write the report, then run the command." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      checkpointer: "sqlite",
+      script: () => ({
+        turns: [
+          {
+            text: "Writing the report.",
+            toolCalls: [{ id: WRITE_CALL_ID, name: "write_file", args: { file_path: `/${REPORT}`, content: REPORT_BODY } }],
+            usage: { inputTokens: 1_300, outputTokens: 60 },
+          },
+          { text: "Now the command.", toolCalls: [EXECUTE_CALL_A], usage: { inputTokens: 1_500, outputTokens: 40 } },
+          CLOSING_TURN,
+        ],
+      }),
+    });
+
+    // ── Turn 1 ───────────────────────────────────────────────────────────────
+    const turn1 = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+    expect(turn1.outcome.kind).toBe("returned");
+    expect(record.persistedPhases.at(-1)).toBe(ExecutionPhase.EXECUTION_WAITING_FOR_APPROVAL);
+    const run1 = record.lastFullStatus!;
+    expect(run1.artifacts.map((a) => [a.name, a.storageKey])).toEqual([
+      [REPORT, `artifacts/${FIXTURE.executionId}/${REPORT}`],
+    ]);
+    expect(run1.artifacts[0].contentHash, "sha256 of the bytes").toMatch(/^[0-9a-f]{64}$/);
+    expect(existsSync(join(env.artifactPath, run1.artifacts[0].storageKey)), "the bytes reached the store").toBe(true);
+    const writeRow = record.toolCalls().find((tc) => tc.id === WRITE_CALL_ID)!;
+    expect(writeRow.status, "capture mode let the write flow").toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
+    expect(record.waitingToolCalls().map((tc) => tc.id)).toEqual([EXECUTE_CALL_A.id]);
+    await expect(statusJson(run1)).toMatchFileSnapshot("./goldens/inline-artifact.turn1.status.json");
+
+    // ── Between turns ────────────────────────────────────────────────────────
+    expect(record.decideWaitingToolCalls(ApprovalAction.APPROVE, DECIDED_AT)).toBe(1);
+
+    // ── Turn 2 ───────────────────────────────────────────────────────────────
+    const turn2 = await runDeepAgentTurn(scenario, { turnSeq: 1 });
+    expect(turn2.outcome.kind).toBe("returned");
+    expect((turn2.outcome as { value: Record<string, unknown> }).value.phase).toBe("EXECUTION_COMPLETED");
+    const final = record.lastFullStatus!;
+    expect(final.artifacts.map((a) => [a.name, a.storageKey, a.contentHash]), "the seed carried the artifact").toEqual(
+      run1.artifacts.map((a) => [a.name, a.storageKey, a.contentHash]),
+    );
+    expect(final.messages.flatMap((m) => m.toolCalls).map((tc) => [tc.id, tc.status])).toEqual([
+      [WRITE_CALL_ID, ToolCallStatus.TOOL_CALL_COMPLETED],
+      [EXECUTE_CALL_A.id, ToolCallStatus.TOOL_CALL_COMPLETED],
+    ]);
+
+    // ── Assert: hermeticity ──────────────────────────────────────────────────
+    expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+    const finalJson = statusJson(final);
+    expect(finalJson, "no temp path may reach the golden").not.toContain(env.artifactPath);
+    await expect(finalJson).toMatchFileSnapshot("./goldens/inline-artifact.turn2.status.json");
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hermetic/pause-vs-shutdown.test.ts

@@ -0,0 +1,206 @@
+/**
+ * Hermetic goldens: the two ways a turn is INTERRUPTED from outside, and how
+ * the activity tells them apart — the throw-vs-return table the control
+ * plane's workflow keys on. Since S3 M2a the activity is the turn runtime
+ * over the native adapter, and the shapes below are the runtime's terminal
+ * table (`harness/terminal-table.ts`), ruled at Q-S3-7 (P-1: the pause copy
+ * is Cursor's) and stigmer#1071 ("persist a thrown terminal once").
+ *
+ *  1. USER PAUSE: the activity's `cancellationSignal` aborts. Whether the
+ *     abort lands inside a model call (staged from the scripted model's
+ *     `onTurn`) or while the loop is persisting (staged from the control
+ *     plane's persist channel, the record's `controlSignal`), the runtime's
+ *     stop controller aborts the adapter's signal, the adapter cancels the
+ *     graph run and settles `interrupted`, and the runtime persists PAUSED
+ *     ONCE — WITH the transcript and the row "Execution paused by user. Use
+ *     resume to continue." (P-1: the runtime's pause copy is Cursor's; the
+ *     orchestrator's said "… from this checkpoint.") — then throws
+ *     `CancelledFailure("Activity paused by orchestrator")`. Until M2a the
+ *     orchestrator persisted PAUSED TWICE, the second write bare and erasing
+ *     the transcript (S3 M0 finding F-M0-6, stigmer#1054's double persist);
+ *     `pause.bare.status.json` recorded that write and is gone with it. The
+ *     one write differs by where the cancel landed — after the tool finished
+ *     (`pause.loop.after-tool`) or while it was still RUNNING
+ *     (`pause.loop.mid-tool`, the cancel arriving on the persist that carried
+ *     the row).
+ *  2. WORKER SHUTDOWN: the queue's shutdown signal is aborted before the
+ *     cancel, so the interruption is EXECUTION_FAILED with the interrupted
+ *     copy and one row, thrown as
+ *     `CancelledFailure("Activity cancelled (worker shutdown, not user pause)")`
+ *     (#776) — the workflow re-invokes instead of waiting for a resume. The
+ *     copy is unchanged; what changed at M2a (Q-S3-7, the runtime's arm
+ *     wins) is that the runtime's arm KEEPS the transcript the turn produced
+ *     and appends the row, where the orchestrator's `buildWorkerShutdownStatus`
+ *     replaced the whole status with the row alone.
+ *
+ * Every interruption is staged deterministically — from a model turn or from
+ * a persist, never from a timer.
+ *
+ * Regenerate ONLY after a deliberate behavior change:
+ *   npx vitest run src/activities/execute-deep-agent/__tests__/hermetic -u
+ */
+
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { toJson } from "@bufbuild/protobuf";
+import {
+  AgentExecutionStatusSchema,
+  type AgentExecutionStatus,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import {
+  ExecutionControlSignal,
+  ExecutionPhase,
+  MessageType,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  threwCancelledFailure,
+  type HermeticEnvironment,
+  type InvocationControls,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import {
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+  type DeepAgentScenario,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import type { RoleScript } from "../../__test-utils__/scripted-model.js";
+
+const READ_CALL = { id: "call-hermetic-read-0001", name: "read_file", args: { file_path: "/notes.md" } };
+
+/** A turn that reads, then would speak again — the interruption lands before it can. */
+const SCRIPT: RoleScript = {
+  turns: [
+    { text: "Let me look.", toolCalls: [READ_CALL], usage: { inputTokens: 1_200, outputTokens: 30 } },
+    { text: "Never reaches the record.", usage: { inputTokens: 1_400, outputTokens: 10 } },
+  ],
+};
+
+describe("ExecuteDeepAgent hermetic — pause vs worker shutdown", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+  /** The scenario whose invocation is running — the persist-channel staging cancels it. */
+  let inFlight: DeepAgentScenario | undefined;
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  describe.each([
+    {
+      staging: "inside a model call",
+      begin: (record: ReturnType<typeof deepAgentExecutionRecord>) =>
+        beginDeepAgentScenario({
+          env,
+          clock,
+          record,
+          script: () => SCRIPT,
+          onTurn: (info, controls) => {
+            if (info.round === 1) controls.cancel();
+          },
+        }),
+      controlSignal: undefined as ((status: AgentExecutionStatus) => ExecutionControlSignal) | undefined,
+      loopGolden: "./goldens/pause.loop.after-tool.status.json",
+    },
+    {
+      staging: "between events",
+      begin: (record: ReturnType<typeof deepAgentExecutionRecord>) =>
+        beginDeepAgentScenario({ env, clock, record, script: () => SCRIPT }),
+      // The platform answers the persist that carries the first tool row by
+      // cancelling the activity (the workflow's pause).
+      controlSignal: (status: AgentExecutionStatus) => {
+        if (status.messages.some((m) => m.toolCalls.length > 0)) inFlight?.controls?.cancel();
+        return ExecutionControlSignal.UNSPECIFIED;
+      },
+      loopGolden: "./goldens/pause.loop.mid-tool.status.json",
+    },
+  ])("user pause $staging", ({ begin, controlSignal, loopGolden }) => {
+    it("persists PAUSED once, with the transcript and the row, and throws as a pause", async () => {
+      clock.reset();
+      const record = deepAgentExecutionRecord({ message: "Look at my notes.", controlSignal });
+      inFlight = begin(record);
+
+      const invocation = await runDeepAgentTurn(inFlight);
+
+      expect(threwCancelledFailure(invocation.outcome), "a pause THROWS CancelledFailure").toBe(true);
+      expect((invocation.outcome as { error: Error }).error.message).toBe("Activity paused by orchestrator");
+      expect(record.persistedPhases).toEqual([ExecutionPhase.EXECUTION_IN_PROGRESS, ExecutionPhase.EXECUTION_PAUSED]);
+
+      const pausedWrites = record.persisted.filter((s) => s.phase === ExecutionPhase.EXECUTION_PAUSED);
+      expect(pausedWrites, "PAUSED is persisted exactly once (stigmer#1071; F-M0-6 closed at M2a)").toHaveLength(1);
+      const [paused] = pausedWrites;
+      expect(paused.messages.filter((m) => m.type === MessageType.MESSAGE_SYSTEM).map((m) => m.content)).toEqual([
+        "Execution paused by user. Use resume to continue.",
+      ]);
+      expect(
+        paused.messages.flatMap((m) => m.toolCalls).map((tc) => tc.name),
+        "the read that landed before the pause is in the one write",
+      ).toEqual(["read_file"]);
+      expect(paused.completedAt).toBe("");
+      expect(record.lastFullStatus, "the wire ends on the write that carries the transcript").toEqual(paused);
+
+      const json = JSON.stringify(toJson(AgentExecutionStatusSchema, paused), null, 2) + "\n";
+      await expect(json).toMatchFileSnapshot(loopGolden);
+    });
+  });
+
+  it("worker shutdown: FAILED with the interrupted copy, thrown as a shutdown", async () => {
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: "Look at my notes." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      script: () => SCRIPT,
+      onTurn: (info, controls: InvocationControls) => {
+        if (info.round === 1) {
+          controls.signalWorkerShutdown();
+          controls.cancel();
+        }
+      },
+    });
+
+    const invocation = await runDeepAgentTurn(scenario);
+
+    expect(threwCancelledFailure(invocation.outcome), "a drain THROWS so the workflow re-invokes").toBe(true);
+    expect((invocation.outcome as { error: Error }).error.message).toBe(
+      "Activity cancelled (worker shutdown, not user pause)",
+    );
+    expect(record.persistedPhases.at(-1)).toBe(ExecutionPhase.EXECUTION_FAILED);
+    const final = record.lastFullStatus!;
+    expect(final.error).toBe("Execution interrupted: runner worker was shut down. Retry or resume.");
+    // The transcript the turn produced is kept and the row appended (the
+    // runtime's arm, Q-S3-7); the tool row rides its own empty AI message
+    // (F-M0-1, S4's).
+    expect(final.messages.map((m) => [m.type, m.content])).toEqual([
+      [MessageType.MESSAGE_AI, "Let me look."],
+      [MessageType.MESSAGE_AI, ""],
+      [
+        MessageType.MESSAGE_SYSTEM,
+        "Execution interrupted: the runner worker was shut down while the agent was still running. You can retry or resume.",
+      ],
+    ]);
+    expect(final.messages.flatMap((m) => m.toolCalls).map((tc) => tc.name), "the read that landed before the drain is kept").toEqual(["read_file"]);
+    expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+    const json = JSON.stringify(toJson(AgentExecutionStatusSchema, final), null, 2) + "\n";
+    await expect(json).toMatchFileSnapshot("./goldens/worker-shutdown.status.json");
+  });
+});