@stigmer/runner 0.1.0-dev.20261017020314.g644422724c27

package/src/activities/execute-deep-agent/__tests__/hermetic/resume-across-engine-upgrade.test.ts

@@ -0,0 +1,268 @@
+/**
+ * Hermetic net: a session paused for approval on an OLDER engine resumes on
+ * this one. Every stored checkpoint crosses an engine upgrade (deepagents,
+ * langchain, `@langchain/langgraph` and its checkpoint library move
+ * together), and cloud sessions live in the server's database, so "a session
+ * paused before the upgrade resumes after it" is a property the runner owes on
+ * both savers, not an accident of the serializer.
+ *
+ * The fixtures in `./fixtures/paused-session.deepagents-<version>.json` are
+ * the checkpoint rows an OLDER engine wrote, never regenerated after the fact:
+ * each is recorded on the engine it names, before the upgrade that leaves it
+ * behind, and committed with that upgrade. The scenario it froze:
+ *
+ *  - round 0: the model writes its to-do list (`write_todos`, ungated);
+ *  - round 1: the model proposes a gated `execute`; the gate's `interrupt()`
+ *    pauses the graph and the activity persists WAITING_FOR_APPROVAL.
+ *
+ * A fixture's shape, and the reading and writing of its rows, are
+ * `__test-utils__/paused-session-fixture.ts`'s.
+ *
+ * What each fixture proves on the CURRENT engine:
+ *  - sqlite, end to end: the rows are loaded into the session's checkpoint
+ *    file, the server-held status and the events its turn wrote are
+ *    restored, APPROVE is decided as
+ *    `SubmitApproval` would, and the reinvocation resumes INSIDE the gate — the
+ *    command runs exactly once, the run closes COMPLETED, and the to-do list
+ *    the old engine wrote survives.
+ *  - http: the same stored bytes served through the proxy wire
+ *    (`HttpCheckpointSaver`) load into the same checkpoint tuple the sqlite
+ *    saver loads — the graph resumes from the tuple, so the sqlite resume above
+ *    carries over to cloud sessions.
+ *
+ * Recording a fixture (only on the engine that is about to be left behind,
+ * before its upgrade lands):
+ *   RECORD_ENGINE_UPGRADE_FIXTURE=1 npx vitest run src/activities/execute-deep-agent/__tests__/hermetic/resume-across-engine-upgrade.test.ts
+ * The recording arm names the file after the installed deepagents version and
+ * refuses to overwrite one that exists.
+ *
+ * Resume is one-way: `@langchain/langgraph` 1.4.9 and later write Topic
+ * channels (`__pregel_tasks` among them) as a flat list that 1.3.x reads as
+ * `[seen, values]`, so a checkpoint written by this engine does not resume on
+ * the 1.3 line. Nothing here pretends otherwise.
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import {
+  RunPhase,
+} from "@stigmer/apis/ai/stigmer/agentic/run/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import { getCheckpointDbPath } from "../../../../shared/workspace/platform-dir.js";
+import { SqliteCheckpointSaver } from "../../../../shared/checkpointer/sqlite-saver.js";
+import { HttpCheckpointSaver } from "../../../../shared/checkpointer/http-saver.js";
+import {
+  FIXTURE,
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import type { ScriptedTurn } from "../../__test-utils__/scripted-model.js";
+import { CLOSING_TURN, EXECUTE_CALL_A, GATED_OPENING_TURN } from "../../__test-utils__/hitl-script.js";
+import {
+  engineVersions,
+  installedVersion,
+  loadBlob,
+  loadRows,
+  readRows,
+  type CheckpointRow,
+  type PausedSessionFixture,
+  type StoredBlob,
+  pausedTurnJson,
+  pausedTurnOf,
+} from "../../__test-utils__/paused-session-fixture.js";
+import { ApprovalAction, ToolCallStatus } from "../../../../gen/turn/ai/stigmer/runner/turn/v1/enum_pb.js";
+
+const USER_MESSAGE = "Plan it, then run the command for me.";
+const DECIDED_AT = "2026-01-01T00:00:30.000Z";
+const RESUME_AFTER_MS = 60_000;
+const FIXTURE_DIR = join(dirname(fileURLToPath(import.meta.url)), "fixtures");
+const FIXTURE_PREFIX = "paused-session.deepagents-";
+const RECORDING = process.env.RECORD_ENGINE_UPGRADE_FIXTURE === "1";
+
+const TODOS = [
+  { content: "Run the command", status: "in_progress" },
+  { content: "Report the result", status: "pending" },
+];
+
+/** Round 0: the to-do list, so the fixture carries the `todos` channel an engine writes. */
+const TODO_TURN: ScriptedTurn = {
+  text: "Planning the two steps.",
+  toolCalls: [{ id: "call-hermetic-todo-0001", name: "write_todos", args: { todos: TODOS } }],
+  usage: { inputTokens: 1_200, outputTokens: 40 },
+};
+
+const SCRIPT = () => ({ turns: [TODO_TURN, GATED_OPENING_TURN, CLOSING_TURN] });
+
+/**
+ * The proxy's checkpoint API over a fixture's rows, as `fetch` sees it: the
+ * documents the server stores and returns (`$binary` Extended JSON), keyed as
+ * the saver queries them.
+ */
+function proxyFetch(fixture: PausedSessionFixture): typeof fetch {
+  const binary = (blob: StoredBlob) => ({
+    $binary: { base64: Buffer.from(loadBlob(blob)).toString("base64"), subType: "00" },
+  });
+  const checkpointDoc = (r: CheckpointRow) => ({
+    thread_id: r.thread_id,
+    checkpoint_ns: r.checkpoint_ns,
+    checkpoint_id: r.checkpoint_id,
+    parent_checkpoint_id: r.parent_checkpoint_id,
+    type: r.type,
+    checkpoint: binary(r.checkpoint),
+    metadata_type: r.type,
+    metadata: binary(r.metadata),
+  });
+  const json = (body: unknown) => new Response(JSON.stringify(body), { status: 200 });
+  return (async (input: string | URL | Request) => {
+    const url = new URL(typeof input === "string" ? input : input instanceof URL ? input.href : input.url);
+    const ns = url.searchParams.get("checkpoint_ns") ?? "";
+    const rows = fixture.checkpoints.filter((r) => r.thread_id === url.searchParams.get("thread_id") && r.checkpoint_ns === ns);
+    if (url.pathname.endsWith("/checkpoint")) {
+      const id = url.searchParams.get("checkpoint_id");
+      const row = id ? rows.find((r) => r.checkpoint_id === id) : [...rows].sort((a, b) => b.checkpoint_id.localeCompare(a.checkpoint_id))[0];
+      return row ? json(checkpointDoc(row)) : new Response("", { status: 404 });
+    }
+    if (url.pathname.endsWith("/writes")) {
+      const id = url.searchParams.get("checkpoint_id");
+      const writes = fixture.writes
+        .filter((w) => w.thread_id === url.searchParams.get("thread_id") && w.checkpoint_ns === ns && w.checkpoint_id === id)
+        .map((w) => ({ ...w, value: binary(w.value) }));
+      return json({ writes });
+    }
+    throw new Error(`the fixture proxy serves no ${url.pathname}`);
+  }) as typeof fetch;
+}
+
+function committedFixtures(): string[] {
+  if (!existsSync(FIXTURE_DIR)) return [];
+  return readdirSync(FIXTURE_DIR)
+    .filter((f) => f.startsWith(FIXTURE_PREFIX) && f.endsWith(".json"))
+    .sort();
+}
+
+describe("ExecuteDeepAgent hermetic — a paused session resumes across an engine upgrade", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it.runIf(RECORDING)("records the paused session on the installed engine", async () => {
+    const version = installedVersion("deepagents");
+    const target = join(FIXTURE_DIR, `${FIXTURE_PREFIX}${version}.json`);
+    expect(existsSync(target), `${target} is frozen; a fixture is recorded once, on the engine it names`).toBe(false);
+
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: USER_MESSAGE });
+    const scenario = beginDeepAgentScenario({ env, clock, record, checkpointer: "sqlite", script: SCRIPT });
+    const turn = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+    expect(turn.outcome.kind).toBe("returned");
+    expect(record.lastFullStatus?.phase).toBe(RunPhase.RUN_WAITING_FOR_APPROVAL);
+    expect(record.lastFullRows?.todos).not.toEqual({});
+
+    const fixture: PausedSessionFixture = {
+      recordedWith: engineVersions(),
+      threadId: FIXTURE.threadId,
+      ...readRows(getCheckpointDbPath(FIXTURE.sessionId)),
+      status: pausedTurnJson(record.rowsOf(record.lastFullStatus!), record.lastFullStatus!),
+    };
+    mkdirSync(FIXTURE_DIR, { recursive: true });
+    writeFileSync(target, JSON.stringify(fixture, null, 2) + "\n");
+  });
+
+  describe.skipIf(RECORDING).each(committedFixtures())("%s", (file) => {
+    const fixture = JSON.parse(readFileSync(join(FIXTURE_DIR, file), "utf8")) as PausedSessionFixture;
+
+    it("resumes inside the gate on sqlite: the command runs once, the old to-do list survives", async () => {
+      // ── Arrange: the old engine's rows and the status the server held ─────
+      clock.reset();
+      const record = deepAgentExecutionRecord({ message: USER_MESSAGE });
+      const scenario = beginDeepAgentScenario({ env, clock, record, checkpointer: "sqlite", script: SCRIPT });
+      await loadRows(getCheckpointDbPath(FIXTURE.sessionId), fixture);
+      const paused = pausedTurnOf(fixture.status);
+      await record.restorePausedTurn(paused.rows, paused.status);
+      // The approval arrives after the pause, as it does live. Checkpoint ids
+      // are time-ordered, and the recorded turn ran on this same scripted
+      // clock from its epoch, so the resumed turn starts a minute on: its new
+      // checkpoints must sort after the paused one they continue.
+      clock.tick(RESUME_AFTER_MS);
+      expect(record.waitingToolCalls().map((tc) => tc.id)).toEqual([EXECUTE_CALL_A.id]);
+
+      // ── Between turns: what SubmitApproval does to the row ─────────────────
+      expect(record.decideWaitingToolCalls(ApprovalAction.APPROVE, DECIDED_AT)).toBe(1);
+
+      // ── Act: the reinvocation on THIS engine ───────────────────────────────
+      const turn = await runDeepAgentTurn(scenario, { turnSeq: 1 });
+
+      // ── Assert ─────────────────────────────────────────────────────────────
+      expect(turn.outcome.kind).toBe("returned");
+      expect((turn.outcome as { value: Record<string, unknown> }).value.phase).toBe("RUN_COMPLETED");
+      const final = record.lastFullStatus!;
+      const finalRows = record.rowsOf(final);
+      const rows = finalRows.messages.flatMap((m) => m.toolCalls).filter((tc) => tc.id === EXECUTE_CALL_A.id);
+      expect(rows, "exactly one copy of the gated call — a resume, not a replay").toHaveLength(1);
+      expect(rows[0].status).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
+      expect(rows[0].result, "the command ran on this engine").toContain("hermetic-a");
+      // The to-do list the old engine wrote survives in the log, as the
+      // write_todos call that set it; the resumed rows are their own.
+      const todoWrites = record.sessionEvents.flatMap((e) =>
+        e.event.case === "agentToolUse" && e.event.value.name === "write_todos" ? [e.event.value.input?.["todos"]] : [],
+      );
+      expect(todoWrites, "the to-do list the old engine wrote").toEqual([
+        [
+          { content: "Run the command", status: "in_progress" },
+          { content: "Report the result", status: "pending" },
+        ],
+      ]);
+      expect(finalRows.messages.at(-1)?.content).toBe(CLOSING_TURN.text);
+      expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+    });
+
+    it("loads the same stored bytes into the same tuple through the proxy wire", async () => {
+      const dbPath = join(env.home, "resume-across-engine-upgrade", file.replace(/\.json$/, ".db"));
+      await loadRows(dbPath, fixture);
+      const config = { configurable: { thread_id: fixture.threadId, checkpoint_ns: "" } };
+      const fromSqlite = await new SqliteCheckpointSaver(dbPath).getTuple(config);
+
+      const realFetch = globalThis.fetch;
+      globalThis.fetch = proxyFetch(fixture);
+      try {
+        const fromProxy = await new HttpCheckpointSaver("http://proxy.invalid", { current: "t" }, { maxRetries: 0 }).getTuple(config);
+        expect(fromSqlite, "the sqlite saver loads the old rows").toBeDefined();
+        expect(fromProxy?.checkpoint).toEqual(fromSqlite?.checkpoint);
+        expect(fromProxy?.metadata).toEqual(fromSqlite?.metadata);
+        expect(fromProxy?.parentConfig).toEqual(fromSqlite?.parentConfig);
+        expect(fromProxy?.pendingWrites).toEqual(fromSqlite?.pendingWrites);
+        expect(fromSqlite?.pendingWrites?.length, "the paused turn left its interrupt as a pending write").toBeGreaterThan(0);
+      } finally {
+        globalThis.fetch = realFetch;
+      }
+    });
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hermetic/resume-across-think-retirement.test.ts

@@ -0,0 +1,181 @@
+/**
+ * Hermetic net: a session paused for approval on the engine that still bound
+ * a `think` tool resumes on this one, which binds none (#1976). A saved
+ * conversation is the engine's memory, and the graph reads every earlier
+ * round on resume, so "a history that holds a call to a tool the graph no
+ * longer has" is a state the runner must carry, not one it may refuse.
+ *
+ * The fixture, `./fixtures/paused-session.think-bound.json`, is the checkpoint
+ * rows the last engine that bound `think` wrote, recorded on it before the
+ * retirement and committed with it, never regenerated. The scenario it froze
+ * (thinking left off, the mode that bound the tool):
+ *
+ *  - round 0: the model calls `think`, which the engine answers;
+ *  - round 1: the model proposes a gated `execute`; the gate's `interrupt()`
+ *    pauses the graph and the activity persists WAITING_FOR_APPROVAL.
+ *
+ * What it proves on the CURRENT engine, on sqlite end to end: the rows are
+ * loaded into the session's checkpoint file, the server-held status is seeded,
+ * APPROVE is decided as `SubmitApproval` would, and the reinvocation resumes
+ * inside the gate: the command runs exactly once, the run closes COMPLETED,
+ * and the `think` row of round 0 stays in the record as `TOOL_KIND_THINK`, so
+ * the transcript still renders it. The proxy wire's half of a resume is the
+ * engine-upgrade net's (`resume-across-engine-upgrade.test.ts`): the stored
+ * bytes cross it unchanged whatever graph reads them. The real provider's
+ * acceptance of the same history is the live twin's
+ * (`../resume-across-think-retirement.live.test.ts`).
+ *
+ * Recording the fixture (once, on the engine that binds `think`; the arm
+ * refuses to overwrite it):
+ *   RECORD_THINK_RETIREMENT_FIXTURE=1 npx vitest run src/activities/execute-deep-agent/__tests__/hermetic/resume-across-think-retirement.test.ts
+ */
+
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import {
+  RunPhase,
+} from "@stigmer/apis/ai/stigmer/agentic/run/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import { getCheckpointDbPath } from "../../../../shared/workspace/platform-dir.js";
+import {
+  FIXTURE,
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import type { ScriptedTurn } from "../../__test-utils__/scripted-model.js";
+import { CLOSING_TURN, EXECUTE_CALL_A, GATED_OPENING_TURN } from "../../__test-utils__/hitl-script.js";
+import {
+  THINK_BOUND_FIXTURE_FILE,
+  THINK_CALL_ID,
+  THINK_RETIREMENT_MESSAGE,
+  thinkBoundSession,
+} from "../../__test-utils__/think-bound-session.js";
+import {
+  engineVersions,
+  loadRows,
+  readRows,
+  type PausedSessionFixture,
+  pausedTurnJson,
+  pausedTurnOf,
+} from "../../__test-utils__/paused-session-fixture.js";
+import { ApprovalAction, ToolCallStatus } from "../../../../gen/turn/ai/stigmer/runner/turn/v1/enum_pb.js";
+import { toolUseEventId } from "../../../../harness/events/ids.js";
+
+const DECIDED_AT = "2026-01-01T00:00:30.000Z";
+const RESUME_AFTER_MS = 60_000;
+const FIXTURE_PATH = join(dirname(fileURLToPath(import.meta.url)), "fixtures", THINK_BOUND_FIXTURE_FILE);
+const RECORDING = process.env.RECORD_THINK_RETIREMENT_FIXTURE === "1";
+
+/** Round 0: the reasoning note the old engine answered. */
+const THINK_TURN: ScriptedTurn = {
+  text: "Let me work out the step first.",
+  toolCalls: [{ id: THINK_CALL_ID, name: "think", args: { thought: "One command, then report it." } }],
+  usage: { inputTokens: 1_200, outputTokens: 40 },
+};
+
+const SCRIPT = () => ({ turns: [THINK_TURN, GATED_OPENING_TURN, CLOSING_TURN] });
+
+describe("ExecuteDeepAgent hermetic — a session saved with a think call resumes on an engine that binds none", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it.runIf(RECORDING)("records the paused session on the engine that binds think", async () => {
+    expect(existsSync(FIXTURE_PATH), `${FIXTURE_PATH} is frozen; it is recorded once, before the retirement`).toBe(false);
+
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: THINK_RETIREMENT_MESSAGE });
+    const scenario = beginDeepAgentScenario({ env, clock, record, checkpointer: "sqlite", script: SCRIPT });
+    const turn = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+    expect(turn.outcome.kind).toBe("returned");
+    expect(record.lastFullStatus?.phase).toBe(RunPhase.RUN_WAITING_FOR_APPROVAL);
+    const thinkRow = record.toolCalls().find((tc) => tc.id === THINK_CALL_ID);
+    expect(thinkRow?.status, "the engine bound and answered think").toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
+
+    const fixture: PausedSessionFixture = {
+      recordedWith: engineVersions(),
+      threadId: FIXTURE.threadId,
+      ...readRows(getCheckpointDbPath(FIXTURE.sessionId)),
+      status: pausedTurnJson(record.rowsOf(record.lastFullStatus!), record.lastFullStatus!),
+    };
+    mkdirSync(dirname(FIXTURE_PATH), { recursive: true });
+    writeFileSync(FIXTURE_PATH, JSON.stringify(fixture, null, 2) + "\n");
+  });
+
+  it.skipIf(RECORDING)("resumes inside the gate on sqlite: the command runs once, the think row stays", async () => {
+    const fixture = thinkBoundSession();
+
+    // ── Arrange: the old engine's rows and the status the server held ───────
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: THINK_RETIREMENT_MESSAGE });
+    const scenario = beginDeepAgentScenario({ env, clock, record, checkpointer: "sqlite", script: SCRIPT });
+    await loadRows(getCheckpointDbPath(FIXTURE.sessionId), fixture);
+    const paused = pausedTurnOf(fixture.status);
+    await record.restorePausedTurn(paused.rows, paused.status);
+    // The approval arrives after the pause, as it does live; the resumed
+    // turn's checkpoints must sort after the paused one they continue.
+    clock.tick(RESUME_AFTER_MS);
+    expect(record.waitingToolCalls().map((tc) => tc.id)).toEqual([EXECUTE_CALL_A.id]);
+
+    // ── Between turns: what SubmitApproval does to the row ───────────────────
+    expect(record.decideWaitingToolCalls(ApprovalAction.APPROVE, DECIDED_AT)).toBe(1);
+
+    // ── Act: the reinvocation on THIS engine ─────────────────────────────────
+    const turn = await runDeepAgentTurn(scenario, { turnSeq: 1 });
+
+    // ── Assert ───────────────────────────────────────────────────────────────
+    expect(turn.outcome.kind).toBe("returned");
+    expect((turn.outcome as { value: Record<string, unknown> }).value.phase).toBe("RUN_COMPLETED");
+    const final = record.lastFullStatus!;
+    const finalRows = record.rowsOf(final);
+    const calls = finalRows.messages.flatMap((m) => m.toolCalls);
+    const executed = calls.filter((tc) => tc.id === EXECUTE_CALL_A.id);
+    expect(executed, "exactly one copy of the gated call — a resume, not a replay").toHaveLength(1);
+    expect(executed[0].status).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
+    expect(executed[0].result, "the command ran on this engine").toContain("hermetic-a");
+    // The old engine's think call is the log's: its use and its result were
+    // written once, when the paused turn was, and the resumed rows carry no
+    // copy of it.
+    const thinkUse = toolUseEventId(record.executionId, THINK_CALL_ID);
+    const thinkEvents = record.sessionEvents.flatMap((e) => {
+      if (e.event.case === "agentToolUse" && e.event.value.id === thinkUse) return [["use", e.event.value.name]];
+      if (e.event.case === "agentToolResult" && e.event.value.toolUseId === thinkUse) return [["result", e.event.value.isError]];
+      return [];
+    });
+    expect(thinkEvents, "the old engine's think call stays in the log, once").toEqual([
+      ["use", "think"],
+      ["result", false],
+    ]);
+    expect(calls.some((tc) => tc.id === THINK_CALL_ID), "the resumed rows carry no copy of it").toBe(false);
+    expect(finalRows.messages.at(-1)?.content).toBe(CLOSING_TURN.text);
+    expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hermetic/run-finished-while-waiting.test.ts

@@ -0,0 +1,110 @@
+/**
+ * Hermetic scenario: RUN FINISHED WHILE WAITING — an invocation that waited
+ * for its workspace finds, once it has the lock, that its run has finished,
+ * and runs nothing.
+ *
+ * Invariant pinned: an invocation that had to wait for the workspace lock
+ * reads its run again once it holds it; a run that finished meanwhile
+ * (COMPLETED, FAILED, CANCELLED or TERMINATED) is the turn's outcome. The
+ * engine is never started, no model is asked, nothing is persisted and
+ * nothing is appended to the session's log; the activity returns the
+ * stored phase.
+ *
+ * Why it matters: a pause cancels the running activity, but the
+ * cancellation reaches the runner only at its next heartbeat. The paused
+ * invocation can finish its turn first and persist the run COMPLETED while
+ * the resumed one waits for the workspace it holds. The server then refuses
+ * every event of the finished run, so a resumed invocation that ran on
+ * would spend a model call whose answer the log drops, and leave the
+ * engine's checkpoint holding a reply the session never shows.
+ *
+ * Determinism: the holder is this test's own lock, released only after the
+ * waiting invocation reported its wait and the run was marked finished;
+ * the lock's wait is measured on the scripted clock, so it never times out.
+ *
+ * Regenerate: nothing to regenerate — no snapshot.
+ */
+
+import { mkdirSync } from "node:fs";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { RunPhase } from "@stigmer/apis/ai/stigmer/agentic/run/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import { acquireWorkspaceLock, type ReleaseWorkspaceLock } from "../../../../shared/workspace/workspace-lock.js";
+import {
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+  sessionWorkspaceDir,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import { recordedModelBuilds } from "../../__test-utils__/scripted-model-module.js";
+
+const WAITING = "Waiting for workspace — in use by another session";
+
+describe("ExecuteDeepAgent hermetic — the run finished while the turn waited for its workspace", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it("returns the finished run's phase and runs, persists and appends nothing", async () => {
+    // ── Arrange: another invocation of the run holds the session's workspace ─
+    const record = deepAgentExecutionRecord({ message: "Never streams." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      script: () => {
+        throw new Error("the model must never be asked for a run that already finished");
+      },
+    });
+    const workspaceDir = sessionWorkspaceDir(env);
+    mkdirSync(workspaceDir, { recursive: true });
+    let releaseHeldLock: ReleaseWorkspaceLock | undefined = await acquireWorkspaceLock(workspaceDir);
+
+    try {
+      // ── Act: the turn waits; the holder finishes the run, then lets go ─────
+      const running = runDeepAgentTurn(scenario);
+      await vi.waitFor(() => expect(record.setupProgress).toContain(WAITING), { timeout: 10_000, interval: 10 });
+      record.execution.status!.phase = RunPhase.RUN_COMPLETED;
+      await releaseHeldLock();
+      releaseHeldLock = undefined;
+      const invocation = await running;
+
+      // ── Assert: the stored outcome, and nothing of this invocation's own ───
+      expect(invocation.outcome.kind, "the finished run is the turn's outcome").toBe("returned");
+      const slim = (invocation.outcome as { value: Record<string, unknown> }).value;
+      expect(slim.phase).toBe("RUN_COMPLETED");
+      expect(recordedModelBuilds(), "no model is built for a finished run").toHaveLength(0);
+      expect(record.persistedPhases, "no status is written over the finished run").toEqual([]);
+      expect(record.sessionEvents, "nothing is appended to the finished run's log").toEqual([]);
+      expect(record.refusedAppends).toEqual([]);
+      expect(record.setupProgress.at(-1), "the turn stopped at the lock").toBe(WAITING);
+    } finally {
+      await releaseHeldLock?.();
+    }
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hermetic/run-values-refused.test.ts

@@ -0,0 +1,106 @@
+/**
+ * Hermetic arm: the run's value fetch is refused for a reason its person
+ * fixes (an entry gone from its vault since the run was planned).
+ *
+ * Invariant pinned: the runtime's values phase settles the turn as
+ * RUN_FAILED on the actionable surface with the server's own sentence —
+ * the key, its declarer and the vault — unframed, with one
+ * `Execution failed:` row, and RETURNS (a retry would read the same
+ * vaults). The model is never asked, and no workspace is provisioned. Any
+ * other failure of the fetch (a credential the server refuses) is the
+ * platform's: it takes the runtime's generic error arm, never the
+ * person-facing one.
+ */
+
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { Code, ConnectError } from "@connectrpc/connect";
+import { RunPhase } from "@stigmer/apis/ai/stigmer/agentic/run/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import { ScriptedClock, createHermeticEnvironment, type HermeticEnvironment } from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import { beginDeepAgentScenario, deepAgentExecutionRecord, runDeepAgentTurn } from "../../__test-utils__/hermetic-deep-agent.js";
+import { MessageType } from "../../../../gen/turn/ai/stigmer/runner/turn/v1/enum_pb.js";
+
+const REFUSAL =
+  "the agent support-bot needs ZENDESK_TOKEN from vault 'Support tools', which no longer holds it: " +
+  "save it there again, then recover the run";
+
+describe("ExecuteDeepAgent hermetic — the run's values are refused", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it("fails the turn with the server's sentence and returns, never asking the model", async () => {
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: "Answer the ticket." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      script: () => {
+        throw new Error("the model must never be asked when the run's values are refused");
+      },
+      clientOverrides: {
+        fetchExecutionValues: vi.fn(async () => {
+          throw new ConnectError(REFUSAL, Code.FailedPrecondition);
+        }),
+      },
+    });
+
+    const turn = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+
+    expect(turn.outcome.kind, "a refusal the person fixes RETURNS").toBe("returned");
+    const final = record.lastFullStatus!;
+    const finalRows = record.rowsOf(final);
+    expect(final.phase).toBe(RunPhase.RUN_FAILED);
+    expect(final.error).toBe(REFUSAL);
+    expect(finalRows.messages.filter((m) => m.type === MessageType.MESSAGE_SYSTEM).map((m) => m.content)).toEqual([
+      `Execution failed: ${REFUSAL}`,
+    ]);
+  });
+
+  it("takes the generic error arm for a fetch the server refuses as the platform's", async () => {
+    clock.reset();
+    const record = deepAgentExecutionRecord({ message: "Answer the ticket." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      script: () => {
+        throw new Error("the model must never be asked when the run's values cannot be fetched");
+      },
+      clientOverrides: {
+        fetchExecutionValues: vi.fn(async () => {
+          throw new ConnectError("not bound to this execution", Code.PermissionDenied);
+        }),
+      },
+    });
+
+    const turn = await runDeepAgentTurn(scenario, { turnSeq: 0 });
+
+    expect(turn.outcome.kind).toBe("returned");
+    const final = record.lastFullStatus!;
+    expect(final.phase).toBe(RunPhase.RUN_FAILED);
+    expect(final.error).toContain("not bound to this execution");
+    expect(final.error, "never the person-facing sentence").not.toBe("not bound to this execution");
+  });
+});