@stigmer/runner 3.15.0 → 3.15.2-dev.20260913223433

package/src/activities/execute-deep-agent/__tests__/hermetic/workspace-lock-timeout.test.ts

@@ -0,0 +1,152 @@
+/**
+ * Hermetic scenario: WORKSPACE LOCK TIMEOUT — a turn that cannot acquire its
+ * workspace within the configured wait fails with an actionable reason and
+ * RETURNS.
+ *
+ * Invariant pinned: the activity serializes on the workspace root
+ * (`shared/workspace/workspace-lock.ts`); while another turn holds the lock it
+ * reports a visible waiting state and heartbeats; past
+ * `Config.workspaceLockTimeoutMs` it maps `WorkspaceLockTimeoutError` to
+ * EXECUTION_FAILED with the lock's own message (which names the directory)
+ * and one system row, persists ONCE, and returns — a Temporal retry would only
+ * queue behind the same holder.
+ *
+ * The order, ruled at Q-S3-8 and landed at S3 M2a: the runtime takes the
+ * lock right after provisioning the workspace, BEFORE the tool surface, the
+ * skills, the attachments and the graph — so a turn that cannot have the
+ * tree builds no model and compiles no sub-agent. Until then the
+ * orchestrator took the lock AFTER the whole of `performSetup`. The setup
+ * labels are the runtime's resolution phases'; the adapter's own labels
+ * ("Connecting tools…", "Configuring sub-agents…", "Creating agent…") never
+ * appear, because the turn ends before the adapter runs.
+ *
+ * Why this is the one terminal arm WITHOUT a file golden (owner ruling,
+ * 2026-09-11, entry 20260911.03 M0; carried for native): the wire copy embeds
+ * the absolute workspace path, a per-run temp directory. The S0 rule is to
+ * control a volatile source at its origin or escalate — never redact — so
+ * this scenario pins every byte the path does not touch with explicit
+ * assertions and builds the expected message with the SAME error class over
+ * the real resolved path.
+ *
+ * Determinism: the lock wait is `waitedMs + pollIntervalMs > timeoutMs`, so a
+ * timeout below the 1 s poll interval fails on the FIRST contended attempt with
+ * no sleep; `waitedMs` is measured on the scripted clock, so it is 0.
+ *
+ * Regenerate: nothing to regenerate — no snapshot.
+ */
+
+import { mkdirSync } from "node:fs";
+import { realpath } from "node:fs/promises";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { ExecutionPhase, MessageType } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+
+vi.mock("../../../../shared/model-client.js", async () =>
+  (await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
+);
+vi.mock("../../../../client/stigmer-client.js", async () =>
+  (await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
+);
+
+import {
+  ScriptedClock,
+  createHermeticEnvironment,
+  type HermeticEnvironment,
+} from "../../../../__test-utils__/hermetic-activity.js";
+import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
+import {
+  WorkspaceLockTimeoutError,
+  acquireWorkspaceLock,
+  type ReleaseWorkspaceLock,
+} from "../../../../shared/workspace/workspace-lock.js";
+import {
+  beginDeepAgentScenario,
+  deepAgentExecutionRecord,
+  runDeepAgentTurn,
+  sessionWorkspaceDir,
+} from "../../__test-utils__/hermetic-deep-agent.js";
+import { recordedModelBuilds } from "../../__test-utils__/scripted-model-module.js";
+
+/** Below the lock's 1 s poll interval: the first contended attempt times out. */
+const LOCK_TIMEOUT_MS = 250;
+
+describe("ExecuteDeepAgent hermetic — workspace lock timeout", () => {
+  let env: HermeticEnvironment;
+  let registry: ReturnType<typeof stubRegistryFetch>;
+  const clock = new ScriptedClock();
+
+  beforeAll(() => {
+    env = createHermeticEnvironment();
+    registry = stubRegistryFetch();
+    clock.install();
+  });
+
+  afterAll(() => {
+    clock.uninstall();
+    registry.restore();
+    env.dispose();
+  });
+
+  it("fails the turn with the lock's message when another turn holds the workspace", async () => {
+    // ── Arrange: another turn already holds this session's workspace ─────────
+    const record = deepAgentExecutionRecord({ message: "Never streams." });
+    const scenario = beginDeepAgentScenario({
+      env,
+      clock,
+      record,
+      script: () => {
+        throw new Error("the model must never be asked while the workspace is held");
+      },
+      config: { workspaceLockTimeoutMs: LOCK_TIMEOUT_MS },
+    });
+    // Created AFTER the scenario begins (which clears the session workspace)
+    // and BEFORE either side locks it, so both resolve the same realpath
+    // (macOS's /var -> /private/var) and contend on ONE key.
+    const workspaceDir = sessionWorkspaceDir(env);
+    mkdirSync(workspaceDir, { recursive: true });
+    const resolvedDir = await realpath(workspaceDir);
+    let releaseHeldLock: ReleaseWorkspaceLock | undefined = await acquireWorkspaceLock(workspaceDir);
+
+    try {
+      // ── Act ────────────────────────────────────────────────────────────────
+      const invocation = await runDeepAgentTurn(scenario);
+
+      // ── Assert: outcome and phases ─────────────────────────────────────────
+      expect(invocation.outcome.kind, "a lock timeout RETURNS — a retry would queue behind the same holder").toBe(
+        "returned",
+      );
+      const slim = (invocation.outcome as { value: Record<string, unknown> }).value;
+      expect(slim.phase).toBe("EXECUTION_FAILED");
+      expect(record.persistedPhases, "the one and only full persist").toEqual([ExecutionPhase.EXECUTION_FAILED]);
+
+      // ── Assert: the copy, built with the same class over the real path ─────
+      const expectedMessage = new WorkspaceLockTimeoutError(resolvedDir, 0).message;
+      const final = record.lastFullStatus!;
+      expect(final.error).toBe(expectedMessage);
+      expect(final.completedAt).not.toBe("");
+      expect(final.messages.filter((m) => m.type === MessageType.MESSAGE_SYSTEM).map((m) => m.content)).toEqual([
+        `Execution failed: ${expectedMessage}`,
+      ]);
+      expect(final.messages, "the system row is the whole transcript").toHaveLength(1);
+
+      // ── Assert: the order — the lock is tried before the tool surface ─────
+      expect(record.setupProgress).toEqual([
+        "Fetching execution",
+        "Resolving agent blueprint",
+        "Resolving environment",
+        "Provisioning workspace",
+        "Waiting for workspace — in use by another session",
+      ]);
+      expect(recordedModelBuilds(), "no model is built for a turn that cannot have the tree (Q-S3-8)").toHaveLength(0);
+
+      // ── Assert: hermeticity ────────────────────────────────────────────────
+      expect(registry.urls.every((u) => u.includes("/model-registry"))).toBe(true);
+      // The runtime's heartbeat posture (Q-S3-8, parent Q8): every resolution
+      // phase pulses on entry and the terminal write pulses once, so a turn
+      // that ends at the lock has heartbeated before any stream existed.
+      expect(invocation.heartbeats.length, "the runtime heartbeats through resolution").toBeGreaterThan(0);
+    } finally {
+      await releaseHeldLock?.();
+      releaseHeldLock = undefined;
+    }
+  });
+});

package/src/activities/execute-deep-agent/__tests__/hitl.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect } from "vitest";
 import { Command } from "@langchain/langgraph";
-import { resolveResumeInput, reconcileNonExecutingDecisions, reconcileUnattendedSkips } from "../hitl.js";
+import { detectPendingInterrupts, resolveResumeInput, reconcileUnattendedSkips } from "../hitl.js";
 import { ApprovalAction, ApprovalPolicySource, ToolCallStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { create } from "@bufbuild/protobuf";
 import {
@@ -8,50 +8,19 @@ import {
   ToolCallSchema,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
 import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
-import { MessageType } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
-import type { AgentExecution } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import type { GraphStateSnapshot } from "../hitl.js";
 
-function makeExecution(
-  toolCalls: Array<{
-    id: string;
-    status: ToolCallStatus;
-    approvalAction: ApprovalAction;
-  }>,
-): AgentExecution {
-  const msg = create(AgentMessageSchema, {
-    type: MessageType.MESSAGE_AI,
-    content: "test",
-    toolCalls: toolCalls.map(tc =>
-      create(ToolCallSchema, {
-        id: tc.id,
-        name: "test_tool",
-        status: tc.status,
-        approvalAction: tc.approvalAction,
-      }),
-    ),
-  });
-
-  return {
-    $typeName: "ai.stigmer.agentic.agentexecution.v1.AgentExecution",
-    $unknown: undefined,
-    metadata: undefined,
-    spec: {
-      $typeName: "ai.stigmer.agentic.agentexecution.v1.AgentExecutionSpec",
-      $unknown: undefined,
-      sessionId: "session-1",
-      agentId: "agent-1",
-      message: "test message",
-      executionConfig: undefined,
-      workspaceFileRefs: [],
-      callbackToken: new Uint8Array(),
-      autoApproveAll: false,
-      parentWorkflowId: "",
-    },
-    status: create(AgentExecutionStatusSchema, {
-      messages: [msg],
-    }),
-  } as unknown as AgentExecution;
+/** The runtime's reading of the adjudicated rows (`approvalDecisionsOf`): a decision per WAITING tool-call id. */
+function makeDecisions(
+  toolCalls: Array<{ id: string; status: ToolCallStatus; approvalAction: ApprovalAction }>,
+): ReadonlyMap<string, ApprovalAction> {
+  const decisions = new Map<string, ApprovalAction>();
+  for (const tc of toolCalls) {
+    if (tc.status === ToolCallStatus.TOOL_CALL_WAITING_APPROVAL && tc.approvalAction !== ApprovalAction.UNSPECIFIED) {
+      decisions.set(tc.id, tc.approvalAction);
+    }
+  }
+  return decisions;
 }
 
 function makeGraphState(
@@ -70,29 +39,29 @@ function makeGraphState(
 }
 
 describe("resolveResumeInput", () => {
-  it("reports not-a-resume when no interrupts exist (caller falls back to setup.langgraphInput)", () => {
-    const execution = makeExecution([]);
+  it("reports not-a-resume when no interrupts exist (the caller sends the turn's user message)", () => {
+    const decisions = makeDecisions([]);
     const state: GraphStateSnapshot = { values: {}, tasks: [] };
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     expect(result.isResumeFromApproval).toBe(false);
     // No graphInput on this branch — the turn's user message has exactly ONE
-    // construction site (setup.langgraphInput), never a second copy here.
+    // construction site (the caller's), never a second copy here.
     expect("graphInput" in result).toBe(false);
   });
 
   it("reports not-a-resume when interrupts exist but no decisions", () => {
-    const execution = makeExecution([]);
+    const decisions = makeDecisions([]);
     const state = makeGraphState([{ taskId: "task-1", toolCallId: "call-1" }]);
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     expect(result.isResumeFromApproval).toBe(false);
   });
 
   it("builds Command(resume) when interrupts match decisions", () => {
-    const execution = makeExecution([{
+    const decisions = makeDecisions([{
       id: "call-1",
       status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
       approvalAction: ApprovalAction.APPROVE,
@@ -100,14 +69,14 @@ describe("resolveResumeInput", () => {
 
     const state = makeGraphState([{ taskId: "task-1", toolCallId: "call-1" }]);
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     if (!result.isResumeFromApproval) throw new Error("expected an approval resume");
     expect(result.graphInput).toBeInstanceOf(Command);
   });
 
   it("builds a resume Command for a REJECT decision (denies the tool, does not fail the run)", () => {
-    const execution = makeExecution([{
+    const decisions = makeDecisions([{
       id: "call-1",
       status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
       approvalAction: ApprovalAction.REJECT,
@@ -115,7 +84,7 @@ describe("resolveResumeInput", () => {
 
     const state = makeGraphState([{ taskId: "task-1", toolCallId: "call-1" }]);
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     // REJECT resumes the gate like any other decision — the gate returns a
     // denial ToolMessage and the run continues. There is no execution-level
@@ -126,7 +95,7 @@ describe("resolveResumeInput", () => {
   });
 
   it("handles multiple interrupts with mixed decisions", () => {
-    const execution = makeExecution([
+    const decisions = makeDecisions([
       {
         id: "call-1",
         status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
@@ -144,13 +113,13 @@ describe("resolveResumeInput", () => {
       { taskId: "task-2", toolCallId: "call-2" },
     ]);
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     expect(result.isResumeFromApproval).toBe(true);
   });
 
   it("skips already-resumed interrupts", () => {
-    const execution = makeExecution([{
+    const decisions = makeDecisions([{
       id: "call-1",
       status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
       approvalAction: ApprovalAction.APPROVE,
@@ -160,82 +129,41 @@ describe("resolveResumeInput", () => {
       { taskId: "task-1", toolCallId: "call-1", hasResume: true },
     ]);
 
-    const result = resolveResumeInput(execution, state);
+    const result = resolveResumeInput(decisions, state);
 
     expect(result.isResumeFromApproval).toBe(false);
   });
 });
 
-describe("reconcileNonExecutingDecisions", () => {
-  function statusWith(
-    toolCalls: Array<{ id: string; status: ToolCallStatus; approvalAction: ApprovalAction }>,
-  ) {
-    return create(AgentExecutionStatusSchema, {
-      messages: [
-        create(AgentMessageSchema, {
-          toolCalls: toolCalls.map(tc =>
-            create(ToolCallSchema, {
-              id: tc.id,
-              name: "test_tool",
-              status: tc.status,
-              approvalAction: tc.approvalAction,
-            }),
-          ),
-        }),
+describe("detectPendingInterrupts", () => {
+  it("normalizes each un-resumed interrupt into a pending approval and skips resumed ones", () => {
+    const state: GraphStateSnapshot = {
+      values: {},
+      tasks: [
+        {
+          id: "task-1",
+          interrupts: [{
+            value: { tool_call_id: "call-1", tool_name: "shell", mcp_server_slug: "", message: "Run it?", policy_source: "classifier_default" },
+          }],
+        },
+        {
+          id: "task-2",
+          interrupts: [{ value: { tool_call_id: "call-2", tool_name: "write_file", message: "Write?" }, resumeValue: { action: "approve" } }],
+        },
       ],
-    });
-  }
+    };
 
-  it("terminalizes a REJECT decision to SKIPPED with a reason (does not FAIL)", () => {
-    const status = statusWith([{
-      id: "call-reject",
-      status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
-      approvalAction: ApprovalAction.REJECT,
-    }]);
-
-    reconcileNonExecutingDecisions(status);
-
-    const tc = status.messages[0].toolCalls[0];
-    expect(tc.status).toBe(ToolCallStatus.TOOL_CALL_SKIPPED);
-    expect(tc.error).toContain("Rejected by user");
-    // The REJECT decision is preserved so the audit trail stays honest.
-    expect(tc.approvalAction).toBe(ApprovalAction.REJECT);
-  });
-
-  it("terminalizes a SKIP decision to SKIPPED (fixes the stuck-WAITING for skip too)", () => {
-    const status = statusWith([{
-      id: "call-skip",
-      status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
-      approvalAction: ApprovalAction.SKIP,
-    }]);
-
-    reconcileNonExecutingDecisions(status);
-
-    expect(status.messages[0].toolCalls[0].status).toBe(ToolCallStatus.TOOL_CALL_SKIPPED);
-  });
-
-  it("leaves APPROVE / APPROVE_ALL untouched (the tool executes)", () => {
-    const status = statusWith([
-      { id: "call-approve", status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL, approvalAction: ApprovalAction.APPROVE },
-      { id: "call-approve-all", status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL, approvalAction: ApprovalAction.APPROVE_ALL },
+    expect(detectPendingInterrupts(state)).toEqual([
+      { toolCallId: "call-1", toolName: "shell", mcpServerSlug: "", message: "Run it?", policySource: "classifier_default" },
     ]);
-
-    reconcileNonExecutingDecisions(status);
-
-    // Still WAITING — real tool events (not this reconciler) terminalize them.
-    expect(status.messages[0].toolCalls[0].status).toBe(ToolCallStatus.TOOL_CALL_WAITING_APPROVAL);
-    expect(status.messages[0].toolCalls[1].status).toBe(ToolCallStatus.TOOL_CALL_WAITING_APPROVAL);
   });
 
-  it("is a no-op when there are no decided tool calls", () => {
-    const status = statusWith([{
-      id: "call-pending",
-      status: ToolCallStatus.TOOL_CALL_WAITING_APPROVAL,
-      approvalAction: ApprovalAction.UNSPECIFIED,
-    }]);
-
-    expect(() => reconcileNonExecutingDecisions(status)).not.toThrow();
-    expect(status.messages[0].toolCalls[0].status).toBe(ToolCallStatus.TOOL_CALL_WAITING_APPROVAL);
+  it("reads an absent policy source as undefined", () => {
+    const state: GraphStateSnapshot = {
+      values: {},
+      tasks: [{ id: "task-1", interrupts: [{ value: { tool_call_id: "call-1", tool_name: "shell", message: "Run it?" } }] }],
+    };
+    expect(detectPendingInterrupts(state)[0].policySource).toBeUndefined();
   });
 });
 

package/src/activities/execute-deep-agent/__tests__/inline-publisher.test.ts

@@ -7,14 +7,16 @@ import { create } from "@bufbuild/protobuf";
 import { AgentExecutionStatusSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
 import { ExecutionArtifactKind } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { InlinePublisher } from "../inline-publisher.js";
-import { StatusBuilder } from "../status-builder.js";
+import { V3StatusBuilder } from "../v3-status-builder.js";
 import { LocalWorkspaceBackend } from "../../../shared/workspace/local-backend.js";
 import type { ArtifactStorage } from "../../../shared/artifact-storage.js";
 import { makeInMemoryArtifactStorage } from "../../../__test-utils__/fake-artifact-storage.js";
 import type { WorkspaceBackend } from "../../../shared/workspace/types.js";
 
-function makeStatusBuilder(): StatusBuilder {
-  return new StatusBuilder("exec-test", create(AgentExecutionStatusSchema, {}));
+// The publisher writes artifacts through the transcript builder's
+// `ExecutionStatusWriter` face (`addArtifact`); the one builder since S3 M2b.
+function makeStatusBuilder(): V3StatusBuilder {
+  return new V3StatusBuilder("exec-test", create(AgentExecutionStatusSchema, {}));
 }
 
 function mockWorkspaceBackend(files: Record<string, string>): WorkspaceBackend {
@@ -53,7 +55,7 @@ function sha256(content: string): string {
 }
 
 describe("InlinePublisher", () => {
-  let sb: StatusBuilder;
+  let sb: V3StatusBuilder;
   let storage: ReturnType<typeof mockArtifactStorage>;
   let backend: WorkspaceBackend;
   let publisher: InlinePublisher;
@@ -207,7 +209,7 @@ describe("InlinePublisher with LocalWorkspaceBackend (disk-backed)", () => {
     await mkdir(join(dir, "src"), { recursive: true });
     await writeFile(join(dir, "src/app.ts"), "export const x = 42;", "utf-8");
 
-    const sb = new StatusBuilder("exec-disk", create(AgentExecutionStatusSchema, {}));
+    const sb = new V3StatusBuilder("exec-disk", create(AgentExecutionStatusSchema, {}));
     const storage = mockArtifactStorage();
     const backend = new LocalWorkspaceBackend(dir);
 
@@ -235,7 +237,7 @@ describe("InlinePublisher with LocalWorkspaceBackend (disk-backed)", () => {
     const dir = join(tmpdir(), `stigmer-publisher-test-${Date.now()}`);
     await mkdir(dir, { recursive: true });
 
-    const sb = new StatusBuilder("exec-miss", create(AgentExecutionStatusSchema, {}));
+    const sb = new V3StatusBuilder("exec-miss", create(AgentExecutionStatusSchema, {}));
     const storage = mockArtifactStorage();
     const backend = new LocalWorkspaceBackend(dir);
 

package/src/activities/execute-deep-agent/__tests__/plan-mode-path-normalization.test.ts

@@ -14,7 +14,7 @@
  * the workspace — an out-of-root name is simply nonexistent, and the
  * `.stigmer` symlink keeps platform-dir reads addressable in-root.
  *
- * The graph here is composed exactly the way setup.ts composes the parent:
+ * The graph here is composed exactly the way turn-setup.ts composes the parent:
  * the PRODUCTION buildMiddlewareStack (pathNormalization present on every
  * graph) + the CAS capture backend + the PRODUCTION buildPlanModePermissions
  * rules. These tests are also the empirical proof that langchain's
@@ -44,7 +44,7 @@ const OUTSIDE_CONTENT = "OUT_OF_ROOT_SECRET_TOKEN: must never cross the boundary
 const PLATFORM_CONTENT = "PLATFORM_SKILL_TOKEN: reached through the .stigmer symlink";
 
 /**
- * Build a plan-mode parent graph the way setup.ts does: the production
+ * Build a plan-mode parent graph the way turn-setup.ts does: the production
  * middleware stack with pathNormalization set (derived, like the graph's
  * permissions, from the plan-mode rules), a filesystem-only CAS capture
  * backend (no shellEnv — plan mode clears it), and the production