@stigmer/server 3.37.0 → 3.38.0

package/src/domain/agentexecution/__tests__/store-faults.test.ts

@@ -0,0 +1,315 @@
+/**
+ * Pins the store-fault contract every agent-execution load shares: a typed
+ * ResourceNotFoundError answers NotFound with the surface's own pinned copy,
+ * and any other store failure is an infrastructure fault answered as a
+ * sanitized Internal (`failed to load agent execution`), never a NotFound
+ * that tells a client the execution does not exist (stigmer/stigmer#859,
+ * stigmer/stigmer#1337).
+ *
+ * The surfaces: the five lifecycle verbs (their shared LoadExecutionById
+ * step), subscribe's snapshot read, the two artifact reads' existence
+ * checks, and the usage report's LoadExecution. The composed suites reach
+ * only a real store, which cannot fail selectively, so each surface runs
+ * here against a store whose one read throws. Every other dependency is
+ * untouchable: a load that fails must stop the call before any of them.
+ *
+ * Out of scope: the NotFound copy itself (wire contract, pinned where each
+ * surface is otherwise tested) and loads outside this domain.
+ */
+import { create } from "@bufbuild/protobuf";
+import type { HandlerContext } from "@connectrpc/connect";
+import { Code, ConnectError, createContextValues } from "@connectrpc/connect";
+import { describe, expect, it } from "vitest";
+
+import {
+  AgentExecutionIdSchema,
+  CancelAgentExecutionInputSchema,
+  GetArtifactContentRequestSchema,
+  GetArtifactDownloadUrlRequestSchema,
+  PauseAgentExecutionInputSchema,
+  RecoverAgentExecutionInputSchema,
+  ResumeAgentExecutionInputSchema,
+  TerminateAgentExecutionInputSchema,
+} from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/io_pb";
+import { AgentExecutionQueryController } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/query_pb";
+
+import type { ArtifactStorage } from "../../../artifactstorage/artifact-storage.js";
+import { createLogger } from "../../../boot/logger.js";
+import { callerIdentityKey } from "../../../pipeline/interceptors/auth.js";
+import { testCallerIdentity } from "../../../pipeline/__tests__/support.js";
+import { newPermissiveSingleTeamAuthorizer } from "../../../pipeline/steps/authorize.js";
+import { ResourceNotFoundError } from "../../../store/interface.js";
+import type { Store } from "../../../store/interface.js";
+
+import { getArtifactContent, getArtifactDownloadUrl } from "../artifacts.js";
+import type { LifecycleDeps } from "../lifecycle.js";
+import {
+  cancelExecution,
+  pauseExecution,
+  recoverExecution,
+  resumeExecution,
+  terminateExecution,
+} from "../lifecycle.js";
+import { StreamBroker } from "../stream-broker.js";
+import { subscribeExecution } from "../subscribe.js";
+import { getExecutionUsageReport } from "../usage.js";
+
+const silentLogger = createLogger({
+  level: "error",
+  pretty: false,
+  write: () => {},
+});
+
+const EXECUTION_ID = "aex_storefault";
+
+const LOAD_FAULT_COPY = "failed to load agent execution";
+
+/** A dependency the call must never reach once its load has failed. */
+function untouchable<T extends object>(name: string): T {
+  return new Proxy({} as T, {
+    get(_target, prop) {
+      throw new Error(`${name}.${String(prop)} reached after a failed load`);
+    },
+  });
+}
+
+/** A store whose every read fails with the given error. */
+function failingStore(error: Error): Store {
+  return {
+    getResource: () => Promise.reject(error),
+  } as unknown as Store;
+}
+
+const MISSING = (): Error =>
+  new ResourceNotFoundError(`agent_execution/${EXECUTION_ID}`);
+const LOCKED = (): Error => new Error("SQLITE_BUSY: database is locked");
+
+async function errorOf(call: () => Promise<unknown>): Promise<ConnectError> {
+  const error = await call().then(
+    () => undefined,
+    (e: unknown) => e,
+  );
+  expect(error, "the call must fail").toBeInstanceOf(ConnectError);
+  return error as ConnectError;
+}
+
+function lifecycleDeps(store: Store): LifecycleDeps {
+  return {
+    store,
+    logger: silentLogger,
+    authorizer: newPermissiveSingleTeamAuthorizer(),
+    broker: untouchable("broker"),
+    engineState: untouchable("engineState"),
+    executionContextBuilder: untouchable("executionContextBuilder"),
+    gateSteps: new Map(),
+    statusObservers: [],
+    sandboxLane: untouchable("sandboxLane"),
+    temporalConfig: untouchable("temporalConfig"),
+  };
+}
+
+const LIFECYCLE_VERBS: ReadonlyArray<
+  readonly [string, (deps: LifecycleDeps) => Promise<unknown>]
+> = [
+  [
+    "cancel",
+    (deps) =>
+      cancelExecution(
+        deps,
+        create(CancelAgentExecutionInputSchema, { id: EXECUTION_ID }),
+        testCallerIdentity(),
+      ),
+  ],
+  [
+    "terminate",
+    (deps) =>
+      terminateExecution(
+        deps,
+        create(TerminateAgentExecutionInputSchema, { id: EXECUTION_ID }),
+        testCallerIdentity(),
+      ),
+  ],
+  [
+    "pause",
+    (deps) =>
+      pauseExecution(
+        deps,
+        create(PauseAgentExecutionInputSchema, { id: EXECUTION_ID }),
+        testCallerIdentity(),
+      ),
+  ],
+  [
+    "resume",
+    (deps) =>
+      resumeExecution(
+        deps,
+        create(ResumeAgentExecutionInputSchema, { id: EXECUTION_ID }),
+        testCallerIdentity(),
+      ),
+  ],
+  [
+    "recover",
+    (deps) =>
+      recoverExecution(
+        deps,
+        create(RecoverAgentExecutionInputSchema, { id: EXECUTION_ID }),
+        testCallerIdentity(),
+      ),
+  ],
+];
+
+describe.each(LIFECYCLE_VERBS)("%s — LoadExecutionById", (_verb, run) => {
+  it("a missing execution answers NotFound with the lifecycle copy", async () => {
+    const error = await errorOf(() =>
+      run(lifecycleDeps(failingStore(MISSING()))),
+    );
+
+    expect(error.code).toBe(Code.NotFound);
+    expect(error.rawMessage).toBe(`agent_execution not found: ${EXECUTION_ID}`);
+  });
+
+  it("any other store failure answers a sanitized Internal", async () => {
+    const error = await errorOf(() =>
+      run(lifecycleDeps(failingStore(LOCKED()))),
+    );
+
+    expect(error.code).toBe(Code.Internal);
+    expect(error.rawMessage).toBe(LOAD_FAULT_COPY);
+  });
+});
+
+describe("subscribe — the snapshot read", () => {
+  function handlerContext(): HandlerContext {
+    const values = createContextValues();
+    values.set(callerIdentityKey, testCallerIdentity());
+    return { signal: new AbortController().signal, values } as HandlerContext;
+  }
+
+  async function subscribeError(
+    store: Store,
+    broker: StreamBroker,
+  ): Promise<ConnectError> {
+    return errorOf(() =>
+      subscribeExecution(
+        {
+          store,
+          logger: silentLogger,
+          broker,
+          authorizer: newPermissiveSingleTeamAuthorizer(),
+        },
+        create(AgentExecutionIdSchema, { value: EXECUTION_ID }),
+        handlerContext(),
+      ).next(),
+    );
+  }
+
+  it("a missing execution answers NotFound with the subscribe copy", async () => {
+    const broker = new StreamBroker(silentLogger);
+    const error = await subscribeError(failingStore(MISSING()), broker);
+
+    expect(error.code).toBe(Code.NotFound);
+    expect(error.rawMessage).toBe(`AgentExecution not found: ${EXECUTION_ID}`);
+    expect(broker.getSubscriberCount(EXECUTION_ID)).toBe(0);
+  });
+
+  it("any other store failure answers a sanitized Internal and releases the subscription", async () => {
+    const broker = new StreamBroker(silentLogger);
+    const error = await subscribeError(failingStore(LOCKED()), broker);
+
+    expect(error.code).toBe(Code.Internal);
+    expect(error.rawMessage).toBe(LOAD_FAULT_COPY);
+    expect(broker.getSubscriberCount(EXECUTION_ID)).toBe(0);
+  });
+});
+
+describe("artifact reads — the existence check", () => {
+  function artifactDeps(store: Store) {
+    return {
+      store,
+      logger: silentLogger,
+      artifactStorage: untouchable<ArtifactStorage>("artifactStorage"),
+      authorizer: newPermissiveSingleTeamAuthorizer(),
+    };
+  }
+
+  // getArtifactContent checks the key's prefix before its load, so the key
+  // sits under the execution; getArtifactDownloadUrl loads first.
+  const ARTIFACT_READS: ReadonlyArray<
+    readonly [string, (store: Store) => Promise<unknown>]
+  > = [
+    [
+      "getArtifactContent",
+      (store) =>
+        getArtifactContent(
+          artifactDeps(store),
+          create(GetArtifactContentRequestSchema, {
+            executionId: EXECUTION_ID,
+            storageKey: `artifacts/${EXECUTION_ID}/out.txt`,
+          }),
+          testCallerIdentity(),
+        ),
+    ],
+    [
+      "getArtifactDownloadUrl",
+      (store) =>
+        getArtifactDownloadUrl(
+          artifactDeps(store),
+          create(GetArtifactDownloadUrlRequestSchema, {
+            executionId: EXECUTION_ID,
+            storageKey: `artifacts/${EXECUTION_ID}/out.txt`,
+          }),
+          testCallerIdentity(),
+        ),
+    ],
+  ];
+
+  describe.each(ARTIFACT_READS)("%s", (_read, run) => {
+    it("a missing execution answers NotFound with the artifact copy", async () => {
+      const error = await errorOf(() => run(failingStore(MISSING())));
+
+      expect(error.code).toBe(Code.NotFound);
+      expect(error.rawMessage).toBe(`execution not found: ${EXECUTION_ID}`);
+    });
+
+    it("any other store failure answers a sanitized Internal", async () => {
+      const error = await errorOf(() => run(failingStore(LOCKED())));
+
+      expect(error.code).toBe(Code.Internal);
+      expect(error.rawMessage).toBe(LOAD_FAULT_COPY);
+    });
+  });
+});
+
+describe("getExecutionUsageReport — LoadExecution", () => {
+  function reportError(store: Store): Promise<ConnectError> {
+    return errorOf(() =>
+      getExecutionUsageReport(
+        {
+          store,
+          logger: silentLogger,
+          authorizer: newPermissiveSingleTeamAuthorizer(),
+          listReadScope: undefined,
+        },
+        create(
+          AgentExecutionQueryController.method.getExecutionUsageReport.input,
+          { executionId: EXECUTION_ID },
+        ),
+        testCallerIdentity(),
+      ),
+    );
+  }
+
+  it("a missing execution answers NotFound with the domain's copy", async () => {
+    const error = await reportError(failingStore(MISSING()));
+
+    expect(error.code).toBe(Code.NotFound);
+    expect(error.rawMessage).toBe(`agent_execution not found: ${EXECUTION_ID}`);
+  });
+
+  it("any other store failure answers a sanitized Internal", async () => {
+    const error = await reportError(failingStore(LOCKED()));
+
+    expect(error.code).toBe(Code.Internal);
+    expect(error.rawMessage).toBe(LOAD_FAULT_COPY);
+  });
+});

package/src/domain/agentexecution/__tests__/update-status.test.ts

@@ -5,6 +5,10 @@
  * transcript regression guard (shrink + front-truncation-with-append +
  * the terminal exemption), the equal-length approval finalize, and the
  * presence-guarded runner-owned field pattern (recalled_memories_report).
+ * Also pins the platform-row rules (platform-rows.ts, stigmer#980): a
+ * platform marker write is appended to the transcript and never replaces
+ * it, idempotently, and a PAUSED or CANCELLED execution's stop row stays
+ * last across the runner's late writes, whatever order they land in.
  * These exercise applyUpdateStatusMerge directly on a clone — mirroring
  * the freshly-loaded resource the merge mutates in place inside the
  * updateResource write lock.
@@ -31,6 +35,7 @@ import { AgentMessageSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexec
 
 import { createLogger } from "../../../boot/logger.js";
 import { findToolCallInExecution } from "../submit-approval.js";
+import { CANCEL_STOP_ROW, PAUSE_STOP_ROW } from "../platform-rows.js";
 import { applyUpdateStatusMerge } from "../update-status.js";
 
 const silentLogger = createLogger({
@@ -367,3 +372,215 @@ describe("the presence-guarded runner-owned field pattern (recalled_memories_rep
     expect(merged.status?.recalledMemoriesReport).toBeUndefined();
   });
 });
+
+describe("platform rows (stigmer#980)", () => {
+  // The runner's transcript: AI and tool turns, then (for a failure it
+  // reports itself) its own system rows.
+  function runnerTranscript(...contents: string[]) {
+    return contents.map((content) => ({ type: MessageType.MESSAGE_AI, content }));
+  }
+  function system(...contents: string[]) {
+    return contents.map((content) => ({ type: MessageType.MESSAGE_SYSTEM, content }));
+  }
+  function executionAt(
+    phase: ExecutionPhase,
+    msgs: MessageInitShape<typeof AgentMessageSchema>[],
+  ): AgentExecution {
+    return create(AgentExecutionSchema, {
+      metadata: { id: "exec-rows", name: "exec-rows" },
+      spec: {},
+      status: { phase, messages: msgs },
+    });
+  }
+  function contents(execution: AgentExecution): string[] {
+    return (execution.status?.messages ?? []).map((m) => m.content);
+  }
+
+  describe("a platform marker write is appended, never substituted", () => {
+    it("appends the cancel row to a transcript already persisted CANCELLED", () => {
+      // The Cancel RPC persisted CANCELLED first, then the workflow's
+      // cancellation cleanup lands its row: today it replaced the whole
+      // transcript with that one line.
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_CANCELLED, runnerTranscript("a1", "a2")),
+        { phase: ExecutionPhase.EXECUTION_CANCELLED, messages: system(CANCEL_STOP_ROW) },
+      );
+      expect(contents(merged)).toEqual(["a1", "a2", CANCEL_STOP_ROW]);
+    });
+
+    it("appends the broken-flow failure lines to a FAILED transcript", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_FAILED, runnerTranscript("a1", "a2", "a3")),
+        {
+          phase: ExecutionPhase.EXECUTION_FAILED,
+          error: "boom",
+          messages: system("Internal failure.", "Error details: boom"),
+        },
+      );
+      expect(contents(merged)).toEqual([
+        "a1",
+        "a2",
+        "a3",
+        "Internal failure.",
+        "Error details: boom",
+      ]);
+    });
+
+    it("appends a marker landing before the phase is terminal instead of dropping it to the shrink guard", () => {
+      // The runner died without persisting a terminal: the stored phase is
+      // still IN_PROGRESS, and the failure lines are shorter than the
+      // transcript. Today the shrink guard dropped them.
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_IN_PROGRESS, runnerTranscript("a1", "a2", "a3")),
+        {
+          phase: ExecutionPhase.EXECUTION_FAILED,
+          messages: system("Internal failure.", "Error details: boom"),
+        },
+      );
+      expect(contents(merged)).toEqual([
+        "a1",
+        "a2",
+        "a3",
+        "Internal failure.",
+        "Error details: boom",
+      ]);
+      expect(merged.status?.phase).toBe(ExecutionPhase.EXECUTION_FAILED);
+    });
+
+    it("is idempotent: an activity retry of the same marker appends nothing", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_FAILED, [
+          ...runnerTranscript("a1"),
+          ...system("Internal failure.", "Error details: boom"),
+        ]),
+        {
+          phase: ExecutionPhase.EXECUTION_FAILED,
+          messages: system("Internal failure.", "Error details: boom"),
+        },
+      );
+      expect(contents(merged)).toEqual(["a1", "Internal failure.", "Error details: boom"]);
+    });
+
+    it("keeps an all-system write that extends the stored transcript a replacement (the runner's own growth)", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_IN_PROGRESS, system("s1")),
+        { phase: ExecutionPhase.EXECUTION_IN_PROGRESS, messages: system("s1", "s2") },
+      );
+      expect(contents(merged)).toEqual(["s1", "s2"]);
+    });
+
+    it("keeps a system write onto an empty transcript a replacement", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_IN_PROGRESS, []),
+        { phase: ExecutionPhase.EXECUTION_FAILED, messages: system("Internal failure.") },
+      );
+      expect(contents(merged)).toEqual(["Internal failure."]);
+    });
+
+    it("keeps the runner's full terminal write a replacement (the terminal exemption is untouched)", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_FAILED, runnerTranscript("a1", "a2")),
+        {
+          phase: ExecutionPhase.EXECUTION_FAILED,
+          messages: [...runnerTranscript("a1-final"), ...system("Execution failed: x")],
+        },
+      );
+      expect(contents(merged)).toEqual(["a1-final", "Execution failed: x"]);
+    });
+  });
+
+  describe("a stop row stays last across the runner's late writes", () => {
+    it("re-appends the cancel row after a late runner write that extends the transcript", () => {
+      // Cancel's three writers race: the RPC's CANCELLED persist and the
+      // workflow's row landed; the runner's settle write (it learns of the
+      // cancel on its next heartbeat) lands last, carrying no row.
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_CANCELLED, [
+          ...runnerTranscript("a1"),
+          ...system(CANCEL_STOP_ROW),
+        ]),
+        { messages: runnerTranscript("a1", "a2") },
+      );
+      expect(contents(merged)).toEqual(["a1", "a2", CANCEL_STOP_ROW]);
+      expect(merged.status?.phase).toBe(ExecutionPhase.EXECUTION_CANCELLED);
+    });
+
+    it("keeps the pause row last when the runner's settle write lands after it", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_PAUSED, [
+          ...runnerTranscript("a1"),
+          ...system(PAUSE_STOP_ROW),
+        ]),
+        { messages: runnerTranscript("a1") },
+      );
+      expect(
+        contents(merged),
+        "the settle write does not trip the shrink guard on the row, and does not drop it",
+      ).toEqual(["a1", PAUSE_STOP_ROW]);
+    });
+
+    it("merges a longer settle write under the pause row", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_PAUSED, [
+          ...runnerTranscript("a1"),
+          ...system(PAUSE_STOP_ROW),
+        ]),
+        { messages: runnerTranscript("a1", "a2", "a3") },
+      );
+      expect(contents(merged)).toEqual(["a1", "a2", "a3", PAUSE_STOP_ROW]);
+    });
+
+    it("still guards a paused transcript against a real shrink", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_PAUSED, [
+          ...runnerTranscript("a1", "a2"),
+          ...system(PAUSE_STOP_ROW),
+        ]),
+        { messages: runnerTranscript("a1") },
+      );
+      expect(contents(merged)).toEqual(["a1", "a2", PAUSE_STOP_ROW]);
+    });
+
+    it("does not duplicate the row when a retry of the stop marker lands on it", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_CANCELLED, [
+          ...runnerTranscript("a1"),
+          ...system(CANCEL_STOP_ROW),
+        ]),
+        { phase: ExecutionPhase.EXECUTION_CANCELLED, messages: system(CANCEL_STOP_ROW) },
+      );
+      expect(contents(merged)).toEqual(["a1", CANCEL_STOP_ROW]);
+    });
+
+    it("treats the pause row as ordinary history once a resume moved the phase on", () => {
+      // The resumed runner seeds the persisted transcript, row included,
+      // and grows it; nothing is held back or re-appended.
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_IN_PROGRESS, [
+          ...runnerTranscript("a1"),
+          ...system(PAUSE_STOP_ROW),
+        ]),
+        {
+          phase: ExecutionPhase.EXECUTION_IN_PROGRESS,
+          messages: [
+            ...runnerTranscript("a1"),
+            ...system(PAUSE_STOP_ROW),
+            ...runnerTranscript("a2"),
+          ],
+        },
+      );
+      expect(contents(merged)).toEqual(["a1", PAUSE_STOP_ROW, "a2"]);
+    });
+
+    it("never holds a row the phase does not own (a pause row on a CANCELLED execution is history)", () => {
+      const merged = runBuildStep(
+        executionAt(ExecutionPhase.EXECUTION_CANCELLED, [
+          ...runnerTranscript("a1"),
+          ...system(PAUSE_STOP_ROW),
+        ]),
+        { messages: [...runnerTranscript("a1"), ...system(PAUSE_STOP_ROW), ...runnerTranscript("a2")] },
+      );
+      expect(contents(merged)).toEqual(["a1", PAUSE_STOP_ROW, "a2"]);
+    });
+  });
+});

package/src/domain/agentexecution/artifacts.ts

@@ -57,6 +57,7 @@ import {
   invalidArgumentError,
 } from "../../pipeline/errors.js";
 import { authorizeDirect } from "../../pipeline/steps/authorize.js";
+import { ResourceNotFoundError } from "../../store/interface.js";
 import type { Store } from "../../store/interface.js";
 import { casBlobContentMismatch } from "./filereview/cas-blob.js";
 
@@ -254,18 +255,22 @@ export async function getArtifactContent(
     throw invalidArgumentError("storage_key does not belong to this execution");
   }
 
-  // Existence check (Go answers NotFound for any load failure here).
+  // Existence check: only a missing execution is NotFound; any other store
+  // failure is an infrastructure fault.
   try {
     await deps.store.getResource(
       ApiResourceKind.agent_execution,
       req.executionId,
       AgentExecutionSchema,
     );
-  } catch {
-    throw new ConnectError(
-      `execution not found: ${req.executionId}`,
-      Code.NotFound,
-    );
+  } catch (error) {
+    if (error instanceof ResourceNotFoundError) {
+      throw new ConnectError(
+        `execution not found: ${req.executionId}`,
+        Code.NotFound,
+      );
+    }
+    throw internalError(error, "failed to load agent execution");
   }
 
   let maxBytes = Number(req.maxBytes);
@@ -424,7 +429,8 @@ export async function getArtifactDownloadUrl(
   );
 
   // Load BEFORE the key check: the attachment arm needs spec.attachments,
-  // and this doubles as the existence check.
+  // and this doubles as the existence check (a missing execution is
+  // NotFound; any other store failure is an infrastructure fault).
   let execution: AgentExecution;
   try {
     execution = await deps.store.getResource(
@@ -432,11 +438,14 @@ export async function getArtifactDownloadUrl(
       req.executionId,
       AgentExecutionSchema,
     );
-  } catch {
-    throw new ConnectError(
-      `execution not found: ${req.executionId}`,
-      Code.NotFound,
-    );
+  } catch (error) {
+    if (error instanceof ResourceNotFoundError) {
+      throw new ConnectError(
+        `execution not found: ${req.executionId}`,
+        Code.NotFound,
+      );
+    }
+    throw internalError(error, "failed to load agent execution");
   }
 
   // Same normalize-before-check posture as getArtifactContent

package/src/domain/agentexecution/lifecycle.ts

@@ -135,9 +135,11 @@ function newLoadExecutionByIdStep<Desc extends DescMessage>(
           executionId,
           AgentExecutionSchema,
         );
-      } catch {
-        // Go converts every load failure here to the same NotFound.
-        throw notFoundError("agent_execution", executionId);
+      } catch (error) {
+        if (error instanceof ResourceNotFoundError) {
+          throw notFoundError("agent_execution", executionId);
+        }
+        throw internalError(error, "failed to load agent execution");
       }
       ctx.set(LOADED_EXECUTION_KEY, execution);
     },