@stigmer/server 3.37.0 → 3.38.0

package/src/pipeline/__tests__/blind-not-found.test.ts

@@ -0,0 +1,297 @@
+/**
+ * Holds the error contract's store-fault rule structurally: no error handler
+ * in the server answers NotFound without looking at the error it caught.
+ * Only a typed not-found (`ResourceNotFoundError` and its kin) means the
+ * record is missing; any other failure is an infrastructure fault answered
+ * through `internalError` (`src/pipeline/errors.ts`), because a client told
+ * "not found" may discard state that still exists
+ * (`.agents/skills/ts-server-dev-guidelines/SKILL.md`, "Errors").
+ *
+ * A *blind NotFound* is a `throw` of a NotFound that is a direct statement of
+ * a `catch` block or of a `.catch(...)` callback — not nested under any
+ * condition. That is the defect's exact shape: a discriminating handler throws
+ * its NotFound under `if (error instanceof ...)` or a code check, so it passes
+ * whatever error class it tests, and the guard needs no list of them. A throw
+ * constructs a NotFound when it is `new ConnectError(..., Code.NotFound)` or
+ * a call to a NotFound constructor; the constructors are derived from the
+ * source (`notFoundError`, plus every function whose body is a single
+ * `return` of a NotFound construction, to a fixed point), so a domain's own
+ * wrapper cannot hide a blind site, and a discriminating mapper is not one.
+ *
+ * The scan parses each module with the TypeScript compiler API rather than a
+ * regular expression, because a catch block's extent (nested braces, a log
+ * line before the throw) is a syntax question. `PENDING` is the list of known
+ * blind sites still to be fixed; the scan must equal it exactly, so a new
+ * blind site fails here, and so does a fixed site still listed.
+ */
+import { readFileSync, readdirSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+import ts from "typescript";
+import { describe, expect, it } from "vitest";
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const SRC = path.resolve(HERE, "../..");
+
+/** Known blind sites by module (relative to `src`), with their count. */
+const PENDING: ReadonlyMap<string, number> = new Map([
+  // Store loads, stigmer/stigmer#1345 (fixed domain by domain).
+  ["domain/agent/controller.ts", 1],
+  ["domain/agentchannel/controller.ts", 1],
+  ["domain/agentinstance/controller.ts", 1],
+  ["domain/agentshare/controller.ts", 1],
+  ["domain/artifact/controller.ts", 1],
+  ["domain/environment/controller.ts", 1],
+  ["domain/environment/steps.ts", 1],
+  ["domain/mcpserver/complete-oauth-connect.ts", 1],
+  ["domain/mcpserver/connect.ts", 1],
+  ["domain/mcpserver/controller.ts", 1],
+  ["domain/mcpserver/initiate-oauth-connect.ts", 1],
+  ["domain/mcpserver/start-connect.ts", 2],
+  ["domain/plugin/controller.ts", 1],
+  ["domain/skill/controller.ts", 1],
+  ["domain/workflow/controller.ts", 1],
+  ["domain/workflowinstance/controller.ts", 1],
+  // Parent loads through the in-process chain that fold every RPC error,
+  // validation included, stigmer/stigmer#1351.
+  ["domain/agentinstance/steps.ts", 1],
+  ["domain/workflowinstance/steps.ts", 1],
+]);
+
+interface BlindSite {
+  readonly module: string;
+  readonly line: number;
+}
+
+function isCodeNotFound(node: ts.Expression): boolean {
+  return (
+    ts.isPropertyAccessExpression(node) &&
+    ts.isIdentifier(node.expression) &&
+    node.expression.text === "Code" &&
+    node.name.text === "NotFound"
+  );
+}
+
+function isNotFoundConstruction(
+  node: ts.Expression | undefined,
+  constructors: ReadonlySet<string>,
+): boolean {
+  if (node === undefined) {
+    return false;
+  }
+  if (ts.isCallExpression(node)) {
+    return (
+      ts.isIdentifier(node.expression) && constructors.has(node.expression.text)
+    );
+  }
+  if (ts.isNewExpression(node)) {
+    return (
+      ts.isIdentifier(node.expression) &&
+      node.expression.text === "ConnectError" &&
+      (node.arguments ?? []).some(isCodeNotFound)
+    );
+  }
+  return false;
+}
+
+function forEachNode(node: ts.Node, visit: (node: ts.Node) => void): void {
+  visit(node);
+  ts.forEachChild(node, (child) => forEachNode(child, visit));
+}
+
+/** `notFoundError` plus every single-`return` wrapper of a NotFound. */
+function notFoundConstructors(
+  sources: readonly ts.SourceFile[],
+): ReadonlySet<string> {
+  const constructors = new Set(["notFoundError"]);
+  let grew = true;
+  while (grew) {
+    grew = false;
+    for (const source of sources) {
+      forEachNode(source, (node) => {
+        if (
+          !ts.isFunctionDeclaration(node) ||
+          node.name === undefined ||
+          node.body === undefined ||
+          node.body.statements.length !== 1 ||
+          constructors.has(node.name.text)
+        ) {
+          return;
+        }
+        const only = node.body.statements[0];
+        if (
+          only !== undefined &&
+          ts.isReturnStatement(only) &&
+          isNotFoundConstruction(only.expression, constructors)
+        ) {
+          constructors.add(node.name.text);
+          grew = true;
+        }
+      });
+    }
+  }
+  return constructors;
+}
+
+/** The handler block of a `.catch(fn)` call, when `fn` has a block body. */
+function catchCallbackBlock(node: ts.Node): ts.Block | undefined {
+  if (
+    !ts.isCallExpression(node) ||
+    !ts.isPropertyAccessExpression(node.expression) ||
+    node.expression.name.text !== "catch"
+  ) {
+    return undefined;
+  }
+  const handler = node.arguments[0];
+  if (
+    handler !== undefined &&
+    (ts.isArrowFunction(handler) || ts.isFunctionExpression(handler)) &&
+    ts.isBlock(handler.body)
+  ) {
+    return handler.body;
+  }
+  return undefined;
+}
+
+function blindSites(
+  source: ts.SourceFile,
+  module: string,
+  constructors: ReadonlySet<string>,
+): BlindSite[] {
+  const sites: BlindSite[] = [];
+  forEachNode(source, (node) => {
+    const block = ts.isCatchClause(node)
+      ? node.block
+      : catchCallbackBlock(node);
+    if (block === undefined) {
+      return;
+    }
+    for (const statement of block.statements) {
+      if (
+        ts.isThrowStatement(statement) &&
+        isNotFoundConstruction(statement.expression, constructors)
+      ) {
+        const { line } = source.getLineAndCharacterOfPosition(
+          statement.getStart(source),
+        );
+        sites.push({ module, line: line + 1 });
+      }
+    }
+  });
+  return sites;
+}
+
+function parse(fileName: string, text: string): ts.SourceFile {
+  return ts.createSourceFile(fileName, text, ts.ScriptTarget.Latest, true);
+}
+
+/** Every non-test module under `src`, parsed. */
+function serverSources(): ts.SourceFile[] {
+  const sources: ts.SourceFile[] = [];
+  const walk = (dir: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name !== "__tests__") {
+          walk(full);
+        }
+      } else if (
+        entry.name.endsWith(".ts") &&
+        !entry.name.endsWith(".d.ts") &&
+        !entry.name.endsWith(".test.ts")
+      ) {
+        sources.push(parse(full, readFileSync(full, "utf8")));
+      }
+    }
+  };
+  walk(SRC);
+  return sources;
+}
+
+/** The rule applied to one inline module, for the rule's own cases. */
+function blindLinesOf(text: string): number[] {
+  const source = parse("fixture.ts", text);
+  return blindSites(source, "fixture.ts", notFoundConstructors([source])).map(
+    (site) => site.line,
+  );
+}
+
+describe("the blind-NotFound rule", () => {
+  it("flags an unconditional NotFound in a catch block, bound or not", () => {
+    expect(
+      blindLinesOf(`
+        try { load(); } catch { throw notFoundError("agent", id); }
+        try { load(); } catch (error) {
+          log(error);
+          throw new ConnectError(copy(id), Code.NotFound);
+        }
+      `),
+    ).toEqual([2, 5]);
+  });
+
+  it("flags a NotFound built by a derived wrapper and one in a .catch callback", () => {
+    expect(
+      blindLinesOf(`
+        function missing(id: string) { return notFoundError("agent", id); }
+        function missingAgain(id: string) { return missing(id); }
+        try { load(); } catch { throw missingAgain(id); }
+        load().catch(() => { throw missing(id); });
+      `),
+    ).toEqual([4, 5]);
+  });
+
+  it("passes a NotFound thrown only under a test of the error", () => {
+    expect(
+      blindLinesOf(`
+        try { load(); } catch (error) {
+          if (error instanceof ResourceNotFoundError) {
+            throw notFoundError("agent", id);
+          }
+          throw internalError(error, "failed to load agent");
+        }
+      `),
+    ).toEqual([]);
+  });
+
+  it("does not treat a discriminating mapper as a constructor", () => {
+    expect(
+      blindLinesOf(`
+        function mapNotFound(error: unknown, id: string): unknown {
+          if (error instanceof WorkflowNotFoundError) {
+            return notFoundError("workflow", id);
+          }
+          return error;
+        }
+        try { load(); } catch (error) { throw mapNotFound(error, id); }
+      `),
+    ).toEqual([]);
+  });
+});
+
+describe("the server source", () => {
+  it("has no blind NotFound beyond the pending list, and the list names only live sites", () => {
+    const sources = serverSources();
+    const constructors = notFoundConstructors(sources);
+    const found = new Map<string, number[]>();
+    for (const source of sources) {
+      const module = path
+        .relative(SRC, source.fileName)
+        .split(path.sep)
+        .join("/");
+      for (const site of blindSites(source, module, constructors)) {
+        found.set(module, [...(found.get(module) ?? []), site.line]);
+      }
+    }
+
+    const counts = Object.fromEntries(
+      [...found].map(([module, lines]) => [module, lines.length]),
+    );
+    const where = [...found]
+      .flatMap(([module, lines]) => lines.map((line) => `${module}:${line}`))
+      .join("\n");
+    expect(counts, `blind NotFound sites found:\n${where}`).toEqual(
+      Object.fromEntries(PENDING),
+    );
+  });
+});

package/src/pipeline/steps/authorization-tuples.ts

@@ -330,8 +330,12 @@ export function newCreateAuthorizationTuplesStep<Desc extends DescMessage>(
 /**
  * CleanupIamPolicies — after the store delete in every delete chain.
  * Best-effort by contract: a driver failure is logged and the delete
- * succeeds (Java's DeleteOperationCleanupIamPoliciesStep swallows all —
- * the cloud side owns convergence for orphaned grants).
+ * succeeds (Java's DeleteOperationCleanupIamPoliciesStep swallows all),
+ * because a deleted resource's rows grant nothing once it is gone. The
+ * organization is the one kind whose id another caller can take next, so
+ * its chain revokes the organization's rows before the row, failing
+ * closed (RevokeOrganizationPolicies, domain/organization/steps.ts), and
+ * this step is only its backstop there.
  */
 export function newCleanupIamPoliciesStep<Desc extends DescMessage>(
   lifecycle: ResourceAuthorizationLifecycle | undefined,

package/src/temporal/agentexecution/__tests__/invoke-workflow.test.ts

@@ -16,6 +16,10 @@
  *   - bounded auto-recovery on worker-shutdown shapes (#776: IN_PROGRESS
  *     persisted, honest status copy on exhaustion);
  *   - EXECUTION_FAILED propagation with the fallback persist;
+ *   - the stop and failure copy each has one writer (stigmer#980): the
+ *     pause persist carries the pause row, a runner-reported failure's
+ *     write carries no lines (the runner explained it), and only a broken
+ *     flow gets the platform's two lines;
  *   - external cancellation cleanup (CANCELLED persisted quietly —
  *     stigmer#282 — and the EC deleted);
  *   - callback-token completion on success AND failure (the DD-001 lane,
@@ -50,9 +54,15 @@ import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/ag
 import {
   ExecutionPhase,
   FileChangeSetStatus,
+  MessageType,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { Harness } from "@stigmer/protos/ai/stigmer/agentic/session/v1/enum_pb";
 
+import {
+  CANCEL_STOP_ROW,
+  PAUSE_STOP_ROW,
+} from "../../../domain/agentexecution/platform-rows.js";
+
 import {
   INVOKE_AGENT_EXECUTION_WORKFLOW_NAME,
   MEMO_ACTIVITY_TASK_QUEUE,
@@ -565,6 +575,12 @@ describe("invoke-agent-execution workflow (TestWorkflowEnvironment)", () => {
         .includes(ExecutionPhase.EXECUTION_IN_PROGRESS),
       "resume re-asserts IN_PROGRESS after the PAUSED persist",
     ).toBe(true);
+    // The workflow is the pause row's one writer: the runner cannot tell
+    // this stop from a cancel, so it settles without a row.
+    expect(
+      script.persistedStatuses[pausedAt]!.status.messages.map((m) => [m.type, m.content]),
+      "the PAUSED persist carries the pause row",
+    ).toEqual([[MessageType.MESSAGE_SYSTEM, PAUSE_STOP_ROW]]);
   }, 60_000);
 
   it("auto-recovers a worker-shutdown interruption and persists IN_PROGRESS (#776)", async (testCtx) => {    if (!envReady) return testCtx.skip();
@@ -611,6 +627,40 @@ describe("invoke-agent-execution workflow (TestWorkflowEnvironment)", () => {
     );
     expect(failed.length).toBeGreaterThan(0);
     expect(failed[0]!.status.error).toBe("agent blew up");
+    // The runner already explained its own failure in the transcript, so
+    // no FAILED write of the workflow adds a second explanation.
+    expect(
+      failed.flatMap((entry) => entry.status.messages),
+      "a runner-reported failure adds no platform lines",
+    ).toEqual([]);
+    expectExecutionContextDeleted();
+  }, 30_000);
+
+  it("explains a flow that broke without a runner verdict with the platform's two lines", async (testCtx) => {    if (!envReady) return testCtx.skip();
+    const { ApplicationFailure } = await import("@temporalio/common");
+    script.executeBehaviors = [
+      async () => {
+        // Not recoverable (not a cancellation, heartbeat timeout or worker
+        // shutdown), so the flow breaks with no FAILED result to read.
+        throw ApplicationFailure.create({ message: "runner crashed", nonRetryable: true });
+      },
+    ];
+
+    const handle = await startWorkflow(workflowInput());
+    await expect(handle.result()).rejects.toThrow(/Workflow execution failed/);
+
+    const failed = script.persistedStatuses.filter(
+      (entry) => entry.status.phase === ExecutionPhase.EXECUTION_FAILED,
+    );
+    expect(failed.length).toBeGreaterThan(0);
+    const lines = failed.at(-1)!.status.messages;
+    expect(lines.map((m) => m.type)).toEqual([
+      MessageType.MESSAGE_SYSTEM,
+      MessageType.MESSAGE_SYSTEM,
+    ]);
+    // The details carry the wrapped activity error (wrapActivityError's
+    // operator-actionable prefix), not a runner-authored sentence.
+    expect(lines[1]!.content).toMatch(/^Error details: activity 'ExecuteDeepAgent'/);
     expectExecutionContextDeleted();
   }, 30_000);
 
@@ -638,7 +688,7 @@ describe("invoke-agent-execution workflow (TestWorkflowEnvironment)", () => {
     // system message is the durable marker.
     expect(cancelled.status.error).toBe("");
     expect(cancelled.status.messages.map((message) => message.content)).toEqual([
-      "Execution was cancelled.",
+      CANCEL_STOP_ROW,
     ]);
     expectExecutionContextDeleted();
   }, 60_000);

package/src/temporal/agentexecution/__tests__/own-behalf-status-writes.test.ts

@@ -20,19 +20,24 @@
  * execution status` from authorize.ts — the dark production composition's
  * exact log line.
  *
- * The three payload shapes are the workflow's real ones
+ * The payload shapes are the workflow's real ones
  * (invoke-agent-execution.ts): updateStatusOnFailure's FAILED with two
- * MESSAGE_SYSTEM lines, updateStatusOnCancellation's CANCELLED with one,
+ * MESSAGE_SYSTEM lines (a flow that broke) or none (a runner-reported
+ * failure), updateStatusOnCancellation's CANCELLED with its stop row,
  * and the phase-only persistFinalStatus / persistInterruptedStatus /
- * persistResumedStatus shape. Transcript shape after a late fallback is
- * deliberately NOT asserted here — stigmer#980 owns that cross-edition
- * behavior.
+ * persistResumedStatus shape.
+ *
+ * The last arms pin the transcript a stopped execution keeps over this
+ * production lane (stigmer#980): the platform's lines are appended to the
+ * runner's transcript, never substituted for it, and the cancel row stays
+ * last when the runner's settle write lands after it. The merge's full
+ * case matrix is domain/agentexecution/__tests__/update-status.test.ts.
  */
 import { mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { create, toJson } from "@bufbuild/protobuf";
-import type { JsonValue } from "@bufbuild/protobuf";
+import type { JsonValue, MessageInitShape } from "@bufbuild/protobuf";
 import { createClient } from "@connectrpc/connect";
 import type { Client } from "@temporalio/client";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
@@ -46,9 +51,12 @@ import {
   ExecutionPhase,
   MessageType,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import { AgentExecutionUpdateStatusInputSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/io_pb";
+import type { AgentMessageSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
 import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
 
 import { loadConfig } from "../../../boot/config.js";
+import { CANCEL_STOP_ROW } from "../../../domain/agentexecution/platform-rows.js";
 import { composeServer } from "../../../boot/compose.js";
 import type { ComposedServer } from "../../../boot/compose.js";
 import { createLogger } from "../../../boot/logger.js";
@@ -147,7 +155,11 @@ describe("own-behalf status writes under an enforcing Authorizer", () => {
     transitions.length = 0;
   });
 
-  async function seedExecution(id: string): Promise<void> {
+  async function seedExecution(
+    id: string,
+    phase: ExecutionPhase = ExecutionPhase.EXECUTION_IN_PROGRESS,
+    messages: MessageInitShape<typeof AgentMessageSchema>[] = [],
+  ): Promise<void> {
     // Seeded through the store: the create path is not under test, and
     // the denying Authorizer would refuse it over the wire.
     await server.store.saveResource(
@@ -159,7 +171,7 @@ describe("own-behalf status writes under an enforcing Authorizer", () => {
         kind: "AgentExecution",
         metadata: { id, name: "own-behalf", org: "acme" },
         spec: { agentId: "agt_1", sessionId: "ses_1" },
-        status: { phase: ExecutionPhase.EXECUTION_IN_PROGRESS },
+        status: { phase, messages },
       }),
     );
   }
@@ -253,7 +265,7 @@ describe("own-behalf status writes under an enforcing Authorizer", () => {
           messages: [
             {
               type: MessageType.MESSAGE_SYSTEM,
-              content: "Execution was cancelled.",
+              content: CANCEL_STOP_ROW,
             },
           ],
         }),
@@ -296,6 +308,134 @@ describe("own-behalf status writes under an enforcing Authorizer", () => {
     expect(authorizerCalls).toHaveLength(0);
   });
 
+  describe("the transcript a stopped execution keeps (stigmer#980)", () => {
+    const runnerTurns = [
+      { type: MessageType.MESSAGE_AI, content: "a1" },
+      { type: MessageType.MESSAGE_AI, content: "a2" },
+    ];
+
+    async function transcriptOf(id: string): Promise<string[]> {
+      return ((await persisted(id)).status?.messages ?? []).map(
+        (m) => m.content,
+      );
+    }
+
+    it("appends the cancel row to a transcript the Cancel RPC already stamped CANCELLED, and keeps it last across the runner's late settle write", async () => {
+      await seedExecution(
+        "aex_ownbehalf_cancel_race",
+        ExecutionPhase.EXECUTION_CANCELLED,
+        runnerTurns,
+      );
+
+      await updateStatus(
+        "aex_ownbehalf_cancel_race",
+        toJson(
+          AgentExecutionStatusSchema,
+          create(AgentExecutionStatusSchema, {
+            phase: ExecutionPhase.EXECUTION_CANCELLED,
+            messages: [
+              { type: MessageType.MESSAGE_SYSTEM, content: CANCEL_STOP_ROW },
+            ],
+          }),
+        ),
+      );
+      expect(await transcriptOf("aex_ownbehalf_cancel_race")).toEqual([
+        "a1",
+        "a2",
+        CANCEL_STOP_ROW,
+      ]);
+
+      // The runner learns of the cancel on its next heartbeat and settles
+      // its transcript over the same handler, with no row and no phase.
+      await statusWriter.updateStatus(
+        create(AgentExecutionUpdateStatusInputSchema, {
+          executionId: "aex_ownbehalf_cancel_race",
+          status: {
+            messages: [
+              ...runnerTurns,
+              { type: MessageType.MESSAGE_AI, content: "a3" },
+            ],
+          },
+        }),
+      );
+      expect(await transcriptOf("aex_ownbehalf_cancel_race")).toEqual([
+        "a1",
+        "a2",
+        "a3",
+        CANCEL_STOP_ROW,
+      ]);
+    });
+
+    it("appends a broken flow's two lines to the runner's transcript", async () => {
+      await seedExecution(
+        "aex_ownbehalf_broken_flow",
+        ExecutionPhase.EXECUTION_IN_PROGRESS,
+        runnerTurns,
+      );
+
+      await updateStatus(
+        "aex_ownbehalf_broken_flow",
+        toJson(
+          AgentExecutionStatusSchema,
+          create(AgentExecutionStatusSchema, {
+            phase: ExecutionPhase.EXECUTION_FAILED,
+            error: "activity crashed",
+            messages: [
+              {
+                type: MessageType.MESSAGE_SYSTEM,
+                content: "Internal system error occurred during execution.",
+              },
+              {
+                type: MessageType.MESSAGE_SYSTEM,
+                content: "Error details: activity crashed",
+              },
+            ],
+          }),
+        ),
+      );
+
+      const execution = await persisted("aex_ownbehalf_broken_flow");
+      expect(execution.status?.phase).toBe(ExecutionPhase.EXECUTION_FAILED);
+      expect(await transcriptOf("aex_ownbehalf_broken_flow")).toEqual([
+        "a1",
+        "a2",
+        "Internal system error occurred during execution.",
+        "Error details: activity crashed",
+      ]);
+    });
+
+    it("leaves a runner-reported failure's transcript as the runner wrote it", async () => {
+      await seedExecution(
+        "aex_ownbehalf_runner_failed",
+        ExecutionPhase.EXECUTION_FAILED,
+        [
+          ...runnerTurns,
+          {
+            type: MessageType.MESSAGE_SYSTEM,
+            content: "Execution failed: the tool refused",
+          },
+        ],
+      );
+
+      await updateStatus(
+        "aex_ownbehalf_runner_failed",
+        toJson(
+          AgentExecutionStatusSchema,
+          create(AgentExecutionStatusSchema, {
+            phase: ExecutionPhase.EXECUTION_FAILED,
+            error: "agent execution failed: the tool refused",
+          }),
+        ),
+      );
+
+      expect(await transcriptOf("aex_ownbehalf_runner_failed")).toEqual([
+        "a1",
+        "a2",
+        "Execution failed: the tool refused",
+      ]);
+    });
+  });
+
   it("surfaces the lane's NotFound as the activity's failure for a deleted execution", async () => {
     await expect(
       updateStatus("aex_ownbehalf_missing", { phase: "EXECUTION_FAILED" }),