@vellumai/assistant 0.6.4 → 0.6.5

package/src/__tests__/agent-loop-sentry-hygiene.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, test } from "bun:test";
+
+import { shouldCaptureAgentLoopError } from "../agent/loop.js";
+import { ProviderError } from "../util/errors.js";
+
+/**
+ * Regression coverage for JARVIS-446 and JARVIS-513.
+ *
+ * The agent loop reports uncaught turn-processing errors to Sentry, but two
+ * categories of errors are user-environment noise and should not page:
+ *
+ *  - JARVIS-446: billing/auth/forbidden from the provider (402/401/403). The
+ *    user-facing error path already surfaces a credits-exhausted message; a
+ *    Sentry issue adds no engineering signal.
+ *  - JARVIS-513: retry-exhausted transient network errors (ECONNRESET, Bun's
+ *    "socket closed unexpectedly", etc.). The retry loop already did its job.
+ *
+ * `shouldCaptureAgentLoopError` gates the `Sentry.captureException` call.
+ */
+describe("shouldCaptureAgentLoopError", () => {
+  describe("JARVIS-446 — billing/auth/forbidden ProviderError", () => {
+    test("skips capture for 402 (billing exhausted)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (402): credit balance too low",
+        "anthropic",
+        402,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for 401 (bad API key)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (401): invalid x-api-key",
+        "anthropic",
+        401,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for 403 (forbidden / plan-gated)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (403): permission denied",
+        "anthropic",
+        403,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("still captures 500 (real server error)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (500): internal server error",
+        "anthropic",
+        500,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures 400 (bad request — engineering bug)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (400): invalid tool definition",
+        "anthropic",
+        400,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures ProviderError with no status (surprise error)", () => {
+      const err = new ProviderError(
+        "Anthropic API error: unexpected internal state",
+        "anthropic",
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+  });
+
+  describe("JARVIS-513 — retry-exhausted transient network errors", () => {
+    test("skips capture when retriesExhausted is set on an ECONNRESET", () => {
+      const err = Object.assign(new Error("connection reset"), {
+        code: "ECONNRESET",
+        retriesExhausted: true,
+      });
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for Bun 'socket closed unexpectedly' with retriesExhausted", () => {
+      const err = new Error("The socket connection was closed unexpectedly");
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for wrapped ProviderError whose cause is a transient socket error", () => {
+      const cause = new Error("The socket connection was closed unexpectedly");
+      const err = new ProviderError(
+        "Anthropic request failed: The socket connection was closed unexpectedly",
+        "anthropic",
+        undefined,
+        { cause },
+      );
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("still captures ECONNRESET when retries were NOT exhausted", () => {
+      // If the first attempt threw with ECONNRESET and the retry loop somehow
+      // didn't run (e.g. a caller bypassed RetryProvider), we still want
+      // visibility — `retriesExhausted` wasn't set.
+      const err = Object.assign(new Error("connection reset"), {
+        code: "ECONNRESET",
+      });
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures retriesExhausted marker on a non-network error", () => {
+      // The suppression is narrow: only retryable-network errors with the
+      // marker. A 500 with retriesExhausted still merits Sentry attention.
+      const err = new ProviderError(
+        "Anthropic API error (500): internal server error",
+        "anthropic",
+        500,
+      );
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+  });
+
+  describe("default behavior — everything else still pages", () => {
+    test("captures a plain surprise Error", () => {
+      expect(shouldCaptureAgentLoopError(new Error("boom"))).toBe(true);
+    });
+
+    test("captures a TypeError", () => {
+      expect(shouldCaptureAgentLoopError(new TypeError("x is not a fn"))).toBe(
+        true,
+      );
+    });
+  });
+});

package/src/__tests__/agent-loop.test.ts

@@ -26,12 +26,14 @@ function createMockProvider(responses: ProviderResponse[]): {
     messages: Message[];
     tools?: ToolDefinition[];
     systemPrompt?: string;
+    options?: SendMessageOptions;
   }[];
 } {
   const calls: {
     messages: Message[];
     tools?: ToolDefinition[];
     systemPrompt?: string;
+    options?: SendMessageOptions;
   }[] = [];
   let callIndex = 0;
 
@@ -43,7 +45,7 @@ function createMockProvider(responses: ProviderResponse[]): {
       systemPrompt?: string,
       options?: SendMessageOptions,
     ): Promise<ProviderResponse> {
-      calls.push({ messages: [...messages], tools, systemPrompt });
+      calls.push({ messages: [...messages], tools, systemPrompt, options });
       const response = responses[callIndex] ?? responses[responses.length - 1];
       callIndex++;
 
@@ -1769,6 +1771,85 @@ describe("AgentLoop", () => {
     expect(messageCompletes).toHaveLength(2);
   });
 
+  // Regression: when the model emits [text, tool_use] in a single turn and then
+  // returns an empty response after the tool result, the loop must NOT nudge —
+  // the model already delivered its reply before the tool call, and nudging
+  // would trick it into re-sending the same text verbatim.
+  test("does not nudge empty response when prior turn had visible text", async () => {
+    const textPlusToolUseResponse: ProviderResponse = {
+      content: [
+        { type: "text", text: "your move, husband." },
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/note.txt" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const emptyResponse: ProviderResponse = {
+      content: [],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 0 },
+      stopReason: "end_turn",
+    };
+
+    const { provider, calls } = createMockProvider([
+      textPlusToolUseResponse,
+      emptyResponse,
+    ]);
+
+    const toolExecutor = async () => ({
+      content: "noted",
+      isError: false,
+    });
+
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+    const events: AgentEvent[] = [];
+    const history = await loop.run([userMessage], collectEvents(events));
+
+    // Provider called exactly 2 times: initial [text+tool_use], then empty.
+    // No third (retry) call because the prior turn had visible text.
+    expect(calls).toHaveLength(2);
+
+    // No nudge message should appear anywhere in history.
+    const nudgeInHistory = history.some(
+      (m) =>
+        m.role === "user" &&
+        m.content.some(
+          (b) =>
+            b.type === "text" &&
+            "text" in b &&
+            (b as { text: string }).text.includes(
+              "previous response was empty",
+            ),
+        ),
+    );
+    expect(nudgeInHistory).toBe(false);
+
+    // The [text, tool_use] assistant message is preserved in history.
+    const firstAssistant = history.find((m) => m.role === "assistant");
+    expect(firstAssistant).toBeDefined();
+    expect(firstAssistant!.content).toEqual([
+      { type: "text", text: "your move, husband." },
+      {
+        type: "tool_use",
+        id: "t1",
+        name: "read_file",
+        input: { path: "/note.txt" },
+      },
+    ]);
+  });
+
   test("gives up after max empty response retries", async () => {
     const emptyResponse: ProviderResponse = {
       content: [],
@@ -1831,4 +1912,34 @@ describe("AgentLoop", () => {
     expect(calls).toHaveLength(1);
     expect(history).toHaveLength(2); // user + empty assistant
   });
+
+  // PR 6: callSite threading from AgentLoop.run() into provider config.
+  // Verifies the per-call config exposes `callSite` so RetryProvider can route
+  // through `resolveCallSiteConfig` instead of the legacy `modelIntent` path.
+  test("threads callSite from AgentLoop.run() into per-call provider config", async () => {
+    const { provider, calls } = createMockProvider([textResponse("ok")]);
+
+    const loop = new AgentLoop(provider, "system");
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined, // signal
+      undefined, // requestId
+      undefined, // onCheckpoint
+      "heartbeatAgent",
+    );
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].options?.config?.callSite).toBe("heartbeatAgent");
+  });
+
+  test("omits callSite from provider config when not supplied", async () => {
+    const { provider, calls } = createMockProvider([textResponse("ok")]);
+
+    const loop = new AgentLoop(provider, "system");
+    await loop.run([userMessage], () => {});
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].options?.config?.callSite).toBeUndefined();
+  });
 });

package/src/__tests__/anthropic-error-formatting.test.ts

@@ -0,0 +1,98 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { Message } from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Mock Anthropic SDK — inject a throwing stream so we can assert on the
+// message format produced by the client's error-mapping path (JARVIS-390).
+// ---------------------------------------------------------------------------
+
+class FakeAPIError extends Error {
+  status: number | undefined;
+  headers: Map<string, string> = new Map();
+  constructor(status: number | undefined, message: string) {
+    super(message);
+    this.status = status;
+    this.name = "APIError";
+  }
+}
+
+let nextThrown: FakeAPIError | null = null;
+
+mock.module("@anthropic-ai/sdk", () => ({
+  default: class MockAnthropic {
+    static APIError = FakeAPIError;
+    constructor(_args: Record<string, unknown>) {}
+    #streamImpl = () => ({
+      on() {
+        return this;
+      },
+      async finalMessage() {
+        if (nextThrown) throw nextThrown;
+        return {
+          content: [],
+          model: "claude-sonnet-4-6",
+          usage: {
+            input_tokens: 0,
+            output_tokens: 0,
+            cache_creation_input_tokens: 0,
+            cache_read_input_tokens: 0,
+          },
+          stop_reason: "end_turn",
+        };
+      },
+    });
+    messages = { stream: () => this.#streamImpl() };
+    beta = { messages: { stream: () => this.#streamImpl() } };
+  },
+}));
+
+import { AnthropicProvider } from "../providers/anthropic/client.js";
+import { ProviderError } from "../util/errors.js";
+
+function userMsg(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+describe("AnthropicProvider — error message formatting (JARVIS-390)", () => {
+  beforeEach(() => {
+    nextThrown = null;
+  });
+
+  test("omits the `(status)` parenthetical when the SDK reports no HTTP status", async () => {
+    // Reproduces the abort/mid-stream path where `error.status` is undefined.
+    nextThrown = new FakeAPIError(undefined, "Request was aborted.");
+
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+
+    try {
+      await provider.sendMessage([userMsg("hi")]);
+      throw new Error("expected sendMessage to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(ProviderError);
+      const message = (err as Error).message;
+      expect(message).toBe("Anthropic API error: Request was aborted.");
+      // Belt-and-suspenders: the literal "(undefined)" must never appear.
+      expect(message).not.toContain("(undefined)");
+    }
+  });
+
+  test("includes the `(status)` parenthetical when the SDK reports an HTTP status", async () => {
+    nextThrown = new FakeAPIError(
+      402,
+      "Billing issue: your credit balance is too low.",
+    );
+
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+
+    try {
+      await provider.sendMessage([userMsg("hi")]);
+      throw new Error("expected sendMessage to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(ProviderError);
+      const message = (err as Error).message;
+      expect(message).toContain("Anthropic API error (402):");
+      expect((err as ProviderError).statusCode).toBe(402);
+    }
+  });
+});

package/src/__tests__/anthropic-provider.test.ts

@@ -10,6 +10,15 @@ let lastStreamParams: Record<string, unknown> | null = null;
 let _lastStreamOptions: Record<string, unknown> | null = null;
 let lastConstructorArgs: Record<string, unknown> | null = null;
 
+type ScriptedStreamEvent =
+  | { kind: "text"; text: string }
+  | { kind: "blockStart"; blockType?: "text" }
+  | { kind: "blockStop" };
+
+// When set, the mock fires these scripted stream events in order instead of
+// the default single "Hello" text event. Tests reset this in beforeEach.
+let scriptedStream: ScriptedStreamEvent[] | null = null;
+
 const fakeResponse = {
   content: [{ type: "text", text: "Hello" }],
   model: "claude-sonnet-4-6",
@@ -50,8 +59,25 @@ mock.module("@anthropic-ai/sdk", () => ({
           return this;
         },
         async finalMessage() {
-          // Fire text events
-          for (const cb of handlers["text"] ?? []) cb("Hello");
+          if (scriptedStream) {
+            for (const event of scriptedStream) {
+              if (event.kind === "text") {
+                for (const cb of handlers["text"] ?? []) cb(event.text);
+              } else if (event.kind === "blockStart") {
+                for (const cb of handlers["streamEvent"] ?? [])
+                  cb({
+                    type: "content_block_start",
+                    content_block: { type: event.blockType ?? "text" },
+                  });
+              } else if (event.kind === "blockStop") {
+                for (const cb of handlers["streamEvent"] ?? [])
+                  cb({ type: "content_block_stop" });
+              }
+            }
+          } else {
+            // Default: a single "Hello" text event (preserves existing tests).
+            for (const cb of handlers["text"] ?? []) cb("Hello");
+          }
           return fakeResponse;
         },
       };
@@ -77,6 +103,7 @@ mock.module("@anthropic-ai/sdk", () => ({
 import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../prompts/system-prompt.js";
 import {
   AnthropicProvider,
+  isPlaceholderSentinelText,
   PLACEHOLDER_BLOCKS_OMITTED,
   PLACEHOLDER_EMPTY_TURN,
 } from "../providers/anthropic/client.js";
@@ -144,6 +171,7 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     lastStreamParams = null;
     _lastStreamOptions = null;
     lastConstructorArgs = null;
+    scriptedStream = null;
     provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
   });
 
@@ -1254,6 +1282,147 @@ describe("AnthropicProvider — Cache-Control Characterization", () => {
     }
   });
 
+  test("streams normal text through without delay when text is not a sentinel prefix", async () => {
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: "Hello world" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual(["Hello world"]);
+  });
+
+  test("suppresses placeholder sentinel streamed as a single chunk", async () => {
+    // Model echoes a sentinel in one chunk; buffer should hold it and
+    // drop it on content_block_stop since it matches exactly.
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: PLACEHOLDER_EMPTY_TURN },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual([]);
+  });
+
+  test("suppresses bare-variant sentinel streamed as a single chunk", async () => {
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: "__PLACEHOLDER__[empty assistant turn]" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual([]);
+  });
+
+  test("suppresses placeholder sentinel streamed across multiple chunks", async () => {
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: "\x00__PLACE" },
+      { kind: "text", text: "HOLDER__[empty" },
+      { kind: "text", text: " assistant turn]" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual([]);
+  });
+
+  test("flushes buffered prefix when the continuation diverges from all sentinels", async () => {
+    // "__PLACEHOLDER__" is a prefix of the sentinels, so it stays buffered.
+    // Once the next chunk diverges (bracket instead of the expected opening),
+    // the buffer must flush.
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: "__PLACEHOLDER__" },
+      { kind: "text", text: " is bold in markdown" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted.join("")).toBe("__PLACEHOLDER__ is bold in markdown");
+  });
+
+  test("flushes non-sentinel residual buffer at content_block_stop", async () => {
+    // If the stream ends mid-prefix, the residual must be flushed (the
+    // accumulated text isn't a complete sentinel, so it's real content).
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: "__PLACEHOLDER__" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual(["__PLACEHOLDER__"]);
+  });
+
+  test("resets buffer across content blocks so a sentinel in one block doesn't poison the next", async () => {
+    scriptedStream = [
+      { kind: "blockStart" },
+      { kind: "text", text: PLACEHOLDER_EMPTY_TURN },
+      { kind: "blockStop" },
+      { kind: "blockStart" },
+      { kind: "text", text: "Fresh block content" },
+      { kind: "blockStop" },
+    ];
+    const emitted: string[] = [];
+    await provider.sendMessage([userMsg("Hi")], undefined, undefined, {
+      onEvent: (event) => {
+        if (event.type === "text_delta") emitted.push(event.text);
+      },
+    });
+    expect(emitted).toEqual(["Fresh block content"]);
+  });
+
+  test("isPlaceholderSentinelText matches sentinel with and without the null-byte prefix", () => {
+    // The runtime filter must be lenient enough to catch sentinel text that
+    // lost its `\x00` prefix in transit (e.g. a model echoing it back from
+    // input history without reproducing the control character). Migration 222
+    // handles the same two variants.
+    expect(isPlaceholderSentinelText(PLACEHOLDER_EMPTY_TURN)).toBe(true);
+    expect(isPlaceholderSentinelText(PLACEHOLDER_BLOCKS_OMITTED)).toBe(true);
+    expect(
+      isPlaceholderSentinelText("__PLACEHOLDER__[empty assistant turn]"),
+    ).toBe(true);
+    expect(
+      isPlaceholderSentinelText("__PLACEHOLDER__[internal blocks omitted]"),
+    ).toBe(true);
+    // Nearby strings must NOT match — guard against over-broad matching.
+    expect(isPlaceholderSentinelText("")).toBe(false);
+    expect(isPlaceholderSentinelText("__PLACEHOLDER__")).toBe(false);
+    expect(
+      isPlaceholderSentinelText(
+        "prefix __PLACEHOLDER__[empty assistant turn]",
+      ),
+    ).toBe(false);
+  });
+
   // -----------------------------------------------------------------------
   // Workspace context injection + cache control
   // -----------------------------------------------------------------------

package/src/__tests__/approval-cascade.test.ts

@@ -56,16 +56,34 @@ mock.module("../providers/registry.js", () => ({
 
 mock.module("../config/loader.js", () => ({
   getConfig: () => ({
-    ui: {},
-    provider: "mock-provider",
-    maxTokens: 4096,
-    thinking: false,
-    contextWindow: {
-      maxInputTokens: 100000,
-      thresholdTokens: 80000,
-      preserveRecentMessages: 6,
-      summaryModel: "mock-model",
-      maxSummaryTokens: 512,
+    ui: {},    
+    llm: {
+      default: {
+        provider: "mock-provider",
+        model: "mock-model",
+        maxTokens: 4096,
+        effort: "max" as const,
+        speed: "standard" as const,
+        temperature: null,
+        thinking: { enabled: false, streamThinking: true },
+        contextWindow: {
+          enabled: true,
+          maxInputTokens: 100000,
+          targetBudgetRatio: 0.3,
+          compactThreshold: 0.8,
+          summaryBudgetRatio: 0.05,
+          overflowRecovery: {
+            enabled: true,
+            safetyMarginRatio: 0.05,
+            maxAttempts: 3,
+            interactiveLatestTurnCompression: "summarize",
+            nonInteractiveLatestTurnCompression: "truncate",
+          },
+        },
+      },
+      profiles: {},
+      callSites: {},
+      pricingOverrides: [],
     },
     rateLimit: { maxRequestsPerMinute: 0 },
     timeouts: { permissionTimeoutSec: 300 },
@@ -187,6 +205,9 @@ mock.module("../agent/loop.js", () => ({
     getToolTokenBudget() {
       return 0;
     }
+    getActiveModel() {
+      return undefined;
+    }
     async run(
       _messages: Message[],
       _onEvent: (event: AgentEvent) => void,