@vellumai/assistant 0.6.4 → 0.6.6

package/src/__tests__/agent-loop-callsite-precedence.test.ts

@@ -0,0 +1,318 @@
+/**
+ * Verifies the field-precedence contract for `AgentLoop.run(..., callSite)`.
+ *
+ * When `callSite` is set, the loop must NOT pre-set the
+ * `max_tokens`/`thinking`/`effort`/`speed` fields from `this.config`
+ * (sourced from `llm.default`) because the downstream
+ * `RetryProvider.normalizeSendMessageOptions` only fills these fields when
+ * they're undefined. If the loop pre-sets them, every per-call-site override
+ * for these knobs is silently ignored.
+ *
+ * Precedence (highest wins):
+ *   1. Per-turn explicit (from `resolveSystemPrompt`'s
+ *      `resolved.maxTokens` / `resolved.model`)
+ *   2. Call-site resolved values (from `resolveCallSiteConfig` via the
+ *      normalizer)
+ *   3. Conversation defaults (`this.config.*`, from `llm.default`)
+ *
+ * The tests pipe the loop's per-call options through `RetryProvider` so we
+ * can observe the final, post-resolution config that downstream provider
+ * clients consume.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, { get: () => () => {} }),
+}));
+
+let mockLlmConfig: Record<string, unknown> = {};
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({ llm: mockLlmConfig }),
+}));
+
+import type { ResolvedSystemPrompt } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import { LLMSchema } from "../config/schemas/llm.js";
+import { RetryProvider } from "../providers/retry.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "hi" }],
+};
+
+function setLlmConfig(raw: unknown): void {
+  mockLlmConfig = LLMSchema.parse(raw) as Record<string, unknown>;
+}
+
+beforeEach(() => {
+  mockLlmConfig = LLMSchema.parse({}) as Record<string, unknown>;
+});
+
+/**
+ * Build a provider that captures the final `config` it receives. Wrap it in
+ * `RetryProvider` so the call-site resolver runs over whatever the agent loop
+ * emits — exactly mirroring production wiring.
+ */
+function makePipeline(providerName: string): {
+  provider: Provider;
+  lastConfig: () => Record<string, unknown> | undefined;
+} {
+  let captured: Record<string, unknown> | undefined;
+  const inner: Provider = {
+    name: providerName,
+    async sendMessage(
+      _messages: Message[],
+      _tools?: ToolDefinition[],
+      _systemPrompt?: string,
+      options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      captured = options?.config as Record<string, unknown> | undefined;
+      return {
+        content: [{ type: "text", text: "ok" }],
+        model: "mock-model",
+        usage: { inputTokens: 1, outputTokens: 1 },
+        stopReason: "end_turn",
+      };
+    },
+  };
+  return {
+    provider: new RetryProvider(inner),
+    lastConfig: () => captured,
+  };
+}
+
+describe("AgentLoop — call-site precedence", () => {
+  test("call-site maxTokens wins over conversation default when callSite is set", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        maxTokens: 64000,
+      },
+      callSites: { mainAgent: { maxTokens: 4096 } },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", { maxTokens: 64000 });
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    expect(lastConfig()!.max_tokens).toBe(4096);
+  });
+
+  test("call-site effort wins over conversation default when callSite is set", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        effort: "high",
+      },
+      callSites: { mainAgent: { effort: "low" } },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", {
+      maxTokens: 64000,
+      effort: "high",
+    });
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    expect(lastConfig()!.effort).toBe("low");
+  });
+
+  test("call-site speed wins over conversation default when callSite is set", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        speed: "standard",
+      },
+      callSites: { mainAgent: { speed: "fast" } },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", {
+      maxTokens: 64000,
+      effort: "high",
+      // Conversation default is "fast" (which would normally be applied) —
+      // ensure the call-site value is the one that ends up on the wire.
+      speed: "fast",
+    });
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    expect(lastConfig()!.speed).toBe("fast");
+  });
+
+  test("call-site thinking wins over conversation default when callSite is set", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        // Default thinking enabled.
+        thinking: { enabled: true, streamThinking: true },
+      },
+      callSites: {
+        // Call site disables thinking — must be honoured even though the
+        // conversation default has it on.
+        mainAgent: { thinking: { enabled: false } },
+      },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", {
+      maxTokens: 64000,
+      // Conversation default also has thinking on — without the fix, this
+      // would pre-set `thinking: { type: "adaptive" }` and mask the
+      // call-site override.
+      thinking: { enabled: true },
+    });
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    // Call-site override resolves `thinking.enabled: false`, so the
+    // RetryProvider normalizer must omit `thinking` entirely (matching the
+    // legacy non-callSite path which only sets `providerConfig.thinking`
+    // when enabled). Without the fix at agent/loop.ts, the conversation
+    // default's `thinking: { type: "adaptive" }` would be pre-set and mask
+    // the call-site override.
+    expect(lastConfig()!.thinking).toBeUndefined();
+  });
+
+  test("call-site thinking is converted to Anthropic wire-format when enabled", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        thinking: { enabled: true, streamThinking: true },
+      },
+      callSites: { mainAgent: {} },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", { maxTokens: 64000 });
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    // Must be wire-format `{ type: "adaptive" }` so the Anthropic SDK's
+    // `ThinkingConfigParam` accepts it. The schema-shape `{ enabled,
+    // streamThinking }` would be a runtime API error.
+    expect(lastConfig()!.thinking).toEqual({ type: "adaptive" });
+  });
+
+  test("conversation defaults still apply when callSite is absent", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        // Resolver values that would *normally* be filled — but we don't
+        // pass a callSite, so they must not surface.
+        maxTokens: 999,
+        effort: "low",
+      },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const loop = new AgentLoop(provider, "system", {
+      maxTokens: 64000,
+      effort: "high",
+      speed: "fast",
+      thinking: { enabled: true },
+    });
+
+    await loop.run([userMessage], () => {}, undefined, undefined, undefined);
+
+    const config = lastConfig()!;
+    expect(config.max_tokens).toBe(64000);
+    expect(config.effort).toBe("high");
+    expect(config.speed).toBe("fast");
+    // No callSite → loop sets the wire-format thinking directly.
+    expect(config.thinking).toEqual({ type: "adaptive" });
+  });
+
+  test("per-turn resolveSystemPrompt.maxTokens wins over both call-site and default", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+        maxTokens: 64000,
+      },
+      callSites: { mainAgent: { maxTokens: 4096 } },
+    });
+
+    const { provider, lastConfig } = makePipeline("anthropic");
+    const resolveSystemPrompt = (): ResolvedSystemPrompt => ({
+      systemPrompt: "per-turn system",
+      maxTokens: 8192,
+    });
+
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      { maxTokens: 64000 },
+      [],
+      undefined,
+      undefined,
+      resolveSystemPrompt,
+    );
+
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined,
+      undefined,
+      undefined,
+      "mainAgent",
+    );
+
+    // Per-turn explicit value beats both the call-site (4096) and the
+    // default (64000).
+    expect(lastConfig()!.max_tokens).toBe(8192);
+  });
+});

package/src/__tests__/agent-loop-sentry-hygiene.test.ts

@@ -0,0 +1,137 @@
+import { describe, expect, test } from "bun:test";
+
+import { shouldCaptureAgentLoopError } from "../agent/loop.js";
+import { ProviderError } from "../util/errors.js";
+
+/**
+ * Regression coverage for JARVIS-446 and JARVIS-513.
+ *
+ * The agent loop reports uncaught turn-processing errors to Sentry, but two
+ * categories of errors are user-environment noise and should not page:
+ *
+ *  - JARVIS-446: billing/auth/forbidden from the provider (402/401/403). The
+ *    user-facing error path already surfaces a credits-exhausted message; a
+ *    Sentry issue adds no engineering signal.
+ *  - JARVIS-513: retry-exhausted transient network errors (ECONNRESET, Bun's
+ *    "socket closed unexpectedly", etc.). The retry loop already did its job.
+ *
+ * `shouldCaptureAgentLoopError` gates the `Sentry.captureException` call.
+ */
+describe("shouldCaptureAgentLoopError", () => {
+  describe("JARVIS-446 — billing/auth/forbidden ProviderError", () => {
+    test("skips capture for 402 (billing exhausted)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (402): credit balance too low",
+        "anthropic",
+        402,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for 401 (bad API key)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (401): invalid x-api-key",
+        "anthropic",
+        401,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for 403 (forbidden / plan-gated)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (403): permission denied",
+        "anthropic",
+        403,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("still captures 500 (real server error)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (500): internal server error",
+        "anthropic",
+        500,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures 400 (bad request — engineering bug)", () => {
+      const err = new ProviderError(
+        "Anthropic API error (400): invalid tool definition",
+        "anthropic",
+        400,
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures ProviderError with no status (surprise error)", () => {
+      const err = new ProviderError(
+        "Anthropic API error: unexpected internal state",
+        "anthropic",
+      );
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+  });
+
+  describe("JARVIS-513 — retry-exhausted transient network errors", () => {
+    test("skips capture when retriesExhausted is set on an ECONNRESET", () => {
+      const err = Object.assign(new Error("connection reset"), {
+        code: "ECONNRESET",
+        retriesExhausted: true,
+      });
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for Bun 'socket closed unexpectedly' with retriesExhausted", () => {
+      const err = new Error("The socket connection was closed unexpectedly");
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("skips capture for wrapped ProviderError whose cause is a transient socket error", () => {
+      const cause = new Error("The socket connection was closed unexpectedly");
+      const err = new ProviderError(
+        "Anthropic request failed: The socket connection was closed unexpectedly",
+        "anthropic",
+        undefined,
+        { cause },
+      );
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(false);
+    });
+
+    test("still captures ECONNRESET when retries were NOT exhausted", () => {
+      // If the first attempt threw with ECONNRESET and the retry loop somehow
+      // didn't run (e.g. a caller bypassed RetryProvider), we still want
+      // visibility — `retriesExhausted` wasn't set.
+      const err = Object.assign(new Error("connection reset"), {
+        code: "ECONNRESET",
+      });
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+
+    test("still captures retriesExhausted marker on a non-network error", () => {
+      // The suppression is narrow: only retryable-network errors with the
+      // marker. A 500 with retriesExhausted still merits Sentry attention.
+      const err = new ProviderError(
+        "Anthropic API error (500): internal server error",
+        "anthropic",
+        500,
+      );
+      (err as Error & { retriesExhausted?: boolean }).retriesExhausted = true;
+      expect(shouldCaptureAgentLoopError(err)).toBe(true);
+    });
+  });
+
+  describe("default behavior — everything else still pages", () => {
+    test("captures a plain surprise Error", () => {
+      expect(shouldCaptureAgentLoopError(new Error("boom"))).toBe(true);
+    });
+
+    test("captures a TypeError", () => {
+      expect(shouldCaptureAgentLoopError(new TypeError("x is not a fn"))).toBe(
+        true,
+      );
+    });
+  });
+});

package/src/__tests__/agent-loop.test.ts

@@ -26,12 +26,14 @@ function createMockProvider(responses: ProviderResponse[]): {
     messages: Message[];
     tools?: ToolDefinition[];
     systemPrompt?: string;
+    options?: SendMessageOptions;
   }[];
 } {
   const calls: {
     messages: Message[];
     tools?: ToolDefinition[];
     systemPrompt?: string;
+    options?: SendMessageOptions;
   }[] = [];
   let callIndex = 0;
 
@@ -43,7 +45,7 @@ function createMockProvider(responses: ProviderResponse[]): {
       systemPrompt?: string,
       options?: SendMessageOptions,
     ): Promise<ProviderResponse> {
-      calls.push({ messages: [...messages], tools, systemPrompt });
+      calls.push({ messages: [...messages], tools, systemPrompt, options });
       const response = responses[callIndex] ?? responses[responses.length - 1];
       callIndex++;
 
@@ -1769,6 +1771,85 @@ describe("AgentLoop", () => {
     expect(messageCompletes).toHaveLength(2);
   });
 
+  // Regression: when the model emits [text, tool_use] in a single turn and then
+  // returns an empty response after the tool result, the loop must NOT nudge —
+  // the model already delivered its reply before the tool call, and nudging
+  // would trick it into re-sending the same text verbatim.
+  test("does not nudge empty response when prior turn had visible text", async () => {
+    const textPlusToolUseResponse: ProviderResponse = {
+      content: [
+        { type: "text", text: "your move, husband." },
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/note.txt" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const emptyResponse: ProviderResponse = {
+      content: [],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 0 },
+      stopReason: "end_turn",
+    };
+
+    const { provider, calls } = createMockProvider([
+      textPlusToolUseResponse,
+      emptyResponse,
+    ]);
+
+    const toolExecutor = async () => ({
+      content: "noted",
+      isError: false,
+    });
+
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+    const events: AgentEvent[] = [];
+    const history = await loop.run([userMessage], collectEvents(events));
+
+    // Provider called exactly 2 times: initial [text+tool_use], then empty.
+    // No third (retry) call because the prior turn had visible text.
+    expect(calls).toHaveLength(2);
+
+    // No nudge message should appear anywhere in history.
+    const nudgeInHistory = history.some(
+      (m) =>
+        m.role === "user" &&
+        m.content.some(
+          (b) =>
+            b.type === "text" &&
+            "text" in b &&
+            (b as { text: string }).text.includes(
+              "previous response was empty",
+            ),
+        ),
+    );
+    expect(nudgeInHistory).toBe(false);
+
+    // The [text, tool_use] assistant message is preserved in history.
+    const firstAssistant = history.find((m) => m.role === "assistant");
+    expect(firstAssistant).toBeDefined();
+    expect(firstAssistant!.content).toEqual([
+      { type: "text", text: "your move, husband." },
+      {
+        type: "tool_use",
+        id: "t1",
+        name: "read_file",
+        input: { path: "/note.txt" },
+      },
+    ]);
+  });
+
   test("gives up after max empty response retries", async () => {
     const emptyResponse: ProviderResponse = {
       content: [],
@@ -1831,4 +1912,34 @@ describe("AgentLoop", () => {
     expect(calls).toHaveLength(1);
     expect(history).toHaveLength(2); // user + empty assistant
   });
+
+  // PR 6: callSite threading from AgentLoop.run() into provider config.
+  // Verifies the per-call config exposes `callSite` so RetryProvider can route
+  // through `resolveCallSiteConfig` instead of the legacy `modelIntent` path.
+  test("threads callSite from AgentLoop.run() into per-call provider config", async () => {
+    const { provider, calls } = createMockProvider([textResponse("ok")]);
+
+    const loop = new AgentLoop(provider, "system");
+    await loop.run(
+      [userMessage],
+      () => {},
+      undefined, // signal
+      undefined, // requestId
+      undefined, // onCheckpoint
+      "heartbeatAgent",
+    );
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].options?.config?.callSite).toBe("heartbeatAgent");
+  });
+
+  test("omits callSite from provider config when not supplied", async () => {
+    const { provider, calls } = createMockProvider([textResponse("ok")]);
+
+    const loop = new AgentLoop(provider, "system");
+    await loop.run([userMessage], () => {});
+
+    expect(calls).toHaveLength(1);
+    expect(calls[0].options?.config?.callSite).toBeUndefined();
+  });
 });

package/src/__tests__/anthropic-error-formatting.test.ts

@@ -0,0 +1,98 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import type { Message } from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Mock Anthropic SDK — inject a throwing stream so we can assert on the
+// message format produced by the client's error-mapping path (JARVIS-390).
+// ---------------------------------------------------------------------------
+
+class FakeAPIError extends Error {
+  status: number | undefined;
+  headers: Map<string, string> = new Map();
+  constructor(status: number | undefined, message: string) {
+    super(message);
+    this.status = status;
+    this.name = "APIError";
+  }
+}
+
+let nextThrown: FakeAPIError | null = null;
+
+mock.module("@anthropic-ai/sdk", () => ({
+  default: class MockAnthropic {
+    static APIError = FakeAPIError;
+    constructor(_args: Record<string, unknown>) {}
+    #streamImpl = () => ({
+      on() {
+        return this;
+      },
+      async finalMessage() {
+        if (nextThrown) throw nextThrown;
+        return {
+          content: [],
+          model: "claude-sonnet-4-6",
+          usage: {
+            input_tokens: 0,
+            output_tokens: 0,
+            cache_creation_input_tokens: 0,
+            cache_read_input_tokens: 0,
+          },
+          stop_reason: "end_turn",
+        };
+      },
+    });
+    messages = { stream: () => this.#streamImpl() };
+    beta = { messages: { stream: () => this.#streamImpl() } };
+  },
+}));
+
+import { AnthropicProvider } from "../providers/anthropic/client.js";
+import { ProviderError } from "../util/errors.js";
+
+function userMsg(text: string): Message {
+  return { role: "user", content: [{ type: "text", text }] };
+}
+
+describe("AnthropicProvider — error message formatting (JARVIS-390)", () => {
+  beforeEach(() => {
+    nextThrown = null;
+  });
+
+  test("omits the `(status)` parenthetical when the SDK reports no HTTP status", async () => {
+    // Reproduces the abort/mid-stream path where `error.status` is undefined.
+    nextThrown = new FakeAPIError(undefined, "Request was aborted.");
+
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+
+    try {
+      await provider.sendMessage([userMsg("hi")]);
+      throw new Error("expected sendMessage to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(ProviderError);
+      const message = (err as Error).message;
+      expect(message).toBe("Anthropic API error: Request was aborted.");
+      // Belt-and-suspenders: the literal "(undefined)" must never appear.
+      expect(message).not.toContain("(undefined)");
+    }
+  });
+
+  test("includes the `(status)` parenthetical when the SDK reports an HTTP status", async () => {
+    nextThrown = new FakeAPIError(
+      402,
+      "Billing issue: your credit balance is too low.",
+    );
+
+    const provider = new AnthropicProvider("sk-ant-test", "claude-sonnet-4-6");
+
+    try {
+      await provider.sendMessage([userMsg("hi")]);
+      throw new Error("expected sendMessage to throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(ProviderError);
+      const message = (err as Error).message;
+      expect(message).toContain("Anthropic API error (402):");
+      expect((err as ProviderError).statusCode).toBe(402);
+    }
+  });
+});