@vellumai/assistant 0.8.2 → 0.8.3

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -0,0 +1,272 @@
+/**
+ * Tests for the `agent_loop_exit` instrumentation added in this PR.
+ *
+ * Coverage targets:
+ *  1. **One emit per run** — the idempotency guard fires once, even if the
+ *     code path would otherwise reach two emit sites (the empty-response
+ *     throw → catch-block fallback case).
+ *  2. **Reason matches break site** — for each reachable break site, the
+ *     emitted reason is the one documented in `AgentLoopExitReason`.
+ *  3. **Always the last AgentEvent of the run** — consumers can rely on
+ *     positional ordering to find it.
+ *
+ * Sites not exercised here (`empty_response_exhausted`, `aborted_via_error`)
+ * require deeper provider fakery and are best covered by integration tests
+ * once we wire up the empty-response pipeline mock.
+ */
+import { describe, expect, test } from "bun:test";
+
+import type {
+  AgentEvent,
+  CheckpointDecision,
+  CheckpointInfo,
+} from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrored from agent-loop.test.ts so this file is self-contained)
+// ---------------------------------------------------------------------------
+
+function createMockProvider(responses: ProviderResponse[]): {
+  provider: Provider;
+} {
+  let callIndex = 0;
+  const provider: Provider = {
+    name: "mock",
+    async sendMessage(
+      _messages: Message[],
+      _tools?: ToolDefinition[],
+      _systemPrompt?: string,
+      _options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      const response = responses[callIndex] ?? responses[responses.length - 1];
+      callIndex++;
+      return response;
+    },
+  };
+  return { provider };
+}
+
+function textResponse(text: string): ProviderResponse {
+  return {
+    content: [{ type: "text", text }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "end_turn",
+  };
+}
+
+function toolUseResponse(
+  id: string,
+  name: string,
+  input: Record<string, unknown>,
+): ProviderResponse {
+  return {
+    content: [{ type: "tool_use", id, name, input }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "tool_use",
+  };
+}
+
+const dummyTools: ToolDefinition[] = [
+  {
+    name: "read_file",
+    description: "Read a file",
+    input_schema: { type: "object", properties: { path: { type: "string" } } },
+  },
+];
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "Hello" }],
+};
+
+function lastExitEvent(
+  events: AgentEvent[],
+): Extract<AgentEvent, { type: "agent_loop_exit" }> | undefined {
+  return events.find(
+    (e): e is Extract<AgentEvent, { type: "agent_loop_exit" }> =>
+      e.type === "agent_loop_exit",
+  );
+}
+
+function countExitEvents(events: AgentEvent[]): number {
+  return events.filter((e) => e.type === "agent_loop_exit").length;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("AgentLoop exit-reason instrumentation", () => {
+  test("emits exit event exactly once with 'no_tool_calls' on plain text response", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(countExitEvents(events)).toBe(1);
+    const exit = lastExitEvent(events);
+    expect(exit?.reason).toBe("no_tool_calls");
+  });
+
+  test("agent_loop_exit is the last event emitted", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(events.length).toBeGreaterThan(0);
+    expect(events[events.length - 1].type).toBe("agent_loop_exit");
+  });
+
+  test("emits 'aborted_pre_call' when signal is already aborted at run start", async () => {
+    const { provider } = createMockProvider([textResponse("never sent")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const controller = new AbortController();
+    controller.abort();
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_pre_call");
+  });
+
+  test("emits 'yield_to_user' when tool result requests yieldToUser", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("yield_to_user");
+  });
+
+  test("emits 'checkpoint_yield' when onCheckpoint returns 'yield'", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const onCheckpoint = (_info: CheckpointInfo): CheckpointDecision =>
+      "yield";
+
+    const events: AgentEvent[] = [];
+    await loop.run(
+      [userMessage],
+      (e) => { events.push(e); },
+      undefined,
+      undefined,
+      onCheckpoint,
+    );
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("checkpoint_yield");
+  });
+
+  test("emits 'error' when provider throws an unhandled error", async () => {
+    const provider: Provider = {
+      name: "broken",
+      async sendMessage(): Promise<ProviderResponse> {
+        throw new Error("provider exploded");
+      },
+    };
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("error");
+  });
+
+  test("does not double-emit when multiple exit conditions stack", async () => {
+    // Tool returns yieldToUser AND the controller is aborted post-response —
+    // the first reached condition wins, but the guard prevents a second
+    // emit even if subsequent code paths attempt one.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+  });
+
+  test("emits 'aborted_during_tools' when signal aborts after tool execution", async () => {
+    const controller = new AbortController();
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    // Abort the signal inside the tool executor so by the time the loop
+    // re-checks signal.aborted post-tools the abort has landed.
+    const toolExecutor = async () => {
+      controller.abort();
+      return { content: "ok", isError: false };
+    };
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_during_tools");
+  });
+});

package/src/__tests__/agent-loop-provider-error-recording.test.ts

@@ -0,0 +1,195 @@
+/**
+ * Integration tests for the agent loop's `provider_error` recording path.
+ *
+ * When the `llmCall` pipeline throws (provider rejected the request before
+ * returning a usable response), the loop must emit a `provider_error` event
+ * carrying the loop-level raw request and the thrown error so downstream
+ * consumers can persist an `llm_request_logs` row. Without this, rejected
+ * calls leave nothing in the LLM inspector — only a pino log line.
+ *
+ * Coverage:
+ *  - Emits `provider_error` with `rawRequest`, `error`, and `actualProvider`
+ *    when the provider throws a `ProviderError`.
+ *  - `rawRequest` carries the message history, tools, and system prompt the
+ *    loop attempted to send — so the row replays/debugs cleanly.
+ *  - `actualProvider` echoes `ProviderError.provider` when available, falling
+ *    back to `provider.name` for non-ProviderError throws.
+ *  - The error is still re-thrown internally (the existing `error` event
+ *    still fires after the new `provider_error` event), preserving the
+ *    outer-catch behavior (abort/Sentry/break).
+ *  - Skips emission on user-aborted runs — there is no provider rejection
+ *    worth recording when the user cancelled.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import type { AgentEvent } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+import { ProviderError } from "../util/errors.js";
+
+/**
+ * Build a provider that throws on every `sendMessage` call. Records what
+ * the loop attempted to send so the test can assert `rawRequest` carries
+ * the right payload.
+ */
+function makeThrowingProvider(
+  name: string,
+  throwFn: () => Error,
+): {
+  provider: Provider;
+  calls: Array<{
+    messages: Message[];
+    tools?: ToolDefinition[];
+    systemPrompt?: string;
+  }>;
+} {
+  const calls: Array<{
+    messages: Message[];
+    tools?: ToolDefinition[];
+    systemPrompt?: string;
+  }> = [];
+  const provider: Provider = {
+    name,
+    async sendMessage(
+      messages: Message[],
+      tools?: ToolDefinition[],
+      systemPrompt?: string,
+      _options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      calls.push({ messages: [...messages], tools, systemPrompt });
+      throw throwFn();
+    },
+  };
+  return { provider, calls };
+}
+
+describe("AgentLoop provider_error event emission", () => {
+  test("emits provider_error with loop-level rawRequest when provider throws ProviderError", async () => {
+    const thrown = new ProviderError(
+      "Anthropic API error (429): rate limited",
+      "anthropic",
+      429,
+      { retryAfterMs: 1500 },
+    );
+    const { provider, calls } = makeThrowingProvider("anthropic", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "you are a helpful assistant");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    expect(calls).toHaveLength(1);
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    expect(providerErrorEvent).toBeDefined();
+    if (providerErrorEvent?.type !== "provider_error") {
+      throw new Error("type narrowing");
+    }
+    expect(providerErrorEvent.error).toBe(thrown);
+    expect(providerErrorEvent.actualProvider).toBe("anthropic");
+
+    // rawRequest should carry the loop-level abstract shape: messages,
+    // tools, systemPrompt, and the provider name we tried to dispatch
+    // through. The provider-specific shape (e.g. Gemini's `contents`) is
+    // never built because the provider threw before returning it.
+    const raw = providerErrorEvent.rawRequest as Record<string, unknown>;
+    expect(raw.provider).toBe("anthropic");
+    expect(raw.systemPrompt).toBe("you are a helpful assistant");
+    expect(Array.isArray(raw.messages)).toBe(true);
+    expect((raw.messages as Message[])[0].role).toBe("user");
+  });
+
+  test("error event still fires after provider_error (outer catch behavior unchanged)", async () => {
+    const thrown = new ProviderError(
+      "Gemini API error (500): internal",
+      "gemini",
+      500,
+    );
+    const { provider } = makeThrowingProvider("gemini", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    const providerErrorIdx = events.findIndex(
+      (e) => e.type === "provider_error",
+    );
+    const errorIdx = events.findIndex((e) => e.type === "error");
+    expect(providerErrorIdx).toBeGreaterThanOrEqual(0);
+    expect(errorIdx).toBeGreaterThanOrEqual(0);
+    // Recording-first ordering is load-bearing: a consumer that sees the
+    // generic `error` event and shuts the stream down must have already
+    // received the `provider_error` row for the rejected call.
+    expect(providerErrorIdx).toBeLessThan(errorIdx);
+  });
+
+  test("falls back to provider.name when a non-ProviderError is thrown", async () => {
+    const thrown = new Error("unexpected SDK boom");
+    const { provider } = makeThrowingProvider("openai", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    expect(providerErrorEvent).toBeDefined();
+    if (providerErrorEvent?.type !== "provider_error") {
+      throw new Error("type narrowing");
+    }
+    // The thrown Error has no `.provider` field, so the event falls back to
+    // the dispatching provider's `name` — keeps the persisted log row's
+    // `provider` column populated even for surprise errors.
+    expect(providerErrorEvent.actualProvider).toBe("openai");
+    expect(providerErrorEvent.error).toBe(thrown);
+  });
+
+  test("does NOT emit provider_error on user-aborted runs", async () => {
+    const controller = new AbortController();
+    const thrown = new Error("aborted");
+    const { provider } = makeThrowingProvider("anthropic", () => {
+      // Pre-abort then throw so the loop's catch sees `signal.aborted === true`.
+      controller.abort();
+      return thrown;
+    });
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+      controller.signal,
+    );
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    // Cancellation should never produce a recording row — there's no
+    // provider rejection worth logging when the user pulled the plug.
+    expect(providerErrorEvent).toBeUndefined();
+  });
+});

package/src/__tests__/compactor-tail-resolution.test.ts

@@ -1,6 +1,46 @@
 import { describe, expect, it } from "bun:test";
 
-import { canonicalDateTimeKey } from "../context/compactor.js";
+import {
+  adjustTailIndexForToolPairing,
+  canonicalDateTimeKey,
+} from "../context/compactor.js";
+import type { Message } from "../providers/types.js";
+
+const userText = (text: string): Message => ({
+  role: "user",
+  content: [{ type: "text", text }],
+});
+
+const userToolResult = (toolUseId: string): Message => ({
+  role: "user",
+  content: [{ type: "tool_result", tool_use_id: toolUseId, content: "ok" }],
+});
+
+const userToolResultAndText = (toolUseId: string, text: string): Message => ({
+  role: "user",
+  content: [
+    { type: "tool_result", tool_use_id: toolUseId, content: "ok" },
+    { type: "text", text },
+  ],
+});
+
+const assistantToolUse = (id: string): Message => ({
+  role: "assistant",
+  content: [{ type: "tool_use", id, name: "Bash", input: {} }],
+});
+
+const assistantText = (text: string): Message => ({
+  role: "assistant",
+  content: [{ type: "text", text }],
+});
+
+const assistantWebSearch = (id: string): Message => ({
+  role: "assistant",
+  content: [
+    { type: "server_tool_use", id, name: "web_search", input: {} },
+    { type: "web_search_tool_result", tool_use_id: id, content: [] },
+  ],
+});
 
 describe("canonicalDateTimeKey", () => {
   const stored = "2026-04-02 (Thursday) 01:52:33 -05:00 (America/Chicago)";
@@ -39,3 +79,69 @@ describe("canonicalDateTimeKey", () => {
     expect(canonicalDateTimeKey("01:52:33")).toBeNull();
   });
 });
+
+describe("adjustTailIndexForToolPairing", () => {
+  it("returns tailIndex unchanged when the tail starts on a clean user turn", () => {
+    const messages: Message[] = [
+      userText("hi"),
+      assistantText("hello"),
+      userText("how are you"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 2)).toBe(2);
+  });
+
+  it("walks back past the orphan tool_result cluster to the prior user turn", () => {
+    const messages: Message[] = [
+      userText("setup"),
+      assistantText("ok"),
+      userText("run a command"),
+      assistantToolUse("X"),
+      userToolResult("X"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 4)).toBe(2);
+  });
+
+  it("keeps walking when the candidate also leads with tool_result", () => {
+    const messages: Message[] = [
+      userText("first"),
+      userText("second"),
+      assistantToolUse("X1"),
+      userToolResult("X1"),
+      assistantToolUse("X2"),
+      userToolResult("X2"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 5)).toBe(1);
+  });
+
+  it("returns 0 when the walk falls off the front of the array", () => {
+    const messages: Message[] = [
+      userToolResult("X"),
+      assistantToolUse("Y"),
+      userToolResult("Y"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 2)).toBe(0);
+  });
+
+  it("ignores server-side web_search_tool_result blocks", () => {
+    const messages: Message[] = [
+      userText("look it up"),
+      assistantWebSearch("WS1"),
+      userText("thanks"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 2)).toBe(2);
+  });
+
+  it("treats mixed tool_result + text user messages as unsafe", () => {
+    const messages: Message[] = [
+      userText("kick off"),
+      assistantToolUse("X"),
+      userToolResultAndText("X", "and here is more text"),
+    ];
+    expect(adjustTailIndexForToolPairing(messages, 2)).toBe(0);
+  });
+
+  it("returns tailIndex unchanged when tailIndex is 0", () => {
+    const messages: Message[] = [userText("only one")];
+    expect(adjustTailIndexForToolPairing(messages, 0)).toBe(0);
+  });
+});