@vellumai/assistant 0.8.2 → 0.8.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -0,0 +1,271 @@
+/**
+ * Tests for the `agent_loop_exit` instrumentation added in this PR.
+ *
+ * Coverage targets:
+ *  1. **One emit per run** — the idempotency guard fires once, even if the
+ *     code path would otherwise reach two emit sites (the empty-response
+ *     throw → catch-block fallback case).
+ *  2. **Reason matches break site** — for each reachable break site, the
+ *     emitted reason is the one documented in `AgentLoopExitReason`.
+ *  3. **Always the last AgentEvent of terminal runs** — consumers can rely on
+ *     positional ordering to find it when a run reaches a terminal state.
+ *
+ * Sites not exercised here (`empty_response_exhausted`, `aborted_via_error`)
+ * require deeper provider fakery and are best covered by integration tests
+ * once we wire up the empty-response pipeline mock.
+ */
+import { describe, expect, test } from "bun:test";
+
+import type {
+  AgentEvent,
+  CheckpointDecision,
+  CheckpointInfo,
+} from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrored from agent-loop.test.ts so this file is self-contained)
+// ---------------------------------------------------------------------------
+
+function createMockProvider(responses: ProviderResponse[]): {
+  provider: Provider;
+} {
+  let callIndex = 0;
+  const provider: Provider = {
+    name: "mock",
+    async sendMessage(
+      _messages: Message[],
+      _tools?: ToolDefinition[],
+      _systemPrompt?: string,
+      _options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      const response = responses[callIndex] ?? responses[responses.length - 1];
+      callIndex++;
+      return response;
+    },
+  };
+  return { provider };
+}
+
+function textResponse(text: string): ProviderResponse {
+  return {
+    content: [{ type: "text", text }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "end_turn",
+  };
+}
+
+function toolUseResponse(
+  id: string,
+  name: string,
+  input: Record<string, unknown>,
+): ProviderResponse {
+  return {
+    content: [{ type: "tool_use", id, name, input }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "tool_use",
+  };
+}
+
+const dummyTools: ToolDefinition[] = [
+  {
+    name: "read_file",
+    description: "Read a file",
+    input_schema: { type: "object", properties: { path: { type: "string" } } },
+  },
+];
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "Hello" }],
+};
+
+function lastExitEvent(
+  events: AgentEvent[],
+): Extract<AgentEvent, { type: "agent_loop_exit" }> | undefined {
+  return events.find(
+    (e): e is Extract<AgentEvent, { type: "agent_loop_exit" }> =>
+      e.type === "agent_loop_exit",
+  );
+}
+
+function countExitEvents(events: AgentEvent[]): number {
+  return events.filter((e) => e.type === "agent_loop_exit").length;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("AgentLoop exit-reason instrumentation", () => {
+  test("emits exit event exactly once with 'no_tool_calls' on plain text response", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(countExitEvents(events)).toBe(1);
+    const exit = lastExitEvent(events);
+    expect(exit?.reason).toBe("no_tool_calls");
+  });
+
+  test("agent_loop_exit is the last event emitted", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(events.length).toBeGreaterThan(0);
+    expect(events[events.length - 1].type).toBe("agent_loop_exit");
+  });
+
+  test("emits 'aborted_pre_call' when signal is already aborted at run start", async () => {
+    const { provider } = createMockProvider([textResponse("never sent")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const controller = new AbortController();
+    controller.abort();
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_pre_call");
+  });
+
+  test("emits 'yield_to_user' when tool result requests yieldToUser", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("yield_to_user");
+  });
+
+  test("does not emit agent_loop_exit when onCheckpoint yields control", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const onCheckpoint = (_info: CheckpointInfo): CheckpointDecision =>
+      "yield";
+
+    const events: AgentEvent[] = [];
+    await loop.run(
+      [userMessage],
+      (e) => { events.push(e); },
+      undefined,
+      undefined,
+      onCheckpoint,
+    );
+
+    expect(countExitEvents(events)).toBe(0);
+  });
+
+  test("emits 'error' when provider throws an unhandled error", async () => {
+    const provider: Provider = {
+      name: "broken",
+      async sendMessage(): Promise<ProviderResponse> {
+        throw new Error("provider exploded");
+      },
+    };
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("error");
+  });
+
+  test("does not double-emit when multiple exit conditions stack", async () => {
+    // Tool returns yieldToUser AND the controller is aborted post-response —
+    // the first reached condition wins, but the guard prevents a second
+    // emit even if subsequent code paths attempt one.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+  });
+
+  test("emits 'aborted_during_tools' when signal aborts after tool execution", async () => {
+    const controller = new AbortController();
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    // Abort the signal inside the tool executor so by the time the loop
+    // re-checks signal.aborted post-tools the abort has landed.
+    const toolExecutor = async () => {
+      controller.abort();
+      return { content: "ok", isError: false };
+    };
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_during_tools");
+  });
+});

package/src/__tests__/agent-loop-override-profile.test.ts

@@ -257,7 +257,7 @@ const anthropicStub = { name: "anthropic" };
 mock.module("../providers/registry.js", () => ({
   getProvider: () => anthropicStub,
   listProviders: () => ["anthropic"],
-  initializeProviders: () => {},
+  initializeProviders: async () => {},
   resolveProviderFromConnection: async () => anthropicStub,
 }));
 

package/src/__tests__/agent-loop-provider-error-recording.test.ts

@@ -0,0 +1,195 @@
+/**
+ * Integration tests for the agent loop's `provider_error` recording path.
+ *
+ * When the `llmCall` pipeline throws (provider rejected the request before
+ * returning a usable response), the loop must emit a `provider_error` event
+ * carrying the loop-level raw request and the thrown error so downstream
+ * consumers can persist an `llm_request_logs` row. Without this, rejected
+ * calls leave nothing in the LLM inspector — only a pino log line.
+ *
+ * Coverage:
+ *  - Emits `provider_error` with `rawRequest`, `error`, and `actualProvider`
+ *    when the provider throws a `ProviderError`.
+ *  - `rawRequest` carries the message history, tools, and system prompt the
+ *    loop attempted to send — so the row replays/debugs cleanly.
+ *  - `actualProvider` echoes `ProviderError.provider` when available, falling
+ *    back to `provider.name` for non-ProviderError throws.
+ *  - The error is still re-thrown internally (the existing `error` event
+ *    still fires after the new `provider_error` event), preserving the
+ *    outer-catch behavior (abort/Sentry/break).
+ *  - Skips emission on user-aborted runs — there is no provider rejection
+ *    worth recording when the user cancelled.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import type { AgentEvent } from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+import { ProviderError } from "../util/errors.js";
+
+/**
+ * Build a provider that throws on every `sendMessage` call. Records what
+ * the loop attempted to send so the test can assert `rawRequest` carries
+ * the right payload.
+ */
+function makeThrowingProvider(
+  name: string,
+  throwFn: () => Error,
+): {
+  provider: Provider;
+  calls: Array<{
+    messages: Message[];
+    tools?: ToolDefinition[];
+    systemPrompt?: string;
+  }>;
+} {
+  const calls: Array<{
+    messages: Message[];
+    tools?: ToolDefinition[];
+    systemPrompt?: string;
+  }> = [];
+  const provider: Provider = {
+    name,
+    async sendMessage(
+      messages: Message[],
+      tools?: ToolDefinition[],
+      systemPrompt?: string,
+      _options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      calls.push({ messages: [...messages], tools, systemPrompt });
+      throw throwFn();
+    },
+  };
+  return { provider, calls };
+}
+
+describe("AgentLoop provider_error event emission", () => {
+  test("emits provider_error with loop-level rawRequest when provider throws ProviderError", async () => {
+    const thrown = new ProviderError(
+      "Anthropic API error (429): rate limited",
+      "anthropic",
+      429,
+      { retryAfterMs: 1500 },
+    );
+    const { provider, calls } = makeThrowingProvider("anthropic", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "you are a helpful assistant");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    expect(calls).toHaveLength(1);
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    expect(providerErrorEvent).toBeDefined();
+    if (providerErrorEvent?.type !== "provider_error") {
+      throw new Error("type narrowing");
+    }
+    expect(providerErrorEvent.error).toBe(thrown);
+    expect(providerErrorEvent.actualProvider).toBe("anthropic");
+
+    // rawRequest should carry the loop-level abstract shape: messages,
+    // tools, systemPrompt, and the provider name we tried to dispatch
+    // through. The provider-specific shape (e.g. Gemini's `contents`) is
+    // never built because the provider threw before returning it.
+    const raw = providerErrorEvent.rawRequest as Record<string, unknown>;
+    expect(raw.provider).toBe("anthropic");
+    expect(raw.systemPrompt).toBe("you are a helpful assistant");
+    expect(Array.isArray(raw.messages)).toBe(true);
+    expect((raw.messages as Message[])[0].role).toBe("user");
+  });
+
+  test("error event still fires after provider_error (outer catch behavior unchanged)", async () => {
+    const thrown = new ProviderError(
+      "Gemini API error (500): internal",
+      "gemini",
+      500,
+    );
+    const { provider } = makeThrowingProvider("gemini", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    const providerErrorIdx = events.findIndex(
+      (e) => e.type === "provider_error",
+    );
+    const errorIdx = events.findIndex((e) => e.type === "error");
+    expect(providerErrorIdx).toBeGreaterThanOrEqual(0);
+    expect(errorIdx).toBeGreaterThanOrEqual(0);
+    // Recording-first ordering is load-bearing: a consumer that sees the
+    // generic `error` event and shuts the stream down must have already
+    // received the `provider_error` row for the rejected call.
+    expect(providerErrorIdx).toBeLessThan(errorIdx);
+  });
+
+  test("falls back to provider.name when a non-ProviderError is thrown", async () => {
+    const thrown = new Error("unexpected SDK boom");
+    const { provider } = makeThrowingProvider("openai", () => thrown);
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+    );
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    expect(providerErrorEvent).toBeDefined();
+    if (providerErrorEvent?.type !== "provider_error") {
+      throw new Error("type narrowing");
+    }
+    // The thrown Error has no `.provider` field, so the event falls back to
+    // the dispatching provider's `name` — keeps the persisted log row's
+    // `provider` column populated even for surprise errors.
+    expect(providerErrorEvent.actualProvider).toBe("openai");
+    expect(providerErrorEvent.error).toBe(thrown);
+  });
+
+  test("does NOT emit provider_error on user-aborted runs", async () => {
+    const controller = new AbortController();
+    const thrown = new Error("aborted");
+    const { provider } = makeThrowingProvider("anthropic", () => {
+      // Pre-abort then throw so the loop's catch sees `signal.aborted === true`.
+      controller.abort();
+      return thrown;
+    });
+
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop(provider, "system");
+
+    await loop.run(
+      [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+      (e) => {
+        events.push(e);
+      },
+      controller.signal,
+    );
+
+    const providerErrorEvent = events.find((e) => e.type === "provider_error");
+    // Cancellation should never produce a recording row — there's no
+    // provider rejection worth logging when the user pulled the plug.
+    expect(providerErrorEvent).toBeUndefined();
+  });
+});

package/src/__tests__/agent-loop.test.ts

@@ -181,6 +181,88 @@ describe("AgentLoop", () => {
     ]);
   });
 
+  test("re-resolves override profile before each provider call", async () => {
+    const toolCallId = "tool-1";
+    const { provider, calls } = createMockProvider([
+      toolUseResponse(toolCallId, "read_file", { path: "/tmp/test.txt" }),
+      textResponse("File contents received."),
+    ]);
+
+    let overrideProfile: string | undefined;
+    const toolExecutor = async () => {
+      overrideProfile = "quality-optimized";
+      return { content: "ok", isError: false };
+    };
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    await loop.run(
+      [userMessage],
+      collectEvents([]),
+      undefined,
+      "req-1",
+      undefined,
+      "mainAgent",
+      undefined,
+      undefined,
+      undefined,
+      () => overrideProfile,
+    );
+
+    expect(calls).toHaveLength(2);
+    expect(calls[0].options?.config?.overrideProfile).toBeUndefined();
+    expect(calls[1].options?.config?.overrideProfile).toBe("quality-optimized");
+  });
+
+  test("re-resolves max input tokens before truncating tool results", async () => {
+    const toolCallId = "tool-1";
+    const toolOutput = "x".repeat(2_500);
+    const { provider, calls } = createMockProvider([
+      toolUseResponse(toolCallId, "read_file", { path: "/tmp/test.txt" }),
+      textResponse("File contents received."),
+    ]);
+
+    let maxInputTokens = 1_000;
+    const toolExecutor = async () => {
+      maxInputTokens = 10_000;
+      return { content: toolOutput, isError: false };
+    };
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    await loop.run(
+      [userMessage],
+      collectEvents([]),
+      undefined,
+      "req-1",
+      undefined,
+      "mainAgent",
+      undefined,
+      undefined,
+      1_000,
+      undefined,
+      () => maxInputTokens,
+    );
+
+    const secondCallMessages = calls[1].messages;
+    const lastMsg = secondCallMessages[secondCallMessages.length - 1];
+    const toolResultBlock = lastMsg.content.find(
+      (b): b is Extract<ContentBlock, { type: "tool_result" }> =>
+        b.type === "tool_result",
+    );
+    expect(toolResultBlock?.content).toBe(toolOutput);
+  });
+
   // 3. Multi-turn tool loop
   test("supports multi-turn tool execution", async () => {
     const { provider, calls } = createMockProvider([
@@ -470,7 +552,6 @@ describe("AgentLoop", () => {
     ).toBe(false);
   });
 
-
   // 9. Tool executor error results are forwarded correctly
   test("forwards tool error results to provider", async () => {
     const { provider, calls } = createMockProvider([
@@ -1767,7 +1848,9 @@ describe("AgentLoop", () => {
     ]);
 
     // message_complete emitted for tool_use response + retry text response (not the empty one)
-    const messageCompletes = events.filter((e) => e.type === "message_complete");
+    const messageCompletes = events.filter(
+      (e) => e.type === "message_complete",
+    );
     expect(messageCompletes).toHaveLength(2);
   });
 
@@ -1883,7 +1966,9 @@ describe("AgentLoop", () => {
     expect(calls).toHaveLength(3);
 
     // message_complete: tool_use response + final empty response (retry exhausted)
-    const messageCompletes = events.filter((e) => e.type === "message_complete");
+    const messageCompletes = events.filter(
+      (e) => e.type === "message_complete",
+    );
     expect(messageCompletes).toHaveLength(2);
 
     // The last assistant message in history is the empty one