@vellumai/assistant 0.8.7 → 0.8.8-dev.202606052332.17fc8ea

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.7",
+  "version": "0.8.8-dev.202606052332.17fc8ea",
   "license": "MIT",
   "type": "module",
   "exports": {
@@ -33,7 +33,7 @@
     "prepack": "node ../scripts/prepack-bundled-deps.mjs"
   },
   "dependencies": {
-    "@agentclientprotocol/sdk": "0.16.1",
+    "@agentclientprotocol/sdk": "0.25.0",
     "@anthropic-ai/sdk": "0.78.0",
     "@google/genai": "1.45.0",
     "@modelcontextprotocol/sdk": "1.27.1",

package/scripts/generate-openapi.ts

@@ -56,12 +56,24 @@ const RouteBodySchemaSchema = z.any().refine(
   { message: "Expected a Zod schema or a plain JSON Schema object" },
 );
 
-const RouteRequestBodyVariantSchema = z.object({
+/** Explicit `{ contentType, schema }` body for non-JSON media types. */
+const RouteBodyWithContentTypeSchema = z.object({
   contentType: z.string(),
   /** Zod schema OR plain JSON Schema fragment. */
   schema: z.any(),
 });
 
+/**
+ * A route's request or success-response body: either a bare Zod/JSON schema
+ * (advertised as `application/json`) or an explicit `{ contentType, schema }`
+ * pair for non-JSON media (e.g. an `application/octet-stream` upload or binary
+ * download).
+ */
+const RouteContentBodySchema = z.union([
+  RouteBodyWithContentTypeSchema,
+  RouteBodySchemaSchema,
+]);
+
 const RouteAdditionalResponseSchema = z.object({
   description: z.string(),
   schema: z.any().optional(),
@@ -79,12 +91,10 @@ const RouteEntrySchema = z.object({
   tags: z.array(z.string()).optional(),
   /** Query parameter definitions. */
   queryParams: z.array(RouteQueryParamSchema).optional(),
-  /** JSON Schema for the request body. */
-  requestBody: RouteBodySchemaSchema.optional(),
-  /** Multi-content-type request body variants (overrides `requestBody` when present). */
-  requestBodies: z.array(RouteRequestBodyVariantSchema).optional(),
-  /** JSON Schema for the success response body. */
-  responseBody: RouteBodySchemaSchema.optional(),
+  /** Request body: a bare Zod/JSON schema (JSON) or `{ contentType, schema }`. */
+  requestBody: RouteContentBodySchema.optional(),
+  /** Success response body: a bare Zod/JSON schema (JSON) or `{ contentType, schema }`. */
+  responseBody: RouteContentBodySchema.optional(),
   /** HTTP status code for the success response. Defaults to "200".
    * Callable responseStatus values (used at runtime) are ignored here. */
   responseStatus: z.preprocess(
@@ -292,6 +302,11 @@ interface OpenApiParameter {
   description?: string;
 }
 
+interface OpenApiResponse {
+  description: string;
+  content?: Record<string, { schema: JSONSchemaObject }>;
+}
+
 interface OpenApiOperation {
   operationId: string;
   summary?: string;
@@ -302,13 +317,27 @@ interface OpenApiOperation {
     required: boolean;
     content: Record<string, { schema: JSONSchemaObject }>;
   };
-  responses: Record<
-    string,
-    {
-      description: string;
-      content?: Record<string, { schema: JSONSchemaObject }>;
-    }
-  >;
+  responses: Record<string, OpenApiResponse>;
+}
+
+/**
+ * Resolve a body declaration (request or success response) into its media type
+ * and the schema source to convert. A bare Zod/JSON schema is advertised as
+ * `application/json`; the explicit `{ contentType, schema }` form carries its
+ * own media type (e.g. `application/octet-stream` for binary bodies).
+ */
+function resolveBodyContent(body: unknown): {
+  contentType: string;
+  schemaSource: unknown;
+} {
+  const hasContentType =
+    typeof body === "object" && body !== null && "contentType" in body;
+  return {
+    contentType: hasContentType
+      ? (body as { contentType: string }).contentType
+      : "application/json",
+    schemaSource: hasContentType ? (body as { schema: unknown }).schema : body,
+  };
 }
 
 interface OpenApiPathItem {
@@ -420,22 +449,27 @@ function buildSpec(
     // that enqueue a job and return immediately set responseStatus: "202"
     // so the generated spec matches the handler's actual response code.
     const successStatus = entry.responseStatus ?? "200";
+    let successResponse: OpenApiResponse = {
+      description: "Successful response",
+    };
+    if (entry.responseBody) {
+      const { contentType, schemaSource } = resolveBodyContent(
+        entry.responseBody,
+      );
+      successResponse = {
+        description: "Successful response",
+        content: {
+          [contentType]: { schema: toJSONSchemaObject(schemaSource) },
+        },
+      };
+    }
     const operation: OpenApiOperation = {
       operationId,
       ...(entry.summary ? { summary: entry.summary } : {}),
       ...(entry.description ? { description: entry.description } : {}),
       ...(tags ? { tags } : {}),
       responses: {
-        [successStatus]: entry.responseBody
-          ? {
-              description: "Successful response",
-              content: {
-                "application/json": {
-                  schema: toJSONSchemaObject(entry.responseBody),
-                },
-              },
-            }
-          : { description: "Successful response" },
+        [successStatus]: successResponse,
       },
     };
 
@@ -443,26 +477,19 @@ function buildSpec(
       operation.parameters = parameters;
     }
 
-    // Multi-content-type request bodies take precedence over the single
-    // application/json requestBody. This lets an endpoint advertise a
-    // `oneOf`-style choice between `application/octet-stream`,
-    // `multipart/form-data`, and `application/json` on the same URL.
-    if (entry.requestBodies && entry.requestBodies.length > 0) {
-      const content: Record<string, { schema: JSONSchemaObject }> = {};
-      for (const variant of entry.requestBodies) {
-        content[variant.contentType] = {
-          schema: toJSONSchemaObject(variant.schema, {
-            stripRequiredDefaults: true,
-          }),
-        };
-      }
-      operation.requestBody = { required: true, content };
-    } else if (entry.requestBody) {
+    // A bare Zod/JSON schema is advertised as `application/json`; the
+    // explicit `{ contentType, schema }` form lets a route declare a non-JSON
+    // body (e.g. a raw `application/octet-stream` upload) so the generated SDK
+    // describes a real body type instead of `never`.
+    if (entry.requestBody) {
+      const { contentType, schemaSource } = resolveBodyContent(
+        entry.requestBody,
+      );
       operation.requestBody = {
         required: true,
         content: {
-          "application/json": {
-            schema: toJSONSchemaObject(entry.requestBody, {
+          [contentType]: {
+            schema: toJSONSchemaObject(schemaSource, {
               stripRequiredDefaults: true,
             }),
           },

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -1,20 +1,19 @@
 /**
- * Tests for the `agent_loop_exit` instrumentation added in this PR.
+ * Tests for the `agent_loop_exit` instrumentation.
  *
  * Coverage targets:
- *  1. **One emit per run** — the idempotency guard fires once, even if the
- *     code path would otherwise reach two emit sites (the empty-response
- *     throw → catch-block fallback case).
+ *  1. **One emit per run** — the idempotency guard fires once, even when
+ *     multiple exit conditions stack and the code path would otherwise
+ *     reach a second emit site.
  *  2. **Reason matches break site** — for each reachable break site, the
  *     emitted reason is the one documented in `AgentLoopExitReason`.
  *  3. **Always the last AgentEvent of terminal runs** — consumers can rely on
  *     positional ordering to find it when a run reaches a terminal state.
  *
- * Sites not exercised here (`empty_response_exhausted`, `aborted_via_error`)
- * require deeper provider fakery and are best covered by integration tests
- * once we wire up the empty-response pipeline mock.
+ * Sites not exercised here (`aborted_via_error`) require deeper provider
+ * fakery and are best covered by integration tests.
  */
-import { describe, expect, spyOn, test } from "bun:test";
+import { describe, expect, test } from "bun:test";
 
 import type {
   AgentEvent,
@@ -24,7 +23,6 @@ import type {
 } from "../agent/loop.js";
 import { AgentLoop, isMaxTokensStopReason } from "../agent/loop.js";
 import type { TurnContext } from "../plugins/types.js";
-import { PluginTimeoutError } from "../plugins/types.js";
 import type {
   Message,
   Provider,
@@ -103,8 +101,8 @@ const userMessage: Message = {
 };
 
 // A turn context whose `contextWindowManager.maybeCompact` returns a canned
-// result, so the loop's native compaction pipeline runs without the real
-// orchestrator machinery.
+// result, so the loop's compaction call runs without the real orchestrator
+// machinery.
 function fakeCompactionTurnContext(result: {
   compacted: boolean;
   exhausted: boolean;
@@ -120,22 +118,6 @@ function fakeCompactionTurnContext(result: {
   } as unknown as TurnContext;
 }
 
-// A turn context whose compaction call times out, exercising the loop's
-// PluginTimeoutError handling.
-function timeoutCompactionTurnContext(): TurnContext {
-  return {
-    requestId: "req-compact",
-    conversationId: "conv-compact",
-    turnIndex: 0,
-    trust: { sourceChannel: "vellum", trustClass: "unknown" },
-    contextWindowManager: {
-      maybeCompact: async () => {
-        throw new PluginTimeoutError("compaction", "default-compaction", 1);
-      },
-    },
-  } as unknown as TurnContext;
-}
-
 function lastExitEvent(
   events: AgentEvent[],
 ): Extract<AgentEvent, { type: "agent_loop_exit" }> | undefined {
@@ -379,82 +361,44 @@ describe("AgentLoop exit-reason instrumentation", () => {
       toolExecutor: toolExecutor,
     });
 
-    let prepared = false;
-    let appliedResult = false;
     let reinjected = false;
+    const events: AgentEvent[] = [];
     const compaction: MidLoopCompaction = {
-      prepare: (history) => {
-        prepared = true;
-        return { rawHistory: history, options: undefined };
-      },
-      applyResult: async () => {
-        appliedResult = true;
-      },
-      reinject: async () => {
+      postCompactionHook: async () => {
         reinjected = true;
         return [userMessage];
       },
     };
 
     // WHEN the in-loop budget gate trips at the checkpoint
-    const result = await loop.run([userMessage], () => {}, {
-      resolveContextWindow: () => ({
-        maxInputTokens: 10,
-        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
-      }),
-      compaction,
-      turnContext: fakeCompactionTurnContext({
-        compacted: true,
-        exhausted: false,
-      }),
-    });
+    const result = await loop.run(
+      [userMessage],
+      (event) => {
+        events.push(event);
+      },
+      {
+        resolveContextWindow: () => ({
+          maxInputTokens: 10,
+          overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
+        }),
+        compaction,
+        turnContext: fakeCompactionTurnContext({
+          compacted: true,
+          exhausted: false,
+        }),
+      },
+    );
 
     // THEN the loop runs the compaction ceremony in place and continues to a
-    // clean exit instead of yielding for budget.
-    expect(prepared).toBe(true);
-    expect(appliedResult).toBe(true);
+    // clean exit instead of yielding for budget. The durable commit is
+    // signalled via a `compaction_completed` event rather than an injected hook.
+    expect(events.some((event) => event.type === "compaction_completed")).toBe(
+      true,
+    );
     expect(reinjected).toBe(true);
     expect(result.exitReason).not.toBe("budget");
   });
 
-  test("yields 'budget' when inline compaction times out", async () => {
-    const { provider } = createMockProvider([
-      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
-      textResponse("never reached"),
-    ]);
-    const toolExecutor = async () => ({ content: "ok", isError: false });
-    const loop = new AgentLoop(provider, "system", {
-      tools: dummyTools,
-      toolExecutor: toolExecutor,
-    });
-
-    const compaction: MidLoopCompaction = {
-      prepare: (history) => ({ rawHistory: history, options: undefined }),
-      applyResult: async () => {},
-      reinject: async () => {
-        throw new Error("reinject must not run after a timeout");
-      },
-    };
-    const recordOutcomeSpy = spyOn(loop.compactionCircuit, "recordOutcome");
-
-    // WHEN the compaction pipeline throws a PluginTimeoutError
-    const result = await loop.run([userMessage], () => {}, {
-      resolveContextWindow: () => ({
-        maxInputTokens: 10,
-        overflowRecovery: { enabled: true, safetyMarginRatio: 0 },
-      }),
-      compaction,
-      turnContext: timeoutCompactionTurnContext(),
-    });
-
-    // THEN the loop records the timeout as a compaction failure against its
-    // own circuit breaker, and yields for budget so the orchestrator can
-    // escalate.
-    expect(recordOutcomeSpy).toHaveBeenCalledTimes(1);
-    expect(recordOutcomeSpy.mock.calls[0]?.[1]).toBe(true);
-    expect(result.exitReason).toBe("budget");
-  });
-
   test("yields 'budget' when inline compaction reports exhausted", async () => {
     const { provider } = createMockProvider([
       toolUseResponse("t1", "read_file", { path: "/a.txt" }),
@@ -467,10 +411,8 @@ describe("AgentLoop exit-reason instrumentation", () => {
     });
 
     const compaction: MidLoopCompaction = {
-      prepare: (history) => ({ rawHistory: history, options: undefined }),
-      applyResult: async () => {},
-      reinject: async () => {
-        throw new Error("reinject must not run when exhausted");
+      postCompactionHook: async () => {
+        throw new Error("postCompactionHook must not run when exhausted");
       },
     };
 

package/src/__tests__/agent-loop-provider-error-recording.test.ts

@@ -1,7 +1,7 @@
 /**
  * Integration tests for the agent loop's `provider_error` recording path.
  *
- * When the `llmCall` pipeline throws (provider rejected the request before
+ * When the provider call throws (provider rejected the request before
  * returning a usable response), the loop must emit a `provider_error` event
  * carrying the loop-level raw request and the thrown error so downstream
  * consumers can persist an `llm_request_logs` row. Without this, rejected

package/src/__tests__/agent-loop.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { beforeEach, describe, expect, test } from "bun:test";
 
 import type {
   AgentEvent,
@@ -6,90 +6,24 @@ import type {
   CheckpointInfo,
 } from "../agent/loop.js";
 import { AgentLoop } from "../agent/loop.js";
+import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
 import type {
   ContentBlock,
   Message,
   Provider,
   ProviderResponse,
-  SendMessageOptions,
   ToolDefinition,
 } from "../providers/types.js";
+import {
+  createMockProvider,
+  textResponse,
+  toolUseResponse,
+} from "./helpers/mock-provider.js";
 
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------
 
-/** A mock provider that returns pre-configured responses in sequence. */
-function createMockProvider(responses: ProviderResponse[]): {
-  provider: Provider;
-  calls: {
-    messages: Message[];
-    tools?: ToolDefinition[];
-    systemPrompt?: string;
-    options?: SendMessageOptions;
-  }[];
-} {
-  const calls: {
-    messages: Message[];
-    tools?: ToolDefinition[];
-    systemPrompt?: string;
-    options?: SendMessageOptions;
-  }[] = [];
-  let callIndex = 0;
-
-  const provider: Provider = {
-    name: "mock",
-    async sendMessage(
-      messages: Message[],
-      options?: SendMessageOptions,
-    ): Promise<ProviderResponse> {
-      calls.push({
-        messages: [...messages],
-        tools: options?.tools,
-        systemPrompt: options?.systemPrompt,
-        options,
-      });
-      const response = responses[callIndex] ?? responses[responses.length - 1];
-      callIndex++;
-
-      // Emit streaming events if the response has text blocks
-      if (options?.onEvent) {
-        for (const block of response.content) {
-          if (block.type === "text") {
-            options.onEvent({ type: "text_delta", text: block.text });
-          }
-        }
-      }
-
-      return response;
-    },
-  };
-
-  return { provider, calls };
-}
-
-function textResponse(text: string): ProviderResponse {
-  return {
-    content: [{ type: "text", text }],
-    model: "mock-model",
-    usage: { inputTokens: 10, outputTokens: 5 },
-    stopReason: "end_turn",
-  };
-}
-
-function toolUseResponse(
-  id: string,
-  name: string,
-  input: Record<string, unknown>,
-): ProviderResponse {
-  return {
-    content: [{ type: "tool_use", id, name, input }],
-    model: "mock-model",
-    usage: { inputTokens: 10, outputTokens: 5 },
-    stopReason: "tool_use",
-  };
-}
-
 const dummyTools: ToolDefinition[] = [
   {
     name: "read_file",
@@ -112,6 +46,13 @@ function collectEvents(events: AgentEvent[]): (event: AgentEvent) => void {
 // ---------------------------------------------------------------------------
 
 describe("AgentLoop", () => {
+  // The agent loop fires the `post-tool-use` hook for tool-result
+  // truncation, which only runs when the default plugin is registered.
+  // Register the defaults so the loop behaves as it does in production.
+  beforeEach(() => {
+    resetPluginRegistryAndRegisterDefaults();
+  });
+
   // 1. Basic text response
   test("returns history with assistant message for simple text response", async () => {
     const { provider } = createMockProvider([textResponse("Hi there!")]);
@@ -1569,9 +1510,11 @@ describe("AgentLoop", () => {
     expect(textBlock!.text).toBe("Normal response with no placeholders.");
   });
 
-  // Tool error retry nudge — when a tool returns isError: true, the loop
-  // should inject a system_notice nudging the LLM to retry instead of ending.
-  test("injects retry nudge system_notice when tool returns an error", async () => {
+  // Tool error retry coaching — when a tool returns isError: true, the coaching
+  // notice is appended as a separate text block in the tool-result user message
+  // (not into the tool_result content), nudging the LLM to retry instead of
+  // ending the turn while keeping the tool's actual output clean.
+  test("appends retry coaching as a separate block when a tool returns an error", async () => {
     const { provider, calls } = createMockProvider([
       // First turn: LLM calls a tool that errors
       toolUseResponse("t1", "read_file", { path: "/missing.txt" }),
@@ -1606,7 +1549,8 @@ describe("AgentLoop", () => {
     // Provider should have been called 3 times (error -> retry -> final text)
     expect(calls).toHaveLength(3);
 
-    // The second call's messages should contain the retry nudge system_notice
+    // The second call's messages should carry the retry coaching as a separate
+    // text block, with the tool_result's own content left untouched.
     const secondCallMessages = calls[1].messages;
     const toolResultMessage = secondCallMessages[secondCallMessages.length - 1];
     expect(toolResultMessage.role).toBe("user");
@@ -1617,7 +1561,14 @@ describe("AgentLoop", () => {
     );
     expect(retryNudge).toBeDefined();
 
-    // The third call should NOT have the retry nudge (successful tool result)
+    // The errored tool_result itself stays the tool's actual output.
+    const erroredToolResult = toolResultMessage.content.find(
+      (b): b is Extract<ContentBlock, { type: "tool_result" }> =>
+        b.type === "tool_result",
+    );
+    expect(erroredToolResult?.content).not.toContain("looks recoverable");
+
+    // The third call should NOT have the retry coaching (successful tool result)
     const thirdCallMessages = calls[2].messages;
     const thirdToolResultMessage =
       thirdCallMessages[thirdCallMessages.length - 1];
@@ -1628,8 +1579,9 @@ describe("AgentLoop", () => {
     expect(noRetryNudge).toBeUndefined();
   });
 
-  // Retry nudge stops after MAX_CONSECUTIVE_ERROR_NUDGES (3) consecutive errors
-  test("stops injecting retry nudge after 3 consecutive error turns", async () => {
+  // Retry coaching stops after a tool fails 3 times in a row — past that the
+  // error is likely unrecoverable and further coaching only burns tokens.
+  test("stops appending retry coaching after 3 consecutive failures of a tool", async () => {
     const { provider, calls } = createMockProvider([
       // 4 consecutive error turns, then final text
       toolUseResponse("t1", "read_file", { path: "/a" }),
@@ -1654,23 +1606,21 @@ describe("AgentLoop", () => {
 
     expect(calls).toHaveLength(5);
 
-    // Helper to check if a call's last user message has the retry nudge
+    // Helper to check if a call's last user message has the retry coaching
+    // (a separate text block, not part of the tool_result content).
     const hasRetryNudge = (callIndex: number): boolean => {
       const msgs = calls[callIndex].messages;
       const lastMsg = msgs[msgs.length - 1];
       return lastMsg.content.some(
-        (b) =>
-          b.type === "text" &&
-          "text" in b &&
-          (b as { text: string }).text.includes("looks recoverable"),
+        (b) => b.type === "text" && b.text.includes("looks recoverable"),
       );
     };
 
-    // Turns 1-3 should have the nudge
+    // Turns 1-3 should have the coaching
     expect(hasRetryNudge(1)).toBe(true);
     expect(hasRetryNudge(2)).toBe(true);
     expect(hasRetryNudge(3)).toBe(true);
-    // Turn 4 should NOT have the nudge (exceeded limit)
+    // Turn 4 should NOT have the coaching (exceeded limit)
     expect(hasRetryNudge(4)).toBe(false);
   });
 

package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts

@@ -77,6 +77,8 @@ function makeTarget(): WakeTarget {
       run: (async (messages: Message[]) => ({
         history: messages,
         exitReason: null,
+        appendedNewMessages: false,
+        newMessages: [],
       })) as WakeTarget["agentLoop"]["run"],
     },
     getMessages: () => history,