@vellumai/assistant 0.8.1 → 0.8.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.1",
+  "version": "0.8.3",
   "license": "MIT",
   "type": "module",
   "exports": {
@@ -62,6 +62,7 @@
     "openai": "6.29.0",
     "pino": "9.14.0",
     "pino-pretty": "13.1.3",
+    "semver": "7.8.0",
     "playwright": "1.58.2",
     "postgres": "3.4.8",
     "rrule": "2.8.1",
@@ -94,6 +95,7 @@
     "@types/archiver": "7.0.0",
     "@types/bun": "1.3.10",
     "@types/node": "25.5.0",
+    "@types/semver": "7.5.8",
     "@types/uuid": "10.0.0",
     "drizzle-kit": "0.30.6",
     "eslint": "10.0.3",

package/scripts/generate-openapi.ts

@@ -183,14 +183,26 @@ function toJSONSchemaObject(
 async function collectRoutesFromModules(): Promise<RouteEntry[]> {
   const routes: RouteEntry[] = [];
 
-  const files = (await readdir(ROUTES_DIR, { recursive: true })).filter(
-    (f) =>
-      typeof f === "string" &&
-      f.endsWith(".ts") &&
-      !f.endsWith(".test.ts") &&
-      !f.endsWith(".benchmark.test.ts") &&
-      !f.includes("node_modules"),
-  );
+  // Skip the `index.ts` barrel: it re-exports every other route module's
+  // ROUTES into a single combined array, so importing it would double-count
+  // every entry. The duplicate `method:endpoint` keys are deduped later by
+  // first-seen, but the surviving entry's `sourceModule` (used to derive
+  // OpenAPI `tags`) depends on `readdir` order — which is filesystem
+  // dependent and diverges between local sandbox and the CI runner, making
+  // the generator non-reproducible. Sort the file list as well so directory
+  // entry order can never affect the output.
+  const files = (await readdir(ROUTES_DIR, { recursive: true }))
+    .filter(
+      (f) =>
+        typeof f === "string" &&
+        f.endsWith(".ts") &&
+        !f.endsWith(".test.ts") &&
+        !f.endsWith(".benchmark.test.ts") &&
+        !f.includes("node_modules") &&
+        f !== "index.ts" &&
+        !f.endsWith("/index.ts"),
+    )
+    .sort();
 
   for (const file of files) {
     const filePath = join(ROUTES_DIR, file);
@@ -204,8 +216,20 @@ async function collectRoutesFromModules(): Promise<RouteEntry[]> {
       continue;
     }
 
-    if ("ROUTES" in mod && Array.isArray(mod.ROUTES)) {
-      for (const raw of mod.ROUTES) {
+    // Collect every export whose name is `ROUTES` or ends in `_ROUTES`.
+    // A handful of route files (e.g. `channel-route-definitions.ts`,
+    // `contact-prompt-routes.ts`) export under domain-prefixed names like
+    // `CHANNEL_ROUTES` and `CONTACT_PROMPT_ROUTES` rather than the
+    // canonical `ROUTES`. Without this fan-out the only way those routes
+    // reached the spec was via the `index.ts` barrel — which is excluded
+    // above for reproducibility.
+    const exportNames = Object.keys(mod)
+      .filter((k) => k === "ROUTES" || k.endsWith("_ROUTES"))
+      .sort();
+    for (const name of exportNames) {
+      const arr = mod[name];
+      if (!Array.isArray(arr)) continue;
+      for (const raw of arr) {
         const result = RouteEntrySchema.safeParse({
           ...(typeof raw === "object" && raw !== null ? raw : {}),
           sourceModule: file,
@@ -545,6 +569,55 @@ async function main() {
     }
     if (existing !== yamlOutput) {
       console.error("openapi.yaml is stale. Run: bun run generate:openapi");
+      // Emit the first byte-level divergence and a windowed diff around it
+      // so CI logs are actionable without a follow-up local repro.
+      const maxLen = Math.max(existing.length, yamlOutput.length);
+      let firstDiff = -1;
+      for (let i = 0; i < maxLen; i++) {
+        if (existing[i] !== yamlOutput[i]) {
+          firstDiff = i;
+          break;
+        }
+      }
+      if (firstDiff >= 0) {
+        const lineNo = (existing.slice(0, firstDiff).match(/\n/g) ?? []).length + 1;
+        const winStart = Math.max(0, firstDiff - 120);
+        const winEnd = Math.min(maxLen, firstDiff + 120);
+        console.error(`First divergence at byte ${firstDiff} (~line ${lineNo}):`);
+        console.error(`  existing[${winStart}..${winEnd}]:`);
+        console.error(`    ${JSON.stringify(existing.slice(winStart, winEnd))}`);
+        console.error(`  generated[${winStart}..${winEnd}]:`);
+        console.error(`    ${JSON.stringify(yamlOutput.slice(winStart, winEnd))}`);
+      }
+      // Also flag which path operations are present in one but not the other —
+      // the common failure mode is a missing or duplicated route entry, and
+      // the path keys are the actionable thing for the human reading the log.
+      const pathsRe = /^\s\s(\/\S+):/gm;
+      const existingPaths = new Set(
+        Array.from(existing.matchAll(pathsRe), (m) => m[1]),
+      );
+      const generatedPaths = new Set(
+        Array.from(yamlOutput.matchAll(pathsRe), (m) => m[1]),
+      );
+      const inExistingOnly = [...existingPaths].filter(
+        (p) => !generatedPaths.has(p),
+      );
+      const inGeneratedOnly = [...generatedPaths].filter(
+        (p) => !existingPaths.has(p),
+      );
+      if (inExistingOnly.length || inGeneratedOnly.length) {
+        console.error(
+          `Path set drift: existing has ${existingPaths.size} paths, generated has ${generatedPaths.size}`,
+        );
+        if (inGeneratedOnly.length) {
+          console.error(`  Only in generated (missing from committed yaml):`);
+          for (const p of inGeneratedOnly.slice(0, 20)) console.error(`    + ${p}`);
+        }
+        if (inExistingOnly.length) {
+          console.error(`  Only in existing (stale entries in committed yaml):`);
+          for (const p of inExistingOnly.slice(0, 20)) console.error(`    - ${p}`);
+        }
+      }
       process.exit(1);
     }
     console.log("openapi.yaml is up to date.");

package/scripts/sync-llm-catalog.ts

@@ -97,8 +97,8 @@ function projectProvider(entry: ProviderCatalogEntry): Record<string, unknown> {
     projected.apiKeyPlaceholder = entry.apiKeyPlaceholder;
   if (entry.credentialsGuide !== undefined)
     projected.credentialsGuide = entry.credentialsGuide;
-  if (entry.supportsManagedAuth !== undefined)
-    projected.supportsManagedAuth = entry.supportsManagedAuth;
+  if (entry.supportsPlatformAuth !== undefined)
+    projected.supportsPlatformAuth = entry.supportsPlatformAuth;
   projected.defaultModel = entry.defaultModel;
   projected.models = entry.models.map(projectModel);
   // NOTE: `apiKeyUrl` intentionally omitted — clients use

package/scripts/sync-web-search-catalog.ts

@@ -1,17 +1,24 @@
 #!/usr/bin/env bun
 /**
- * Generate `meta/web-search-provider-catalog.json` from the canonical
- * `SEARCH_PROVIDER_CATALOG` in
+ * Generate the client-facing web-search-provider catalog JSON from the
+ * canonical `SEARCH_PROVIDER_CATALOG` in
  * `assistant/src/providers/search-provider-catalog.ts`.
  *
- * Companion to `sync-llm-catalog.ts`. The meta JSON is the cross-package
- * artifact consumed by:
+ * Two byte-identical copies are written:
+ *   - `meta/web-search-provider-catalog.json` — primary checked-in artifact,
+ *      consumed by:
+ *        - `cli/src/__tests__/search-provider-env-var-parity.test.ts`
+ *          (drift guard for the CLI's hardcoded env-var mirror).
+ *        - Downstream `vellum-assistant-platform/web/src/lib/generated/
+ *          web-search-provider-catalog.json` (manually sync'd today; the
+ *          scheduled sync workflow is a planned follow-up).
+ *   - `clients/shared/Resources/web-search-provider-catalog.json` — SwiftPM
+ *      resource bundled into `VellumAssistantShared`. SwiftPM cannot reach
+ *      files outside a target's source directory, so this mirror is
+ *      necessary; both files are produced by the same generator and
+ *      asserted equal by the parity test, making drift impossible.
  *
- *   - `cli/src/__tests__/search-provider-env-var-parity.test.ts`
- *     (drift guard for the CLI's hardcoded env-var mirror).
- *   - Downstream `vellum-assistant-platform/web/src/lib/generated/
- *     web-search-provider-catalog.json` (manually sync'd today; scheduled
- *     sync workflow is a planned follow-up).
+ * Companion to `sync-llm-catalog.ts`; same dual-write pattern.
  *
  * Usage:
  *   cd assistant && bun run scripts/sync-web-search-catalog.ts
@@ -20,7 +27,7 @@
  */
 
 import { readFile, writeFile } from "node:fs/promises";
-import { join, resolve } from "node:path";
+import { join, relative, resolve } from "node:path";
 
 import {
   SEARCH_PROVIDER_CATALOG,
@@ -28,7 +35,10 @@ import {
 } from "../src/providers/search-provider-catalog.js";
 
 const ROOT = resolve(import.meta.dir, "../..");
-const OUTPUT_PATH = join(ROOT, "meta/web-search-provider-catalog.json");
+const OUTPUT_PATHS = [
+  join(ROOT, "meta/web-search-provider-catalog.json"),
+  join(ROOT, "clients/shared/Resources/web-search-provider-catalog.json"),
+] as const;
 
 /**
  * Bumped when the *shape* of the client catalog JSON changes in a way
@@ -84,24 +94,36 @@ async function main(): Promise<void> {
   const next = generate();
 
   if (checkMode) {
-    let current = "";
-    try {
-      current = await readFile(OUTPUT_PATH, "utf8");
-    } catch {
-      // File doesn't exist yet — treat as stale.
-    }
-    if (current !== next) {
-      console.error(
-        `\n${OUTPUT_PATH} is out of sync with SEARCH_PROVIDER_CATALOG.\n` +
-          `Run: cd assistant && bun run sync:web-search-catalog\n`,
-      );
-      process.exit(1);
+    let anyStale = false;
+    for (const path of OUTPUT_PATHS) {
+      const rel = relative(ROOT, path);
+      let existing = "";
+      try {
+        existing = await readFile(path, "utf-8");
+      } catch {
+        console.error(
+          `${rel} does not exist. Run: bun run sync:web-search-catalog`,
+        );
+        anyStale = true;
+        continue;
+      }
+      if (existing !== next) {
+        console.error(
+          `${rel} is stale. Run: bun run sync:web-search-catalog`,
+        );
+        anyStale = true;
+        continue;
+      }
+      console.log(`${rel} is up to date.`);
     }
+    if (anyStale) process.exit(1);
     return;
   }
 
-  await writeFile(OUTPUT_PATH, next, "utf8");
-  console.log(`Wrote ${OUTPUT_PATH}`);
+  for (const path of OUTPUT_PATHS) {
+    await writeFile(path, next, "utf-8");
+    console.log(`Wrote ${relative(ROOT, path)}`);
+  }
 }
 
 await main();

package/src/__tests__/agent-image-optimize.test.ts

@@ -14,10 +14,18 @@ describe("shouldRescaleImage", () => {
     expect(shouldRescaleImage({ width: 1200, height: 800 }, 50_000)).toBe(
       false,
     );
-    // Even a large file is fine as long as dimensions are within limits —
-    // Anthropic's constraint is per-side pixels, not bytes.
+  });
+
+  it("rescales when raw bytes would inflate past Anthropic's 5 MB base64 cap", () => {
+    // Regression: an oversized image retained across compaction was rejected
+    // with "image.source.base64: image exceeds 5 MB maximum" even though
+    // dimensions were within the 1568 px cap. base64 inflates raw bytes by
+    // 4/3, so anything over ~3.5 MB raw risks crossing the 5 MB API limit.
     expect(shouldRescaleImage({ width: 1568, height: 1568 }, 5_000_000)).toBe(
-      false,
+      true,
+    );
+    expect(shouldRescaleImage({ width: 1200, height: 800 }, 4_000_000)).toBe(
+      true,
     );
   });
 

package/src/__tests__/agent-loop-exit-reason.test.ts

@@ -0,0 +1,272 @@
+/**
+ * Tests for the `agent_loop_exit` instrumentation added in this PR.
+ *
+ * Coverage targets:
+ *  1. **One emit per run** — the idempotency guard fires once, even if the
+ *     code path would otherwise reach two emit sites (the empty-response
+ *     throw → catch-block fallback case).
+ *  2. **Reason matches break site** — for each reachable break site, the
+ *     emitted reason is the one documented in `AgentLoopExitReason`.
+ *  3. **Always the last AgentEvent of the run** — consumers can rely on
+ *     positional ordering to find it.
+ *
+ * Sites not exercised here (`empty_response_exhausted`, `aborted_via_error`)
+ * require deeper provider fakery and are best covered by integration tests
+ * once we wire up the empty-response pipeline mock.
+ */
+import { describe, expect, test } from "bun:test";
+
+import type {
+  AgentEvent,
+  CheckpointDecision,
+  CheckpointInfo,
+} from "../agent/loop.js";
+import { AgentLoop } from "../agent/loop.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "../providers/types.js";
+
+// ---------------------------------------------------------------------------
+// Helpers (mirrored from agent-loop.test.ts so this file is self-contained)
+// ---------------------------------------------------------------------------
+
+function createMockProvider(responses: ProviderResponse[]): {
+  provider: Provider;
+} {
+  let callIndex = 0;
+  const provider: Provider = {
+    name: "mock",
+    async sendMessage(
+      _messages: Message[],
+      _tools?: ToolDefinition[],
+      _systemPrompt?: string,
+      _options?: SendMessageOptions,
+    ): Promise<ProviderResponse> {
+      const response = responses[callIndex] ?? responses[responses.length - 1];
+      callIndex++;
+      return response;
+    },
+  };
+  return { provider };
+}
+
+function textResponse(text: string): ProviderResponse {
+  return {
+    content: [{ type: "text", text }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "end_turn",
+  };
+}
+
+function toolUseResponse(
+  id: string,
+  name: string,
+  input: Record<string, unknown>,
+): ProviderResponse {
+  return {
+    content: [{ type: "tool_use", id, name, input }],
+    model: "mock-model",
+    usage: { inputTokens: 10, outputTokens: 5 },
+    stopReason: "tool_use",
+  };
+}
+
+const dummyTools: ToolDefinition[] = [
+  {
+    name: "read_file",
+    description: "Read a file",
+    input_schema: { type: "object", properties: { path: { type: "string" } } },
+  },
+];
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "Hello" }],
+};
+
+function lastExitEvent(
+  events: AgentEvent[],
+): Extract<AgentEvent, { type: "agent_loop_exit" }> | undefined {
+  return events.find(
+    (e): e is Extract<AgentEvent, { type: "agent_loop_exit" }> =>
+      e.type === "agent_loop_exit",
+  );
+}
+
+function countExitEvents(events: AgentEvent[]): number {
+  return events.filter((e) => e.type === "agent_loop_exit").length;
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("AgentLoop exit-reason instrumentation", () => {
+  test("emits exit event exactly once with 'no_tool_calls' on plain text response", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(countExitEvents(events)).toBe(1);
+    const exit = lastExitEvent(events);
+    expect(exit?.reason).toBe("no_tool_calls");
+  });
+
+  test("agent_loop_exit is the last event emitted", async () => {
+    const { provider } = createMockProvider([textResponse("Hi there!")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => {
+      events.push(e);
+    });
+
+    expect(events.length).toBeGreaterThan(0);
+    expect(events[events.length - 1].type).toBe("agent_loop_exit");
+  });
+
+  test("emits 'aborted_pre_call' when signal is already aborted at run start", async () => {
+    const { provider } = createMockProvider([textResponse("never sent")]);
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const controller = new AbortController();
+    controller.abort();
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_pre_call");
+  });
+
+  test("emits 'yield_to_user' when tool result requests yieldToUser", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("yield_to_user");
+  });
+
+  test("emits 'checkpoint_yield' when onCheckpoint returns 'yield'", async () => {
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+      textResponse("never reached"),
+    ]);
+    const toolExecutor = async () => ({ content: "ok", isError: false });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const onCheckpoint = (_info: CheckpointInfo): CheckpointDecision =>
+      "yield";
+
+    const events: AgentEvent[] = [];
+    await loop.run(
+      [userMessage],
+      (e) => { events.push(e); },
+      undefined,
+      undefined,
+      onCheckpoint,
+    );
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("checkpoint_yield");
+  });
+
+  test("emits 'error' when provider throws an unhandled error", async () => {
+    const provider: Provider = {
+      name: "broken",
+      async sendMessage(): Promise<ProviderResponse> {
+        throw new Error("provider exploded");
+      },
+    };
+    const loop = new AgentLoop(provider, "system prompt");
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("error");
+  });
+
+  test("does not double-emit when multiple exit conditions stack", async () => {
+    // Tool returns yieldToUser AND the controller is aborted post-response —
+    // the first reached condition wins, but the guard prevents a second
+    // emit even if subsequent code paths attempt one.
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    const toolExecutor = async () => ({
+      content: "ok",
+      isError: false,
+      yieldToUser: true,
+    });
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); });
+
+    expect(countExitEvents(events)).toBe(1);
+  });
+
+  test("emits 'aborted_during_tools' when signal aborts after tool execution", async () => {
+    const controller = new AbortController();
+    const { provider } = createMockProvider([
+      toolUseResponse("t1", "read_file", { path: "/a.txt" }),
+    ]);
+    // Abort the signal inside the tool executor so by the time the loop
+    // re-checks signal.aborted post-tools the abort has landed.
+    const toolExecutor = async () => {
+      controller.abort();
+      return { content: "ok", isError: false };
+    };
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+
+    const events: AgentEvent[] = [];
+    await loop.run([userMessage], (e) => { events.push(e); }, controller.signal);
+
+    expect(countExitEvents(events)).toBe(1);
+    expect(lastExitEvent(events)?.reason).toBe("aborted_during_tools");
+  });
+});