@vellumai/assistant 0.12.1 → 0.12.2-dev.202609162314.7778bc8

package/scripts/smoke-desktop-window-theme.ts

@@ -0,0 +1,186 @@
+import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { writeDesktopWindowTheme } from "../src/desktop/desktop-window-theme.js";
+
+if (process.platform !== "linux") {
+  throw new Error(
+    "Run in a disposable Linux environment with Xvfb, Openbox, xterm, xdotool, xwininfo and xmllint installed",
+  );
+}
+
+const directory = await mkdtemp(join(tmpdir(), "window-theme-smoke-"));
+const env = { ...process.env, DISPLAY: ":97" };
+const children: ReturnType<typeof Bun.spawn>[] = [];
+const start = (cmd: string[]) => {
+  const child = Bun.spawn(cmd, {
+    env,
+    stdout: "ignore",
+    stderr: "inherit",
+    windowsHide: true,
+  });
+  children.push(child);
+  return child;
+};
+const run = (...cmd: string[]) => {
+  const result = Bun.spawnSync(cmd, {
+    env,
+
+    windowsHide: true,
+  });
+  assert.equal(result.exitCode, 0, result.stderr.toString());
+  return result.stdout.toString();
+};
+const waitFor = async (predicate: () => boolean) => {
+  for (let attempt = 0; attempt < 100; attempt++) {
+    try {
+      if (predicate()) {
+        return;
+      }
+    } catch {}
+    await Bun.sleep(50);
+  }
+  throw new Error("Window theme smoke condition timed out");
+};
+const pointer = async (...args: string[]) => {
+  run("xdotool", ...args);
+  await Bun.sleep(200);
+};
+const geometry = () => {
+  const info = run("xwininfo", "-name", "Theme smoke");
+  return [
+    "Absolute upper-left X",
+    "Absolute upper-left Y",
+    "Width",
+    "Height",
+  ].map((label) =>
+    Number(new RegExp(`${label}:\\s+(-?\\d+)`).exec(info)?.[1]),
+  ) as [number, number, number, number];
+};
+const state = () => run("xprop", "-name", "Theme smoke", "_NET_WM_STATE");
+const button = async (offset: number) => {
+  const [x, y, width] = geometry();
+  await pointer(
+    "mousemove",
+    String(x + width - offset),
+    String(y - 16),
+    "click",
+    "1",
+  );
+};
+
+try {
+  const home = join(directory, "home");
+  const sourceDir = join(home, ".config", "openbox");
+  await mkdir(sourceDir, { recursive: true });
+  const original = await readFile("/etc/xdg/openbox/rc.xml", "utf8");
+  const mouse = original.match(/<mouse>[^]*?<\/mouse>/)![0];
+  const keyboard = original.match(/<keyboard>[^]*?<\/keyboard>/)![0];
+  const source = original
+    .replace(
+      mouse,
+      `<xi:include href="bindings.xml" xpointer="xpointer(/*/*)"/>`,
+    )
+    .replace(keyboard, "");
+  const sourcePath = join(sourceDir, "rc.xml");
+  await writeFile(sourcePath, source);
+  await writeFile(
+    join(sourceDir, "bindings.xml"),
+    `<bindings xmlns="http://openbox.org/3.4/rc">${mouse}${keyboard}</bindings>`,
+  );
+  const configDir = join(directory, "managed & theme");
+  const config = writeDesktopWindowTheme(configDir, home, "#53b7aa");
+  const expanded = run("xmllint", "--xinclude", config);
+  assert(
+    expanded.includes("NextWindow") && expanded.includes("Move"),
+    "Relative includes must retain the actual keyboard and mouse bindings",
+  );
+  assert.equal(await readFile(sourcePath, "utf8"), source);
+  writeDesktopWindowTheme(configDir, home, "#bc7799");
+  writeDesktopWindowTheme(configDir, home, "#53b7aa");
+  assert.equal(await readFile(sourcePath, "utf8"), source);
+  const blocked = join(directory, "blocked");
+  await writeFile(blocked, "not a directory");
+  assert.throws(() => writeDesktopWindowTheme(blocked, home));
+  assert.equal(await readFile(sourcePath, "utf8"), source);
+  const customBase = source.replace(
+    "<openbox_config",
+    '<openbox_config xml:base="./custom/"',
+  );
+  await writeFile(sourcePath, customBase);
+  assert.throws(() => writeDesktopWindowTheme(configDir, home));
+  assert.equal(await readFile(sourcePath, "utf8"), customBase);
+  await writeFile(sourcePath, source);
+
+  start(["Xvfb", ":97", "-screen", "0", "1100x700x24"]);
+  await waitFor(
+    () => run("xdotool", "getdisplaygeometry").trim() === "1100 700",
+  );
+  start(["openbox", "--config-file", config]);
+  await waitFor(() =>
+    run("xprop", "-root", "_OB_THEME").includes("window-theme"),
+  );
+  start([
+    "xterm",
+    "-title",
+    "Theme smoke",
+    "-geometry",
+    "48x12+450+250",
+    "-e",
+    "sleep",
+    "60",
+  ]);
+  await waitFor(
+    () => run("xdotool", "search", "--name", "^Theme smoke$").trim().length > 0,
+  );
+  await Bun.sleep(300);
+  const id = run("xdotool", "search", "--name", "^Theme smoke$").trim();
+  await button(39);
+  assert(
+    state().includes("MAXIMIZED_HORZ") && state().includes("MAXIMIZED_VERT"),
+  );
+  await button(39);
+  assert(!state().includes("MAXIMIZED"));
+  await button(62);
+  assert(state().includes("HIDDEN"));
+  await pointer("windowactivate", id);
+  assert(!state().includes("HIDDEN"));
+  let [x, y, width, height] = geometry();
+  await pointer("mousemove", String(x + 100), String(y - 16), "mousedown", "1");
+  await pointer("mousemove", String(x + 90), String(y - 26));
+  await pointer("mousemove", String(x + 45), String(y - 65));
+  await pointer("mouseup", "1");
+  const [movedX, movedY] = geometry();
+  assert(Math.abs(movedX - x) >= 30 && Math.abs(movedY - y) >= 30);
+  [x, y, width, height] = geometry();
+  await pointer(
+    "mousemove",
+    String(x + width - 2),
+    String(y + height + 2),
+    "mousedown",
+    "1",
+  );
+  await pointer("mousemove", String(x + width + 10), String(y + height + 10));
+  await pointer("mousemove", String(x + width + 60), String(y + height + 40));
+  await pointer("mouseup", "1");
+  const [, , resizedWidth, resizedHeight] = geometry();
+  assert(resizedWidth > width && resizedHeight > height);
+  await pointer("key", "alt+F4");
+  await waitFor(
+    () =>
+      !run("xprop", "-root", "_NET_CLIENT_LIST").includes(
+        `0x${Number(id).toString(16)}`,
+      ),
+  );
+  console.log(
+    "PASS: relative includes, preserved source, repeated theme updates, maximize/restore, minimize/activation, dragging, resizing, and keyboard close",
+  );
+} finally {
+  for (const child of children.reverse()) {
+    child.kill();
+  }
+  await Promise.all(children.map((child) => child.exited));
+  await rm(directory, { recursive: true, force: true });
+}

package/scripts/sync-llm-catalog.ts

@@ -86,6 +86,9 @@ function projectModel(model: CatalogModel): Record<string, unknown> {
   if (model.supportsToolUse !== undefined) {
     projected.supportsToolUse = model.supportsToolUse;
   }
+  if (model.supportsText !== undefined) {
+    projected.supportsText = model.supportsText;
+  }
   if (model.pricing !== undefined) {
     projected.pricing = model.pricing;
   }

package/scripts/test.ts

@@ -62,26 +62,28 @@ const EXPERIMENTAL_FILES = new Set([
 const KNOWN_BROKEN_FILES = new Set<string>([]);
 
 // ---------------------------------------------------------------------------
-// Feature-flag registry sync
+// Bundled catalog sync
 // ---------------------------------------------------------------------------
 
-// Ensure the bundled feature-flag-registry.json exists before running tests.
-// The canonical copy lives at meta/feature-flags/feature-flag-registry.json and
-// is synced into assistant/src/ and gateway/src/. CI runs this as a dedicated
-// step; locally postinstall handles it — but when node_modules is symlinked
-// (e.g. worktrees) postinstall never fires, so the bundled copy can be missing
-// and feature-flag-registry-bundled.test.ts fails. Running the sync here is
-// idempotent and cheap (two file copies).
-function syncFeatureFlagRegistry(): void {
+// Ensure generated catalog inputs exist before tests. Locally, postinstall may
+// not run in worktrees that share node_modules, so the static JSON imports can
+// otherwise be missing.
+function syncBundledCatalog(): void {
   const syncScript = join("..", "meta", "sync-bundled-copies.ts");
   if (!existsSync(syncScript)) {
     return;
   }
-  Bun.spawnSync(["bun", "run", "meta/sync-bundled-copies.ts"], {
-    cwd: join(process.cwd(), ".."),
+  const result = Bun.spawnSync(["bun", "run", "sync:bundled-catalog"], {
+    cwd: process.cwd(),
     stdout: "ignore",
-    stderr: "ignore",
+    stderr: "pipe",
+    windowsHide: true,
   });
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `Failed to generate bundled catalog inputs: ${result.stderr.toString()}`,
+    );
+  }
 }
 
 // ---------------------------------------------------------------------------
@@ -514,7 +516,7 @@ async function buildMigratedFixture(outWorkspaceDir: string): Promise<void> {
 // ---------------------------------------------------------------------------
 
 async function main(): Promise<void> {
-  syncFeatureFlagRegistry();
+  syncBundledCatalog();
 
   let testFiles = collectTestFiles();
   if (testFiles.length === 0) {

package/scripts/voice-ttft-spike.ts

@@ -15,7 +15,7 @@
  *   full-quality  quality-optimized, full prompt + tools (pre-flag brain)
  *
  * Run with the daemon's environment so the managed connection resolves:
- *   VELLUM_WORKSPACE_DIR=<daemon workspace> CES_LOCAL_SOCKET=<socket> \
+ *   VELLUM_WORKSPACE_DIR=<daemon workspace> \
  *     bun run scripts/voice-ttft-spike.ts [--trials 3] [--variants a,b]
  *
  * Outside the daemon env, credentials fall back to the encrypted file
@@ -217,7 +217,7 @@ async function main(): Promise<void> {
     if (!provider) {
       console.error(
         `${spec.name}: no provider resolved (managed credentials missing? ` +
-          `run with the daemon's VELLUM_WORKSPACE_DIR + CES_LOCAL_SOCKET)`,
+          `run with the daemon's VELLUM_WORKSPACE_DIR)`,
       );
       continue;
     }

package/src/__tests__/agent-loop.test.ts

@@ -1977,6 +1977,130 @@ describe("AgentLoop", () => {
     expect(calls[0].tools).not.toEqual(dummyTools);
   });
 
+  test("onToolsSent observes the exact tool array of every provider call", async () => {
+    const perCall: ToolDefinition[][] = [
+      [
+        {
+          name: "first",
+          description: "First",
+          input_schema: { type: "object" },
+        },
+      ],
+      [
+        {
+          name: "second",
+          description: "Second",
+          input_schema: { type: "object" },
+        },
+      ],
+    ];
+    let resolveCount = 0;
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([
+      toolUseResponse("t1", "first", {}),
+      textResponse("Done"),
+    ]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      toolExecutor: async () => ({ content: "result", isError: false }),
+      resolveTools: () => perCall[resolveCount++]!,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // One observation per provider call, each the array that call sent.
+    expect(sent).toHaveLength(2);
+    expect(calls[0].tools).toEqual(sent[0]);
+    expect(calls[1].tools).toEqual(sent[1]);
+    expect(sent.map((tools) => tools.map((t) => t.name))).toEqual([
+      ["first"],
+      ["second"],
+    ]);
+  });
+
+  test("onToolsSent sees the provider-native web_search tool the loop appends", async () => {
+    const dynamicTools: ToolDefinition[] = [
+      {
+        name: "dynamic_tool",
+        description: "Dynamic",
+        input_schema: { type: "object" },
+      },
+    ];
+    const sent: ToolDefinition[][] = [];
+
+    const { provider, calls } = createMockProvider([textResponse("Hi")]);
+    Object.assign(provider, { supportsNativeWebSearch: true });
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      config: { enableNativeWebSearch: true },
+      resolveTools: () => dynamicTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // The resolver never saw web_search; the observer sees the wire array.
+    expect(sent).toHaveLength(1);
+    expect(sent[0]!.map((t) => t.name)).toEqual(["dynamic_tool", "web_search"]);
+    expect(calls[0].tools).toEqual(sent[0]);
+  });
+
+  test("onToolsSent does not fire for a call cancelled before the request leaves", async () => {
+    // A pre-model-call hook runs after tool resolution and before the send,
+    // so aborting there is a call whose tools are resolved but never sent.
+    const controller = new AbortController();
+    registerPlugin({
+      manifest: { name: "abort-before-send", version: "0.0.1" },
+      hooks: {
+        "pre-model-call": async (ctx) => {
+          controller.abort();
+          return ctx;
+        },
+      },
+    });
+    const sent: ToolDefinition[][] = [];
+    const { provider } = createMockProvider([textResponse("never")]);
+    const loop = new AgentLoop({
+      provider: provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      resolveTools: () => dummyTools,
+      onToolsSent: (tools) => {
+        sent.push(tools);
+      },
+    });
+
+    await loop
+      .run({
+        requestId: "test-request",
+        messages: [userMessage],
+        onEvent: () => {},
+        trust: { sourceChannel: "vellum", trustClass: "unknown" },
+        signal: controller.signal,
+      })
+      .catch(() => {});
+
+    expect(sent).toEqual([]);
+  });
+
   // 28. Tool list can change between turns
   test("resolveTools can return different tools on each turn", async () => {
     const toolsPerTurn: ToolDefinition[][] = [
@@ -2426,6 +2550,141 @@ describe("AgentLoop", () => {
     expect(noRetryNudge).toBeUndefined();
   });
 
+  // A native web search left deferred by the same assistant turn (a
+  // server_tool_use with no result, alongside the client tool call) runs on
+  // the next request only if that request's tool-result message holds
+  // tool_result blocks alone. The coaching then rides inside the errored
+  // tool_result instead of as a trailing text block.
+  test("folds retry coaching into the tool_result when the assistant turn left a server tool deferred", async () => {
+    const mixedTurn: ProviderResponse = {
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/missing.txt" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 10, outputTokens: 5 },
+      stopReason: "tool_use",
+    };
+    const { provider, calls } = createMockProvider([
+      mixedTurn,
+      textResponse("Reported the missing file."),
+    ]);
+    const toolExecutor = async () => ({
+      content: "Error: HTTP 404",
+      isError: true,
+    });
+
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+      tools: dummyTools,
+      toolExecutor,
+    });
+    await loop.run({
+      requestId: "test-request",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    const followUpMessages = calls[1].messages;
+    const followUp = followUpMessages[followUpMessages.length - 1];
+    expect(followUp.role).toBe("user");
+    expect(followUp.content.map((b) => b.type)).toEqual(["tool_result"]);
+    const erroredToolResult = followUp.content[0] as Extract<
+      ContentBlock,
+      { type: "tool_result" }
+    >;
+    expect(erroredToolResult.is_error).toBe(true);
+    expect(erroredToolResult.content).toContain("Error: HTTP 404");
+    expect(erroredToolResult.content).toContain("looks recoverable");
+  });
+
+  // A deferred search answered by a message with text after the results
+  // (history assembled elsewhere) is rejected by the provider as unpaired.
+  // The ordering-repair retry stamps the synthetic result rather than
+  // re-sending the same history.
+  test("repairs a deferred search rejected as unpaired instead of re-sending the same history", async () => {
+    const mixedTurn: Message = {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id: "t1",
+          name: "read_file",
+          input: { path: "/a" },
+        },
+        {
+          type: "server_tool_use",
+          id: "srvtoolu_1",
+          name: "web_search",
+          input: { query: "news" },
+        },
+      ],
+    };
+    const answeredWithText: Message = {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "t1",
+          content: "Error: HTTP 404",
+          is_error: true,
+        },
+        { type: "text", text: "<system_notice>retry</system_notice>" },
+      ],
+    };
+    const { provider, calls } = createMockProvider([
+      new Error(
+        "Anthropic API error (400): messages.1: `web_search` tool use with id `srvtoolu_1` was found without a corresponding `web_search_tool_result` block",
+      ),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    const { history } = await loop.run({
+      requestId: "test-request",
+      messages: [userMessage, mixedTurn, answeredWithText],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(2);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+    // Deep repair stamped the synthetic result on the orphaned search, and the
+    // outbound sanitizer then summarized the answered pair as text, so the
+    // retry carries no unanswered server_tool_use.
+    expect(calls[1].messages[1].content.map((b) => b.type)).toEqual([
+      "tool_use",
+      "text",
+    ]);
+    expect(calls[1].messages[1].content[1]).toMatchObject({
+      type: "text",
+      text: expect.stringContaining("results unavailable"),
+    });
+    expect(history[history.length - 1]).toEqual({
+      role: "assistant",
+      content: [{ type: "text", text: "recovered" }],
+    });
+  });
+
   // Retry coaching stops after a tool fails 3 times in a row — past that the
   // error is likely unrecoverable and further coaching only burns tokens.
   test("stops appending retry coaching after 3 consecutive failures of a tool", async () => {

package/src/__tests__/always-loaded-tools-guard.test.ts

@@ -26,7 +26,7 @@ afterAll(() => {
 });
 
 describe("always-loaded tool count", () => {
-  test("should be exactly 11 with recall occupying the existing slot", async () => {
+  test("should be exactly 19 with recall occupying the existing slot", async () => {
     await initializeTools();
     const allDefs = getAllToolDefinitions();
 
@@ -44,23 +44,31 @@ describe("always-loaded tool count", () => {
     );
     const activeNames = activeTools.map((t) => t.name).sort();
 
-    // Host tools (host_bash, host_file_*) are excluded when no client is
-    // connected — without a human in the loop, the guardian auto-approve
-    // path would allow unchecked host command execution.
-    //
-    // `watch_retro_report` is here for the same reason the ui_surface tools are
-    // NOT: a watch retrospective runs clientless, so it can only report through
-    // a tool that survives this baseline. Its description and schema are kept
-    // deliberately terse because that is the cost of the slot.
+    // `watch_retro_report` survives this baseline because a watch
+    // retrospective runs clientless and can only report through a tool that
+    // stays on this baseline. The ui_* tools stay here because background
+    // surfaces persist and return instead of awaiting action, so a connected
+    // client is not required for the definitions to be present.
+    // Host tool definitions stay on the wire for background turns: schemas
+    // are cache-stable, and the shared execution gate rejects host tools on
+    // explicitly non-interactive turns before any proxy dispatch.
     const expectedNames = [
       "bash",
       "file_edit",
       "file_read",
       "file_write",
+      "host_bash",
+      "host_file_edit",
+      "host_file_read",
+      "host_file_transfer",
+      "host_file_write",
       "recall",
       "remember",
       "skill_execute",
       "skill_load",
+      "ui_dismiss",
+      "ui_show",
+      "ui_update",
       "watch_retro_report",
       "web_fetch",
       "web_search",
@@ -68,6 +76,6 @@ describe("always-loaded tool count", () => {
 
     expect(activeNames).toEqual(expectedNames);
     expect(activeNames.filter((name) => name === "recall")).toHaveLength(1);
-    expect(activeTools.length).toBe(11);
+    expect(activeTools.length).toBe(19);
   });
 });