@vellumai/assistant 0.8.10-dev.202606100925.28c1cfb → 0.8.10-dev.202606101122.0de2aff

package/openapi.yaml

@@ -24689,13 +24689,64 @@ paths:
   /v1/tools:
     get:
       operationId: tools_get
-      summary: List tools
-      description: Return available tool names with their descriptions, risk levels, and categories.
+      summary: List registered tools with metadata and schemas
+      description:
+        "Return every registered tool. `tools` carries per-tool metadata (description, author-asserted risk level,
+        category, and contributing source: core, skill, plugin, or MCP server). `names`/`schemas` additionally cover
+        skill tools whose manifests are present but not yet loaded, for the permission-simulator catalog."
       tags:
         - tools
       responses:
         "200":
           description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  names:
+                    type: array
+                    items:
+                      type: string
+                  schemas:
+                    type: object
+                    propertyNames:
+                      type: string
+                    additionalProperties:
+                      type: object
+                      propertyNames:
+                        type: string
+                      additionalProperties: {}
+                  tools:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        name:
+                          type: string
+                        description:
+                          type: string
+                        riskLevel:
+                          type: string
+                        category:
+                          type: string
+                        source:
+                          type: string
+                          description:
+                            'Tool origin: "core" for built-ins, otherwise "<kind>:<id>" (e.g. "plugin:echo", "skill:my-skill",
+                            "mcp:server").'
+                      required:
+                        - name
+                        - description
+                        - riskLevel
+                        - category
+                        - source
+                      additionalProperties: false
+                required:
+                  - names
+                  - schemas
+                  - tools
+                additionalProperties: false
   /v1/tools/simulate-permission:
     post:
       operationId: tools_simulatepermission_post

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.8.10-dev.202606100925.28c1cfb",
+  "version": "0.8.10-dev.202606101122.0de2aff",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/agent-loop.test.ts

@@ -14,6 +14,7 @@ import type {
   ProviderResponse,
   ToolDefinition,
 } from "../providers/types.js";
+import { ContextOverflowError } from "../providers/types.js";
 import {
   createMockProvider,
   textResponse,
@@ -311,6 +312,106 @@ describe("AgentLoop", () => {
     ).toBe("API rate limit exceeded");
   });
 
+  // 5b. Reactive context-overflow recovery
+  test("recovers from a context-overflow rejection by re-running the call", async () => {
+    /**
+     * The provider rejects the first call with `context_too_large`; the loop
+     * should calibrate the estimator from the error and re-run instead of
+     * surfacing an error, succeeding on the retry.
+     */
+
+    // GIVEN a provider that rejects the first call as too large, then succeeds
+    const { provider, calls } = createMockProvider([
+      new ContextOverflowError("prompt is too long", "mock", {
+        actualTokens: 250_000,
+      }),
+      textResponse("recovered"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    const { history } = await loop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN the call was retried and the recovery response landed in history
+    expect(calls).toHaveLength(2);
+    expect(history).toHaveLength(2);
+    expect(history[1].content).toEqual([{ type: "text", text: "recovered" }]);
+
+    // AND no error event was emitted — the overflow was handled in-loop
+    expect(events.filter((e) => e.type === "error")).toHaveLength(0);
+  });
+
+  test("surfaces the overflow error after exhausting reactive retries", async () => {
+    /**
+     * When every re-run still overflows, the loop must stop retrying after a
+     * bounded number of attempts and surface the error rather than looping
+     * forever.
+     */
+
+    // GIVEN a provider that rejects every call as too large
+    const { provider, calls } = createMockProvider([
+      new ContextOverflowError("prompt is too long", "mock", {
+        actualTokens: 250_000,
+      }),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    await loop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN it tried the initial call plus the bounded retries, then gave up
+    expect(calls).toHaveLength(3);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(1);
+  });
+
+  test("does not retry an overflow with no recoverable token count", async () => {
+    /**
+     * Without a parseable actual-token count there is nothing to calibrate
+     * against, so the loop surfaces the error immediately instead of
+     * re-running blindly.
+     */
+
+    // GIVEN an overflow error carrying neither a typed count nor a parseable one
+    const { provider, calls } = createMockProvider([
+      new ContextOverflowError("context window exceeded", "mock"),
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "test-conversation",
+    });
+    const events: AgentEvent[] = [];
+
+    // WHEN the loop runs
+    await loop.run({
+      messages: [userMessage],
+      onEvent: collectEvents(events),
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    // THEN it surfaced the error on the first call without retrying
+    expect(calls).toHaveLength(1);
+    expect(events.filter((e) => e.type === "error")).toHaveLength(1);
+  });
+
   // 6. Abort signal — verify the loop respects AbortSignal
   test("stops when abort signal is triggered before provider call", async () => {
     const controller = new AbortController();

package/src/__tests__/catalog-install-normalize.test.ts

@@ -0,0 +1,141 @@
+/**
+ * Regression tests for catalog entry normalization in catalog-install.ts.
+ *
+ * The platform `/v1/skills/` API flattens skill metadata into top-level
+ * fields (`category`, `display_name`, `icon`) rather than the nested
+ * `metadata.vellum.*` shape used by the local `catalog.json`. Both
+ * `fetchCatalog` (remote) and `readLocalCatalog` (local file) must produce a
+ * canonical `CatalogSkill` whose `metadata.vellum.category` is populated, so
+ * downstream category assignment doesn't fall back to "system" for every skill.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, mock, test } from "bun:test";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+mock.module("../config/env.js", () => ({
+  getPlatformBaseUrl: () => "https://platform.example.com",
+}));
+
+import { fetchCatalog, readLocalCatalog } from "../skills/catalog-install.js";
+
+const originalFetch = globalThis.fetch;
+
+afterEach(() => {
+  globalThis.fetch = originalFetch;
+});
+
+/**
+ * Build a `fetch` stub that responds with `payload` serialized as JSON.
+ *
+ * `Object.assign` attaches the `preconnect` static that Bun's `fetch` type
+ * requires, so the stub satisfies `typeof fetch` without an `as unknown` cast.
+ */
+function stubFetchJson(payload: unknown): typeof fetch {
+  const respond = async () =>
+    new Response(JSON.stringify(payload), {
+      status: 200,
+      headers: { "content-type": "application/json" },
+    });
+  return Object.assign(respond, { preconnect: () => {} });
+}
+
+describe("fetchCatalog normalization", () => {
+  test("re-nests the platform's flattened category under metadata.vellum", async () => {
+    globalThis.fetch = stubFetchJson({
+      skills: [
+        {
+          id: "amazon",
+          name: "amazon",
+          display_name: "Amazon",
+          description: "Shop on Amazon",
+          icon: "🛒",
+          category: "commerce",
+          updated_at: "2026-04-19T19:26:17Z",
+        },
+      ],
+    });
+
+    const catalog = await fetchCatalog();
+
+    expect(catalog).toHaveLength(1);
+    const [skill] = catalog;
+    expect(skill.id).toBe("amazon");
+    expect(skill.metadata?.vellum?.category).toBe("commerce");
+    expect(skill.metadata?.vellum?.["display-name"]).toBe("Amazon");
+    expect(skill.icon).toBe("🛒");
+    expect(skill.metadata?.icon).toBe("🛒");
+    expect(skill.updatedAt).toBe("2026-04-19T19:26:17Z");
+  });
+
+  test("leaves category undefined when the API omits it", async () => {
+    globalThis.fetch = stubFetchJson({
+      skills: [{ id: "no-cat", name: "no-cat", description: "d" }],
+    });
+
+    const catalog = await fetchCatalog();
+
+    expect(catalog).toHaveLength(1);
+    expect(catalog[0].metadata?.vellum?.category).toBeUndefined();
+  });
+
+  test("drops malformed entries without throwing", async () => {
+    globalThis.fetch = stubFetchJson({
+      skills: [
+        null,
+        undefined,
+        "not-an-object",
+        { name: "missing-id", description: "d" },
+        { id: 42 },
+        { id: "valid", name: "valid", description: "d", category: "email" },
+      ],
+    });
+
+    const catalog = await fetchCatalog();
+
+    expect(catalog).toHaveLength(1);
+    expect(catalog[0].id).toBe("valid");
+    expect(catalog[0].metadata?.vellum?.category).toBe("email");
+  });
+});
+
+describe("readLocalCatalog normalization", () => {
+  test("reads the nested metadata.vellum.category from catalog.json", () => {
+    const dir = mkdtempSync(join(tmpdir(), "vellum-catalog-"));
+    try {
+      writeFileSync(
+        join(dir, "catalog.json"),
+        JSON.stringify({
+          version: 1,
+          skills: [
+            {
+              id: "tasks",
+              name: "tasks",
+              description: "Task management",
+              metadata: {
+                icon: "✅",
+                vellum: { "display-name": "Tasks", category: "productivity" },
+              },
+            },
+          ],
+        }),
+      );
+
+      const catalog = readLocalCatalog(dir);
+
+      expect(catalog).toHaveLength(1);
+      expect(catalog[0].metadata?.vellum?.category).toBe("productivity");
+      expect(catalog[0].metadata?.vellum?.["display-name"]).toBe("Tasks");
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/__tests__/credential-security-invariants.test.ts

@@ -230,8 +230,7 @@ describe("Invariant 2: no generic plaintext secret read API", () => {
       "runtime/routes/platform-routes.ts", // CLI platform connect/disconnect/status routes (CLI-migrated to IPC)
       "ipc/skill-routes/providers.ts", // host.providers.secureKeys.getProviderKey IPC route (out-of-process SkillHost companion)
       "daemon/external-plugins-bootstrap.ts", // reads credentials at plugin init (manifest.requiresCredential) via the CES-mediated getSecureKeyAsync path
-      "plugins/external-api.ts", // globalThis runtime bridge that exposes getSecureKeyAsync to dynamically-imported workspace plugins (compiled-binary plugin loading)
-      "plugin-api/index.ts", // public @vellumai/plugin-api surface re-exports getSecureKeyAsync to workspace plugins via the boot-time shim (same exposure as the external-api bridge, routed through the typed public package)
+      "plugin-api/index.ts", // public @vellumai/plugin-api surface re-exports getSecureKeyAsync to dynamically-imported workspace plugins via the boot-time shim (compiled-binary plugin loading)
       "inbound/platform-callback-registration.ts", // managed credential lookup for platform base URL, assistant ID, and API key
       "tts/providers/elevenlabs-provider.ts", // ElevenLabs TTS API key lookup
       "tts/providers/deepgram-provider.ts", // Deepgram TTS API key lookup

package/src/__tests__/tools-get-route.test.ts

@@ -0,0 +1,126 @@
+/**
+ * Tests for the `tools` route handler (operationId `tools_get`), which backs
+ * both the macOS permission-simulator catalog (`names`/`schemas`) and the
+ * `assistant tools list` CLI command (`tools`). The `tools` array carries
+ * every registered tool with its description, author-asserted risk band,
+ * category, and the source (core / skill / plugin / mcp) that contributed it
+ * — with the source read from the registry's ownership map rather than off
+ * the tool object.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { RiskLevel } from "../permissions/types.js";
+import { ROUTES } from "../runtime/routes/settings-routes.js";
+import {
+  __resetRegistryForTesting,
+  registerMcpTools,
+  registerPluginTools,
+  registerSkillTools,
+  registerTool,
+} from "../tools/registry.js";
+import type { Tool, ToolContext, ToolExecutionResult } from "../tools/types.js";
+
+interface ToolListEntry {
+  name: string;
+  description: string;
+  riskLevel: string;
+  category: string;
+  source: string;
+}
+
+interface ToolsGetResponse {
+  names: string[];
+  schemas: Record<string, unknown>;
+  tools: ToolListEntry[];
+}
+
+function makeFakeTool(name: string, extras: Partial<Tool> = {}): Tool {
+  return {
+    name,
+    description: `Fake ${name}`,
+    defaultRiskLevel: RiskLevel.Low,
+    executionTarget: "sandbox",
+    input_schema: { type: "object", properties: {}, required: [] },
+    category: "",
+    async execute(
+      _input: Record<string, unknown>,
+      _context: ToolContext,
+    ): Promise<ToolExecutionResult> {
+      return { content: "ok", isError: false };
+    },
+    ...extras,
+  };
+}
+
+const handler = (() => {
+  const route = ROUTES.find(
+    (r) => r.endpoint === "tools" && r.method === "GET",
+  );
+  if (!route) {
+    throw new Error("No route found for GET tools");
+  }
+  return route.handler;
+})();
+
+describe("GET /tools", () => {
+  beforeEach(() => {
+    __resetRegistryForTesting();
+  });
+
+  test("returns each registered tool's metadata, sorted by name, with core source", async () => {
+    // GIVEN two core tools registered out of alphabetical order
+    registerTool(
+      makeFakeTool("b_core_tool", {
+        description: "Beta tool",
+        defaultRiskLevel: RiskLevel.High,
+        category: "testing",
+      }),
+    );
+    registerTool(makeFakeTool("a_core_tool"));
+
+    // WHEN the tools handler runs
+    const { names, schemas, tools } = (await handler({})) as ToolsGetResponse;
+
+    // THEN the catalog fields the permission simulator reads are present
+    expect(names).toContain("a_core_tool");
+    expect(names).toContain("b_core_tool");
+    expect(schemas.a_core_tool).toBeDefined();
+
+    // AND the metadata array is sorted by name with full per-tool metadata
+    const toolNames = tools.map((t) => t.name);
+    expect(toolNames.indexOf("a_core_tool")).toBeLessThan(
+      toolNames.indexOf("b_core_tool"),
+    );
+    expect([...toolNames]).toEqual(
+      [...toolNames].sort((a, b) => a.localeCompare(b)),
+    );
+
+    // AND each core tool reports "core" as its source plus its metadata
+    const beta = tools.find((t) => t.name === "b_core_tool");
+    expect(beta).toEqual({
+      name: "b_core_tool",
+      description: "Beta tool",
+      riskLevel: RiskLevel.High,
+      category: "testing",
+      source: "core",
+    });
+  });
+
+  test("reports plugin, mcp, and skill tools as <kind>:<id>", async () => {
+    // GIVEN one tool from each non-core source
+    registerPluginTools("echo", [makeFakeTool("p_tool")]);
+    registerMcpTools("linear", [makeFakeTool("m_tool")]);
+    registerSkillTools("my-skill", [makeFakeTool("s_tool")]);
+
+    // WHEN the tools handler runs
+    const { tools } = (await handler({})) as ToolsGetResponse;
+
+    // THEN each tool's source encodes its owning extension
+    const sourceOf = (name: string) =>
+      tools.find((t) => t.name === name)?.source;
+    expect(sourceOf("p_tool")).toBe("plugin:echo");
+    expect(sourceOf("m_tool")).toBe("mcp:linear");
+    expect(sourceOf("s_tool")).toBe("skill:my-skill");
+  });
+});

package/src/agent/loop.ts

@@ -3,6 +3,7 @@ import * as Sentry from "@sentry/node";
 import { isAssistantFeatureFlagEnabled } from "../config/assistant-feature-flags.js";
 import { getConfig } from "../config/loader.js";
 import type { LLMCallSite } from "../config/schemas/llm.js";
+import { recordEstimate } from "../context/estimator-calibration.js";
 import { stripInjectionsForCompaction } from "../context/strip-injections.js";
 import {
   estimatePromptTokensRaw,
@@ -11,6 +12,7 @@ import {
   getCalibrationProviderKey,
 } from "../context/token-estimator.js";
 import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
+import { parseActualTokensFromError } from "../daemon/parse-actual-tokens-from-error.js";
 import type { TrustContext } from "../daemon/trust-context.js";
 import { stripHistoricalWebSearchResults } from "../daemon/web-search-history.js";
 import { HOOKS } from "../plugin-api/constants.js";
@@ -35,6 +37,7 @@ import type {
   ToolDefinition,
   ToolResultContent,
 } from "../providers/types.js";
+import { isContextOverflowError } from "../providers/types.js";
 import type { SensitiveOutputBinding } from "../tools/sensitive-output-placeholders.js";
 import {
   applyStreamingSubstitution,
@@ -54,6 +57,15 @@ const MID_LOOP_YIELD_THRESHOLD_RATIO = 0.85;
 const LONG_HISTORY_MESSAGE_THRESHOLD = 50;
 const LONG_HISTORY_SAFETY_MARGIN_FLOOR = 0.15;
 
+/**
+ * Consecutive context-overflow rejections the loop recovers from in place
+ * before letting the error propagate. Each recovery calibrates the estimator
+ * from the provider's real token count and re-fires the budget gate, which
+ * compacts before the next call; the cap stops an unbounded
+ * reject → compact → reject cycle when compaction cannot get under the limit.
+ */
+const MAX_REACTIVE_OVERFLOW_RETRIES = 2;
+
 export interface AgentLoopConfig {
   maxTokens: number;
   maxInputTokens?: number; // context window size for tool result truncation
@@ -805,6 +817,14 @@ export class AgentLoop {
     // prior post-call placement, plus the first call when
     // `compactInPlace` is set (the primary run's turn-start compaction).
     let budgetGateArmed = compactInPlace;
+    // Raw pre-send estimate for the most recent provider call, captured so the
+    // overflow catch can calibrate the estimator against the provider's actual
+    // token count. Reset to the success path's value on every call.
+    let lastPreSendEstimatedTokens = 0;
+    // Consecutive context-overflow rejections recovered in place this run,
+    // reset after any successful provider call. Bounded by
+    // MAX_REACTIVE_OVERFLOW_RETRIES.
+    let reactiveOverflowRetries = 0;
     const rlog = requestId ? log.child({ requestId }) : log;
 
     // Resolve the inference-profile override that applies right now. The
@@ -1078,6 +1098,7 @@ export class AgentLoop {
             toolTokenBudget,
           },
         );
+        lastPreSendEstimatedTokens = preSendEstimatedTokens;
         rlog.info({ turn: toolUseTurns }, "LLM call start");
 
         // Sanitize the outbound history right before sending: drop accumulated
@@ -1246,6 +1267,10 @@ export class AgentLoop {
           throw llmCallError;
         }
 
+        // The call fit — clear the reactive-overflow budget so a later
+        // independent overflow this run gets its own recovery attempts.
+        reactiveOverflowRetries = 0;
+
         const providerDurationMs = Date.now() - providerStart;
 
         onEvent({
@@ -1716,6 +1741,42 @@ export class AgentLoop {
           await emitExit("aborted_via_error");
           break;
         }
+
+        // Reactive context-overflow recovery. The provider rejected the call
+        // because the prompt exceeded its window — ground truth the estimator
+        // under-counted. Fold the provider's actual token count into the
+        // per-provider calibration so the next iteration's budget gate sees a
+        // corrected (higher) estimate, crosses its threshold, and compacts in
+        // place before re-issuing the call. Bounded so an overflow that
+        // compaction cannot resolve falls through to the error path below
+        // instead of looping forever.
+        if (isContextOverflowError(error)) {
+          const actualTokens = parseActualTokensFromError(error);
+          if (
+            actualTokens !== null &&
+            reactiveOverflowRetries < MAX_REACTIVE_OVERFLOW_RETRIES
+          ) {
+            reactiveOverflowRetries++;
+            recordEstimate(
+              getCalibrationProviderKey(this.provider),
+              "",
+              lastPreSendEstimatedTokens,
+              actualTokens,
+            );
+            budgetGateArmed = true;
+            rlog.warn(
+              {
+                turn: toolUseTurns,
+                attempt: reactiveOverflowRetries,
+                estimated: lastPreSendEstimatedTokens,
+                actualTokens,
+              },
+              "Context too large — calibrated estimator and re-running to compact",
+            );
+            continue;
+          }
+        }
+
         const err = error instanceof Error ? error : new Error(String(error));
         rlog.error(
           { err, turn: toolUseTurns, messageCount: history.length },

package/src/cli/commands/tools.ts

@@ -0,0 +1,92 @@
+/**
+ * `assistant tools` — inspect the tools registered with the running
+ * assistant (core built-ins plus skill-, plugin-, and MCP-contributed
+ * tools).
+ *
+ * Thin `ipc` wrapper: each subcommand forwards to a single daemon route and
+ * renders the response. The registry lives in the daemon, so these commands
+ * require the daemon to be running.
+ */
+
+import type { Command } from "commander";
+
+import { cliIpcCall, exitFromIpcResult } from "../../ipc/cli-client.js";
+import { registerCommand } from "../lib/register-command.js";
+
+interface ToolListEntry {
+  name: string;
+  description: string;
+  riskLevel: string;
+  category: string;
+  source: string;
+}
+
+interface ToolsGetResponse {
+  names: string[];
+  schemas: Record<string, unknown>;
+  tools: ToolListEntry[];
+}
+
+export function registerToolsCommand(program: Command): void {
+  registerCommand(program, {
+    name: "tools",
+    transport: "ipc",
+    description: "Inspect tools registered with the running assistant",
+    build: (tools) => {
+      tools.addHelpText(
+        "after",
+        `
+Tools are registered with the daemon from four sources: core built-ins,
+skills, external plugins, and MCP servers. The "source" column reports the
+origin as "core" or "<kind>:<id>" (e.g. "plugin:echo", "skill:my-skill",
+"mcp:linear"). The risk level is the author-asserted default band used for
+permission gating, not the runtime-classified risk of a specific call.
+
+Examples:
+  $ assistant tools list
+  $ assistant tools ls
+  $ assistant tools list --json`,
+      );
+
+      tools
+        .command("list")
+        .alias("ls")
+        .description(
+          "List all registered tools with their source and risk level",
+        )
+        .option("--json", "Emit machine-readable JSON instead of a table")
+        .action(async (opts: { json?: boolean }) => {
+          const response = await cliIpcCall<ToolsGetResponse>("tools_get");
+          if (!response.ok) {
+            return exitFromIpcResult(response);
+          }
+
+          const tools = response.result?.tools ?? [];
+
+          if (opts.json) {
+            console.log(JSON.stringify(tools, null, 2));
+            return;
+          }
+
+          if (tools.length === 0) {
+            console.log("No tools registered.");
+            return;
+          }
+
+          const nameW = Math.max(4, ...tools.map((t) => t.name.length));
+          const sourceW = Math.max(6, ...tools.map((t) => t.source.length));
+          const riskW = Math.max(4, ...tools.map((t) => t.riskLevel.length));
+
+          console.log(
+            `${"NAME".padEnd(nameW)}  ${"SOURCE".padEnd(sourceW)}  ${"RISK".padEnd(riskW)}  DESCRIPTION`,
+          );
+          for (const t of tools) {
+            console.log(
+              `${t.name.padEnd(nameW)}  ${t.source.padEnd(sourceW)}  ${t.riskLevel.padEnd(riskW)}  ${t.description}`,
+            );
+          }
+          console.log(`\n${tools.length} tool(s)`);
+        });
+    },
+  });
+}