@vellumai/assistant 0.11.4 → 0.11.5-staging.2

package/src/__tests__/run-conversation-turn-persistence.test.ts

@@ -110,9 +110,12 @@ function holdTheTurnOpen(): void {
 // gate it never opened.
 afterEach(() => releaseGate?.());
 
+const providerCalls: { tools?: { name: string }[] }[] = [];
+
 const provider: Provider = {
   name: "scripted",
   async sendMessage(messages, options): Promise<ProviderResponse> {
+    providerCalls.push({ tools: options?.tools });
     await gate;
     return scriptedProvider.sendMessage(messages, options);
   },
@@ -169,6 +172,7 @@ function resetDb(): void {
   db.run("DELETE FROM conversation_keys");
   db.run("DELETE FROM conversations");
   broadcasts.length = 0;
+  providerCalls.length = 0;
 }
 
 describe("runConversationTurn persistence", () => {
@@ -399,12 +403,32 @@ describe("runConversationTurn channel binding", () => {
     expect(metadataOf(rows[0])).toMatchObject({
       userMessageChannel: "plugin",
       assistantMessageChannel: "plugin",
+      userMessageInterface: "plugin",
+      assistantMessageInterface: "plugin",
     });
     expect(rows[1].role).toBe("assistant");
     expect(metadataOf(rows[1])).toMatchObject({
       userMessageChannel: "plugin",
       assistantMessageChannel: "plugin",
+      userMessageInterface: "plugin",
+      assistantMessageInterface: "plugin",
+    });
+  });
+
+  test("puts core tools on the wire for a plugin-channel turn", async () => {
+    // Plugin-driven iMessage turns are non-interactive, so host tools stay
+    // off, but sandbox tools (bash, web_search, ...) have to remain on the
+    // request. Missing them is what makes the model dump `to=bash code:`
+    // into the SMS as prose. The create path initializes the registry
+    // before constructing the Conversation that snapshots it.
+    await runConversationTurn({
+      channel: CHANNEL,
+      content: [{ type: "text", text: "hello" }],
     });
+
+    const names = providerCalls[0]?.tools?.map((tool) => tool.name) ?? [];
+    expect(names.length).toBeGreaterThan(0);
+    expect(names).toContain("bash");
   });
 
   test("a queued turn carries its own channel rather than inheriting one", async () => {
@@ -441,6 +465,8 @@ describe("runConversationTurn channel binding", () => {
     expect(metadataOf(userRows(conversationId)[1])).toMatchObject({
       userMessageChannel: "plugin",
       assistantMessageChannel: "plugin",
+      userMessageInterface: "plugin",
+      assistantMessageInterface: "plugin",
     });
   }, 20_000);
 
@@ -466,7 +492,7 @@ describe("runConversationTurn channel binding", () => {
       userMessageChannel: "vellum",
       assistantMessageChannel: "vellum",
     });
-  });
+  }, 20_000);
 
   test("keeps a known sender when a later turn omits it", async () => {
     // A plugin that knows only the chat coordinates this time is silent about

package/src/__tests__/secret-routes-platform-proxy.test.ts

@@ -103,6 +103,7 @@ mock.module("../daemon/conversation-store.js", () => ({
   },
 }));
 
+import { initializeDb } from "../persistence/db-init.js";
 import {
   getProviderRoutingSource,
   initializeProviders,
@@ -113,6 +114,12 @@ import {
   ROUTES,
 } from "../runtime/routes/secret-routes.js";
 
+// A delete looks up the provider connections that resolve their auth through
+// the credential, so the inference schema has to exist. No connection is
+// created here: the refusal is covered by
+// runtime/routes/__tests__/credential-delete-in-use.test.ts.
+await initializeDb();
+
 const addRoute = ROUTES.find(
   (r) => r.method === "POST" && r.endpoint === "secrets",
 )!;

package/src/__tests__/skill-tool-factory.test.ts

@@ -470,6 +470,75 @@ describe("createSkillTool — required/type/enum validation", () => {
     );
   });
 
+  test("repairs array shapes before validation and passes the repaired value to the executor", async () => {
+    const hash = computeSkillVersionHash(tempDir);
+    const tool = createSkillTool(
+      makeEntry({
+        executor: "echo.ts",
+        input_schema: {
+          type: "object",
+          properties: {
+            activation_hints: { type: "array", items: { type: "string" } },
+            avoid_when: { type: "array", items: { type: "string" } },
+            files: { type: "array", items: { type: "object" } },
+            name: { type: "string" },
+          },
+          required: ["activation_hints"],
+        },
+      }),
+      tempDir,
+      hash,
+      BUNDLED,
+    );
+
+    const result = await tool.execute(
+      {
+        activation_hints: '["user asks to deploy staging","needs a rollback"]',
+        avoid_when: "the repo is dirty",
+        files: '[{"path":"references/notes.md","content":"hi"}]',
+        name: "x",
+      },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    const parsed = JSON.parse(result.content);
+    expect(parsed.input).toEqual({
+      activation_hints: ["user asks to deploy staging", "needs a rollback"],
+      avoid_when: ["the repo is dirty"],
+      files: [{ path: "references/notes.md", content: "hi" }],
+      name: "x",
+    });
+  });
+
+  test("rejects a truncated array with a self-correcting message", async () => {
+    const hash = computeSkillVersionHash(tempDir);
+    const tool = createSkillTool(
+      makeEntry({
+        executor: "echo.ts",
+        input_schema: {
+          type: "object",
+          properties: {
+            activation_hints: { type: "array", items: { type: "string" } },
+          },
+        },
+      }),
+      tempDir,
+      hash,
+      BUNDLED,
+    );
+
+    const result = await tool.execute(
+      { activation_hints: "[user asks to deploy staging" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain(
+      "activation_hints must be an array: pass a JSON array, not a string",
+    );
+  });
+
   test("passes valid input through to the executor unchanged", async () => {
     const hash = computeSkillVersionHash(tempDir);
     const tool = createSkillTool(
@@ -680,6 +749,98 @@ function subagentSpawnTool() {
  * than the resolver silently deletes the resolver's behavior, and every test
  * that calls `executeSubagentSpawn` directly still passes.
  */
+describe("createSkillTool: bundled input repairs", () => {
+  const SCAFFOLD_SCHEMA = {
+    type: "object",
+    properties: {
+      skill_id: { type: "string" },
+      body_markdown: { type: "string" },
+      files: {
+        type: "array",
+        items: {
+          type: "object",
+          properties: {
+            path: { type: "string" },
+            content: { type: "string" },
+            copy_from: { type: "string" },
+          },
+          required: ["path"],
+        },
+      },
+    },
+    required: ["skill_id", "body_markdown"],
+  };
+
+  function makeScaffoldTool(bundled: boolean) {
+    return createSkillTool(
+      makeEntry({
+        name: "scaffold_managed_skill",
+        executor: "echo.ts",
+        input_schema: SCAFFOLD_SCHEMA,
+      }),
+      tempDir,
+      computeSkillVersionHash(tempDir),
+      bundled,
+    );
+  }
+
+  test("reads `body` as `body_markdown` and reaches the executor", async () => {
+    const result = await makeScaffoldTool(true).execute(
+      { skill_id: "deploy", body: "# Deploy\n" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(JSON.parse(result.content).input).toEqual({
+      skill_id: "deploy",
+      body_markdown: "# Deploy\n",
+    });
+  });
+
+  test("reads a path-keyed files map as the declared array", async () => {
+    const result = await makeScaffoldTool(true).execute(
+      {
+        skill_id: "deploy",
+        body_markdown: "# Deploy\n",
+        files: { "references/notes.md": "hi" },
+      },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(JSON.parse(result.content).input.files).toEqual([
+      { path: "references/notes.md", content: "hi" },
+    ]);
+  });
+
+  test("accepts the injected activity field the model is shown", async () => {
+    const result = await makeScaffoldTool(true).execute(
+      {
+        skill_id: "deploy",
+        body_markdown: "# Deploy\n",
+        activity: "Writing the deploy skill",
+      },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(JSON.parse(result.content).input.activity).toBe(
+      "Writing the deploy skill",
+    );
+  });
+
+  test("a non-bundled skill reusing the name keeps its own contract", async () => {
+    const result = await makeScaffoldTool(false).execute(
+      { skill_id: "deploy", body: "# Deploy\n" },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("body_markdown is required");
+    expect(result.content).toContain('Unknown parameter "body"');
+  });
+});
+
 describe("createSkillTool: subagent_spawn role validation", () => {
   const ROLE_CASES: ReadonlyArray<readonly [string, SubagentRole]> = [
     ["researcher", "researcher"],

package/src/__tests__/tool-execution-pipeline.benchmark.test.ts

@@ -93,7 +93,7 @@ mock.module("../skills/version-hash.js", () => ({
 // In CI there's no gateway, so we return a sensible default.
 mock.module("../ipc/gateway-client.js", () => ({
   ipcClassifyRisk: async (): Promise<
-    import("../permissions/ipc-risk-types.js").ClassificationResult
+    import("@vellumai/gateway-client").ClassifyRiskIpcResponse
   > => ({
     risk: "low",
     reason: "mock",

package/src/__tests__/tool-executor-lifecycle-events.test.ts

@@ -6,9 +6,11 @@ import type { UsageAttributionSnapshot } from "../usage/attribution.js";
 let checkerDecision: "allow" | "prompt" | "deny" = "allow";
 let checkerReason = "allowed";
 let checkerRisk = "low";
+let classifyThrow: Error | null = null;
 let promptDecision: "allow" | "deny" = "allow";
 let fakeToolResult: ToolExecutionResult = { content: "ok", isError: false };
 let toolThrow: Error | null = null;
+let grantConsumeOk = false;
 
 // ── audit-terminal captures ───────────────────────────────
 // The executor and its permission/approval collaborators no longer emit
@@ -86,13 +88,38 @@ mock.module("../platform/consent-cache.js", () => ({
 
 mock.module("../permissions/checker.js", () => ({
   isDynamicSkillLoadInvocation: () => false,
-  classifyRisk: async () => ({ level: checkerRisk }),
+  // Mirrors the real classifier's two failure modes: it throws on an aborted
+  // signal before doing anything, and it throws when the gateway is
+  // unreachable.
+  classifyRisk: async (
+    _name: string,
+    _input: Record<string, unknown>,
+    _workingDir?: string,
+    _preParsed?: unknown,
+    _manifestOverride?: unknown,
+    signal?: AbortSignal,
+  ) => {
+    signal?.throwIfAborted();
+    if (classifyThrow) {
+      throw classifyThrow;
+    }
+    return { level: checkerRisk };
+  },
   check: async () => ({ decision: checkerDecision, reason: checkerReason }),
-  generateAllowlistOptions: () => [
-    { label: "exact", description: "exact", pattern: "exact" },
-  ],
   generateScopeOptions: () => [{ label: "/tmp", scope: "/tmp" }],
-  getCachedAssessment: () => undefined,
+}));
+
+// The scoped-grant consume the gate performs for a sensitive tool invoked by
+// an unknown actor. `mintGrantFromDecision` is stubbed only so the module's
+// other importer links; nothing here mints.
+mock.module("../approvals/approval-primitive.js", () => ({
+  consumeGrantForInvocation: async () =>
+    grantConsumeOk
+      ? { ok: true, grant: { id: "grant-1" } }
+      : { ok: false, reason: "no_match" },
+  mintGrantFromDecision: async () => {
+    throw new Error("mintGrantFromDecision should not be called");
+  },
 }));
 
 mock.module("../persistence/conversation-crud.js", () => ({
@@ -255,9 +282,11 @@ describe("ToolExecutor audit terminals", () => {
     checkerDecision = "allow";
     checkerReason = "allowed";
     checkerRisk = "low";
+    classifyThrow = null;
     promptDecision = "allow";
     fakeToolResult = { content: "ok", isError: false };
     toolThrow = null;
+    grantConsumeOk = false;
     executedCaptures.length = 0;
     errorCaptures.length = 0;
     deniedCaptures.length = 0;
@@ -786,4 +815,102 @@ describe("ToolExecutor audit terminals", () => {
     expect(executedCaptures).toHaveLength(1);
     expect(executedCaptures[0].toolName).toBe("file_edit");
   });
+
+  // ── audit riskLevel on rows the permission check never produces ─────────
+  // The classifier runs before the pre-execution gates, so a gate denial, a
+  // gate error, and a grant-consumed execution all record the call's real
+  // risk rather than a placeholder (LUM-3159).
+
+  test("a gate-denied call records the classified risk, not a placeholder", async () => {
+    checkerRisk = "high";
+    const executor = new ToolExecutor(makePrompter());
+
+    // Verification control-plane paths are guardian-only; an unknown actor is
+    // denied by the gate before any permission check.
+    const result = await executor.execute(
+      "bash",
+      { command: "curl http://host/v1/channel-verification-sessions" },
+      makeContext({ trustClass: "unknown" }),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(deniedCaptures).toHaveLength(1);
+    expect(deniedCaptures[0].riskLevel).toBe("high");
+    expect(deniedCaptures[0].wasPrompted).toBe(false);
+  });
+
+  test("a gate-errored call records the classified risk", async () => {
+    checkerRisk = "medium";
+    const executor = new ToolExecutor(makePrompter());
+
+    await executor.execute("unknown_tool", { test: true }, makeContext());
+
+    expect(errorCaptures).toHaveLength(1);
+    expect(errorCaptures[0].errorMessage).toContain("Unknown tool");
+    expect(errorCaptures[0].riskLevel).toBe("medium");
+  });
+
+  test("a grant-consumed execution records the classified risk", async () => {
+    checkerRisk = "high";
+    grantConsumeOk = true;
+    const executor = new ToolExecutor(makePrompter());
+
+    // A sensitive tool from an unknown actor routes through the scoped-grant
+    // consume; a consumed grant skips the permission check entirely, so this
+    // is the one executed row whose risk the permission check never supplies.
+    const result = await executor.execute(
+      "file_write",
+      { path: "/tmp/project/out.txt", content: "x" },
+      makeContext({ trustClass: "unknown" }),
+    );
+
+    expect(result).toMatchObject({
+      isError: false,
+      approvalReason: "grant_scoped_consumed",
+    });
+    expect(executedCaptures).toHaveLength(1);
+    expect(executedCaptures[0].decision).toBe("allow");
+    expect(executedCaptures[0].riskLevel).toBe("high");
+  });
+
+  test("a call whose classification never completes records 'unclassified', not 'low'", async () => {
+    classifyThrow = new Error("gateway unreachable");
+    const executor = new ToolExecutor(makePrompter());
+
+    // Gate error path: the row says the risk was never assessed.
+    await executor.execute("unknown_tool", { test: true }, makeContext());
+    expect(errorCaptures).toHaveLength(1);
+    expect(errorCaptures[0].riskLevel).toBe("unclassified");
+
+    // Past the gates, the permission check still requires a classification
+    // and fails closed; that error row is likewise honest about the risk.
+    errorCaptures.length = 0;
+    const result = await executor.execute(
+      "file_read",
+      { path: "README.md" },
+      makeContext(),
+    );
+    expect(result.isError).toBe(true);
+    expect(executedCaptures).toHaveLength(0);
+    expect(errorCaptures).toHaveLength(1);
+    expect(errorCaptures[0].errorMessage).toContain("gateway unreachable");
+    expect(errorCaptures[0].riskLevel).toBe("unclassified");
+  });
+
+  test("a call aborted before it starts records 'unclassified' on its cancelled row", async () => {
+    checkerRisk = "high";
+    const executor = new ToolExecutor(makePrompter());
+    const controller = new AbortController();
+    controller.abort();
+
+    const result = await executor.execute(
+      "bash",
+      { command: "rm -rf /tmp" },
+      makeContext({ signal: controller.signal }),
+    );
+
+    expect(result).toEqual({ content: "Cancelled", isError: true });
+    expect(errorCaptures).toHaveLength(1);
+    expect(errorCaptures[0].riskLevel).toBe("unclassified");
+  });
 });