@vellumai/assistant 0.11.4 → 0.11.5-staging.1

package/src/__tests__/guardian-prompt-notice-privacy.test.ts

@@ -0,0 +1,133 @@
+/**
+ * Where a guardian's own gated-tool prompt is delivered.
+ *
+ * The prompt is raised by a turn the guardian is having, and that turn can be
+ * running in a room: a shared Slack channel, a Telegram group. The card
+ * carries the tool name and a preview of the command, so delivering it where
+ * the turn is shows both to everyone there. It goes to the guardian's own
+ * bound chat instead.
+ *
+ * The address is only half of it. A callback URL addresses the turn it came
+ * from, and a param left on one either fails the send or wins over the new
+ * address outright, which is indistinguishable from never having redirected.
+ * The assertions below are written against params the transports read today
+ * *and* against one nothing reads, because the guarantee is that everything is
+ * dropped rather than that a known list is.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import {
+  resolveGuardianPromptDelivery,
+  stripTurnDestination,
+} from "../approvals/guardian-channel-delivery.js";
+
+import "./test-preload.js";
+
+const ROOM = "C0SHARED123";
+const GUARDIAN_CHAT = "D0GUARDIANDM";
+
+describe("a prompt raised where the guardian is not alone", () => {
+  test("is addressed to the guardian's own chat", () => {
+    expect(
+      resolveGuardianPromptDelivery({
+        turnChatId: ROOM,
+        turnCallbackUrl: "https://gw.test/deliver/slack",
+        guardianChatId: GUARDIAN_CHAT,
+      }).chatId,
+    ).toBe(GUARDIAN_CHAT);
+  });
+
+  test("applies to any channel, including ones not built yet", () => {
+    // A Telegram group carries the same exposure as a shared Slack channel,
+    // and so will whatever is added next. The rule names no channel, so there
+    // is no per-channel list to keep in step with the transports.
+    for (const [turnChatId, guardianChatId] of [
+      ["-1001234567890", "555001"],
+      ["guild-channel-1", "dm-channel-1"],
+      ["room@some-future-channel", "direct@some-future-channel"],
+    ]) {
+      expect(
+        resolveGuardianPromptDelivery({
+          turnChatId,
+          turnCallbackUrl: "https://gw.test/deliver/whatever",
+          guardianChatId,
+        }).chatId,
+      ).toBe(guardianChatId);
+    }
+  });
+});
+
+describe("a prompt raised in the guardian's own chat", () => {
+  const turnCallbackUrl = "https://gw.test/deliver/slack?threadTs=111.222";
+
+  test("does not move, and keeps the turn's callback intact", () => {
+    expect(
+      resolveGuardianPromptDelivery({
+        turnChatId: GUARDIAN_CHAT,
+        turnCallbackUrl,
+        guardianChatId: GUARDIAN_CHAT,
+      }),
+    ).toEqual({ chatId: GUARDIAN_CHAT, callbackUrl: turnCallbackUrl });
+  });
+
+  test("does not move when no binding resolved", () => {
+    // The trust context falls back to the turn's own chat, so this is the same
+    // comparison as above: delivery is left exactly as it is today rather than
+    // dropped on the floor.
+    expect(
+      resolveGuardianPromptDelivery({
+        turnChatId: ROOM,
+        turnCallbackUrl,
+        guardianChatId: undefined,
+      }),
+    ).toEqual({ chatId: ROOM, callbackUrl: turnCallbackUrl });
+  });
+});
+
+describe("a redirected delivery leaves the turn behind entirely", () => {
+  const redirected = (turnCallbackUrl: string) =>
+    resolveGuardianPromptDelivery({
+      turnChatId: ROOM,
+      turnCallbackUrl,
+      guardianChatId: GUARDIAN_CHAT,
+    }).callbackUrl;
+
+  test("drops params the transports read today", () => {
+    // threadTs: Slack's thread, absent from the new chat.
+    // threadId: a Telegram forum topic, and on Discord the destination itself
+    // (`sendTarget` returns it in place of the chatId).
+    // dm: changes how Discord reads the address.
+    // channel: the turn's own chat, which the gateway hangs on every Slack
+    // callback.
+    expect(
+      redirected(
+        "https://gw.test/deliver/slack?channel=C1&threadTs=1.2&threadId=9&dm=1&messageTs=3.4",
+      ),
+    ).toBe("https://gw.test/deliver/slack");
+  });
+
+  test("drops a param nothing reads, which is the actual guarantee", () => {
+    // If this only dropped a known list, a transport that starts reading a new
+    // param would silently undo the redirect and no test would notice. The
+    // contract is that the callback keeps its channel and nothing else.
+    expect(redirected("https://gw.test/deliver/x?roomId=42")).toBe(
+      "https://gw.test/deliver/x",
+    );
+  });
+
+  test("keeps the channel, which is how the transport is resolved", () => {
+    // `isDirectDelivery` picks a transport off the path, so the path has to
+    // survive for the redirect to be deliverable at all.
+    expect(redirected("https://gw.test/deliver/telegram?threadId=7")).toContain(
+      "/deliver/telegram",
+    );
+  });
+});
+
+describe("stripTurnDestination", () => {
+  test("returns relative or malformed urls untouched", () => {
+    // They carry no params, and `new URL` throws on them.
+    expect(stripTurnDestination("/deliver/slack")).toBe("/deliver/slack");
+  });
+});

package/src/__tests__/guardian-verify-setup-skill-regression.test.ts

@@ -1,8 +1,12 @@
 /**
  * Regression tests for the guardian-verify-setup skill.
  *
- * Ensures the voice verification flow includes proactive auto-check polling
- * so the user does not have to manually ask whether verification succeeded.
+ * The behaviour under guard is proactive auto-check polling: after a code is
+ * delivered, the skill must poll for completion rather than leaving the user
+ * to ask whether it worked. Originally that existed only for voice, and these
+ * tests pinned the voice section. The three per-channel polling sections were
+ * later merged into one parameterised section, so the assertions here pin the
+ * behaviour in its shared form and cover every channel that polls.
  */
 
 import { readFileSync } from "node:fs";
@@ -23,128 +27,170 @@ const SKILL_PATH = resolve(
 
 const skillContent = readFileSync(SKILL_PATH, "utf-8");
 
+/**
+ * The channels that poll. Telegram is deliberately absent: it confirms through
+ * its own bot-driven flow, so polling it reports nothing.
+ */
+const POLLED_CHANNELS = ["phone", "slack", "discord", "email"] as const;
+
+function section(from: string, to: string): string {
+  const body = skillContent.split(from)[1]?.split(to)[0];
+  // An empty slice would make every `toContain` below fail loudly rather than
+  // silently pass, which is the intent: a renamed heading must break this.
+  return body ?? "";
+}
+
+const pollingSection = () => section("## Auto-Check Polling", "## Step 6");
+
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
 
-describe("guardian-verify-setup skill — voice auto-followup", () => {
-  test("voice path in Step 3 references the auto-check polling loop", () => {
-    // The voice success instruction in Step 3 must direct the assistant to
-    // begin the polling loop rather than waiting for the user to report back.
-    expect(skillContent).toContain(
-      "immediately begin the voice auto-check polling loop",
-    );
+describe("guardian-verify-setup skill: proactive auto-check polling", () => {
+  test("the polling section exists", () => {
+    expect(skillContent).toContain("## Auto-Check Polling");
+  });
+
+  test("Step 3 sends every polled channel into the loop", () => {
+    const step3 = section("## Step 3", "## Step 4");
+    for (const channel of POLLED_CHANNELS) {
+      const bullet = step3
+        .split("\n")
+        .filter((line) =>
+          new RegExp(
+            `^\\s*-\\s+\\*\\*(Phone|Slack|Discord|Email)\\*\\*`,
+            "i",
+          ).test(line),
+        )
+        .find((line) => new RegExp(channel, "i").test(line));
+      expect(bullet, `Step 3 has no bullet for ${channel}`).toBeDefined();
+      expect(bullet).toContain("auto-check polling loop");
+    }
   });
 
-  test("voice path in Step 4 (resend) references the auto-check polling loop", () => {
-    // After a voice resend, the same auto-check behavior must kick in.
-    const resendSection =
-      skillContent.split("## Step 4")[1]?.split("## Step 5")[0] ?? "";
-    expect(resendSection).toContain("voice auto-check polling loop");
+  test("Step 4 resend sends every polled channel back into the loop", () => {
+    const resend = section("## Step 4", "## Step 5");
+    for (const channel of POLLED_CHANNELS) {
+      expect(
+        new RegExp(`\\*\\*${channel}\\*\\*`, "i").test(resend),
+        `Step 4 has no bullet for ${channel}`,
+      ).toBe(true);
+    }
+    expect(resend).toContain("auto-check polling loop");
   });
 
-  test("contains a Voice Auto-Check Polling section", () => {
-    expect(skillContent).toContain("## Voice Auto-Check Polling");
+  test("the polling section names every polled channel and excludes Telegram", () => {
+    const body = pollingSection();
+    for (const channel of POLLED_CHANNELS) {
+      expect(body, `polling section omits ${channel}`).toContain(channel);
+    }
+    // Telegram may only appear as an exclusion, never as a channel to poll.
+    expect(body).toContain("Never Telegram");
   });
 
-  test("polling section specifies the correct status command for voice", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain(
-      "assistant channel-verification-sessions status --channel phone --json",
+  test("the polling command is guarded against an unset channel", () => {
+    const body = pollingSection();
+    // The command is parameterised, so the channel has to be assigned in the
+    // same block. An unset CHANNEL would call the CLI with an empty value.
+    expect(body).toContain('CHANNEL=""');
+    expect(body).toContain('if [ -z "$CHANNEL" ]');
+    expect(body).toContain(
+      'channel-verification-sessions status --channel "$CHANNEL" --json',
     );
   });
 
-  test("polling section includes ~15 second interval", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("~15 seconds");
+  test("the polling section states an interval and a timeout per channel", () => {
+    const body = pollingSection();
+    expect(body).toContain("15s");
+    expect(body).toContain("20s");
+    expect(body).toContain("2 minutes");
+    expect(body).toContain("3 minutes");
   });
 
-  test("polling section includes 2-minute timeout", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("2 minutes");
+  test("the polling section checks for bound: true", () => {
+    expect(pollingSection()).toContain("bound: true");
   });
 
-  test("polling section checks for bound: true", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("bound: true");
+  test("success is reported proactively, not on request", () => {
+    const body = pollingSection();
+    expect(body).toContain("success message");
+    expect(body).toContain("Do NOT require the user to ask");
   });
 
-  test("polling section includes proactive success confirmation", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("proactive success message");
+  test("timeout offers a resend rather than stopping silently", () => {
+    const body = pollingSection();
+    expect(body).toContain("timeout");
+    expect(body).toContain("resend");
   });
 
-  test("polling section includes timeout fallback with resend/restart offer", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("timeout");
-    expect(pollingSection).toContain("resend");
+  test("the rebind guard survives, with its false-success reasoning", () => {
+    const body = pollingSection();
+    expect(body).toContain("Rebind guard");
+    expect(body).toContain("verificationSessionId");
+    expect(body).toContain("Non-rebind flows");
   });
 
-  test("polling section includes rebind guard against false-success from pre-existing binding", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    // Must mention rebind guard concept
-    expect(pollingSection).toContain("Rebind guard");
-    // Must instruct not to trust bound: true alone in a rebind flow
-    expect(pollingSection).toContain(
-      "do NOT treat `bound: true` alone as success",
-    );
-    // Must reference verificationSessionId as the mechanism to detect fresh binding
-    expect(pollingSection).toContain("verificationSessionId");
-    // Must clarify non-rebind flows are unaffected
-    expect(pollingSection).toContain("Non-rebind flows");
+  test("no polled channel's Step 3 bullet tells the assistant to wait", () => {
+    const step3 = section("## Step 3", "## Step 4");
+    // Narrowed to the polled bullets: Telegram's "wait for the user to confirm
+    // they clicked the link" is a real instruction for its own bootstrap flow.
+    const bullets = step3
+      .split("\n")
+      .filter((line) =>
+        /^\s*-\s+\*\*(Phone|Slack|Discord|Email)\*\*/i.test(line),
+      );
+    expect(bullets.length).toBe(POLLED_CHANNELS.length);
+    for (const bullet of bullets) {
+      expect(bullet).not.toContain("wait for the user to confirm");
+      expect(bullet).not.toContain("ask the user if it worked");
+    }
+  });
+});
+
+describe("guardian-verify-setup skill: channel coverage", () => {
+  test("every channel the CLI accepts is offered in Step 1", () => {
+    const step1 = section("## Step 1", "## Step 2");
+    for (const channel of [...POLLED_CHANNELS, "telegram"]) {
+      expect(step1, `Step 1 does not offer ${channel}`).toContain(
+        `**${channel}**`,
+      );
+    }
+  });
+
+  test("Step 2 collects a destination for every channel", () => {
+    const step2 = section("## Step 2", "## Step 3");
+    for (const channel of ["Phone", "Telegram", "Slack", "Discord", "Email"]) {
+      expect(step2, `Step 2 has no destination for ${channel}`).toContain(
+        `**${channel}**`,
+      );
+    }
   });
 
-  test("polling is voice-only — does not apply to Telegram", () => {
-    const pollingSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(pollingSection).toContain("voice-only");
-    expect(pollingSection).toContain("Do NOT poll for Telegram");
+  test("the missing-secret guardrail names only its exception", () => {
+    const guardrail = skillContent
+      .split("\n")
+      .find((line) => line.includes("Missing `secret` guardrail"));
+    expect(guardrail).toBeDefined();
+    // The rule covers every flow but one, so the line states that exception.
+    // Naming any other channel makes it an enumeration, and an enumeration
+    // drifts out of date the next time a channel is added.
+    expect(guardrail).toContain("except");
+    expect(guardrail).toContain("Telegram");
+    const lower = guardrail!.toLowerCase();
+    for (const channel of ["phone", "voice", "slack", "discord", "email"]) {
+      expect(lower, `guardrail enumerates ${channel}`).not.toContain(channel);
+    }
   });
 
-  test('no instruction requires waiting for user to ask "did it work?"', () => {
-    // The skill should never instruct the assistant to wait for the user to
-    // confirm that voice verification worked. The auto-check polling loop
-    // makes this unnecessary.
-    const voiceAutoCheckSection =
-      skillContent
-        .split("## Voice Auto-Check Polling")[1]
-        ?.split("## Step 6")[0] ?? "";
-    expect(voiceAutoCheckSection).toContain("Do NOT require the user to ask");
-    // The voice bullet in Step 3 should not instruct the assistant to wait
-    // for the user to confirm or ask if it worked. Narrow to just the voice
-    // bullet line to avoid false positives from Telegram's "wait for the
-    // user to confirm they clicked the link" which is unrelated to voice.
-    const step3Section =
-      skillContent.split("## Step 3")[1]?.split("## Step 4")[0] ?? "";
-    const voiceBullet = step3Section
+  test("the status and revoke guards allow every supported channel", () => {
+    // These enumerate valid CHANNEL values. A channel missing here reads to
+    // the assistant as unsupported at exactly the point a user asks for it.
+    const guards = skillContent
       .split("\n")
-      .filter((line) => /^\s*-\s+\*\*Phone\*\*/.test(line))
-      .join("\n");
-    expect(voiceBullet).not.toHaveLength(0);
-    expect(voiceBullet).not.toContain("wait for the user to confirm");
-    expect(voiceBullet).not.toContain("ask the user if it worked");
+      .filter((line) => line.includes("MUST set to one of"));
+    expect(guards.length).toBeGreaterThanOrEqual(2);
+    for (const guard of guards) {
+      expect(guard).toContain("discord");
+    }
   });
 });

package/src/__tests__/helpers/gateway-classify-mock.ts

@@ -17,7 +17,7 @@
  *   installIpcMock();
  *
  *   mockIpcResponse("classify_risk",
- *     { risk: "low", reason: "ls", matchType: "shell" });
+ *     { risk: "low", reason: "ls", matchType: "registry", scopeOptions: [] });
  *
  *   const result = await classifyRisk("bash", { command: "ls" });
  */
@@ -36,12 +36,26 @@ export function mockIpcResponse(method: string, response: unknown): void {
   responses.set(method, response);
 }
 
-/** Clear all registered responses. */
+/** Clear all registered responses and captured params. */
 export function clearIpcMocks(): void {
   responses.clear();
+  lastParams.clear();
 }
 
-function handleCall(method: string): unknown {
+const lastParams = new Map<string, Record<string, unknown> | undefined>();
+
+/**
+ * The params of the most recent call to `method`, for asserting what the
+ * daemon sent rather than what it did with the answer.
+ */
+export function lastIpcParams(
+  method: string,
+): Record<string, unknown> | undefined {
+  return lastParams.get(method);
+}
+
+function handleCall(method: string, params?: Record<string, unknown>): unknown {
+  lastParams.set(method, params);
   const response = responses.get(method);
   if (response !== undefined) {
     return response;
@@ -57,11 +71,18 @@ function handleCall(method: string): unknown {
  */
 export function installIpcMock(): void {
   mock.module("@vellumai/gateway-client/ipc-client", () => ({
-    ipcCall: async (_socketPath: string, method: string) => handleCall(method),
+    ipcCall: async (
+      _socketPath: string,
+      method: string,
+      params?: Record<string, unknown>,
+    ) => handleCall(method, params),
 
     PersistentIpcClient: class MockPersistentIpcClient {
-      async call(method: string): Promise<unknown> {
-        return handleCall(method);
+      async call(
+        method: string,
+        params?: Record<string, unknown>,
+      ): Promise<unknown> {
+        return handleCall(method, params);
       }
       destroy(): void {}
     },

package/src/__tests__/host-proxy-interface.test.ts

@@ -16,6 +16,7 @@ import { isHostProxyTransport } from "../daemon/message-types/conversations.js";
 describe("supportsHostProxy (runtime)", () => {
   test("no-arg form returns true for host-proxy interfaces", () => {
     expect(supportsHostProxy("macos")).toBe(true);
+    expect(supportsHostProxy("windows")).toBe(true);
   });
 
   test("no-arg form returns false for interfaces without host-proxy support", () => {
@@ -42,13 +43,22 @@ describe("supportsHostProxy (runtime)", () => {
     expect(supportsHostProxy("chrome-extension", "host_cu")).toBe(false);
   });
 
-  test("capability form grants all four capabilities to macOS including host_browser", () => {
+  test("capability form grants representative host capabilities to macOS", () => {
     expect(supportsHostProxy("macos", "host_bash")).toBe(true);
     expect(supportsHostProxy("macos", "host_file")).toBe(true);
     expect(supportsHostProxy("macos", "host_cu")).toBe(true);
     expect(supportsHostProxy("macos", "host_browser")).toBe(true);
   });
 
+  test("capability form withholds unimplemented Windows app control", () => {
+    expect(supportsHostProxy("windows", "host_bash")).toBe(true);
+    expect(supportsHostProxy("windows", "host_file")).toBe(true);
+    expect(supportsHostProxy("windows", "host_cu")).toBe(true);
+    expect(supportsHostProxy("windows", "host_browser")).toBe(true);
+    expect(supportsHostProxy("windows", "host_ui_snapshot")).toBe(true);
+    expect(supportsHostProxy("windows", "host_app_control")).toBe(false);
+  });
+
   test("capability form rejects everything for non-host-proxy interfaces", () => {
     expect(supportsHostProxy("ios", "host_bash")).toBe(false);
     expect(supportsHostProxy("cli", "host_file")).toBe(false);

package/src/__tests__/host-shell-tool.test.ts

@@ -678,6 +678,14 @@ describe("host_bash — spawn error handling", () => {
 // ---------------------------------------------------------------------------
 
 describe("host_bash — proxy delegation", () => {
+  // Under tmpdir so the live-workspace guard accepts it as a workspace.
+  const FAKE_INSTANCE_WS = join(
+    tmpdir(),
+    "vellum-instance",
+    ".vellum",
+    "workspace",
+  );
+
   const ROUTING_ENV_KEYS = [
     "VELLUM_WORKSPACE_DIR",
     "VELLUM_DATA_DIR",
@@ -850,6 +858,11 @@ describe("host_bash — proxy delegation", () => {
     for (const key of ROUTING_ENV_KEYS) {
       delete process.env[key];
     }
+    // With VELLUM_WORKSPACE_DIR unset, loadConfig resolves the real ~/.vellum
+    // fallback (read-only here), so bypass the live-workspace guard for it.
+    const originalAllowRealWorkspace =
+      process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS;
+    process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS = "1";
 
     try {
       const proxyResult: ToolExecutionResult = {
@@ -870,14 +883,20 @@ describe("host_bash — proxy delegation", () => {
         __REVEAL_NONCE: expect.any(String),
       });
     } finally {
+      if (originalAllowRealWorkspace === undefined) {
+        delete process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS;
+      } else {
+        process.env.VELLUM_ALLOW_REAL_WORKSPACE_IN_TESTS =
+          originalAllowRealWorkspace;
+      }
       restoreEnv(envSnapshot);
     }
   });
 
   test("propagates daemon routing env vars to proxy for nested assistant CLI calls", async () => {
     const envSnapshot = captureEnv(ROUTING_ENV_KEYS);
-    process.env.VELLUM_WORKSPACE_DIR = "/tmp/vellum-instance/.vellum/workspace";
-    process.env.VELLUM_DATA_DIR = "/tmp/vellum-instance/.vellum/workspace/data";
+    process.env.VELLUM_WORKSPACE_DIR = FAKE_INSTANCE_WS;
+    process.env.VELLUM_DATA_DIR = join(FAKE_INSTANCE_WS, "data");
     process.env.VELLUM_ENVIRONMENT = "local";
     process.env.INTERNAL_GATEWAY_BASE_URL = "http://127.0.0.1:7830";
 
@@ -896,8 +915,8 @@ describe("host_bash — proxy delegation", () => {
       expect(result).toBe(proxyResult);
       expect(calls.length).toBe(1);
       expect(calls[0].input.env).toEqual({
-        VELLUM_WORKSPACE_DIR: "/tmp/vellum-instance/.vellum/workspace",
-        VELLUM_DATA_DIR: "/tmp/vellum-instance/.vellum/workspace/data",
+        VELLUM_WORKSPACE_DIR: FAKE_INSTANCE_WS,
+        VELLUM_DATA_DIR: join(FAKE_INSTANCE_WS, "data"),
         VELLUM_ENVIRONMENT: "local",
         INTERNAL_GATEWAY_BASE_URL: "http://127.0.0.1:7830",
         __CONVERSATION_ID: "test-conversation",