@vellumai/assistant 0.9.0 → 0.10.0-dev.202606192226.8ed8a6a

package/src/__tests__/conversation-title-service.test.ts

@@ -69,10 +69,38 @@ import {
 describe("conversation-title-service", () => {
   beforeEach(() => {
     mockRunBtwSidechain.mockClear();
+    mockRunBtwSidechain.mockImplementation(
+      async (_params: Record<string, unknown>) => ({
+        text: "Project kickoff",
+        hadTextDeltas: true,
+        response: {
+          content: [{ type: "text", text: "Project kickoff" }],
+          model: "test-model",
+          usage: { inputTokens: 10, outputTokens: 5 },
+          stopReason: "end_turn",
+        },
+      }),
+    );
     mockGetConversation.mockClear();
+    mockGetConversation.mockImplementation(
+      (_conversationId: string) =>
+        ({
+          title: "Generating title...",
+          isAutoTitle: 1,
+        }) as {
+          title: string;
+          isAutoTitle: number;
+        },
+    );
     mockGetMessages.mockClear();
+    mockGetMessages.mockImplementation(() => [
+      { role: "user", content: "first message" },
+      { role: "assistant", content: "first reply" },
+      { role: "user", content: "follow-up" },
+    ]);
     mockUpdateConversationTitle.mockClear();
     mockGetConfiguredProvider.mockClear();
+    mockGetConfiguredProvider.mockImplementation(async () => null);
     mockPublishConversationTitleChanged.mockClear();
   });
 
@@ -232,6 +260,30 @@ describe("conversation-title-service", () => {
     );
   });
 
+  test("fallback retry skips regeneration after a successful initial title", async () => {
+    mockGetConversation.mockReturnValueOnce({
+      title: "Project kickoff",
+      isAutoTitle: 1,
+    });
+
+    const provider = {
+      name: "test-provider",
+      sendMessage: mock(async () => {
+        throw new Error("should not call directly");
+      }),
+    };
+
+    const result = await regenerateConversationTitle({
+      conversationId: "conv-1",
+      provider,
+      onlyIfReplaceable: true,
+    });
+
+    expect(result).toEqual({ title: "Project kickoff", updated: false });
+    expect(mockRunBtwSidechain).not.toHaveBeenCalled();
+    expect(mockUpdateConversationTitle).not.toHaveBeenCalled();
+  });
+
   test("rejects meta-failure outputs like 'Missing Context' and uses fallback", async () => {
     mockRunBtwSidechain.mockImplementationOnce(async () => ({
       text: "Missing Context",
@@ -482,6 +534,16 @@ describe("conversation-title-service", () => {
 
     // Both calls went through — failure didn't break the chain
     expect(mockRunBtwSidechain).toHaveBeenCalledTimes(2);
+    const firstUpdate = (
+      mockUpdateConversationTitle.mock.calls as unknown as Array<
+        [string, string, number?]
+      >
+    ).find((c) => c[0] === "conv-1");
+    expect(firstUpdate).toEqual([
+      "conv-1",
+      "Untitled Conversation",
+      AUTO_TITLE_DETERMINISTIC,
+    ]);
     // Second conversation got a proper title
     const secondUpdate = (
       mockUpdateConversationTitle.mock.calls as unknown as string[][]

package/src/__tests__/conversation-workspace-cache-state.test.ts

@@ -139,6 +139,12 @@ mock.module("../memory/retrieval-budget.js", () => ({
 
 mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
+    estimateInputTokens() {
+      return 0;
+    }
+    get tokenCountInputs() {
+      return { systemPrompt: "", tools: undefined };
+    }
     constructor() {}
     updateConfig() {}
     shouldCompact() {

package/src/__tests__/conversation-workspace-injection.test.ts

@@ -178,6 +178,12 @@ mock.module("../memory/retrieval-budget.js", () => ({
 }));
 mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
+    estimateInputTokens() {
+      return 0;
+    }
+    get tokenCountInputs() {
+      return { systemPrompt: "", tools: undefined };
+    }
     updateConfig() {}
     shouldCompact() {
       return { needed: false, estimatedTokens: 0 };

package/src/__tests__/conversation-workspace-tool-tracking.test.ts

@@ -175,6 +175,12 @@ mock.module("../memory/retrieval-budget.js", () => ({
 }));
 mock.module("../plugins/defaults/compaction/window-manager.js", () => ({
   ContextWindowManager: class {
+    estimateInputTokens() {
+      return 0;
+    }
+    get tokenCountInputs() {
+      return { systemPrompt: "", tools: undefined };
+    }
     updateConfig() {}
     shouldCompact() {
       return { needed: false, estimatedTokens: 0 };

package/src/__tests__/credential-broker.test.ts

@@ -1,14 +1,24 @@
-import { mkdtempSync, rmSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 
 import { credentialKey } from "../security/credential-key.js";
+import { _resetBackend, setSecureKeyAsync } from "../security/secure-keys.js";
 import { CredentialBroker } from "../tools/credentials/broker.js";
 import {
   _setMetadataPath,
   upsertCredentialMetadata,
 } from "../tools/credentials/metadata-store.js";
+import { BROWSER_FILL_CAPABILITY } from "../tools/credentials/tool-policy.js";
+import { setStorePathForTesting } from "./encrypted-store-test-helpers.js";
+
+const TEST_DIR = join(
+  tmpdir(),
+  `vellum-broker-test-${randomBytes(4).toString("hex")}`,
+);
+const STORE_PATH = join(TEST_DIR, "keys.enc");
 
 describe("CredentialBroker", () => {
   let broker: CredentialBroker;
@@ -222,3 +232,441 @@ describe("CredentialBroker", () => {
     });
   });
 });
+
+// ---------------------------------------------------------------------------
+// Behavior that reads stored secret values, exercised against the encrypted
+// store backend (transient injection, multi-template serverUseById, revokeAll,
+// and canonical/legacy capability-key resolution).
+// ---------------------------------------------------------------------------
+
+describe("CredentialBroker (encrypted store backend)", () => {
+  let broker: CredentialBroker;
+
+  beforeEach(() => {
+    if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true });
+    mkdirSync(TEST_DIR, { recursive: true });
+    setStorePathForTesting(STORE_PATH);
+    _resetBackend();
+    _setMetadataPath(join(TEST_DIR, "metadata.json"));
+    broker = new CredentialBroker();
+  });
+
+  afterEach(() => {
+    _setMetadataPath(null);
+    setStorePathForTesting(null);
+    _resetBackend();
+    rmSync(TEST_DIR, { recursive: true, force: true });
+  });
+
+  describe("transient credentials", () => {
+    test("consume returns transient value and deletes it", () => {
+      // GIVEN a credential with metadata and an injected one-time transient value
+      upsertCredentialMetadata("svc", "key", { allowedTools: ["tool1"] });
+      broker.injectTransient("svc", "key", "one-time-secret");
+
+      // WHEN the credential is authorized and consumed
+      const auth = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "tool1",
+      });
+      expect(auth.authorized).toBe(true);
+      if (!auth.authorized) return;
+      const result = broker.consume(auth.token.tokenId);
+
+      // THEN the transient value is returned alongside the storage key
+      expect(result.success).toBe(true);
+      expect(result.value).toBe("one-time-secret");
+      expect(result.storageKey).toBe(credentialKey("svc", "key"));
+
+      // AND a subsequent authorize+consume no longer has the transient value
+      const auth2 = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "tool1",
+      });
+      expect(auth2.authorized).toBe(true);
+      if (!auth2.authorized) return;
+      const result2 = broker.consume(auth2.token.tokenId);
+      expect(result2.success).toBe(true);
+      expect(result2.value).toBeUndefined();
+    });
+
+    test("browserFill uses transient value when available", async () => {
+      // GIVEN a credential with a transient value and no stored value
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: ["browser_fill_credential"],
+      });
+      broker.injectTransient("github", "token", "transient-ghp-123");
+
+      // WHEN browserFill runs
+      let filledValue: string | undefined;
+      const result = await broker.browserFill({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+        fill: async (v) => {
+          filledValue = v;
+        },
+      });
+
+      // THEN the transient value is filled
+      expect(result.success).toBe(true);
+      expect(filledValue).toBe("transient-ghp-123");
+    });
+
+    test("browserFill consumes transient value — second fill falls back to stored", async () => {
+      // GIVEN a credential with both a stored value and a transient value
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: ["browser_fill_credential"],
+      });
+      await setSecureKeyAsync(credentialKey("github", "token"), "stored-value");
+      broker.injectTransient("github", "token", "transient-value");
+
+      // WHEN browserFill runs twice
+      let filled1: string | undefined;
+      await broker.browserFill({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+        fill: async (v) => {
+          filled1 = v;
+        },
+      });
+      let filled2: string | undefined;
+      await broker.browserFill({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+        fill: async (v) => {
+          filled2 = v;
+        },
+      });
+
+      // THEN the first fill uses the transient and the second falls back to stored
+      expect(filled1).toBe("transient-value");
+      expect(filled2).toBe("stored-value");
+    });
+
+    test("browserFill preserves transient value on fill failure", async () => {
+      // GIVEN a credential with a transient value
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: ["browser_fill_credential"],
+      });
+      broker.injectTransient("github", "token", "transient-preserved");
+
+      // WHEN the first fill throws
+      const result1 = await broker.browserFill({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+        fill: async () => {
+          throw new Error("Playwright timeout");
+        },
+      });
+      expect(result1.success).toBe(false);
+      let filled: string | undefined;
+      const result2 = await broker.browserFill({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+        fill: async (v) => {
+          filled = v;
+        },
+      });
+
+      // THEN the transient value survives for the second fill
+      expect(result2.success).toBe(true);
+      expect(filled).toBe("transient-preserved");
+    });
+
+    test("serverUse uses transient value when available", async () => {
+      // GIVEN a credential with a transient value
+      upsertCredentialMetadata("vercel", "api_token", {
+        allowedTools: ["deploy"],
+      });
+      broker.injectTransient("vercel", "api_token", "transient-vercel-tok");
+
+      // WHEN serverUse runs
+      const result = await broker.serverUse({
+        service: "vercel",
+        field: "api_token",
+        toolName: "deploy",
+        execute: async (v) => {
+          expect(v).toBe("transient-vercel-tok");
+          return "deployed";
+        },
+      });
+
+      // THEN the transient value is used for execution
+      expect(result.success).toBe(true);
+      expect(result.result).toBe("deployed");
+    });
+
+    test("serverUse consumes transient — subsequent call has no value without stored key", async () => {
+      // GIVEN a credential with only a transient value (no stored value)
+      upsertCredentialMetadata("vercel", "api_token", {
+        allowedTools: ["deploy"],
+      });
+      broker.injectTransient("vercel", "api_token", "transient-only");
+
+      // WHEN serverUse runs twice
+      await broker.serverUse({
+        service: "vercel",
+        field: "api_token",
+        toolName: "deploy",
+        execute: async () => "ok",
+      });
+      const result = await broker.serverUse({
+        service: "vercel",
+        field: "api_token",
+        toolName: "deploy",
+        execute: async () => {
+          throw new Error("should not be called");
+        },
+      });
+
+      // THEN the second call fails because no stored value remains
+      expect(result.success).toBe(false);
+      expect(result.reason).toContain("no stored value");
+    });
+
+    test("injectTransient replaces previous transient for same key", () => {
+      // GIVEN two transient values injected for the same credential
+      upsertCredentialMetadata("svc", "key", { allowedTools: ["t"] });
+      broker.injectTransient("svc", "key", "first");
+      broker.injectTransient("svc", "key", "second");
+
+      // WHEN the credential is authorized and consumed
+      const auth = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "t",
+      });
+      if (!auth.authorized) return;
+      const result = broker.consume(auth.token.tokenId);
+
+      // THEN the most recent transient value wins
+      expect(result.value).toBe("second");
+    });
+
+    test("transient value for one credential does not affect another", () => {
+      // GIVEN a transient value injected for svcA only
+      upsertCredentialMetadata("svcA", "key", { allowedTools: ["t"] });
+      upsertCredentialMetadata("svcB", "key", { allowedTools: ["t"] });
+      broker.injectTransient("svcA", "key", "val-a");
+
+      // WHEN both credentials are authorized and consumed
+      const authB = broker.authorize({
+        service: "svcB",
+        field: "key",
+        toolName: "t",
+      });
+      if (!authB.authorized) return;
+      const resultB = broker.consume(authB.token.tokenId);
+      const authA = broker.authorize({
+        service: "svcA",
+        field: "key",
+        toolName: "t",
+      });
+      if (!authA.authorized) return;
+      const resultA = broker.consume(authA.token.tokenId);
+
+      // THEN only svcA carries the transient value
+      expect(resultB.success).toBe(true);
+      expect(resultB.value).toBeUndefined();
+      expect(resultA.value).toBe("val-a");
+    });
+  });
+
+  describe("serverUseById edge cases", () => {
+    test("serverUseById with multiple injection templates returns all", async () => {
+      // GIVEN a credential with two injection templates and a stored value
+      const meta = upsertCredentialMetadata("multi", "api_key", {
+        allowedTools: ["proxy"],
+        injectionTemplates: [
+          {
+            hostPattern: "*.fal.ai",
+            injectionType: "header",
+            headerName: "Authorization",
+            valuePrefix: "Key ",
+          },
+          {
+            hostPattern: "gateway.fal.ai",
+            injectionType: "header",
+            headerName: "X-Fal-Key",
+          },
+        ],
+      });
+      await setSecureKeyAsync(
+        credentialKey("multi", "api_key"),
+        "multi-secret",
+      );
+
+      // WHEN serverUseById resolves the credential by id
+      const result = await broker.serverUseById({
+        credentialId: meta.credentialId,
+        requestingTool: "proxy",
+      });
+
+      // THEN both injection templates are returned without leaking the secret
+      expect(result.success).toBe(true);
+      if (!result.success) return;
+      expect(result.injectionTemplates).toHaveLength(2);
+      expect(result.injectionTemplates[0].hostPattern).toBe("*.fal.ai");
+      expect(result.injectionTemplates[1].hostPattern).toBe("gateway.fal.ai");
+      expect(JSON.stringify(result)).not.toContain("multi-secret");
+    });
+  });
+
+  describe("revokeAll", () => {
+    test("revokeAll clears all tokens and subsequent consume fails", () => {
+      // GIVEN two active tokens for a credential
+      upsertCredentialMetadata("svc", "key", { allowedTools: ["t1", "t2"] });
+      const a1 = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "t1",
+      });
+      const a2 = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "t2",
+      });
+      expect(broker.activeTokenCount).toBe(2);
+
+      // WHEN revokeAll is called
+      broker.revokeAll();
+
+      // THEN the token count is zero and consuming either token fails
+      expect(broker.activeTokenCount).toBe(0);
+      if (a1.authorized) {
+        expect(broker.consume(a1.token.tokenId).success).toBe(false);
+      }
+      if (a2.authorized) {
+        expect(broker.consume(a2.token.tokenId).success).toBe(false);
+      }
+    });
+
+    test("revokeAll on empty broker is a no-op", () => {
+      // GIVEN a broker with no active tokens
+      expect(broker.activeTokenCount).toBe(0);
+
+      // WHEN revokeAll is called
+      broker.revokeAll();
+
+      // THEN the token count remains zero
+      expect(broker.activeTokenCount).toBe(0);
+    });
+  });
+
+  describe("canonical capability key", () => {
+    test("authorize succeeds with canonical key when metadata has canonical key", () => {
+      // GIVEN metadata allowing the canonical browser-fill capability key
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: [BROWSER_FILL_CAPABILITY],
+      });
+
+      // WHEN authorizing with the canonical key
+      const result = broker.authorize({
+        service: "github",
+        field: "token",
+        toolName: BROWSER_FILL_CAPABILITY,
+      });
+
+      // THEN authorization succeeds
+      expect(result.authorized).toBe(true);
+    });
+
+    test("authorize succeeds with canonical key when metadata has legacy alias", () => {
+      // GIVEN metadata stored under the legacy browser_fill_credential alias
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: ["browser_fill_credential"],
+      });
+
+      // WHEN authorizing with the canonical key
+      const result = broker.authorize({
+        service: "github",
+        field: "token",
+        toolName: BROWSER_FILL_CAPABILITY,
+      });
+
+      // THEN authorization succeeds
+      expect(result.authorized).toBe(true);
+    });
+
+    test("authorize succeeds with legacy alias when metadata has canonical key", () => {
+      // GIVEN metadata allowing the canonical capability key
+      upsertCredentialMetadata("github", "token", {
+        allowedTools: [BROWSER_FILL_CAPABILITY],
+      });
+
+      // WHEN authorizing with the legacy alias
+      const result = broker.authorize({
+        service: "github",
+        field: "token",
+        toolName: "browser_fill_credential",
+      });
+
+      // THEN authorization succeeds
+      expect(result.authorized).toBe(true);
+    });
+
+    test("serverUse with canonical key works when metadata has legacy alias", async () => {
+      // GIVEN metadata stored under the legacy alias with a stored value
+      upsertCredentialMetadata("vercel", "api_token", {
+        allowedTools: ["browser_fill_credential"],
+      });
+      await setSecureKeyAsync(
+        credentialKey("vercel", "api_token"),
+        "vercel-tok",
+      );
+
+      // WHEN serverUse runs with the canonical key
+      const result = await broker.serverUse({
+        service: "vercel",
+        field: "api_token",
+        toolName: BROWSER_FILL_CAPABILITY,
+        execute: async (v) => v,
+      });
+
+      // THEN execution succeeds with the stored value
+      expect(result.success).toBe(true);
+      expect(result.result).toBe("vercel-tok");
+    });
+
+    test("non-aliased tool names are unaffected by alias resolution", () => {
+      // GIVEN metadata allowing a non-aliased custom tool
+      upsertCredentialMetadata("svc", "key", {
+        allowedTools: ["custom_tool"],
+      });
+
+      // WHEN authorizing with that custom tool
+      const result = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "custom_tool",
+      });
+
+      // THEN authorization succeeds
+      expect(result.authorized).toBe(true);
+    });
+
+    test("non-aliased tool denied when only canonical key is allowed", () => {
+      // GIVEN metadata allowing only the canonical capability key
+      upsertCredentialMetadata("svc", "key", {
+        allowedTools: [BROWSER_FILL_CAPABILITY],
+      });
+
+      // WHEN authorizing with an unrelated tool
+      const result = broker.authorize({
+        service: "svc",
+        field: "key",
+        toolName: "unrelated_tool",
+      });
+
+      // THEN authorization is denied
+      expect(result.authorized).toBe(false);
+    });
+  });
+});

package/src/__tests__/credential-execution-shell-lockdown.test.ts

@@ -11,27 +11,33 @@
 
 import { describe, expect, test } from "bun:test";
 
-import { isUntrustedTrustClass } from "../runtime/actor-trust-resolver.js";
+import { resolveCapabilities } from "../runtime/capabilities.js";
 
 // ---------------------------------------------------------------------------
 // Trust class categorization (foundational for lockdown decisions)
 // ---------------------------------------------------------------------------
 
 describe("trust class categorization for CES lockdown", () => {
-  test("guardian is not untrusted", () => {
-    expect(isUntrustedTrustClass("guardian")).toBe(false);
+  test("guardian runs an unsandboxed shell", () => {
+    expect(resolveCapabilities("guardian").canRunUnsandboxedShell).toBe(true);
   });
 
-  test("trusted_contact is untrusted", () => {
-    expect(isUntrustedTrustClass("trusted_contact")).toBe(true);
+  test("trusted_contact shell is sandboxed", () => {
+    expect(resolveCapabilities("trusted_contact").canRunUnsandboxedShell).toBe(false);
   });
 
-  test("unknown is untrusted", () => {
-    expect(isUntrustedTrustClass("unknown")).toBe(true);
+  test("unverified_contact shell is sandboxed", () => {
+    expect(resolveCapabilities("unverified_contact").canRunUnsandboxedShell).toBe(
+      false,
+    );
   });
 
-  test("undefined is untrusted", () => {
-    expect(isUntrustedTrustClass(undefined)).toBe(true);
+  test("unknown shell is sandboxed", () => {
+    expect(resolveCapabilities("unknown").canRunUnsandboxedShell).toBe(false);
+  });
+
+  test("undefined shell is sandboxed", () => {
+    expect(resolveCapabilities(undefined).canRunUnsandboxedShell).toBe(false);
   });
 });
 
@@ -91,7 +97,7 @@ describe("VELLUM_UNTRUSTED_SHELL env flag", () => {
 describe("CES shell lockdown activation", () => {
   test("lockdown is active only when both flag is enabled AND actor is untrusted", () => {
     // Simulates the condition used in shell.ts:
-    // const shellLockdownActive = isCesShellLockdownEnabled(config) && isUntrustedTrustClass(context.trustClass);
+    // const shellLockdownActive = isCesShellLockdownEnabled(config) && !resolveCapabilities(context.trustClass).canRunUnsandboxedShell;
     const cases: Array<{
       flagEnabled: boolean;
       trustClass: "guardian" | "trusted_contact" | "unknown";
@@ -106,7 +112,8 @@ describe("CES shell lockdown activation", () => {
     ];
 
     for (const { flagEnabled, trustClass, expected } of cases) {
-      const active = flagEnabled && isUntrustedTrustClass(trustClass);
+      const active =
+        flagEnabled && !resolveCapabilities(trustClass).canRunUnsandboxedShell;
       expect(active).toBe(expected);
     }
   });

package/src/__tests__/credential-execution-tools.test.ts

@@ -136,7 +136,6 @@ describe("secure tool installation is separate from credential grants", () => {
   test("manage_secure_command_tool does not share tool name with grant tools", () => {
     // The tool name must not collide with existing credential grant tools
     const grantToolNames = [
-      "credential_store",
       "credential_grant",
       "credential_revoke",
       "credential_list",

package/src/__tests__/credential-prompt-route.test.ts

@@ -110,6 +110,7 @@ function slackResult(
     hasAppToken: false,
     hasUserToken: false,
     connected: false,
+    threadMode: "mention_only",
     ...overrides,
   };
 }
@@ -138,10 +139,9 @@ describe("credentials/prompt route", () => {
 
   test("forwards usageDescription as the prompt purpose and to metadata", async () => {
     /**
-     * The CLI `credentials prompt` command exposes `--usage-description` to
-     * match the credential_store tool's prompt action. The route must thread
-     * it to the secure prompt (as `purpose`, shown to the user) and persist it
-     * to credential metadata (as `usageDescription`).
+     * The CLI `credentials prompt` command exposes `--usage-description`. The
+     * route must thread it to the secure prompt (as `purpose`, shown to the
+     * user) and persist it to credential metadata (as `usageDescription`).
      */
     // GIVEN the credentials/prompt route is registered
     expect(promptRoute).toBeDefined();