@vellumai/assistant 0.7.0 → 0.7.1

package/src/__tests__/conversation-usage.test.ts

@@ -7,6 +7,8 @@ const updateConversationUsageCalls: Array<{
   estimatedCost: number;
 }> = [];
 
+let mockLlmConfig = createMockLlmConfig();
+
 mock.module("../util/logger.js", () => ({
   getLogger: () =>
     new Proxy({} as Record<string, unknown>, {
@@ -16,9 +18,7 @@ mock.module("../util/logger.js", () => ({
 
 mock.module("../config/loader.js", () => ({
   getConfig: () => ({
-    llm: {
-      pricingOverrides: [],
-    },
+    llm: mockLlmConfig,
   }),
 }));
 
@@ -42,16 +42,67 @@ import { recordUsage } from "../daemon/conversation-usage.js";
 import { getDb } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
 import { listUsageEvents } from "../memory/llm-usage-store.js";
+import type { Provider, ProviderResponse } from "../providers/types.js";
+import { UsageTrackingProvider } from "../providers/usage-tracking.js";
 import type { PricingUsage } from "../usage/types.js";
 import { resolvePricingForUsageWithOverrides } from "../util/pricing.js";
 
 initializeDb();
 
+function createMockLlmConfig() {
+  return {
+    default: {
+      provider: "anthropic" as const,
+      model: "claude-opus-4-6",
+      maxTokens: 64_000,
+      effort: "max" as const,
+      speed: "standard" as const,
+      verbosity: "medium" as const,
+      temperature: null,
+      thinking: { enabled: true, streamThinking: true },
+      contextWindow: {
+        enabled: true,
+        maxInputTokens: 200_000,
+        targetBudgetRatio: 0.3,
+        compactThreshold: 0.8,
+        summaryBudgetRatio: 0.05,
+        overflowRecovery: {
+          enabled: true,
+          safetyMarginRatio: 0.05,
+          maxAttempts: 3,
+          interactiveLatestTurnCompression: "summarize" as const,
+          nonInteractiveLatestTurnCompression: "truncate" as const,
+        },
+      },
+      openrouter: { only: [] },
+    },
+    profiles: {
+      conversationProfile: {
+        provider: "openai" as const,
+        model: "gpt-4o",
+      },
+      summaryProfile: {
+        provider: "anthropic" as const,
+        model: "claude-haiku-3",
+      },
+    },
+    profileOrder: [],
+    callSites: {
+      conversationSummarization: {
+        profile: "summaryProfile",
+      },
+    },
+    activeProfile: undefined,
+    pricingOverrides: [],
+  };
+}
+
 describe("recordUsage", () => {
   beforeEach(() => {
     const db = getDb();
     db.run(`DELETE FROM llm_usage_events`);
     updateConversationUsageCalls.length = 0;
+    mockLlmConfig = createMockLlmConfig();
   });
 
   test("applies fast mode pricing when any response has speed: fast", () => {
@@ -211,4 +262,203 @@ describe("recordUsage", () => {
       },
     ]);
   });
+
+  test("manual provider usage tracking leaves conversation aggregate recording as the only ledger row", async () => {
+    const response: ProviderResponse = {
+      content: [{ type: "text", text: "ok" }],
+      model: "gpt-5.4-mini",
+      usage: {
+        inputTokens: 1_000,
+        outputTokens: 2_000,
+      },
+      stopReason: "end_turn",
+    };
+    const provider: Provider = {
+      name: "openai",
+      async sendMessage() {
+        return response;
+      },
+    };
+    const wrapped = new UsageTrackingProvider(provider);
+
+    await wrapped.sendMessage(
+      [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
+      undefined,
+      undefined,
+      {
+        config: {
+          callSite: "mainAgent",
+          usageTracking: "manual",
+        },
+      },
+    );
+    expect(listUsageEvents()).toHaveLength(0);
+
+    const usageStats = {
+      inputTokens: 0,
+      outputTokens: 0,
+      estimatedCost: 0,
+    };
+
+    recordUsage(
+      {
+        conversationId: "conv-manual-1",
+        providerName: "openai",
+        usageStats,
+      },
+      response.usage.inputTokens,
+      response.usage.outputTokens,
+      response.model,
+      () => {},
+      "main_agent",
+      "req-manual-1",
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      actor: "main_agent",
+      conversationId: "conv-manual-1",
+      requestId: "req-manual-1",
+      provider: "openai",
+      model: "gpt-5.4-mini",
+      inputTokens: 1_000,
+      outputTokens: 2_000,
+    });
+  });
+
+  test("persists resolved main-agent attribution from input without changing totals", () => {
+    const usageStats = {
+      inputTokens: 0,
+      outputTokens: 0,
+      estimatedCost: 0,
+    };
+
+    recordUsage(
+      {
+        conversationId: "conv-attrib-1",
+        providerName: "openai",
+        usageStats,
+      },
+      100,
+      20,
+      "gpt-4o",
+      () => {},
+      "main_agent",
+      "req-attrib-1",
+      0,
+      0,
+      undefined,
+      1,
+      undefined,
+      {
+        callSite: "mainAgent",
+        overrideProfile: "conversationProfile",
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      actor: "main_agent",
+      provider: "openai",
+      model: "gpt-4o",
+      inputTokens: 100,
+      outputTokens: 20,
+      callSite: "mainAgent",
+      inferenceProfile: "conversationProfile",
+      inferenceProfileSource: "conversation",
+    });
+    expect(usageStats.inputTokens).toBe(100);
+    expect(usageStats.outputTokens).toBe(20);
+  });
+
+  test("persists compaction attribution using the summary call-site profile", () => {
+    const usageStats = {
+      inputTokens: 0,
+      outputTokens: 0,
+      estimatedCost: 0,
+    };
+
+    recordUsage(
+      {
+        conversationId: "conv-attrib-2",
+        providerName: "anthropic",
+        usageStats,
+      },
+      500,
+      80,
+      "claude-haiku-3",
+      () => {},
+      "context_compactor",
+      "req-attrib-2",
+      0,
+      0,
+      undefined,
+      1,
+      undefined,
+      {
+        callSite: "conversationSummarization",
+        overrideProfile: "conversationProfile",
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      actor: "context_compactor",
+      provider: "anthropic",
+      model: "claude-haiku-3",
+      inputTokens: 500,
+      outputTokens: 80,
+      callSite: "conversationSummarization",
+      inferenceProfile: "summaryProfile",
+      inferenceProfileSource: "call_site",
+    });
+  });
+
+  test("persists a pre-resolved attribution snapshot", () => {
+    const usageStats = {
+      inputTokens: 0,
+      outputTokens: 0,
+      estimatedCost: 0,
+    };
+
+    recordUsage(
+      {
+        conversationId: "conv-attrib-3",
+        providerName: "anthropic",
+        usageStats,
+      },
+      25,
+      5,
+      "claude-opus-4-6",
+      () => {},
+      "main_agent",
+      "req-attrib-3",
+      0,
+      0,
+      undefined,
+      1,
+      undefined,
+      {
+        callSite: "mainAgent",
+        activeProfile: null,
+        overrideProfile: null,
+        callSiteProfile: null,
+        appliedProfile: null,
+        profileSource: "default",
+        resolvedProvider: "anthropic",
+        resolvedModel: "claude-opus-4-6",
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      callSite: "mainAgent",
+      inferenceProfile: null,
+      inferenceProfileSource: "default",
+    });
+  });
 });

package/src/__tests__/credential-execution-shell-lockdown.test.ts

@@ -6,15 +6,12 @@
  * - Untrusted bash rejects non-empty credential-ref mode when lockdown is active.
  * - VELLUM_UNTRUSTED_SHELL env flag is injected for untrusted actors.
  * - host_bash sets forcePromptSideEffects for untrusted actors under lockdown.
- * - Protected paths are passed to the sandbox when lockdown is active.
  * - CLI commands deny raw secret/token reveal when VELLUM_UNTRUSTED_SHELL=1.
  */
 
-import { join } from "node:path";
 import { describe, expect, test } from "bun:test";
 
 import { isUntrustedTrustClass } from "../runtime/actor-trust-resolver.js";
-import { getProtectedDir, getWorkspaceDir } from "../util/platform.js";
 
 // ---------------------------------------------------------------------------
 // Trust class categorization (foundational for lockdown decisions)
@@ -114,39 +111,3 @@ describe("CES shell lockdown activation", () => {
     }
   });
 });
-
-// ---------------------------------------------------------------------------
-// Sandbox deny-read paths
-// ---------------------------------------------------------------------------
-
-describe("CES protected paths for sandbox deny-read", () => {
-  test("protected paths include the protected dir and db dir", () => {
-    // The buildCesProtectedPaths function constructs paths from getProtectedDir()
-    // and getWorkspaceDir(). We verify the pattern: paths should be absolute and clean.
-    const expectedPaths = [
-      getProtectedDir(),
-      join(getWorkspaceDir(), "data", "db"),
-    ];
-
-    // Each expected path should be a valid absolute path pattern
-    for (const p of expectedPaths) {
-      expect(p.startsWith("/") || p.match(/^[A-Z]:\\/)).toBeTruthy();
-      expect(p.includes("..")).toBe(false);
-    }
-  });
-});
-
-// ---------------------------------------------------------------------------
-// Native sandbox backend deny-read integration
-// ---------------------------------------------------------------------------
-
-describe("NativeBackend deny-read paths", () => {
-  test("WrapOptions accepts denyReadPaths", () => {
-    // Type-level check: ensure the WrapOptions interface includes denyReadPaths.
-    const opts: import("../tools/terminal/backends/types.js").WrapOptions = {
-      networkMode: "off",
-      denyReadPaths: ["/home/test/.vellum/protected"],
-    };
-    expect(opts.denyReadPaths).toHaveLength(1);
-  });
-});

package/src/__tests__/credential-health-service.test.ts

@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, mock, test } from "bun:test";
 // ── Mutable state for mocks ──────────────────────────────────────────
 
 const secureKeyValues = new Map<string, string>();
+const unreachableKeys = new Set<string>();
 
 let mockProviders: Array<{
   provider: string;
@@ -36,6 +37,10 @@ let mockFetchThrows = false;
 
 mock.module("../security/secure-keys.js", () => ({
   getSecureKeyAsync: async (account: string) => secureKeyValues.get(account),
+  getSecureKeyResultAsync: async (account: string) => ({
+    value: secureKeyValues.get(account),
+    unreachable: unreachableKeys.has(account),
+  }),
   setSecureKeyAsync: async () => {},
   deleteSecureKeyAsync: async () => "deleted",
   listSecureKeysAsync: async () => [],
@@ -140,11 +145,16 @@ function setToken(connectionId: string, token = "mock-token") {
   secureKeyValues.set(`oauth_connection/${connectionId}/access_token`, token);
 }
 
+function markUnreachable(key: string) {
+  unreachableKeys.add(key);
+}
+
 // ── Tests ────────────────────────────────────────────────────────────
 
 describe("credential-health-service", () => {
   beforeEach(() => {
     secureKeyValues.clear();
+    unreachableKeys.clear();
     mockProviders = [];
     mockConnections = new Map();
     mockFetchResponse = { ok: true, status: 200 };
@@ -191,6 +201,30 @@ describe("credential-health-service", () => {
     expect(report.unhealthy).toHaveLength(1);
   });
 
+  test("returns unreachable when credential backend is unreachable", async () => {
+    addProvider("google");
+    addConnection("google", "conn-1");
+    // Don't set token, but mark the path as unreachable
+    markUnreachable("oauth_connection/conn-1/access_token");
+
+    const report = await checkAllCredentials();
+    expect(report.results).toHaveLength(1);
+    expect(report.results[0]!.status).toBe("unreachable");
+    expect(report.results[0]!.canAutoRecover).toBe(true);
+    expect(report.unhealthy).toHaveLength(1);
+  });
+
+  test("returns missing_token (not unreachable) when backend is reachable but token absent", async () => {
+    addProvider("google");
+    addConnection("google", "conn-1");
+    // Don't set token, don't mark unreachable — genuinely missing
+
+    const report = await checkAllCredentials();
+    expect(report.results).toHaveLength(1);
+    expect(report.results[0]!.status).toBe("missing_token");
+    expect(report.results[0]!.canAutoRecover).toBe(false);
+  });
+
   test("returns expired when token is past expiresAt without refresh token", async () => {
     addProvider("google");
     addConnection("google", "conn-1", {
@@ -389,6 +423,40 @@ describe("credential-health-service", () => {
       const report = await checkAllCredentials();
       expect(report.results[0]!.status).toBe("healthy");
     });
+
+    test("slack_channel returns unreachable when credential backend is down", async () => {
+      addProvider("slack_channel", {
+        pingUrl: "https://slack.com/api/auth.test",
+      });
+      addConnection("slack_channel", "conn-slack", {
+        expiresAt: null,
+        hasRefreshToken: false,
+        grantedScopes: [],
+      });
+      // Don't set token, mark the manual-token path as unreachable
+      markUnreachable("credential/slack_channel/bot_token");
+
+      const report = await checkAllCredentials();
+      expect(report.results).toHaveLength(1);
+      expect(report.results[0]!.status).toBe("unreachable");
+      expect(report.results[0]!.canAutoRecover).toBe(true);
+    });
+
+    test("telegram returns unreachable when credential backend is down", async () => {
+      addProvider("telegram");
+      addConnection("telegram", "conn-tg", {
+        expiresAt: null,
+        hasRefreshToken: false,
+        grantedScopes: [],
+      });
+      // Don't set token, mark the manual-token path as unreachable
+      markUnreachable("credential/telegram/bot_token");
+
+      const report = await checkAllCredentials();
+      expect(report.results).toHaveLength(1);
+      expect(report.results[0]!.status).toBe("unreachable");
+      expect(report.results[0]!.canAutoRecover).toBe(true);
+    });
   });
 
   describe("checkCredentialForProvider", () => {

package/src/__tests__/credential-security-e2e.test.ts

@@ -7,8 +7,6 @@ import { beforeEach, describe, expect, mock, test } from "bun:test";
 const mockConfig = {
   secretDetection: {
     enabled: true,
-    action: "block" as "redact" | "warn" | "block",
-    entropyThreshold: 4.0,
     allowOneTimeSend: false,
   },
   timeouts: { permissionTimeoutSec: 300 },
@@ -63,6 +61,10 @@ mock.module("../security/secure-keys.js", () => {
   };
   return {
     getSecureKeyAsync: async (key: string) => storedKeys.get(key) ?? undefined,
+    getSecureKeyResultAsync: async (account: string) => ({
+      value: storedKeys.get(account),
+      unreachable: false,
+    }),
     setSecureKeyAsync: async (key: string, value: string) =>
       syncSet(key, value),
     deleteSecureKeyAsync: async (key: string) => syncDelete(key),
@@ -330,7 +332,6 @@ describe("E2E: cross-cutting secret leak prevention", () => {
   beforeEach(() => {
     storedKeys.clear();
     mockConfig.secretDetection.enabled = true;
-    mockConfig.secretDetection.action = "block";
     mockConfig.secretDetection.allowOneTimeSend = false;
   });
 

package/src/__tests__/credential-security-invariants.test.ts

@@ -189,13 +189,13 @@ describe("Invariant 2: no generic plaintext secret read API", () => {
       "messaging/providers/slack/adapter.ts", // Slack bot token lookup for Socket Mode connectivity check
       "credential-health/credential-health-service.ts", // proactive credential health monitoring
       "daemon/handlers/config-slack-channel.ts", // Slack channel config credential management
-      "daemon/handlers/slack-channel-oauth-install.ts", // Slack OAuth loopback reads client credentials for token exchange
       "providers/managed-proxy/context.ts", // managed proxy API key lookup for provider initialization
       "platform/client.ts", // platform client credential store fallback for standalone CLI auth
       "mcp/mcp-oauth-provider.ts", // MCP OAuth token/client/discovery persistence
       "runtime/routes/integrations/slack/share.ts", // Slack share routes credential lookup
       "mcp/client.ts", // MCP client cached-token lookup
       "oauth/token-persistence.ts", // OAuth token persistence (set/delete tokens)
+      "oauth/credential-token-resolver.ts", // centralized access-token key resolution for OAuth and manual-token providers
       "oauth/connection-resolver.ts", // resolve OAuthConnection from oauth-store (access_token lookup)
       "runtime/routes/secret-routes.ts", // HTTP secret management routes (set/delete secrets)
       "runtime/routes/migration-routes.ts", // migration import credential restore
@@ -583,10 +583,6 @@ describe("One-time send override", () => {
   test("allowOneTimeSend defaults to false in config", () => {
     expect(DEFAULT_CONFIG.secretDetection.allowOneTimeSend).toBe(false);
   });
-
-  test("default secretDetection.action is redact", () => {
-    expect(DEFAULT_CONFIG.secretDetection.action).toBe("redact");
-  });
 });
 
 // ---------------------------------------------------------------------------

package/src/__tests__/credential-token-resolver.test.ts

@@ -0,0 +1,180 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mutable state for mocks ──────────────────────────────────────────
+
+const secureKeyValues = new Map<string, string>();
+const unreachableKeys = new Set<string>();
+
+// ── Module mocks ─────────────────────────────────────────────────────
+
+mock.module("../security/secure-keys.js", () => ({
+  getSecureKeyAsync: async (account: string) => secureKeyValues.get(account),
+  getSecureKeyResultAsync: async (account: string) => ({
+    value: secureKeyValues.get(account),
+    unreachable: unreachableKeys.has(account),
+  }),
+  setSecureKeyAsync: async () => {},
+  deleteSecureKeyAsync: async () => "deleted",
+  listSecureKeysAsync: async () => [],
+  getProviderKeyAsync: async () => undefined,
+  getMaskedProviderKey: () => undefined,
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => ({
+    info: () => {},
+    warn: () => {},
+    debug: () => {},
+    error: () => {},
+  }),
+}));
+
+// ── Import under test ────────────────────────────────────────────────
+
+const { resolveAccessTokenKey, getConnectionAccessTokenResult } =
+  await import("../oauth/credential-token-resolver.js");
+
+// ── Tests ────────────────────────────────────────────────────────────
+
+describe("credential-token-resolver", () => {
+  beforeEach(() => {
+    secureKeyValues.clear();
+    unreachableKeys.clear();
+  });
+
+  describe("resolveAccessTokenKey", () => {
+    test("slack_channel resolves to credential/slack_channel/bot_token", () => {
+      expect(resolveAccessTokenKey("slack_channel", "conn-123")).toBe(
+        "credential/slack_channel/bot_token",
+      );
+    });
+
+    test("telegram resolves to credential/telegram/bot_token", () => {
+      expect(resolveAccessTokenKey("telegram", "conn-456")).toBe(
+        "credential/telegram/bot_token",
+      );
+    });
+
+    test("standard OAuth provider resolves to oauth_connection/<id>/access_token", () => {
+      expect(resolveAccessTokenKey("google", "conn-789")).toBe(
+        "oauth_connection/conn-789/access_token",
+      );
+    });
+
+    test("unknown provider resolves to oauth_connection/<id>/access_token", () => {
+      expect(resolveAccessTokenKey("github", "conn-abc")).toBe(
+        "oauth_connection/conn-abc/access_token",
+      );
+    });
+  });
+
+  describe("getConnectionAccessTokenResult", () => {
+    test("returns token value and key for slack_channel using bot_token path", async () => {
+      secureKeyValues.set("credential/slack_channel/bot_token", "xoxb-valid");
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "slack_channel",
+        connectionId: "conn-slack",
+      });
+
+      expect(result.value).toBe("xoxb-valid");
+      expect(result.unreachable).toBe(false);
+      expect(result.key).toBe("credential/slack_channel/bot_token");
+    });
+
+    test("returns undefined for slack_channel when bot_token is absent even if oauth path is set", async () => {
+      // Simulate the bug scenario: OAuth access-token path is populated but
+      // the manual-token path is not. The resolver must NOT fall through to
+      // the OAuth path for manual-token providers.
+      secureKeyValues.set(
+        "oauth_connection/conn-slack/access_token",
+        "should-be-ignored",
+      );
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "slack_channel",
+        connectionId: "conn-slack",
+      });
+
+      expect(result.value).toBeUndefined();
+      expect(result.key).toBe("credential/slack_channel/bot_token");
+    });
+
+    test("returns token for standard OAuth provider via connection path", async () => {
+      secureKeyValues.set(
+        "oauth_connection/conn-google/access_token",
+        "ya29.token",
+      );
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "google",
+        connectionId: "conn-google",
+      });
+
+      expect(result.value).toBe("ya29.token");
+      expect(result.unreachable).toBe(false);
+      expect(result.key).toBe("oauth_connection/conn-google/access_token");
+    });
+
+    test("returns unreachable when credential backend is down", async () => {
+      unreachableKeys.add("credential/telegram/bot_token");
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "telegram",
+        connectionId: "conn-tg",
+      });
+
+      expect(result.value).toBeUndefined();
+      expect(result.unreachable).toBe(true);
+      expect(result.key).toBe("credential/telegram/bot_token");
+    });
+
+    test("returns undefined (not unreachable) when token is genuinely missing", async () => {
+      const result = await getConnectionAccessTokenResult({
+        provider: "google",
+        connectionId: "conn-missing",
+      });
+
+      expect(result.value).toBeUndefined();
+      expect(result.unreachable).toBe(false);
+      expect(result.key).toBe("oauth_connection/conn-missing/access_token");
+    });
+  });
+
+  describe("regression: oauth ping slack_channel uses bot_token", () => {
+    // The root cause of false credential health alerts was that some code paths
+    // looked up tokens at oauth_connection/<id>/access_token for ALL providers,
+    // while manual-token providers (slack_channel, telegram) actually store
+    // their tokens at credential/<provider>/bot_token. The centralized resolver
+    // ensures ALL consumers agree on the path.
+
+    test("slack_channel token lookup goes to credential/slack_channel/bot_token, not oauth path", async () => {
+      secureKeyValues.set("credential/slack_channel/bot_token", "xoxb-real");
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "slack_channel",
+        connectionId: "any-connection-id",
+      });
+
+      // Must find the token at the manual-token path
+      expect(result.value).toBe("xoxb-real");
+      // Must report the correct key
+      expect(result.key).toBe("credential/slack_channel/bot_token");
+      // Connection ID must be irrelevant for manual-token providers
+      expect(result.key).not.toContain("any-connection-id");
+    });
+
+    test("telegram token lookup goes to credential/telegram/bot_token, not oauth path", async () => {
+      secureKeyValues.set("credential/telegram/bot_token", "tg-token");
+
+      const result = await getConnectionAccessTokenResult({
+        provider: "telegram",
+        connectionId: "any-connection-id",
+      });
+
+      expect(result.value).toBe("tg-token");
+      expect(result.key).toBe("credential/telegram/bot_token");
+      expect(result.key).not.toContain("any-connection-id");
+    });
+  });
+});