@vellumai/assistant 0.8.6 → 0.8.7

package/src/telemetry/usage-telemetry-reporter.test.ts

@@ -98,7 +98,12 @@ mock.module("../config/loader.js", () => ({
 }));
 
 const mockQueryUnreportedLifecycleEvents = mock(
-  () => [] as { id: string; eventName: string; createdAt: number }[],
+  () =>
+    [] as {
+      id: string;
+      eventName: string;
+      createdAt: number;
+    }[],
 );
 
 mock.module("../memory/lifecycle-events-store.js", () => ({
@@ -116,6 +121,7 @@ const mockQueryUnreportedOnboardingEvents = mock(
       tone: string | null;
       googleConnected: boolean | null;
       googleScopesJson: string | null;
+      priorAssistantsJson: string | null;
       abVariant: string | null;
     }[],
 );
@@ -169,6 +175,7 @@ function makeUsageEvent(
     requestId: null,
     estimatedCostUsd: 0.001,
     pricingStatus: "priced",
+    assistantVersion: "test-app-version",
     conversationType: "standard",
     turnIndex: 1,
     ...overrides,
@@ -938,4 +945,134 @@ describe("UsageTelemetryReporter", () => {
     );
     expect(body.events[0].daemon_event_id).toBe("evt-after-reenable");
   });
+
+  // -------------------------------------------------------------------------
+  // Per-event `assistant_version` on the wire
+  //
+  // The envelope's `assistant_version` is upload-time (always the current
+  // binary). The per-event field is record-time (the binary that was running
+  // when the event was persisted to SQLite). In this PR only `llm_usage`
+  // events carry a true record-time value; turn events (and lifecycle /
+  // onboarding events, not asserted here) stamp the running binary's
+  // `APP_VERSION` directly until their respective follow-ups land.
+  // Nullable llm_usage cases (legacy rows from before migration 267 ran)
+  // fall back to the running binary's `APP_VERSION` rather than emitting
+  // explicit `null` — under the platform contract a present-but-null
+  // per-event value would override the envelope, and we'd rather have a
+  // concrete version than no version.
+  // -------------------------------------------------------------------------
+
+  test("llm_usage event carries its recorded assistantVersion on the wire", async () => {
+    const event = makeUsageEvent({
+      id: "evt-version-llm",
+      assistantVersion: "0.8.4",
+    });
+    mockQueryUnreportedUsageEvents.mockReturnValue([event]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":1}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    // Envelope reflects current binary (upload-time).
+    expect(body.assistant_version).toBe("1.2.3-test");
+    // Per-event reflects record-time — a different value here is the
+    // whole point: backlogged events keep the binary they were stamped
+    // with when they were originally recorded.
+    expect(body.events[0].assistant_version).toBe("0.8.4");
+  });
+
+  test("llm_usage event with null assistantVersion falls back to APP_VERSION (pre-migration row)", async () => {
+    const event = makeUsageEvent({
+      id: "evt-version-llm-null",
+      assistantVersion: null,
+    });
+    mockQueryUnreportedUsageEvents.mockReturnValue([event]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":1}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    // No explicit null on the wire — the running binary's APP_VERSION is
+    // stamped instead so a present-but-null value can't override the
+    // envelope under the platform's per-event-wins contract.
+    expect(body.events[0].assistant_version).toBe("1.2.3-test");
+  });
+
+  test("turn event emits assistant_version: APP_VERSION (running binary)", async () => {
+    // Turn events are derived from `messages` + `conversations`, which
+    // don't yet carry a per-event version. Until the follow-up migration
+    // adds a column to `messages`, we stamp the running binary's
+    // APP_VERSION instead of explicit null — matches what the envelope
+    // would have provided but per-event so it survives the platform
+    // contract that treats present per-event values as winning over the
+    // envelope.
+    mockQueryUnreportedUsageEvents.mockReturnValue([]);
+    mockQueryUnreportedTurnEvents.mockReturnValue([
+      {
+        id: "evt-turn-version",
+        createdAt: 1700000300000,
+        conversationId: "conv-1",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":1}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    const turn = body.events.find((e: { type: string }) => e.type === "turn");
+    expect(turn).toBeDefined();
+    expect(turn.assistant_version).toBe("1.2.3-test");
+  });
+
+  test("a single batch can carry mixed per-event versions", async () => {
+    // The whole point of this migration: an old backlogged event keeps
+    // its old version while a freshly recorded event in the same flush
+    // carries the current version. Pre-migration the entire batch would
+    // collapse to the envelope value.
+    const oldEvent = makeUsageEvent({
+      id: "evt-old",
+      assistantVersion: "0.8.3",
+    });
+    const newEvent = makeUsageEvent({
+      id: "evt-new",
+      assistantVersion: "0.8.5",
+    });
+    mockQueryUnreportedUsageEvents.mockReturnValue([oldEvent, newEvent]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":2}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    const versions = body.events.map(
+      (e: { assistant_version: string | null }) => e.assistant_version,
+    );
+    expect(versions.sort()).toEqual(["0.8.3", "0.8.5"]);
+    // Envelope still reflects the running binary, not either event.
+    expect(body.assistant_version).toBe("1.2.3-test");
+  });
 });

package/src/telemetry/usage-telemetry-reporter.ts

@@ -241,6 +241,13 @@ export class UsageTelemetryReporter {
             inference_profile_source: e.inferenceProfileSource,
             cost: e.estimatedCostUsd ?? null,
             recorded_at: e.createdAt,
+            // Record-time version when present; otherwise the running
+            // binary's `APP_VERSION` (a legacy row from before
+            // migration 267 ran). We deliberately don't emit explicit
+            // `null` — under the platform contract a present-but-null
+            // per-event value would override the envelope, and we'd
+            // rather have a concrete version than no version.
+            assistant_version: e.assistantVersion ?? APP_VERSION,
           }),
         ),
         ...turnEvents.map((e): TelemetryEvent => {
@@ -278,6 +285,18 @@ export class UsageTelemetryReporter {
             interface_id: e.interfaceId,
             channel_id: e.channelId,
             client,
+            // Turn events derive from `messages` + `conversations`
+            // rather than a dedicated table. Adding `assistant_version`
+            // to `messages` is a separate (larger) migration; until
+            // then we stamp the running binary's `APP_VERSION` so the
+            // wire value is concrete (matches what the envelope would
+            // have provided, but per-event so it survives the platform
+            // contract that treats present per-event values as winning
+            // over the envelope). Same upload-time attribution risk
+            // for turn events as before this PR — lifecycle, onboarding
+            // and turn events all still rely on the envelope; only
+            // llm_usage is record-time accurate in this PR.
+            assistant_version: APP_VERSION,
           };
         }),
         ...lifecycleEvents.map(
@@ -286,6 +305,12 @@ export class UsageTelemetryReporter {
             daemon_event_id: e.id,
             event_name: e.eventName,
             recorded_at: e.createdAt,
+            // Lifecycle events fall back to the envelope `assistant_version`
+            // — same upload-time attribution risk as before this PR. Adding
+            // the record-time column to `lifecycle_events` (#18112) is a
+            // separate follow-up that mirrors what this PR does for
+            // `llm_usage_events`.
+            assistant_version: APP_VERSION,
           }),
         ),
         ...onboardingEvents.map(
@@ -304,6 +329,12 @@ export class UsageTelemetryReporter {
               ? { google_scopes: JSON.parse(e.googleScopesJson) }
               : {}),
             ...(e.abVariant ? { ab_variant: e.abVariant } : {}),
+            // Onboarding events fall back to the envelope `assistant_version`
+            // — same upload-time attribution risk as before this PR. Adding
+            // the record-time column to `onboarding_events` (#30733) is a
+            // separate follow-up that mirrors what this PR does for
+            // `llm_usage_events`.
+            assistant_version: APP_VERSION,
           }),
         ),
       ];

package/src/tools/acp/spawn.test.ts

@@ -82,23 +82,83 @@ mock.module("../../util/logger.js", () => ({
     }),
 }));
 
-// Stub secure-keys so the `prepareAgentEnv` preflight finds a token without
-// the test having to populate the real OS keyring. Driven via `secureKeyStore`
-// per test in beforeEach; the default seeds a vault token so existing tests
-// (which assume claude spawns succeed) keep passing.
-//
-// The real module's other exports are spread in so transitive importers
-// (e.g. session-manager → pending-interactions → credential-routes, which
-// imports `getSecureKeyResultAsync`) still resolve at parse time. Bun's
-// `mock.module` is process-global and returns *exactly* the keys the factory
-// returns — without the spread, any consumer pulling a non-`getSecureKeyAsync`
-// export errors with "Export named '<X>' not found".
-const secureKeyStore = new Map<string, string>();
-const realSecureKeys = await import("../../security/secure-keys.js");
-
-mock.module("../../security/secure-keys.js", () => ({
-  ...realSecureKeys,
-  getSecureKeyAsync: async (key: string) => secureKeyStore.get(key),
+// Stub credential broker + metadata store so `prepareAgentEnv` can resolve
+// tokens without the real OS keyring. Driven via `vaultStore` per test in
+// beforeEach; the default seeds a vault token so existing tests (which assume
+// claude spawns succeed) keep passing.
+const vaultStore = new Map<string, string>();
+const metadataStore = new Map<
+  string,
+  { allowedTools: string[]; usageDescription?: string }
+>();
+
+mock.module("../../tools/credentials/metadata-store.js", () => ({
+  getCredentialMetadata: (service: string, field: string) => {
+    const key = `${service}/${field}`;
+    const entry = metadataStore.get(key);
+    if (!entry) return undefined;
+    return {
+      credentialId: `cred-${key}`,
+      service,
+      field,
+      allowedTools: entry.allowedTools,
+      allowedDomains: [],
+      usageDescription: entry.usageDescription,
+      createdAt: 0,
+      updatedAt: 0,
+    };
+  },
+  upsertCredentialMetadata: (
+    service: string,
+    field: string,
+    policy?: { allowedTools?: string[]; usageDescription?: string },
+  ) => {
+    const key = `${service}/${field}`;
+    const existing = metadataStore.get(key);
+    metadataStore.set(key, {
+      allowedTools: policy?.allowedTools ?? existing?.allowedTools ?? [],
+      usageDescription:
+        policy?.usageDescription ?? existing?.usageDescription,
+    });
+    return {
+      credentialId: `cred-${key}`,
+      service,
+      field,
+      allowedTools: metadataStore.get(key)!.allowedTools,
+      allowedDomains: [],
+      createdAt: 0,
+      updatedAt: 0,
+    };
+  },
+}));
+
+mock.module("../../tools/credentials/broker.js", () => ({
+  credentialBroker: {
+    serverUse: async <T>(request: {
+      service: string;
+      field: string;
+      toolName: string;
+      execute: (value: string) => Promise<T>;
+    }) => {
+      const key = `${request.service}/${request.field}`;
+      const meta = metadataStore.get(key);
+      if (!meta) {
+        return { success: false, reason: `No credential found for ${key}` };
+      }
+      if (!meta.allowedTools.includes(request.toolName)) {
+        return {
+          success: false,
+          reason: `Tool "${request.toolName}" not allowed`,
+        };
+      }
+      const value = vaultStore.get(key);
+      if (!value) {
+        return { success: false, reason: `No stored value for ${key}` };
+      }
+      const result = await request.execute(value);
+      return { success: true, result };
+    },
+  },
 }));
 
 // Stub session manager so we don't actually spawn child processes.
@@ -153,11 +213,9 @@ beforeEach(() => {
   // Default: vault has a claude token so the preflight in `prepareAgentEnv`
   // succeeds for tests that don't care about env injection. Env-injection
   // tests below clear/override this explicitly.
-  secureKeyStore.clear();
-  secureKeyStore.set(
-    "credential/acp/claude_oauth_token",
-    "default-test-token",
-  );
+  vaultStore.clear();
+  metadataStore.clear();
+  vaultStore.set("acp/claude_oauth_token", "default-test-token");
 });
 
 // ---------------------------------------------------------------------------
@@ -420,12 +478,10 @@ describe("executeAcpSpawn — per-agent resume hint", () => {
 // ---------------------------------------------------------------------------
 
 describe("executeAcpSpawn — CLAUDE_CODE_OAUTH_TOKEN injection", () => {
-  test("injects CLAUDE_CODE_OAUTH_TOKEN from the secure store for the claude agent", async () => {
-    secureKeyStore.clear();
-    secureKeyStore.set(
-      "credential/acp/claude_oauth_token",
-      "tool-vault-token-abc",
-    );
+  test("injects CLAUDE_CODE_OAUTH_TOKEN from the vault via the broker for the claude agent", async () => {
+    vaultStore.clear();
+    metadataStore.clear();
+    vaultStore.set("acp/claude_oauth_token", "tool-vault-token-abc");
     execScripts.set("npm ls", { error: new Error("npm not installed") });
     execScripts.set("npm view", { error: new Error("npm not installed") });
 
@@ -445,10 +501,8 @@ describe("executeAcpSpawn — CLAUDE_CODE_OAUTH_TOKEN injection", () => {
   });
 
   test("accepts CLAUDE_CODE_OAUTH_TOKEN from config.json agent.env without a vault entry", async () => {
-    // Mirrors the route-level precedence test: a config.json env override
-    // is the first-priority provisioning route. The resolver surfaces it
-    // on `resolved.agent.env`, which the helper preserves.
-    secureKeyStore.clear();
+    vaultStore.clear();
+    metadataStore.clear();
     config.setConfig({
       agents: {
         claude: {
@@ -478,11 +532,8 @@ describe("executeAcpSpawn — CLAUDE_CODE_OAUTH_TOKEN injection", () => {
   });
 
   test("returns isError when no token is available from either route (preflight throw mapped to tool result)", async () => {
-    // Both routes empty. The helper throws `FailedDependencyError`; the
-    // tool catches it and returns `{ isError: true, content: <msg> }`
-    // rather than letting it propagate (the tool boundary is a sync
-    // ToolExecutionResult, not an HTTP response).
-    secureKeyStore.clear();
+    vaultStore.clear();
+    metadataStore.clear();
 
     const result = await executeAcpSpawn(
       { agent: "claude", task: "do something" },
@@ -491,7 +542,6 @@ describe("executeAcpSpawn — CLAUDE_CODE_OAUTH_TOKEN injection", () => {
 
     expect(result.isError).toBe(true);
     expect(result.content).toContain("CLAUDE_CODE_OAUTH_TOKEN");
-    // Spawn was NEVER called — preflight fired before the subprocess started.
     expect(spawnMock).not.toHaveBeenCalled();
   });
 });

package/src/tools/apps/definitions.ts

@@ -9,7 +9,11 @@
  */
 
 import { RiskLevel } from "../../permissions/types.js";
-import type { Tool, ToolContext, ToolExecutionResult } from "../types.js";
+import type {
+  ToolContext,
+  ToolDefinition,
+  ToolExecutionResult,
+} from "../types.js";
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -40,7 +44,7 @@ function proxyExecute(toolName: string) {
 // app_open
 // ---------------------------------------------------------------------------
 
-const appOpenTool: Tool = {
+const appOpenTool = {
   name: "app_open",
   description:
     "Open a persistent app in a dynamic_page surface on the connected client.",
@@ -66,10 +70,10 @@ const appOpenTool: Tool = {
   },
 
   execute: proxyExecute("app_open"),
-};
+} satisfies ToolDefinition;
 
 // ---------------------------------------------------------------------------
 // Proxy-only tools registered in the core daemon registry
 // ---------------------------------------------------------------------------
 
-export const coreAppProxyTools: Tool[] = [appOpenTool];
+export const coreAppProxyTools: ToolDefinition[] = [appOpenTool];