@vellumai/assistant 0.7.0 → 0.7.1

package/src/__tests__/provider-usage-tracking.test.ts

@@ -0,0 +1,208 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { makeMockLogger } from "./helpers/mock-logger.js";
+
+let mockLlmConfig: Record<string, unknown> = {};
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => ({
+    llm: mockLlmConfig,
+  }),
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => makeMockLogger(),
+}));
+
+import { LLMSchema } from "../config/schemas/llm.js";
+import { getDb } from "../memory/db-connection.js";
+import { initializeDb } from "../memory/db-init.js";
+import { listUsageEvents } from "../memory/llm-usage-store.js";
+import { CallSiteConfiguredProvider } from "../providers/provider-send-message.js";
+import type { Provider, ProviderResponse } from "../providers/types.js";
+import { UsageTrackingProvider } from "../providers/usage-tracking.js";
+
+initializeDb();
+
+function setLlmConfig(raw: unknown): void {
+  mockLlmConfig = LLMSchema.parse(raw) as Record<string, unknown>;
+}
+
+function makeProvider(response: ProviderResponse): Provider {
+  return {
+    name: "openai",
+    async sendMessage() {
+      return response;
+    },
+  };
+}
+
+describe("UsageTrackingProvider", () => {
+  beforeEach(() => {
+    const db = getDb();
+    db.run(`DELETE FROM llm_usage_events`);
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-opus-4-7",
+      },
+      profiles: {
+        balanced: {
+          provider: "openai",
+          model: "gpt-5.4-mini",
+        },
+      },
+      activeProfile: "balanced",
+      pricingOverrides: [],
+    });
+  });
+
+  test("auto-records attributed non-conversation provider usage", async () => {
+    const provider = new UsageTrackingProvider(
+      makeProvider({
+        content: [{ type: "text", text: "Title" }],
+        model: "gpt-5.4-mini",
+        usage: {
+          inputTokens: 1_000,
+          outputTokens: 2_000,
+        },
+        stopReason: "end_turn",
+      }),
+    );
+
+    await provider.sendMessage(
+      [{ role: "user", content: [{ type: "text", text: "Summarize" }] }],
+      undefined,
+      undefined,
+      {
+        config: {
+          callSite: "conversationTitle",
+        },
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      actor: "llm_call_site",
+      conversationId: null,
+      runId: null,
+      requestId: null,
+      provider: "openai",
+      model: "gpt-5.4-mini",
+      inputTokens: 1_000,
+      outputTokens: 2_000,
+      cacheCreationInputTokens: 0,
+      cacheReadInputTokens: 0,
+      callSite: "conversationTitle",
+      inferenceProfile: "balanced",
+      inferenceProfileSource: "active",
+      pricingStatus: "priced",
+    });
+    expect(events[0].estimatedCostUsd ?? 0).toBeCloseTo(0.00975, 10);
+  });
+
+  test("uses the transport provider when resolved attribution points elsewhere", async () => {
+    setLlmConfig({
+      default: {
+        provider: "openai",
+        model: "gpt-5.4-mini",
+      },
+      callSites: {
+        conversationTitle: {
+          provider: "fireworks",
+          model: "accounts/fireworks/models/deepseek-v3",
+        },
+      },
+      pricingOverrides: [],
+    });
+
+    const provider = new UsageTrackingProvider(
+      makeProvider({
+        content: [{ type: "text", text: "Title" }],
+        model: "gpt-5.4-mini",
+        usage: {
+          inputTokens: 1_000,
+          outputTokens: 2_000,
+        },
+        stopReason: "end_turn",
+      }),
+    );
+
+    await provider.sendMessage(
+      [{ role: "user", content: [{ type: "text", text: "Summarize" }] }],
+      undefined,
+      undefined,
+      {
+        config: {
+          callSite: "conversationTitle",
+        },
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      provider: "openai",
+      model: "gpt-5.4-mini",
+      callSite: "conversationTitle",
+      pricingStatus: "priced",
+    });
+  });
+
+  test("does not record calls without a call site", async () => {
+    const provider = new UsageTrackingProvider(
+      makeProvider({
+        content: [{ type: "text", text: "ok" }],
+        model: "gpt-5.4-mini",
+        usage: {
+          inputTokens: 1,
+          outputTokens: 1,
+        },
+        stopReason: "end_turn",
+      }),
+    );
+
+    await provider.sendMessage([
+      { role: "user", content: [{ type: "text", text: "Hello" }] },
+    ]);
+
+    expect(listUsageEvents()).toHaveLength(0);
+  });
+
+  test("records calls from providers resolved for a call site even when send options omit it", async () => {
+    const provider = new CallSiteConfiguredProvider(
+      new UsageTrackingProvider(
+        makeProvider({
+          content: [{ type: "text", text: "ok" }],
+          model: "gpt-5.4-mini",
+          usage: {
+            inputTokens: 100,
+            outputTokens: 50,
+          },
+          stopReason: "end_turn",
+        }),
+      ),
+      "mainAgent",
+    );
+
+    await provider.sendMessage(
+      [{ role: "user", content: [{ type: "text", text: "Hello" }] }],
+      undefined,
+      undefined,
+      {
+        config: {
+          model: "gpt-5.4-mini",
+        },
+      },
+    );
+
+    const events = listUsageEvents();
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      callSite: "mainAgent",
+      provider: "openai",
+      model: "gpt-5.4-mini",
+    });
+  });
+});

package/src/__tests__/reaction-persistence.test.ts

@@ -26,6 +26,11 @@ mock.module("../config/env.js", () => ({
   getGatewayInternalBaseUrl: () => "http://127.0.0.1:7830",
 }));
 
+const _conversationMocks = new Map<string, unknown>();
+mock.module("../daemon/conversation-store.js", () => ({
+  findConversation: (id: string) => _conversationMocks.get(id),
+}));
+
 mock.module("../tools/credentials/metadata-store.js", () => ({
   getCredentialMetadata: () => undefined,
   upsertCredentialMetadata: () => {},
@@ -39,10 +44,7 @@ mock.module("../runtime/gateway-client.js", () => ({
 
 import { eq } from "drizzle-orm";
 
-import {
-  createGuardianBinding,
-  upsertContactChannel,
-} from "../contacts/contacts-write.js";
+import { upsertContactChannel } from "../contacts/contacts-write.js";
 import type { Conversation } from "../daemon/conversation.js";
 import { getDb } from "../memory/db-connection.js";
 import { initializeDb } from "../memory/db-init.js";
@@ -62,6 +64,7 @@ import {
   parseSlackReactionCallbackData,
 } from "../runtime/routes/inbound-message-handler.js";
 import { handleChannelInbound } from "./helpers/channel-test-adapter.js";
+import { createGuardianBinding } from "./helpers/create-guardian-binding.js";
 
 initializeDb();
 
@@ -435,12 +438,12 @@ function seedPendingGuardianApprovalForReaction(
   // Register a pending interaction so applyGuardianDecision finds the
   // requester-side hook to drive `allow`.
   const handleConfirmationResponse = mock(() => {});
-  const mockSession = {
+  const _mockSession = {
     handleConfirmationResponse,
     ensureActorScopedHistory: async () => {},
   } as unknown as Conversation;
+  _conversationMocks.set(conversationId, _mockSession);
   pendingInteractions.register(requestId, {
-    conversation: mockSession,
     conversationId,
     kind: "confirmation",
     confirmationDetails: {

package/src/__tests__/rebind-secrets-screen.test.ts

@@ -61,12 +61,23 @@ function makeValidateSuccess(): ValidateResponse {
     is_valid: true,
     errors: [],
     manifest: {
-      schema_version: "1.0",
-      created_at: "2025-01-01T00:00:00Z",
-      source: "test",
-      description: "Test bundle",
-      files: [{ path: "config.json", sha256: "abc123", size: 1024 }],
-      manifest_sha256: "manifest-hash",
+      schema_version: 1,
+      bundle_id: "00000000-0000-4000-8000-000000000000",
+      created_at: "2026-03-01T00:00:00Z",
+      assistant: { id: "self", name: "Test", runtime_version: "0.0.0-test" },
+      origin: { mode: "self-hosted-local" },
+      compatibility: {
+        min_runtime_version: "0.0.0-test",
+        max_runtime_version: null,
+      },
+      contents: [{ path: "config.json", sha256: "abc123", size_bytes: 1024 }],
+      checksum: "manifest-hash",
+      secrets_redacted: false,
+      export_options: {
+        include_logs: false,
+        include_browser_state: false,
+        include_memory_vectors: false,
+      },
     },
   };
 }
@@ -93,10 +104,23 @@ function makePreflightSuccess(): ImportPreflightResponse {
     ],
     conflicts: [],
     manifest: {
-      schema_version: "1.0",
-      created_at: "2025-01-01T00:00:00Z",
-      files: [{ path: "config.json", sha256: "abc123", size: 1024 }],
-      manifest_sha256: "manifest-hash",
+      schema_version: 1,
+      bundle_id: "00000000-0000-4000-8000-000000000000",
+      created_at: "2026-03-01T00:00:00Z",
+      assistant: { id: "self", name: "Test", runtime_version: "0.0.0-test" },
+      origin: { mode: "self-hosted-local" },
+      compatibility: {
+        min_runtime_version: "0.0.0-test",
+        max_runtime_version: null,
+      },
+      contents: [{ path: "config.json", sha256: "abc123", size_bytes: 1024 }],
+      checksum: "manifest-hash",
+      secrets_redacted: false,
+      export_options: {
+        include_logs: false,
+        include_browser_state: false,
+        include_memory_vectors: false,
+      },
     },
   };
 }
@@ -122,10 +146,23 @@ function makeImportSuccess(): ImportCommitResponse {
       },
     ],
     manifest: {
-      schema_version: "1.0",
-      created_at: "2025-01-01T00:00:00Z",
-      files: [{ path: "config.json", sha256: "abc123", size: 1024 }],
-      manifest_sha256: "manifest-hash",
+      schema_version: 1,
+      bundle_id: "00000000-0000-4000-8000-000000000000",
+      created_at: "2026-03-01T00:00:00Z",
+      assistant: { id: "self", name: "Test", runtime_version: "0.0.0-test" },
+      origin: { mode: "self-hosted-local" },
+      compatibility: {
+        min_runtime_version: "0.0.0-test",
+        max_runtime_version: null,
+      },
+      contents: [{ path: "config.json", sha256: "abc123", size_bytes: 1024 }],
+      checksum: "manifest-hash",
+      secrets_redacted: false,
+      export_options: {
+        include_logs: false,
+        include_browser_state: false,
+        include_memory_vectors: false,
+      },
     },
     warnings: ["Backup created for config.json"],
   };
@@ -559,8 +596,8 @@ describe("goBackToTransfer", () => {
       exportResult: {
         ok: true,
         filename: "export.vbundle",
-        schemaVersion: "1.0",
-        manifestSha256: "abc",
+        schemaVersion: 1,
+        checksum: "abc",
       },
       importResult: makeImportSuccess(),
     };