@vellumai/assistant 0.8.1 → 0.8.3

package/src/telemetry/usage-telemetry-reporter.test.ts

@@ -34,7 +34,17 @@ mock.module("../memory/llm-usage-store.js", () => ({
 }));
 
 const mockQueryUnreportedTurnEvents = mock(
-  () => [] as { id: string; createdAt: number }[],
+  () =>
+    [] as {
+      id: string;
+      createdAt: number;
+      conversationId: string;
+      conversationType: string;
+      turnIndex: number;
+      interfaceId: string | null;
+      channelId: string | null;
+      clientMetadata: string | null;
+    }[],
 );
 
 mock.module("../memory/turn-events-store.js", () => ({
@@ -95,6 +105,25 @@ mock.module("../memory/lifecycle-events-store.js", () => ({
   queryUnreportedLifecycleEvents: mockQueryUnreportedLifecycleEvents,
 }));
 
+const mockQueryUnreportedOnboardingEvents = mock(
+  () =>
+    [] as {
+      id: string;
+      createdAt: number;
+      screen: string;
+      toolsJson: string | null;
+      tasksJson: string | null;
+      tone: string | null;
+      googleConnected: boolean | null;
+      googleScopesJson: string | null;
+      abVariant: string | null;
+    }[],
+);
+
+mock.module("../memory/onboarding-events-store.js", () => ({
+  queryUnreportedOnboardingEvents: mockQueryUnreportedOnboardingEvents,
+}));
+
 // ---------------------------------------------------------------------------
 // Production import (after mocks)
 // ---------------------------------------------------------------------------
@@ -108,7 +137,18 @@ import { UsageTelemetryReporter } from "./usage-telemetry-reporter.js";
 
 let eventIdCounter = 0;
 
-function makeUsageEvent(overrides: Partial<UsageEvent> = {}): UsageEvent {
+// The reporter consumes `UnreportedUsageEvent` (UsageEvent + the two
+// JOIN-computed fields `conversationType` and `turnIndex`). Build that
+// shape directly so the mock matches `queryUnreportedUsageEvents`'
+// return type exactly.
+type UnreportedUsageEventFixture = UsageEvent & {
+  conversationType: string | null;
+  turnIndex: number | null;
+};
+
+function makeUsageEvent(
+  overrides: Partial<UnreportedUsageEventFixture> = {},
+): UnreportedUsageEventFixture {
   eventIdCounter += 1;
   return {
     id: `evt-${eventIdCounter}`,
@@ -128,6 +168,8 @@ function makeUsageEvent(overrides: Partial<UsageEvent> = {}): UsageEvent {
     requestId: null,
     estimatedCostUsd: 0.001,
     pricingStatus: "priced",
+    conversationType: "standard",
+    turnIndex: 1,
     ...overrides,
   };
 }
@@ -149,6 +191,8 @@ beforeEach(() => {
   mockQueryUnreportedTurnEvents.mockReturnValue([]);
   mockQueryUnreportedLifecycleEvents.mockReset();
   mockQueryUnreportedLifecycleEvents.mockReturnValue([]);
+  mockQueryUnreportedOnboardingEvents.mockReset();
+  mockQueryUnreportedOnboardingEvents.mockReturnValue([]);
   mockPlatformClient = null;
   mockGetPlatformBaseUrl.mockReset();
   mockGetDeviceId.mockReset();
@@ -501,7 +545,16 @@ describe("UsageTelemetryReporter", () => {
     const usageEvent = makeUsageEvent({ id: "evt-mixed-usage" });
     mockQueryUnreportedUsageEvents.mockReturnValue([usageEvent]);
     mockQueryUnreportedTurnEvents.mockReturnValue([
-      { id: "evt-mixed-turn", createdAt: 1700000050000 },
+      {
+        id: "evt-mixed-turn",
+        createdAt: 1700000050000,
+        conversationId: "conv-mixed",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
     ]);
     mockFetch.mockImplementation(() =>
       Promise.resolve(new Response('{"accepted":2}', { status: 200 })),
@@ -532,6 +585,259 @@ describe("UsageTelemetryReporter", () => {
     expect(turnEvent).toBeDefined();
     expect(turnEvent.daemon_event_id).toBe("evt-mixed-turn");
     expect(turnEvent.recorded_at).toBe(1700000050000);
+    expect(turnEvent.conversation_id).toBe("conv-mixed");
+    expect(turnEvent.conversation_type).toBe("standard");
+    expect(turnEvent.turn_index).toBe(1);
+  });
+
+  test("turn events carry conversation_type for background/scheduled conversations", async () => {
+    mockQueryUnreportedUsageEvents.mockReturnValue([]);
+    mockQueryUnreportedTurnEvents.mockReturnValue([
+      {
+        id: "evt-turn-standard",
+        createdAt: 1700000100000,
+        conversationId: "conv-std",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
+      {
+        id: "evt-turn-background",
+        createdAt: 1700000200000,
+        conversationId: "conv-bg",
+        conversationType: "background",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
+      {
+        id: "evt-turn-scheduled",
+        createdAt: 1700000300000,
+        conversationId: "conv-sched",
+        conversationType: "scheduled",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":3}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+
+    const byId: Record<string, { conversation_type: string }> = {};
+    for (const e of body.events as Array<{
+      daemon_event_id: string;
+      conversation_type: string;
+    }>) {
+      byId[e.daemon_event_id] = e;
+    }
+
+    expect(byId["evt-turn-standard"].conversation_type).toBe("standard");
+    expect(byId["evt-turn-background"].conversation_type).toBe("background");
+    expect(byId["evt-turn-scheduled"].conversation_type).toBe("scheduled");
+  });
+
+  test("turn events carry interface_id, channel_id, and parsed client metadata", async () => {
+    // Four turns spanning the relevant cases:
+    //  - macOS in-app turn with full client block (typical interactive user)
+    //  - slack inbound turn with no client block (channel-based, no headers)
+    //  - web turn with malformed client JSON (parsing guard: emit null,
+    //    do not break the batch)
+    //  - historical turn with no metadata at all (pre-rollout row)
+    mockQueryUnreportedUsageEvents.mockReturnValue([]);
+    mockQueryUnreportedTurnEvents.mockReturnValue([
+      {
+        id: "evt-turn-macos",
+        createdAt: 1700000400000,
+        conversationId: "conv-mac",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: "macos",
+        channelId: "vellum",
+        clientMetadata: JSON.stringify({
+          browser_family: null,
+          os: "darwin",
+          interface_version: "0.8.2",
+        }),
+      },
+      {
+        id: "evt-turn-slack",
+        createdAt: 1700000500000,
+        conversationId: "conv-slack",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: "slack",
+        channelId: "slack",
+        clientMetadata: null,
+      },
+      {
+        id: "evt-turn-web-broken",
+        createdAt: 1700000600000,
+        conversationId: "conv-web",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: "web",
+        channelId: "vellum",
+        clientMetadata: "{not valid json",
+      },
+      {
+        id: "evt-turn-legacy",
+        createdAt: 1700000700000,
+        conversationId: "conv-legacy",
+        conversationType: "standard",
+        turnIndex: 1,
+        interfaceId: null,
+        channelId: null,
+        clientMetadata: null,
+      },
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":4}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+
+    const byId: Record<
+      string,
+      {
+        interface_id: string | null;
+        channel_id: string | null;
+        client: Record<string, unknown> | null;
+      }
+    > = {};
+    for (const e of body.events as Array<{
+      daemon_event_id: string;
+      interface_id: string | null;
+      channel_id: string | null;
+      client: Record<string, unknown> | null;
+    }>) {
+      byId[e.daemon_event_id] = e;
+    }
+
+    expect(byId["evt-turn-macos"]).toMatchObject({
+      interface_id: "macos",
+      channel_id: "vellum",
+      client: {
+        os: "darwin",
+        interface_version: "0.8.2",
+      },
+    });
+    expect(byId["evt-turn-slack"]).toMatchObject({
+      interface_id: "slack",
+      channel_id: "slack",
+      client: null,
+    });
+    // Malformed client JSON is downgraded to null without failing the
+    // batch — the interface_id/channel_id from the typed columns still
+    // ride through cleanly.
+    expect(byId["evt-turn-web-broken"]).toMatchObject({
+      interface_id: "web",
+      channel_id: "vellum",
+      client: null,
+    });
+    expect(byId["evt-turn-legacy"]).toMatchObject({
+      interface_id: null,
+      channel_id: null,
+      client: null,
+    });
+  });
+
+  test("llm_usage events carry conversation_id, conversation_type, and turn_index", async () => {
+    // Three LLM calls across the spectrum of the new fields:
+    //  - tied to a conversation, mid-turn (typical foreground)
+    //  - tied to a background conversation, first turn
+    //  - untied (memory consolidation: no conversation, no turn)
+    mockQueryUnreportedUsageEvents.mockReturnValue([
+      makeUsageEvent({
+        id: "evt-fg-call",
+        conversationId: "conv-fg",
+        conversationType: "standard",
+        turnIndex: 4,
+      }),
+      makeUsageEvent({
+        id: "evt-bg-call",
+        conversationId: "conv-bg",
+        conversationType: "background",
+        turnIndex: 1,
+      }),
+      makeUsageEvent({
+        id: "evt-untied-call",
+        conversationId: null,
+        conversationType: null,
+        turnIndex: null,
+      }),
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":3}', { status: 200 })),
+    );
+
+    const reporter = new UsageTelemetryReporter();
+    await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+
+    const byId: Record<
+      string,
+      {
+        type: string;
+        conversation_id: string | null;
+        conversation_type: string | null;
+        turn_index: number | null;
+      }
+    > = {};
+    for (const e of body.events as Array<{
+      type: string;
+      daemon_event_id: string;
+      conversation_id: string | null;
+      conversation_type: string | null;
+      turn_index: number | null;
+    }>) {
+      byId[e.daemon_event_id] = e;
+    }
+
+    expect(byId["evt-fg-call"]).toMatchObject({
+      type: "llm_usage",
+      conversation_id: "conv-fg",
+      conversation_type: "standard",
+      turn_index: 4,
+    });
+    expect(byId["evt-bg-call"]).toMatchObject({
+      type: "llm_usage",
+      conversation_id: "conv-bg",
+      conversation_type: "background",
+      turn_index: 1,
+    });
+    // LLM calls without a parent conversation flush through with all
+    // three conversation-level fields null — the serializer accepts
+    // allow_null and downstream SQL filters can `WHERE conversation_id
+    // IS NOT NULL` to scope to foreground analytics.
+    expect(byId["evt-untied-call"]).toMatchObject({
+      type: "llm_usage",
+      conversation_id: null,
+      conversation_type: null,
+      turn_index: null,
+    });
   });
 
   test("flush is skipped and watermarks advanced when collectUsageData is false", async () => {
@@ -548,15 +854,16 @@ describe("UsageTelemetryReporter", () => {
     // No HTTP call should have been made
     expect(mockFetch).not.toHaveBeenCalled();
 
-    // All 3 timestamp watermarks should have been advanced (IDs left untouched
+    // All 4 timestamp watermarks should have been advanced (IDs left untouched
     // so the compound-cursor branch stays active)
-    expect(mockSetMemoryCheckpoint).toHaveBeenCalledTimes(3);
+    expect(mockSetMemoryCheckpoint).toHaveBeenCalledTimes(4);
 
     const calls = mockSetMemoryCheckpoint.mock.calls;
     const keys = calls.map((c) => c[0]);
     expect(keys).toContain("telemetry:usage:last_reported_at");
     expect(keys).toContain("telemetry:turns:last_reported_at");
     expect(keys).toContain("telemetry:lifecycle:last_reported_at");
+    expect(keys).toContain("telemetry:onboarding:last_reported_at");
   });
 
   test("events sent normally after re-enabling collectUsageData", async () => {

package/src/telemetry/usage-telemetry-reporter.ts

@@ -21,12 +21,13 @@ import {
 } from "../memory/checkpoints.js";
 import { queryUnreportedLifecycleEvents } from "../memory/lifecycle-events-store.js";
 import { queryUnreportedUsageEvents } from "../memory/llm-usage-store.js";
+import { queryUnreportedOnboardingEvents } from "../memory/onboarding-events-store.js";
 import { queryUnreportedTurnEvents } from "../memory/turn-events-store.js";
 import { VellumPlatformClient } from "../platform/client.js";
 import { getDeviceId } from "../util/device-id.js";
 import { getLogger } from "../util/logger.js";
 import { APP_VERSION } from "../version.js";
-import type { TelemetryEvent } from "./types.js";
+import type { TelemetryEvent, TurnTelemetryClientInfo } from "./types.js";
 
 const log = getLogger("usage-telemetry");
 
@@ -42,12 +43,32 @@ const CHECKPOINT_KEY_LIFECYCLE_WATERMARK =
   "telemetry:lifecycle:last_reported_at";
 const CHECKPOINT_KEY_LIFECYCLE_WATERMARK_ID =
   "telemetry:lifecycle:last_reported_id";
+const CHECKPOINT_KEY_ONBOARDING_WATERMARK =
+  "telemetry:onboarding:last_reported_at";
+const CHECKPOINT_KEY_ONBOARDING_WATERMARK_ID =
+  "telemetry:onboarding:last_reported_id";
 const REPORT_INTERVAL_MS = 5 * 60 * 1000;
 const INITIAL_FLUSH_DELAY_MS = 30_000; // Delay first flush to let CES handshake complete
 const BATCH_SIZE = 500;
 const MAX_CONSECUTIVE_BATCHES = 10;
 const TELEMETRY_PATH = "/v1/telemetry/ingest/";
 
+// ---------------------------------------------------------------------------
+// Singleton access
+// ---------------------------------------------------------------------------
+
+let _instance: UsageTelemetryReporter | null = null;
+
+export function getUsageTelemetryReporter(): UsageTelemetryReporter | null {
+  return _instance;
+}
+
+export function setUsageTelemetryReporter(
+  reporter: UsageTelemetryReporter | null,
+): void {
+  _instance = reporter;
+}
+
 // ---------------------------------------------------------------------------
 // Reporter
 // ---------------------------------------------------------------------------
@@ -117,6 +138,7 @@ export class UsageTelemetryReporter {
         setMemoryCheckpoint(CHECKPOINT_KEY_WATERMARK, now);
         setMemoryCheckpoint(CHECKPOINT_KEY_TURN_WATERMARK, now);
         setMemoryCheckpoint(CHECKPOINT_KEY_LIFECYCLE_WATERMARK, now);
+        setMemoryCheckpoint(CHECKPOINT_KEY_ONBOARDING_WATERMARK, now);
         return;
       }
 
@@ -141,6 +163,14 @@ export class UsageTelemetryReporter {
       const lifecycleWatermarkId =
         getMemoryCheckpoint(CHECKPOINT_KEY_LIFECYCLE_WATERMARK_ID) ?? undefined;
 
+      // Read onboarding watermark (compound cursor: createdAt + id)
+      const onboardingWatermark = Number(
+        getMemoryCheckpoint(CHECKPOINT_KEY_ONBOARDING_WATERMARK) ?? "0",
+      );
+      const onboardingWatermarkId =
+        getMemoryCheckpoint(CHECKPOINT_KEY_ONBOARDING_WATERMARK_ID) ??
+        undefined;
+
       // Query unreported events
       const events = queryUnreportedUsageEvents(
         watermark,
@@ -157,11 +187,17 @@ export class UsageTelemetryReporter {
         lifecycleWatermarkId,
         BATCH_SIZE,
       );
+      const onboardingEvents = queryUnreportedOnboardingEvents(
+        onboardingWatermark,
+        onboardingWatermarkId,
+        BATCH_SIZE,
+      );
 
       if (
         events.length === 0 &&
         turnEvents.length === 0 &&
-        lifecycleEvents.length === 0
+        lifecycleEvents.length === 0 &&
+        onboardingEvents.length === 0
       )
         return;
 
@@ -174,6 +210,7 @@ export class UsageTelemetryReporter {
           usageCount: events.length,
           turnCount: turnEvents.length,
           lifecycleCount: lifecycleEvents.length,
+          onboardingCount: onboardingEvents.length,
         },
         "Telemetry flush: resolved auth context",
       );
@@ -184,6 +221,13 @@ export class UsageTelemetryReporter {
           (e): TelemetryEvent => ({
             type: "llm_usage",
             daemon_event_id: e.id,
+            // Conversation-level metadata for analytics joins. All three
+            // are nullable on the wire: `conversation_id` is null for
+            // LLM calls not tied to a conversation (memory consolidation,
+            // background work), and the other two cascade from that.
+            conversation_id: e.conversationId,
+            conversation_type: e.conversationType,
+            turn_index: e.turnIndex,
             provider: e.provider,
             model: e.model,
             input_tokens: e.inputTokens,
@@ -198,13 +242,43 @@ export class UsageTelemetryReporter {
             recorded_at: e.createdAt,
           }),
         ),
-        ...turnEvents.map(
-          (e): TelemetryEvent => ({
+        ...turnEvents.map((e): TelemetryEvent => {
+          // `messages.metadata.client` is a nested JSON object extracted
+          // via `json_extract`; sqlite returns it as a text representation.
+          // Parse defensively — a corrupted blob in the JSON column should
+          // not block the whole batch flush.
+          let client: TurnTelemetryClientInfo | null = null;
+          if (e.clientMetadata) {
+            try {
+              const parsed = JSON.parse(e.clientMetadata) as unknown;
+              if (
+                parsed &&
+                typeof parsed === "object" &&
+                !Array.isArray(parsed)
+              ) {
+                client = parsed as TurnTelemetryClientInfo;
+              }
+            } catch {
+              // Malformed client JSON — emit null rather than fail the
+              // batch. Logged once below for visibility.
+              log.warn(
+                { turnId: e.id, conversationId: e.conversationId },
+                "Telemetry turn: failed to parse messages.metadata.client; emitting null",
+              );
+            }
+          }
+          return {
             type: "turn",
             daemon_event_id: e.id,
             recorded_at: e.createdAt,
-          }),
-        ),
+            conversation_id: e.conversationId,
+            conversation_type: e.conversationType,
+            turn_index: e.turnIndex,
+            interface_id: e.interfaceId,
+            channel_id: e.channelId,
+            client,
+          };
+        }),
         ...lifecycleEvents.map(
           (e): TelemetryEvent => ({
             type: "lifecycle",
@@ -213,6 +287,24 @@ export class UsageTelemetryReporter {
             recorded_at: e.createdAt,
           }),
         ),
+        ...onboardingEvents.map(
+          (e): TelemetryEvent => ({
+            type: "onboarding",
+            daemon_event_id: e.id,
+            recorded_at: e.createdAt,
+            screen: e.screen,
+            ...(e.toolsJson ? { tools: JSON.parse(e.toolsJson) } : {}),
+            ...(e.tasksJson ? { tasks: JSON.parse(e.tasksJson) } : {}),
+            ...(e.tone ? { tone: e.tone } : {}),
+            ...(e.googleConnected != null
+              ? { google_connected: e.googleConnected }
+              : {}),
+            ...(e.googleScopesJson
+              ? { google_scopes: JSON.parse(e.googleScopesJson) }
+              : {}),
+            ...(e.abVariant ? { ab_variant: e.abVariant } : {}),
+          }),
+        ),
       ];
 
       const organizationId = getPlatformOrganizationId() || undefined;
@@ -285,11 +377,25 @@ export class UsageTelemetryReporter {
         );
       }
 
+      // Advance onboarding watermark (compound cursor)
+      if (onboardingEvents.length > 0) {
+        const lastOnboarding = onboardingEvents[onboardingEvents.length - 1];
+        setMemoryCheckpoint(
+          CHECKPOINT_KEY_ONBOARDING_WATERMARK,
+          String(lastOnboarding.createdAt),
+        );
+        setMemoryCheckpoint(
+          CHECKPOINT_KEY_ONBOARDING_WATERMARK_ID,
+          lastOnboarding.id,
+        );
+      }
+
       // If we got a full batch of any type, there may be more — recurse
       if (
         events.length === BATCH_SIZE ||
         turnEvents.length === BATCH_SIZE ||
-        lifecycleEvents.length === BATCH_SIZE
+        lifecycleEvents.length === BATCH_SIZE ||
+        onboardingEvents.length === BATCH_SIZE
       ) {
         await this._doFlush(batchCount + 1);
       }