@vellumai/assistant 0.10.9 → 0.10.10-dev.202607162206.d08e98e

package/src/telemetry/usage-telemetry-reporter.test.ts

@@ -7,6 +7,8 @@
  */
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
+import type { ConsentState } from "../platform/consent-cache.js";
+
 // ---------------------------------------------------------------------------
 // Module mocks (must precede production imports)
 // ---------------------------------------------------------------------------
@@ -91,7 +93,10 @@ mock.module("../version.js", () => ({
   APP_VERSION: "1.2.3-test",
 }));
 
-const mockGetCachedShareAnalytics = mock(() => true);
+// Tri-state `share_analytics` consent (the reporter's flush gate). The
+// boolean accessor used by record-time gates derives from the same knob so a
+// test toggling consent flips both views consistently.
+const mockGetRawShareAnalytics = mock<() => ConsentState>(() => true);
 // Owner's `share_diagnostics` consent — one part of the trace-collection gate.
 const mockGetCachedShareDiagnostics = mock(() => false);
 // Owner's accepted diagnostics-consent version — the disclosing-version part of
@@ -101,7 +106,7 @@ const mockGetCachedShareDiagnostics = mock(() => false);
 const mockGetCachedShareDiagnosticsVersion = mock(() => "2999-01-01");
 
 mock.module("../platform/consent-cache.js", () => ({
-  getCachedShareAnalytics: mockGetCachedShareAnalytics,
+  getRawShareAnalytics: mockGetRawShareAnalytics,
   getCachedShareDiagnostics: mockGetCachedShareDiagnostics,
   getCachedShareDiagnosticsVersion: mockGetCachedShareDiagnosticsVersion,
 }));
@@ -215,6 +220,7 @@ import {
 } from "./telemetry-event-sources.js";
 import {
   insertTelemetryOutboxEvent,
+  OUTBOX_MAX_ROW_AGE_MS,
   queryTelemetryOutboxBatch,
 } from "./telemetry-events-outbox.js";
 import type { OutboxTelemetryEventName, TelemetryEvent } from "./types.js";
@@ -324,8 +330,8 @@ let mockFetch: ReturnType<typeof mock>;
 beforeEach(() => {
   eventIdCounter = 0;
   // Default consent ON so the happy-path send tests exercise the flush.
-  mockGetCachedShareAnalytics.mockReset();
-  mockGetCachedShareAnalytics.mockReturnValue(true);
+  mockGetRawShareAnalytics.mockReset();
+  mockGetRawShareAnalytics.mockReturnValue(true);
   // Default `share_diagnostics` consent OFF — most tests don't expect a trace;
   // the trace-specific tests opt in explicitly.
   mockGetCachedShareDiagnostics.mockReset();
@@ -564,6 +570,126 @@ describe("UsageTelemetryReporter", () => {
     expect(mockFetch).toHaveBeenCalledTimes(10);
   });
 
+  test("flush summary reports sent/persisted/dropped from the ingest response", async () => {
+    mockQueryUnreportedUsageEvents.mockReturnValue([
+      makeUsageEvent(),
+      makeUsageEvent(),
+      makeUsageEvent(),
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(
+        new Response(
+          '{"accepted":3,"persisted":2,"dropped":{"analytics_opt_out":1}}',
+          { status: 200 },
+        ),
+      ),
+    );
+
+    const reporter = makeReporter();
+    const summary = await reporter.flush();
+
+    expect(summary).toEqual({
+      flushed: true,
+      sent: 3,
+      persisted: 2,
+      dropped: 1,
+    });
+  });
+
+  test("flush summary falls back to persisted=sent when the body omits persisted", async () => {
+    // Older platform returns only `accepted` — assume everything landed.
+    mockQueryUnreportedUsageEvents.mockReturnValue([
+      makeUsageEvent(),
+      makeUsageEvent(),
+    ]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response('{"accepted":2}', { status: 200 })),
+    );
+
+    const reporter = makeReporter();
+    const summary = await reporter.flush();
+
+    expect(summary).toEqual({
+      flushed: true,
+      sent: 2,
+      persisted: 2,
+      dropped: 0,
+    });
+  });
+
+  test("flush summary accumulates counts across the batch recursion", async () => {
+    const fullBatch = Array.from({ length: 500 }, (_, i) =>
+      makeUsageEvent({ id: `evt-acc-${i}`, createdAt: 1700000000000 + i }),
+    );
+    // First collect fills a batch (recurse); second returns a short batch (stop).
+    mockQueryUnreportedUsageEvents
+      .mockReturnValueOnce(fullBatch)
+      .mockReturnValue([makeUsageEvent({ id: "evt-acc-tail" })]);
+    // First POST drops 10; second persists its single event.
+    mockFetch
+      .mockImplementationOnce(() =>
+        Promise.resolve(
+          new Response('{"accepted":500,"persisted":490}', { status: 200 }),
+        ),
+      )
+      .mockImplementation(() =>
+        Promise.resolve(new Response('{"persisted":1}', { status: 200 })),
+      );
+
+    const reporter = makeReporter();
+    const summary = await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(2);
+    expect(summary).toEqual({
+      flushed: true,
+      sent: 501,
+      persisted: 491,
+      dropped: 10,
+    });
+  });
+
+  test("flush summary reports skip reasons without shipping", async () => {
+    const reporter = makeReporter();
+
+    // nothing-pending: no events across any source.
+    mockQueryUnreportedUsageEvents.mockReturnValue([]);
+    expect(await reporter.flush()).toEqual({
+      flushed: false,
+      reason: "nothing-pending",
+    });
+
+    // opted-out: confirmed share_analytics opt-out.
+    mockGetRawShareAnalytics.mockReturnValue(false);
+    mockQueryUnreportedUsageEvents.mockReturnValue([makeUsageEvent()]);
+    expect(await reporter.flush()).toEqual({
+      flushed: false,
+      reason: "opted-out",
+    });
+    mockGetRawShareAnalytics.mockReturnValue(true);
+
+    // no-credentials: authenticated-only, no platform client resolved.
+    mockPlatformClient = null;
+    expect(await reporter.flush()).toEqual({
+      flushed: false,
+      reason: "no-credentials",
+    });
+
+    expect(mockFetch).not.toHaveBeenCalled();
+  });
+
+  test("flush summary reports post-failed on a non-2xx response", async () => {
+    mockQueryUnreportedUsageEvents.mockReturnValue([makeUsageEvent()]);
+    mockFetch.mockImplementation(() =>
+      Promise.resolve(new Response("error", { status: 500 })),
+    );
+
+    const reporter = makeReporter();
+    expect(await reporter.flush()).toEqual({
+      flushed: false,
+      reason: "post-failed",
+    });
+  });
+
   test("stop() performs final flush", async () => {
     const events = [makeUsageEvent()];
     mockQueryUnreportedUsageEvents.mockReturnValue(events);
@@ -1164,7 +1290,7 @@ describe("UsageTelemetryReporter", () => {
     // The analytics gate short-circuits the entire flush; trace assembly must
     // never run (and nothing is sent) even when the trace gate is fully on
     // (share_diagnostics true at an eligible version).
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     mockGetCachedShareDiagnostics.mockReturnValue(true);
     mockQueryUnreportedTurnEvents.mockReturnValue(singleTurnEvent());
 
@@ -1565,7 +1691,7 @@ describe("UsageTelemetryReporter", () => {
   });
 
   test("flush is skipped and watermarks advanced when share_analytics consent is off", async () => {
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const events = [makeUsageEvent()];
     mockQueryUnreportedUsageEvents.mockReturnValue(events);
     mockFetch.mockImplementation(() =>
@@ -1608,6 +1734,37 @@ describe("UsageTelemetryReporter", () => {
     expect(keys).not.toContain("telemetry:onboarding:last_reported_id");
   });
 
+  test("warm-up: unknown-consent flushes defer the backlog until consent resolves to true", async () => {
+    // An unresolved consent cache (boot before the first refresh, no platform
+    // session) is not an opt-out: unlike the confirmed-false branch, nothing
+    // is sent, nothing is purged, and no watermark moves, so the backlog
+    // survives the cold-cache window.
+    mockGetRawShareAnalytics.mockReturnValue("unknown");
+    const events = [makeUsageEvent({ id: "evt-warmup-backlog" })];
+    mockQueryUnreportedUsageEvents.mockReturnValue(events);
+
+    const reporter = makeReporter();
+    // Construction initializes the absent tool_executed watermark; clear that
+    // call so the assertions below cover only the flushes.
+    mockSetFlushCheckpoint.mockClear();
+    await reporter.flush();
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(mockSetFlushCheckpoint).not.toHaveBeenCalled();
+
+    // The refresh loop resolves the cache to a confirmed opt-in: the same
+    // backlog now ships.
+    mockGetRawShareAnalytics.mockReturnValue(true);
+    await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    expect(
+      body.events.map((e: { daemon_event_id: string }) => e.daemon_event_id),
+    ).toEqual(["evt-warmup-backlog"]);
+  });
+
   test("platform disabled takes precedence over consent off — watermarks NOT advanced", async () => {
     // VELLUM_DISABLE_PLATFORM keeps the consent cache false (the consent
     // refresh can't create a platform client), so both gates would fire. The
@@ -1615,7 +1772,7 @@ describe("UsageTelemetryReporter", () => {
     // watermarks, preserving the backlog until the flag is cleared — a
     // deployment toggle must not be treated as a privacy opt-out.
     process.env.VELLUM_DISABLE_PLATFORM = "true";
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const events = [makeUsageEvent()];
     mockQueryUnreportedUsageEvents.mockReturnValue(events);
 
@@ -1652,14 +1809,14 @@ describe("UsageTelemetryReporter", () => {
 
   test("events sent normally after re-granting share_analytics consent", async () => {
     // First flush with opt-out — watermarks advance, nothing sent
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const reporter = makeReporter();
     await reporter.flush();
     expect(mockFetch).not.toHaveBeenCalled();
     mockSetFlushCheckpoint.mockReset();
 
     // Re-grant consent and flush with new events
-    mockGetCachedShareAnalytics.mockReturnValue(true);
+    mockGetRawShareAnalytics.mockReturnValue(true);
     const events = [makeUsageEvent({ id: "evt-after-reenable" })];
     mockQueryUnreportedUsageEvents.mockReturnValue(events);
     mockFetch.mockImplementation(() =>
@@ -1969,7 +2126,8 @@ describe("UsageTelemetryReporter", () => {
     insertTelemetryOutboxEvent({
       id: "row-activation-old",
       name: "onboarding",
-      createdAt: 1700000100000,
+      // Row age drives the flush-cycle prune; only the payload is frozen-old.
+      createdAt: Date.now(),
       event: frozen,
     });
     mockFetch.mockImplementation(() =>
@@ -2280,7 +2438,7 @@ describe("UsageTelemetryReporter", () => {
     // constructs and runs the reporter; the always-on audit listener keeps
     // writing rows. Every opted-out flush (5-minute cycle plus the final
     // flush in stop()) advances the watermark past them without sending.
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const optOutRowCreatedAt = Date.now() - 5_000;
     seedToolInvocation({
       id: "ti-opt-out-window",
@@ -2296,7 +2454,7 @@ describe("UsageTelemetryReporter", () => {
 
     // Session 2: the user opts back in and restarts. Only rows recorded
     // after the opt-out epoch ship — the opt-out-window row never does.
-    mockGetCachedShareAnalytics.mockReturnValue(true);
+    mockGetRawShareAnalytics.mockReturnValue(true);
     seedToolInvocation({ id: "ti-after-opt-in", createdAt: advanced + 1000 });
     const reporter = makeReporter();
     await reporter.flush();
@@ -2321,7 +2479,7 @@ describe("UsageTelemetryReporter", () => {
 
     // Opted-out flush: advances the timestamp watermark to Date.now() and
     // must also pin the ID watermark to the high-sorting sentinel.
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const optedOutReporter = makeReporter();
     await optedOutReporter.flush();
     expect(mockFetch).not.toHaveBeenCalled();
@@ -2339,7 +2497,7 @@ describe("UsageTelemetryReporter", () => {
     });
 
     // Re-opt-in: only rows strictly after the opt-out epoch ship.
-    mockGetCachedShareAnalytics.mockReturnValue(true);
+    mockGetRawShareAnalytics.mockReturnValue(true);
     seedToolInvocation({ id: "ti-after-opt-in", createdAt: watermark + 1000 });
     const reporter = makeReporter();
     await reporter.flush();
@@ -2779,7 +2937,7 @@ describe("UsageTelemetryReporter", () => {
   });
 
   test("opt-out: discardPending drops ack-mode rows while watermark sources get the sentinel pin", async () => {
-    mockGetCachedShareAnalytics.mockReturnValue(false);
+    mockGetRawShareAnalytics.mockReturnValue(false);
     const {
       source: ackSource,
       discardPending,
@@ -2805,6 +2963,28 @@ describe("UsageTelemetryReporter", () => {
     expect(keys.some((k) => k.includes("fake_ack"))).toBe(false);
   });
 
+  test("unknown consent: nothing is discarded, acked, or advanced — rows wait for a confirmed state", async () => {
+    mockGetRawShareAnalytics.mockReturnValue("unknown");
+    const {
+      source: ackSource,
+      acknowledge,
+      discardPending,
+      pending,
+    } = makeAckSource("fake_ack", ["row-1"]);
+    const wmSource = makeWatermarkSource("fake_wm", ["wm-1"]);
+    const reporter = new UsageTelemetryReporter(
+      [wmSource, ackSource],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(discardPending).not.toHaveBeenCalled();
+    expect(acknowledge).not.toHaveBeenCalled();
+    expect(pending()).toBe(1);
+    expect(mockSetFlushCheckpoint).not.toHaveBeenCalled();
+  });
+
   test("mixed watermark+ack sources flush in construction order, each acknowledged in its own mode", async () => {
     const wmSource = makeWatermarkSource("fake_wm", ["wm-1", "wm-2"]);
     const { source: ackSource, acknowledge } = makeAckSource("fake_ack", [
@@ -2937,6 +3117,10 @@ describe("UsageTelemetryReporter", () => {
   // shipped event type.
   const OUTBOX_NAME = "test_outbox" as OutboxTelemetryEventName;
 
+  // Recent base for row timestamps: the flush cycle prunes outbox rows older
+  // than OUTBOX_MAX_ROW_AGE_MS, so rows that must ship are seeded fresh.
+  const OUTBOX_BASE = Date.now() - 60_000;
+
   function seedOutboxRow(id: string, createdAt: number): void {
     insertTelemetryOutboxEvent({
       id,
@@ -2951,8 +3135,8 @@ describe("UsageTelemetryReporter", () => {
   }
 
   test("outboxSource ships pending rows and deletes them after a 2xx", async () => {
-    seedOutboxRow("ob-1", 1700000001000);
-    seedOutboxRow("ob-2", 1700000002000);
+    seedOutboxRow("ob-1", OUTBOX_BASE + 1000);
+    seedOutboxRow("ob-2", OUTBOX_BASE + 2000);
     const reporter = new UsageTelemetryReporter(
       [outboxSource(OUTBOX_NAME)],
       fakeFlushCheckpointStore,
@@ -2975,7 +3159,7 @@ describe("UsageTelemetryReporter", () => {
     mockFetch.mockImplementation(() =>
       Promise.resolve(new Response("error", { status: 500 })),
     );
-    seedOutboxRow("ob-1", 1700000001000);
+    seedOutboxRow("ob-1", OUTBOX_BASE + 1000);
     const reporter = new UsageTelemetryReporter(
       [outboxSource(OUTBOX_NAME)],
       fakeFlushCheckpointStore,
@@ -2987,9 +3171,9 @@ describe("UsageTelemetryReporter", () => {
   });
 
   test("outboxSource opt-out discards all pending rows for the name", async () => {
-    mockGetCachedShareAnalytics.mockReturnValue(false);
-    seedOutboxRow("ob-1", 1700000001000);
-    seedOutboxRow("ob-2", 1700000002000);
+    mockGetRawShareAnalytics.mockReturnValue(false);
+    seedOutboxRow("ob-1", OUTBOX_BASE + 1000);
+    seedOutboxRow("ob-2", OUTBOX_BASE + 2000);
     const reporter = new UsageTelemetryReporter(
       [outboxSource(OUTBOX_NAME)],
       fakeFlushCheckpointStore,
@@ -3001,6 +3185,91 @@ describe("UsageTelemetryReporter", () => {
     expect(mockSetFlushCheckpoint).not.toHaveBeenCalled();
   });
 
+  test("outboxSource unknown consent retains all pending rows", async () => {
+    mockGetRawShareAnalytics.mockReturnValue("unknown");
+    seedOutboxRow("ob-1", OUTBOX_BASE + 1000);
+    seedOutboxRow("ob-2", OUTBOX_BASE + 2000);
+    const reporter = new UsageTelemetryReporter(
+      [outboxSource(OUTBOX_NAME)],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pendingOutboxIds()).toEqual(["ob-1", "ob-2"]);
+    expect(mockSetFlushCheckpoint).not.toHaveBeenCalled();
+  });
+
+  // -------------------------------------------------------------------------
+  // Flush-cycle age prune (bounds unknown-consent buffering)
+  // -------------------------------------------------------------------------
+
+  function seedExpiredAndFreshRows(): void {
+    seedOutboxRow("ob-expired", Date.now() - OUTBOX_MAX_ROW_AGE_MS - 60_000);
+    seedOutboxRow("ob-fresh", OUTBOX_BASE);
+  }
+
+  test("opted in: expired rows are pruned rather than shipped; fresh rows ship", async () => {
+    seedExpiredAndFreshRows();
+    const reporter = new UsageTelemetryReporter(
+      [outboxSource(OUTBOX_NAME)],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    const body = JSON.parse(
+      (mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string,
+    );
+    expect(
+      body.events.map((e: { daemon_event_id: string }) => e.daemon_event_id),
+    ).toEqual(["wire-ob-fresh"]);
+    expect(pendingOutboxIds()).toEqual([]);
+  });
+
+  test("unknown consent: expired rows are pruned before the deferral; fresh rows are retained", async () => {
+    mockGetRawShareAnalytics.mockReturnValue("unknown");
+    seedExpiredAndFreshRows();
+    const reporter = new UsageTelemetryReporter(
+      [outboxSource(OUTBOX_NAME)],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    // The unknown-defer branch still sends and acks nothing, but the age
+    // prune has already bounded the backlog — a permanently-unknown state
+    // cannot accumulate rows forever.
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pendingOutboxIds()).toEqual(["ob-fresh"]);
+    expect(mockSetFlushCheckpoint).not.toHaveBeenCalled();
+  });
+
+  test("expired rows are pruned even when the flush skips for platform-disabled", async () => {
+    process.env.VELLUM_DISABLE_PLATFORM = "true";
+    seedExpiredAndFreshRows();
+    const reporter = new UsageTelemetryReporter(
+      [outboxSource(OUTBOX_NAME)],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    expect(mockFetch).not.toHaveBeenCalled();
+    expect(pendingOutboxIds()).toEqual(["ob-fresh"]);
+  });
+
+  test("a reporter without ack-mode sources never prunes the outbox", async () => {
+    // The daemon partition (watermark-only) does not own the outbox; the
+    // prune belongs to the reporter instance that flushes ack-mode sources.
+    seedExpiredAndFreshRows();
+    const reporter = new UsageTelemetryReporter(
+      [makeWatermarkSource("fake_wm")],
+      fakeFlushCheckpointStore,
+    );
+    await reporter.flush();
+
+    expect(pendingOutboxIds()).toEqual(["ob-expired", "ob-fresh"]);
+  });
+
   test("outboxSource purges corrupt-payload rows at collect while valid siblings ship", async () => {
     // Raw inserts: the store API only writes JSON.stringify-ed events, so
     // corrupt payloads (invalid JSON, non-object JSON) go in directly.
@@ -3010,20 +3279,20 @@ describe("UsageTelemetryReporter", () => {
         {
           id: "ob-corrupt-json",
           name: OUTBOX_NAME,
-          createdAt: 1700000001000,
+          createdAt: OUTBOX_BASE + 1000,
           conversationId: null,
           payload: "{not json",
         },
         {
           id: "ob-non-object",
           name: OUTBOX_NAME,
-          createdAt: 1700000002000,
+          createdAt: OUTBOX_BASE + 2000,
           conversationId: null,
           payload: "42",
         },
       ])
       .run();
-    seedOutboxRow("ob-valid", 1700000003000);
+    seedOutboxRow("ob-valid", OUTBOX_BASE + 3000);
     const reporter = new UsageTelemetryReporter(
       [outboxSource(OUTBOX_NAME)],
       fakeFlushCheckpointStore,
@@ -3045,7 +3314,7 @@ describe("UsageTelemetryReporter", () => {
     // 501 rows: the first collect fills BATCH_SIZE (500) and the fullBatch
     // recursion ships the remaining row.
     for (let i = 0; i < 501; i++) {
-      seedOutboxRow(`ob-${String(i).padStart(3, "0")}`, 1700000000000 + i);
+      seedOutboxRow(`ob-${String(i).padStart(3, "0")}`, OUTBOX_BASE + i);
     }
     const reporter = new UsageTelemetryReporter(
       [outboxSource(OUTBOX_NAME)],