@vellumai/assistant 0.10.8 → 0.10.9-dev.202607152225.ec6fdbf

package/src/persistence/schema/infrastructure.ts

@@ -274,45 +274,6 @@ export const llmUsageEvents = sqliteTable(
   ],
 );
 
-export const lifecycleEvents = sqliteTable("lifecycle_events", {
-  id: text("id").primaryKey(),
-  eventName: text("event_name").notNull(), // 'app_open' | 'hatch'
-  createdAt: integer("created_at").notNull(),
-});
-
-export const onboardingEvents = sqliteTable("onboarding_events", {
-  id: text("id").primaryKey(),
-  createdAt: integer("created_at").notNull(),
-  screen: text("screen").notNull(),
-  toolsJson: text("tools_json"),
-  tasksJson: text("tasks_json"),
-  tone: text("tone"),
-  googleConnected: integer("google_connected", { mode: "boolean" }),
-  googleScopesJson: text("google_scopes_json"),
-  priorAssistantsJson: text("prior_assistants_json"),
-  abVariant: text("ab_variant"),
-  sessionId: text("session_id"),
-  stepName: text("step_name"),
-  stepIndex: integer("step_index"),
-  completedAt: text("completed_at"),
-  funnelVersion: text("funnel_version"),
-});
-
-// Aggregated legacy-loopback auth-fallback counts forwarded by the gateway.
-// One row per (guard, path, failure_kind) per flush window; `count` is how many
-// requests fell back to the loopback exemption in that window. Flushed to the
-// platform telemetry endpoint by the usage telemetry reporter.
-export const authFallbackEvents = sqliteTable("auth_fallback_events", {
-  id: text("id").primaryKey(),
-  createdAt: integer("created_at").notNull(),
-  guard: text("guard").notNull(), // 'edge' | 'edge-scoped' | 'edge-guardian'
-  path: text("path").notNull(),
-  failureKind: text("failure_kind").notNull(),
-  count: integer("count").notNull(),
-  windowStart: integer("window_start").notNull(),
-  windowEnd: integer("window_end").notNull(),
-});
-
 // One row per conversation started on the activation-rail bootstrap template.
 // Lets the activation funnel telemetry scope its events to activation
 // conversations without inspecting the bootstrap template at emit time.
@@ -321,47 +282,39 @@ export const activationSessions = sqliteTable("activation_sessions", {
   createdAt: integer("created_at").notNull(),
 });
 
-// One row per `skill_loaded` telemetry event, emitted when a Vellum-produced
-// skill is activated in a conversation — see skill-loaded-events-store.ts for
-// the data contract. Flushed by the usage telemetry reporter.
-export const skillLoadedEvents = sqliteTable(
-  "skill_loaded_events",
+// Key/value store for telemetry flush state — the per-event-type
+// `(last_reported_at, last_reported_id)` watermark cursors advanced by the
+// usage telemetry reporter after each successful upload. Lives on the
+// dedicated telemetry database (assistant-telemetry.db) so flush state
+// stays with the telemetry pipeline; the main DB's `memory_checkpoints`
+// ledger is reserved for DB-migration checkpoints.
+export const flushCheckpoints = sqliteTable("flush_checkpoints", {
+  key: text("key").primaryKey(),
+  value: text("value").notNull(),
+  updatedAt: integer("updated_at").notNull(),
+});
+
+// Generic telemetry outbox. Lives on the dedicated telemetry database
+// (assistant-telemetry.db). Each row's `payload` is the wire `TelemetryEvent`
+// JSON built at record time; `name` is the telemetry source id (e.g.
+// 'lifecycle'). Rows are deleted on successful flush by the usage telemetry
+// reporter. `conversation_id` is populated only for conversation-scoped
+// events so redaction stays an indexed column delete.
+export const telemetryEvents = sqliteTable(
+  "telemetry_events",
   {
     id: text("id").primaryKey(),
+    name: text("name").notNull(),
     createdAt: integer("created_at").notNull(),
     conversationId: text("conversation_id"),
-    skillName: text("skill_name").notNull(),
-    // ISO 8601 timestamp from the merged skill catalog, when known.
-    skillUpdatedAt: text("skill_updated_at"),
-    provider: text("provider"),
-    model: text("model"),
-    inferenceProfile: text("inference_profile"),
-    inferenceProfileSource: text("inference_profile_source"),
+    payload: text("payload").notNull(),
   },
   (table) => [
-    index("idx_skill_loaded_events_created_at_id").on(
+    index("idx_telemetry_events_name_created_at_id").on(
+      table.name,
       table.createdAt,
       table.id,
     ),
-  ],
-);
-
-// One row per `watchdog` telemetry event, emitted when a daemon watchdog
-// check fires (event-loop block, stream-idle stall, restart, ...) — see
-// watchdog-events-store.ts for the data contract. Flushed by the usage
-// telemetry reporter. `value` is a REAL (BQ FLOAT) so the daemon need not
-// distinguish int vs float; the platform serializer coerces ints to float.
-// `detail` is a JSON bag stored as text and forwarded verbatim.
-export const watchdogEvents = sqliteTable(
-  "watchdog_events",
-  {
-    id: text("id").primaryKey(),
-    createdAt: integer("created_at").notNull(),
-    checkName: text("check_name").notNull(),
-    value: real("value"),
-    detail: text("detail"),
-  },
-  (table) => [
-    index("idx_watchdog_events_created_at_id").on(table.createdAt, table.id),
+    index("idx_telemetry_events_conversation_id").on(table.conversationId),
   ],
 );

package/src/persistence/schema/memory-core.ts

@@ -1,6 +1,5 @@
 import {
   blob,
-  index,
   integer,
   sqliteTable,
   text,
@@ -105,27 +104,3 @@ export const memoryRetrospectiveState = sqliteTable(
     rememberedLog: text("remembered_log"),
   },
 );
-
-export const conversationStarters = sqliteTable(
-  "conversation_starters",
-  {
-    id: text("id").primaryKey(),
-    label: text("label").notNull(),
-    prompt: text("prompt").notNull(),
-    generationBatch: integer("generation_batch").notNull(),
-    sourceMemoryKinds: text("source_memory_kinds"),
-    category: text("category"),
-    icon: text("icon"),
-    description: text("description"),
-    tags: text("tags"),
-    cardType: text("card_type").notNull().default("chip"),
-    createdAt: integer("created_at").notNull(),
-  },
-  (table) => [
-    index("idx_conversation_starters_batch").on(
-      table.generationBatch,
-      table.createdAt,
-    ),
-    index("idx_conversation_starters_card_type").on(table.cardType),
-  ],
-);

package/src/persistence/schema/memory-injection.ts

@@ -7,7 +7,9 @@ import {
 } from "drizzle-orm/sqlite-core";
 
 // Time-series of memory-v2 card injections, used by the router to decay a
-// concept's recent injection pressure.
+// concept's recent injection pressure. Lives in the dedicated memory database
+// (`assistant-memory.db`), not main — access it via the memory connection
+// (`getMemoryDb()` / `getMemorySqlite()`).
 export const memoryV2InjectionEvents = sqliteTable(
   "memory_v2_injection_events",
   {

package/src/persistence/schema/tasks.ts

@@ -1,66 +1,7 @@
-import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
 
 import { contacts } from "./contacts.js";
 
-export const tasks = sqliteTable("tasks", {
-  id: text("id").primaryKey(),
-  title: text("title").notNull(),
-  template: text("template").notNull(),
-  inputSchema: text("input_schema"),
-  contextFlags: text("context_flags"),
-  requiredTools: text("required_tools"),
-  createdFromConversationId: text("created_from_conversation_id"),
-  status: text("status").notNull().default("active"),
-  createdAt: integer("created_at").notNull(),
-  updatedAt: integer("updated_at").notNull(),
-});
-
-export const taskRuns = sqliteTable("task_runs", {
-  id: text("id").primaryKey(),
-  taskId: text("task_id")
-    .notNull()
-    .references(() => tasks.id, { onDelete: "cascade" }),
-  conversationId: text("conversation_id"),
-  status: text("status").notNull().default("pending"),
-  startedAt: integer("started_at"),
-  finishedAt: integer("finished_at"),
-  error: text("error"),
-  principalId: text("principal_id"),
-  createdAt: integer("created_at").notNull(),
-});
-
-export const taskCandidates = sqliteTable("task_candidates", {
-  id: text("id").primaryKey(),
-  sourceConversationId: text("source_conversation_id").notNull(),
-  compiledTemplate: text("compiled_template").notNull(),
-  confidence: real("confidence"),
-  requiredTools: text("required_tools"), // JSON array string
-  createdAt: integer("created_at").notNull(),
-  promotedTaskId: text("promoted_task_id"), // set when candidate is promoted to a real task
-});
-
-export const workItems = sqliteTable("work_items", {
-  id: text("id").primaryKey(),
-  taskId: text("task_id")
-    .notNull()
-    .references(() => tasks.id),
-  title: text("title").notNull(),
-  notes: text("notes"),
-  status: text("status").notNull().default("queued"), // queued | running | awaiting_review | failed | cancelled | done | archived
-  priorityTier: integer("priority_tier").notNull().default(1), // 0=high, 1=medium, 2=low
-  sortIndex: integer("sort_index"), // manual ordering within same priority tier; null = fall back to updated_at
-  lastRunId: text("last_run_id"),
-  lastRunConversationId: text("last_run_conversation_id"),
-  lastRunStatus: text("last_run_status"), // 'completed' | 'failed' | null
-  sourceType: text("source_type"), // reserved for future bridge (e.g. 'followup', 'triage')
-  sourceId: text("source_id"), // reserved for future bridge
-  requiredTools: text("required_tools"), // JSON array snapshot of tools needed for this run (null=unknown, []=none, ["bash",...]=specific)
-  approvedTools: text("approved_tools"), // JSON array of pre-approved tool names
-  approvalStatus: text("approval_status").default("none"), // 'none' | 'approved' | 'denied'
-  createdAt: integer("created_at").notNull(),
-  updatedAt: integer("updated_at").notNull(),
-});
-
 export const followups = sqliteTable("followups", {
   id: text("id").primaryKey(),
   channel: text("channel").notNull(), // 'email', 'slack', 'whatsapp', etc.

package/src/persistence/steps.ts

@@ -432,6 +432,17 @@ import { migrateCanonicalGuardianRequestTrigger } from "./migrations/321-canonic
 import { migrateAddProcessingResumeAttempts } from "./migrations/322-add-processing-resume-attempts.js";
 import { migrateDeleteNonDefaultMemoryScopes } from "./migrations/323-delete-non-default-memory-scopes.js";
 import { migrateMessageFinalizedColumn } from "./migrations/324-message-finalized-column.js";
+import { createConfigSettingEventsTable } from "./migrations/325-create-config-setting-events.js";
+import { migrateMoveInjectionEventsToMemoryDb } from "./migrations/326-move-injection-events-to-memory-db.js";
+import { createFlushCheckpointsTable } from "./migrations/327-create-flush-checkpoints.js";
+import { migrateDropMemoryV3LearnedEdgeTables } from "./migrations/328-drop-memory-v3-learned-edge-tables.js";
+import { migrateMoveOnboardingEventsToTelemetryDb } from "./migrations/329-move-onboarding-events-to-telemetry-db.js";
+import { migrateMoveAuthFallbackEventsToTelemetryDb } from "./migrations/330-move-auth-fallback-events-to-telemetry-db.js";
+import { migrateMoveLifecycleEventsToTelemetryDb } from "./migrations/331-move-lifecycle-events-to-telemetry-db.js";
+import { migrateMoveSkillLoadedEventsToTelemetryDb } from "./migrations/332-move-skill-loaded-events-to-telemetry-db.js";
+import { migrateCreateTelemetryEventsTable } from "./migrations/333-create-telemetry-events-table.js";
+import { migrateBackfillTelemetryEventsOutbox } from "./migrations/334-backfill-telemetry-events-outbox.js";
+import { migrateCollapseMemoryEmbedBacklog } from "./migrations/335-collapse-memory-embed-backlog.js";
 import type { MigrationStep } from "./migrations/run-migrations.js";
 
 export const migrationSteps: MigrationStep[] = [
@@ -1335,4 +1346,15 @@ export const migrationSteps: MigrationStep[] = [
   migrateAddProcessingResumeAttempts,
   migrateDeleteNonDefaultMemoryScopes,
   migrateMessageFinalizedColumn,
+  createConfigSettingEventsTable,
+  migrateMoveInjectionEventsToMemoryDb,
+  createFlushCheckpointsTable,
+  migrateDropMemoryV3LearnedEdgeTables,
+  migrateMoveOnboardingEventsToTelemetryDb,
+  migrateMoveAuthFallbackEventsToTelemetryDb,
+  migrateMoveLifecycleEventsToTelemetryDb,
+  migrateMoveSkillLoadedEventsToTelemetryDb,
+  migrateCreateTelemetryEventsTable,
+  migrateBackfillTelemetryEventsOutbox,
+  migrateCollapseMemoryEmbedBacklog,
 ];

package/src/platform/client.test.ts

@@ -265,6 +265,116 @@ describe("VellumPlatformClient", () => {
       expect(await client!.getOwnerConsent()).toBeNull();
     });
 
+    test("null share_analytics (owner never chose) maps to true — telemetry is opt-out", async () => {
+      globalThis.fetch = mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              share_analytics: null,
+              share_diagnostics: true,
+              share_diagnostics_accepted_version: "2026-06-18",
+            }),
+            { status: 200 },
+          ),
+      ) as unknown as typeof globalThis.fetch;
+
+      const client = await VellumPlatformClient.create();
+      const consent = await client!.getOwnerConsent();
+      expect(consent).toEqual({
+        shareAnalytics: true,
+        shareDiagnostics: true,
+        shareDiagnosticsAcceptedVersion: "2026-06-18",
+      });
+    });
+
+    test("coerced false with share_analytics_chosen=false (never chose) maps to true", async () => {
+      globalThis.fetch = mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              share_analytics: false,
+              share_analytics_chosen: false,
+              share_diagnostics: true,
+              share_diagnostics_accepted_version: "2026-06-18",
+            }),
+            { status: 200 },
+          ),
+      ) as unknown as typeof globalThis.fetch;
+
+      const client = await VellumPlatformClient.create();
+      const consent = await client!.getOwnerConsent();
+      expect(consent!.shareAnalytics).toBe(true);
+    });
+
+    test("explicit false with share_analytics_chosen=true stays an opt-out", async () => {
+      globalThis.fetch = mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              share_analytics: false,
+              share_analytics_chosen: true,
+              share_diagnostics: false,
+              share_diagnostics_chosen: true,
+              share_diagnostics_accepted_version: "2026-06-18",
+            }),
+            { status: 200 },
+          ),
+      ) as unknown as typeof globalThis.fetch;
+
+      const client = await VellumPlatformClient.create();
+      const consent = await client!.getOwnerConsent();
+      expect(consent).toEqual({
+        shareAnalytics: false,
+        shareDiagnostics: false,
+        shareDiagnosticsAcceptedVersion: "2026-06-18",
+      });
+    });
+
+    test("false without a *_chosen field (older platform) stays authoritative", async () => {
+      globalThis.fetch = mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              share_analytics: false,
+              share_diagnostics: false,
+              share_diagnostics_accepted_version: "",
+            }),
+            { status: 200 },
+          ),
+      ) as unknown as typeof globalThis.fetch;
+
+      const client = await VellumPlatformClient.create();
+      const consent = await client!.getOwnerConsent();
+      expect(consent).toEqual({
+        shareAnalytics: false,
+        shareDiagnostics: false,
+        shareDiagnosticsAcceptedVersion: "",
+      });
+    });
+
+    test("null share_diagnostics (owner never chose) maps to true with the version gate still closed", async () => {
+      globalThis.fetch = mock(
+        async () =>
+          new Response(
+            JSON.stringify({
+              share_analytics: null,
+              share_diagnostics: null,
+              share_diagnostics_accepted_version: "",
+            }),
+            { status: 200 },
+          ),
+      ) as unknown as typeof globalThis.fetch;
+
+      const client = await VellumPlatformClient.create();
+      const consent = await client!.getOwnerConsent();
+      expect(consent).toEqual({
+        shareAnalytics: true,
+        shareDiagnostics: true,
+        // "" keeps the PII trace-collection version gate fail-closed.
+        shareDiagnosticsAcceptedVersion: "",
+      });
+    });
+
     test("returns null on malformed body (non-boolean fields)", async () => {
       globalThis.fetch = mock(
         async () =>

package/src/platform/client.ts

@@ -17,7 +17,15 @@ const log = getLogger("platform-client");
 let _missingPrereqsWarned = false;
 
 export interface OwnerConsent {
+  /**
+   * Telemetry is opt-out: a null wire value (the owner was never shown the
+   * choice) maps to `true` here — only an explicit opt-out disables sharing.
+   * A `false` accompanied by `share_analytics_chosen: false` is the platform
+   * coercing a never-chose null to a strict boolean, not a real opt-out, so
+   * it also maps to `true`.
+   */
   shareAnalytics: boolean;
+  /** Same opt-out semantics as {@link shareAnalytics}. */
   shareDiagnostics: boolean;
   /**
    * Version of the diagnostics-sharing consent the owner accepted
@@ -148,20 +156,32 @@ export class VellumPlatformClient {
 
       const body = (await res.json()) as {
         share_analytics?: unknown;
+        share_analytics_chosen?: unknown;
         share_diagnostics?: unknown;
+        share_diagnostics_chosen?: unknown;
         share_diagnostics_accepted_version?: unknown;
       };
       if (
-        typeof body.share_analytics !== "boolean" ||
-        typeof body.share_diagnostics !== "boolean"
+        (typeof body.share_analytics !== "boolean" &&
+          body.share_analytics !== null) ||
+        (typeof body.share_diagnostics !== "boolean" &&
+          body.share_diagnostics !== null)
       ) {
         log.debug("owner-consent body malformed — treating as unknown");
         return null;
       }
 
       return {
-        shareAnalytics: body.share_analytics,
-        shareDiagnostics: body.share_diagnostics,
+        // Opt-out: anything but an explicit false enables sharing. A false
+        // with `*_chosen: false` is a platform-coerced never-chose default
+        // (not a real opt-out), so it enables too. An absent `*_chosen`
+        // (older platform) leaves false authoritative.
+        shareAnalytics:
+          body.share_analytics !== false ||
+          body.share_analytics_chosen === false,
+        shareDiagnostics:
+          body.share_diagnostics !== false ||
+          body.share_diagnostics_chosen === false,
         // Back-compat: an older platform that doesn't return this field yields
         // "" → fails the trace-collection version gate → fail-closed (no trace).
         shareDiagnosticsAcceptedVersion:

package/src/platform/consent-cache.test.ts

@@ -1,5 +1,6 @@
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
 
+import { setConfig } from "../__tests__/helpers/set-config.js";
 import type { OwnerConsent } from "./client.js";
 
 // ---------------------------------------------------------------------------
@@ -11,9 +12,6 @@ let mockClient: {
   getOwnerConsent: () => Promise<OwnerConsent | null>;
 } | null = null;
 let createCallCount = 0;
-// Legacy fail-closed opt-out marker surfaced via getConfigReadOnly(). Default
-// off/absent so existing behavior is unchanged.
-let mockLegacyTelemetryOptOut: boolean | undefined = false;
 
 // ---------------------------------------------------------------------------
 // Module mocks (must precede the import under test)
@@ -28,19 +26,6 @@ mock.module("./client.js", () => ({
   },
 }));
 
-mock.module("../config/loader.js", () => ({
-  getConfigReadOnly: () => ({
-    legacyTelemetryOptOut: mockLegacyTelemetryOptOut,
-  }),
-}));
-
-mock.module("../util/logger.js", () => ({
-  getLogger: () =>
-    new Proxy({} as Record<string, unknown>, {
-      get: () => () => {},
-    }),
-}));
-
 // ---------------------------------------------------------------------------
 // Import under test (after mocks)
 // ---------------------------------------------------------------------------
@@ -81,7 +66,9 @@ describe("consent-cache", () => {
   beforeEach(() => {
     mockClient = null;
     createCallCount = 0;
-    mockLegacyTelemetryOptOut = false;
+    // Legacy fail-closed opt-out marker read via getConfigReadOnly(). Default
+    // off so existing behavior is unchanged.
+    setConfig("legacyTelemetryOptOut", false);
     __setCachedShareAnalyticsForTest(false);
     __setCachedShareDiagnosticsForTest(false);
     __setCachedShareDiagnosticsVersionForTest("");
@@ -136,7 +123,7 @@ describe("consent-cache", () => {
   });
 
   test("legacy opt-out marker keeps analytics off despite platform opt-in", async () => {
-    mockLegacyTelemetryOptOut = true;
+    setConfig("legacyTelemetryOptOut", true);
     mockClient = makeClient(makeConsent({ shareAnalytics: true }));
     await refreshConsentCache();
     // Platform reports opt-in, but the fail-closed marker forces off.

package/src/platform/consent-cache.ts

@@ -4,18 +4,20 @@
  * Three values are cached, all refreshed from the same owner-consent fetch:
  *  - `share_analytics`: gates usage telemetry collection.
  *  - `share_diagnostics`: gates crash diagnostics (read by Sentry `beforeSend`),
- *    and composes the per-turn trace-collection gate alongside the
- *    `trace-collection` feature flag.
+ *    and composes the per-turn trace-collection gate alongside the accepted
+ *    consent version.
  *  - `share_diagnostics_accepted_version`: the consent version the owner
  *    accepted; the disclosing-version half of the trace-collection gate (see
  *    telemetry/trace-collection-policy.ts).
  *
  * Hot-path gates (record-time telemetry writes, Sentry `beforeSend`) need a
  * synchronous, I/O-free read, so this module owns the values and refreshes them
- * periodically in the background. Default-off until the first successful fetch:
- * an absent session, a disabled platform, or a transient fetch failure all leave
- * the values untouched (initial `false`), so we never report analytics or send
- * crash diagnostics without a confirmed opt-in.
+ * periodically in the background. Consent is opt-out — an owner who never made
+ * an explicit choice reports as consented (the client maps a never-chose wire
+ * value to `true`) — but UNKNOWN consent stays off: an absent session, a
+ * disabled platform, or a transient fetch failure all leave the values
+ * untouched (initial `false`), so we never report analytics or send crash
+ * diagnostics without a confirmed consent state.
  */
 
 import { getConfigReadOnly } from "../config/loader.js";