@vellumai/assistant 0.9.0 → 0.10.0-staging.2

package/src/memory/migrations/__tests__/291-contact-channels-renormalize-addresses.test.ts

@@ -0,0 +1,341 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { getSqliteFrom } from "../../db-connection.js";
+import * as schema from "../../schema.js";
+import { migrateContactChannelsRenormalizeAddresses } from "../291-contact-channels-renormalize-addresses.js";
+
+interface ChannelRow {
+  id: string;
+  contact_id: string;
+  type: string;
+  address: string;
+  external_user_id: string | null;
+  status: string;
+}
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  sqlite.exec("PRAGMA foreign_keys = ON");
+  return drizzle(sqlite, { schema });
+}
+
+function bootstrap(db: ReturnType<typeof createTestDb>): void {
+  const raw = getSqliteFrom(db);
+  raw.exec(/*sql*/ `
+    CREATE TABLE contacts (
+      id TEXT PRIMARY KEY,
+      display_name TEXT NOT NULL,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL,
+      role TEXT NOT NULL DEFAULT 'contact',
+      contact_type TEXT NOT NULL DEFAULT 'human'
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE TABLE contact_channels (
+      id TEXT PRIMARY KEY,
+      contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
+      type TEXT NOT NULL,
+      address TEXT NOT NULL,
+      is_primary INTEGER NOT NULL DEFAULT 0,
+      external_user_id TEXT,
+      external_chat_id TEXT,
+      status TEXT NOT NULL DEFAULT 'unverified',
+      policy TEXT NOT NULL DEFAULT 'allow',
+      interaction_count INTEGER NOT NULL DEFAULT 0,
+      updated_at INTEGER,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE UNIQUE INDEX idx_contact_channels_type_address
+      ON contact_channels(type, address)
+  `);
+}
+
+function insertContact(raw: Database, id: string): void {
+  const now = Date.now();
+  raw
+    .query(
+      `INSERT INTO contacts (id, display_name, created_at, updated_at) VALUES (?, ?, ?, ?)`,
+    )
+    .run(id, `Contact ${id}`, now, now);
+}
+
+function insertChannel(
+  raw: Database,
+  opts: {
+    id: string;
+    contactId: string;
+    type: string;
+    address: string;
+    externalUserId: string | null;
+    status: string;
+  },
+): void {
+  const now = Date.now();
+  raw
+    .query(
+      `INSERT INTO contact_channels (id, contact_id, type, address, external_user_id, status, updated_at, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+    )
+    .run(
+      opts.id,
+      opts.contactId,
+      opts.type,
+      opts.address,
+      opts.externalUserId,
+      opts.status,
+      now,
+      now,
+    );
+}
+
+describe("migration 291 — renormalize addresses", () => {
+  test("restores Slack address casing from external_user_id", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "slack",
+      address: "u12345abc",
+      externalUserId: "U12345ABC",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("U12345ABC");
+  });
+
+  test("preserves lowercase email addresses (does not overwrite with mixed-case external_user_id)", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "email",
+      address: "user@example.com",
+      // generic-examples:ignore-next-line — reason: testing mixed-case email normalization
+      externalUserId: "User@Example.com",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("user@example.com");
+  });
+
+  test("lowercases email address if it was stored incorrectly", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "email",
+      // generic-examples:ignore-next-line — reason: testing mixed-case email stored incorrectly
+      address: "User@Example.com",
+      // generic-examples:ignore-next-line — reason: testing mixed-case email stored incorrectly
+      externalUserId: "User@Example.com",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("user@example.com");
+  });
+
+  test("removes cross-column blocker before normalization", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertContact(raw, "c2");
+    // Blocker: NULL external_user_id, address matches normalizer's external_user_id
+    insertChannel(raw, {
+      id: "blocker",
+      contactId: "c1",
+      type: "slack",
+      address: "u12345abc",
+      externalUserId: null,
+      status: "unverified",
+    });
+    // Normalizer: has external_user_id that will become its address
+    insertChannel(raw, {
+      id: "normalizer",
+      contactId: "c2",
+      type: "slack",
+      address: "old-address",
+      externalUserId: "U12345ABC",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels ORDER BY id")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe("normalizer");
+    expect(rows[0].address).toBe("U12345ABC");
+  });
+
+  test("idempotent — safe to run twice", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "slack",
+      address: "u12345abc",
+      externalUserId: "U12345ABC",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("U12345ABC");
+  });
+
+  test("does not overwrite canonical E.164 phone address with raw external_user_id", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    // Phone channel whose address was canonicalized to E.164 (+1 prefix)
+    // but external_user_id still holds the raw 10-digit number.
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "phone",
+      // generic-examples:ignore-next-line — reason: testing E.164 canonicalization preservation
+      address: "+15550101234",
+      // generic-examples:ignore-next-line — reason: testing raw 10-digit phone before E.164 normalization
+      externalUserId: "5550101234",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    // generic-examples:ignore-next-line — reason: verifying E.164 address preserved
+    expect(rows[0].address).toBe("+15550101234");
+  });
+
+  test("does not overwrite canonical WhatsApp address with raw external_user_id", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "whatsapp",
+      address: "+447911123456",
+      externalUserId: "447911123456",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("+447911123456");
+  });
+
+  test("no-op when address already matches external_user_id", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "slack",
+      address: "U12345ABC",
+      externalUserId: "U12345ABC",
+      status: "active",
+    });
+
+    migrateContactChannelsRenormalizeAddresses(db);
+
+    const rows = raw
+      .prepare("SELECT * FROM contact_channels")
+      .all() as ChannelRow[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("U12345ABC");
+  });
+
+  test("no-op when external_user_id column is absent (re-run after migration 294)", () => {
+    const db = createTestDb();
+    bootstrap(db);
+    const raw = getSqliteFrom(db);
+
+    insertContact(raw, "c1");
+    insertChannel(raw, {
+      id: "ch1",
+      contactId: "c1",
+      type: "slack",
+      address: "u12345abc",
+      externalUserId: "U12345ABC",
+      status: "active",
+    });
+
+    // Simulate a later startup where migration 294 has already dropped the
+    // index and column. Migration steps re-run on every startup, so this must
+    // tolerate the dropped column rather than throwing "no such column".
+    raw.run("DROP INDEX IF EXISTS idx_contact_channels_type_ext_user");
+    raw.run("ALTER TABLE contact_channels DROP COLUMN external_user_id");
+
+    expect(() => migrateContactChannelsRenormalizeAddresses(db)).not.toThrow();
+
+    const rows = raw
+      .prepare("SELECT id, address FROM contact_channels")
+      .all() as { id: string; address: string }[];
+    expect(rows).toHaveLength(1);
+    expect(rows[0].address).toBe("u12345abc");
+  });
+});

package/src/memory/migrations/__tests__/run-migrations.test.ts

@@ -0,0 +1,52 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import * as schema from "../../schema.js";
+import { type MigrationStep, runMigrationSteps } from "../run-migrations.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  sqlite.exec("PRAGMA foreign_keys = ON");
+  return drizzle(sqlite, { schema });
+}
+
+describe("runMigrationSteps — checkpointing", () => {
+  // Skipped: documents the post-checkpointing contract and fails against the
+  // current behavior, which re-runs every forward step on every boot. Promote
+  // to `test` when true checkpointing of initial migrations lands.
+  test.skip("does not re-execute a step that was already applied on a prior run", () => {
+    /**
+     * Booting an already-migrated database must not re-run forward migration
+     * steps. With true checkpointing each step's body executes at most once
+     * across boots; without it every step re-runs on every startup — the
+     * ~200-step re-probe that floors daemon startup at ~30s.
+     */
+
+    // GIVEN three dummy migration steps that count how often their body runs
+    const calls = { a: 0, b: 0, c: 0 };
+    const steps: MigrationStep[] = [
+      function dummyMigrationA() {
+        calls.a++;
+      },
+      function dummyMigrationB() {
+        calls.b++;
+      },
+      function dummyMigrationC() {
+        calls.c++;
+      },
+    ];
+
+    // AND a database that has already been migrated once
+    const db = createTestDb();
+    runMigrationSteps(db, steps);
+
+    // WHEN the same steps run again against the already-migrated database
+    runMigrationSteps(db, steps);
+
+    // THEN no step body executes a second time
+    expect(calls).toEqual({ a: 1, b: 1, c: 1 });
+  });
+});

package/src/memory/migrations/index.ts

@@ -282,6 +282,12 @@ export { migrateConversationOriginChannelIndex } from "./287-conversation-origin
 export { migrateBackfillOriginChannelFromBindings } from "./288-backfill-origin-channel-from-bindings.js";
 export { migrateContactChannelsUniqueExtUser } from "./289-contact-channels-unique-ext-user.js";
 export { migrateScheduleCapabilities } from "./290-schedule-capabilities.js";
+export { migrateContactChannelsRenormalizeAddresses } from "./291-contact-channels-renormalize-addresses.js";
+export { migrateScheduleDefaultNoReuseConversation } from "./292-schedule-default-no-reuse-conversation.js";
+export { migrateWorkflowJournalLeafTokens } from "./293-workflow-journal-leaf-tokens.js";
+export { migrateDropExternalUserId } from "./294-drop-external-user-id.js";
+export { dropApprovalPromptTsTrackerTable } from "./295-drop-approval-prompt-ts-tracker.js";
+export { migrateRewriteBalancedEconomyProfilePins } from "./296-rewrite-balanced-economy-profile-pins.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/run-migrations.ts

@@ -0,0 +1,41 @@
+import { getLogger } from "../../util/logger.js";
+import type { DrizzleDb } from "../db-connection.js";
+
+const log = getLogger("db-init");
+
+/** A single forward migration step, identified for logging by its `.name`. */
+export type MigrationStep = (database: DrizzleDb) => void;
+
+export interface MigrationRunResult {
+  /** Steps whose body threw. */
+  failed: string[];
+}
+
+/**
+ * Run the ordered list of forward migration steps against the database.
+ *
+ * Individual step failures are caught and logged so one broken migration does
+ * not prevent independent later ones from succeeding.
+ */
+export function runMigrationSteps(
+  database: DrizzleDb,
+  steps: MigrationStep[],
+): MigrationRunResult {
+  const failed: string[] = [];
+
+  for (const step of steps) {
+    try {
+      log.debug({ migration: step.name }, `Starting migration: ${step.name}`);
+      step(database);
+      log.debug({ migration: step.name }, `Migration succeeded: ${step.name}`);
+    } catch (err) {
+      failed.push(step.name);
+      log.error(
+        { err, migration: step.name },
+        `Migration failed: ${step.name}`,
+      );
+    }
+  }
+
+  return { failed };
+}

package/src/memory/migrations/validate-migration-state.ts

@@ -220,7 +220,7 @@ export function validateMigrationState(
   // registry entry — these are from a newer version of the daemon.
   //
   // The memory_checkpoints table is a general-purpose key-value store also
-  // used by non-migration subsystems (e.g., "identity:intro:text",
+  // used by non-migration subsystems (e.g., "empty_state:greeting:text",
   // "conversation_starters:item_count_at_last_gen"). Filter to only keys
   // that follow migration naming conventions before comparing against the
   // registry to avoid false-positive warnings.

package/src/memory/onboarding-events-store.ts

@@ -1,7 +1,7 @@
 import { and, asc, eq, gt, or } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
-import { getConfig } from "../config/loader.js";
+import { getCachedShareAnalytics } from "../platform/consent-cache.js";
 import {
   ACTIVATION_AB_VARIANT,
   ACTIVATION_FUNNEL_VERSION,
@@ -58,7 +58,7 @@ function insertOnboardingEvent(event: OnboardingEvent): OnboardingEvent {
 export function recordOnboardingEvent(
   params: RecordOnboardingEventParams,
 ): OnboardingEvent | null {
-  if (!getConfig().collectUsageData) return null;
+  if (!getCachedShareAnalytics()) return null;
   return insertOnboardingEvent({
     id: uuid(),
     createdAt: Date.now(),
@@ -93,7 +93,7 @@ export function recordActivationEvent(params: {
   userId?: string | null;
   abVariant?: string;
 }): OnboardingEvent | null {
-  if (!getConfig().collectUsageData) return null;
+  if (!getCachedShareAnalytics()) return null;
   const createdAt = Date.now();
   return insertOnboardingEvent({
     id: uuid(),

package/src/memory/schema/contacts.ts

@@ -26,7 +26,6 @@ export const contactChannels = sqliteTable(
     isPrimary: integer("is_primary", { mode: "boolean" })
       .notNull()
       .default(false),
-    externalUserId: text("external_user_id"), // channel-native user ID (e.g., Telegram numeric ID, E.164 phone)
     externalChatId: text("external_chat_id"), // delivery/notification routing address (e.g., Telegram chat ID)
     status: text("status").notNull().default("unverified"), // 'active' | 'pending' | 'revoked' | 'blocked' | 'unverified'
     policy: text("policy").notNull().default("allow"), // 'allow' | 'deny' | 'escalate'
@@ -42,10 +41,6 @@ export const contactChannels = sqliteTable(
     createdAt: integer("created_at").notNull(),
   },
   (table) => [
-    index("idx_contact_channels_type_ext_user").on(
-      table.type,
-      table.externalUserId,
-    ),
     index("idx_contact_channels_type_ext_chat").on(
       table.type,
       table.externalChatId,

package/src/memory/skill-loaded-events-store.test.ts

@@ -8,18 +8,10 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
-let collectUsageData = true;
-
-mock.module("../config/loader.js", () => ({
-  getConfig: () => ({
-    ui: {},
-    model: "test",
-    provider: "test",
-    memory: { enabled: false },
-    rateLimit: { maxRequestsPerMinute: 0 },
-    secretDetection: { enabled: false },
-    collectUsageData,
-  }),
+let shareAnalytics = true;
+
+mock.module("../platform/consent-cache.js", () => ({
+  getCachedShareAnalytics: () => shareAnalytics,
 }));
 
 import { getDb } from "./db-connection.js";
@@ -42,12 +34,12 @@ function insertEvent(
 
 describe("skill-loaded-events-store", () => {
   beforeEach(() => {
-    collectUsageData = true;
+    shareAnalytics = true;
     getDb().delete(skillLoadedEvents).run();
   });
 
-  test("honors the collectUsageData opt-out (records nothing)", () => {
-    collectUsageData = false;
+  test("honors the share_analytics opt-out (records nothing)", () => {
+    shareAnalytics = false;
     recordSkillLoadedEvent({ skillName: "web-research" });
     expect(queryUnreportedSkillLoadedEvents(0, undefined, 10)).toHaveLength(0);
   });

package/src/memory/skill-loaded-events-store.ts

@@ -1,7 +1,7 @@
 import { and, asc, eq, gt, or } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
-import { getConfig } from "../config/loader.js";
+import { getCachedShareAnalytics } from "../platform/consent-cache.js";
 import type { UsageAttributionColumns } from "../usage/attribution.js";
 import { getDb } from "./db-connection.js";
 import { skillLoadedEvents } from "./schema.js";
@@ -38,7 +38,7 @@ export interface SkillLoadedEvent {
  * opt-out, matching the rest of telemetry).
  */
 export function recordSkillLoadedEvent(record: SkillLoadedEventRecord): void {
-  if (!getConfig().collectUsageData) return;
+  if (!getCachedShareAnalytics()) return;
   const db = getDb();
   db.insert(skillLoadedEvents)
     .values({

package/src/memory/tool-executed-events-store.test.ts

@@ -8,12 +8,12 @@ mock.module("../util/logger.js", () => ({
     }),
 }));
 
-// Toggle for the collectUsageData opt-out gate the audit listener consults
+// Toggle for the share_analytics opt-out gate the audit listener consults
 // when populating the telemetry columns.
-let collectUsageData = true;
+let shareAnalytics = true;
 
-mock.module("../config/loader.js", () => ({
-  getConfig: () => ({ collectUsageData }),
+mock.module("../platform/consent-cache.js", () => ({
+  getCachedShareAnalytics: () => shareAnalytics,
 }));
 
 import {
@@ -43,7 +43,7 @@ function insertInvocation(
 
 describe("tool-executed-events-store", () => {
   beforeEach(() => {
-    collectUsageData = true;
+    shareAnalytics = true;
     getDb().delete(toolInvocations).run();
   });
 
@@ -136,10 +136,10 @@ describe("tool-executed-events-store", () => {
 
     listener(executedEvent("t-opted-in-before"));
 
-    collectUsageData = false;
+    shareAnalytics = false;
     listener(executedEvent("t-opted-out"));
 
-    collectUsageData = true;
+    shareAnalytics = true;
     listener(executedEvent("t-opted-in-after"));
 
     // Mid-session opt-out flip: only rows recorded while opted in project.