@vellumai/assistant 0.10.1 → 0.10.2-dev.202606241651.2d2b40d

package/src/memory/migrations/301-create-watchdog-events.ts

@@ -0,0 +1,45 @@
+import { Database } from "bun:sqlite";
+
+import { getTelemetryDbPath } from "../../util/telemetry-db-path.js";
+import {
+  type DrizzleDb,
+  getTelemetrySqlite,
+} from "../db-connection.js";
+
+/**
+ * Create the `watchdog_events` table on the dedicated telemetry database
+ * (`assistant-telemetry.db`), not the main DB. The migration runner passes the
+ * main DrizzleDb, but this table lives on the telemetry connection, so we open
+ * that connection and run DDL against it directly. The dedicated connection
+ * itself performs no DDL on open, so this migration owns the schema.
+ *
+ * Idempotent (`IF NOT EXISTS`). Writes are gated on `share_analytics`
+ * consent at the store level, so opted-out rows never exist and the
+ * reporter's standard 0 watermark default is safe. The index backs the
+ * telemetry reporter's compound `(created_at, id)` watermark cursor.
+ *
+ * The `mainDb` parameter is accepted to satisfy the migration-step signature but
+ * is not used — this migration operates exclusively on the telemetry DB.
+ */
+export function createWatchdogEventsTable(_mainDb: DrizzleDb): void {
+  let raw: Database | null = getTelemetrySqlite();
+  if (!raw) {
+    // The dedicated connection failed to open (logged by openDedicatedDb).
+    // Fall back to opening the file directly so the migration still runs —
+    // the singleton will pick it up on a later access. This mirrors the
+    // fail-soft pattern of the other dedicated-DB migrations.
+    raw = new Database(getTelemetryDbPath());
+  }
+  raw.exec(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS watchdog_events (
+      id TEXT PRIMARY KEY,
+      created_at INTEGER NOT NULL,
+      check_name TEXT NOT NULL,
+      value REAL,
+      detail TEXT
+    )
+  `);
+  raw.exec(
+    /*sql*/ `CREATE INDEX IF NOT EXISTS idx_watchdog_events_created_at_id ON watchdog_events (created_at, id)`,
+  );
+}

package/src/memory/migrations/__tests__/014-backfill-inbox-thread-state.test.ts

@@ -0,0 +1,108 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { getSqliteFrom } from "../../db-connection.js";
+import * as schema from "../../schema.js";
+import { migrateBackfillInboxThreadStateFromBindings } from "../014-backfill-inbox-thread-state.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  return drizzle(sqlite, { schema });
+}
+
+function bootstrap(raw: Database, opts: { withAssistantId: boolean }): void {
+  raw.exec(/*sql*/ `
+    CREATE TABLE memory_checkpoints (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE TABLE external_conversation_bindings (
+      conversation_id TEXT PRIMARY KEY,
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      external_user_id TEXT,
+      display_name TEXT,
+      username TEXT,
+      last_inbound_at INTEGER,
+      last_outbound_at INTEGER,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE TABLE assistant_inbox_thread_state (
+      conversation_id TEXT PRIMARY KEY,
+      ${opts.withAssistantId ? "assistant_id TEXT NOT NULL DEFAULT 'self'," : ""}
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      external_user_id TEXT,
+      display_name TEXT,
+      username TEXT,
+      last_inbound_at INTEGER,
+      last_outbound_at INTEGER,
+      last_message_at INTEGER,
+      unread_count INTEGER NOT NULL DEFAULT 0,
+      pending_escalation_count INTEGER NOT NULL DEFAULT 0,
+      has_pending_escalation INTEGER NOT NULL DEFAULT 0,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.run(
+    `INSERT INTO external_conversation_bindings
+       (conversation_id, source_channel, external_chat_id, external_user_id,
+        display_name, username, last_inbound_at, last_outbound_at,
+        created_at, updated_at)
+     VALUES ('c1','telegram','chat-1','U-1','Name One','one',1000,2000,500,2500)`,
+  );
+}
+
+describe("migration 014 — backfill inbox thread state", () => {
+  test("seeds rows when assistant_id column is present", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    bootstrap(raw, { withAssistantId: true });
+
+    migrateBackfillInboxThreadStateFromBindings(db);
+
+    const row = raw
+      .query(
+        `SELECT assistant_id, last_message_at FROM assistant_inbox_thread_state WHERE conversation_id = 'c1'`,
+      )
+      .get() as { assistant_id: string; last_message_at: number } | undefined;
+    expect(row?.assistant_id).toBe("self");
+    expect(row?.last_message_at).toBe(2000);
+  });
+
+  test("seeds rows without throwing when assistant_id column was dropped", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    // Simulate a DB where drop-assistant-id-columns already removed the column
+    // but the backfill checkpoint was never written.
+    bootstrap(raw, { withAssistantId: false });
+
+    expect(() => migrateBackfillInboxThreadStateFromBindings(db)).not.toThrow();
+
+    const count = raw
+      .query(`SELECT COUNT(*) AS n FROM assistant_inbox_thread_state`)
+      .get() as { n: number };
+    expect(count.n).toBe(1);
+  });
+
+  test("is a no-op once the checkpoint is recorded", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    bootstrap(raw, { withAssistantId: false });
+
+    migrateBackfillInboxThreadStateFromBindings(db);
+    // Removing the source rows would surface any second-run insert attempt.
+    raw.exec(`DELETE FROM external_conversation_bindings`);
+    expect(() => migrateBackfillInboxThreadStateFromBindings(db)).not.toThrow();
+  });
+});

package/src/memory/migrations/__tests__/136-drop-assistant-id-columns.test.ts

@@ -0,0 +1,82 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { getSqliteFrom } from "../../db-connection.js";
+import * as schema from "../../schema.js";
+import { migrateDropAssistantIdColumns } from "../136-drop-assistant-id-columns.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  return drizzle(sqlite, { schema });
+}
+
+function indexExists(raw: Database, name: string): boolean {
+  return !!raw
+    .prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`)
+    .get(name);
+}
+
+/**
+ * Create a minimal assistant_inbox_thread_state carrying assistant_id plus its
+ * composite index, so the column-drop + index-recreate path has something to
+ * act on.
+ */
+function createInboxThreadState(raw: Database): void {
+  raw.exec(/*sql*/ `
+    CREATE TABLE assistant_inbox_thread_state (
+      conversation_id TEXT PRIMARY KEY,
+      assistant_id TEXT NOT NULL DEFAULT 'self',
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      last_message_at INTEGER,
+      has_pending_escalation INTEGER NOT NULL DEFAULT 0
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE UNIQUE INDEX idx_inbox_thread_state_channel
+      ON assistant_inbox_thread_state(assistant_id, source_channel, external_chat_id)
+  `);
+}
+
+describe("migration 136 — drop assistant_id columns", () => {
+  test("drops the column and recreates the index without assistant_id", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    createInboxThreadState(raw);
+
+    migrateDropAssistantIdColumns(db);
+
+    const cols = (
+      raw.query(`PRAGMA table_info(assistant_inbox_thread_state)`).all() as {
+        name: string;
+      }[]
+    ).map((c) => c.name);
+    expect(cols).not.toContain("assistant_id");
+    expect(indexExists(raw, "idx_inbox_thread_state_channel")).toBe(true);
+  });
+
+  test("does not throw when some scoped tables were already dropped", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    // Only the inbox table exists — actor_token_records and friends were
+    // removed by an earlier migration. Index recreation must skip the missing
+    // tables instead of failing with "no such table".
+    createInboxThreadState(raw);
+
+    expect(() => migrateDropAssistantIdColumns(db)).not.toThrow();
+    // Sanity: the index targeting the missing table was not created.
+    expect(indexExists(raw, "idx_actor_tokens_active_device")).toBe(false);
+  });
+
+  test("is idempotent across repeated runs", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    createInboxThreadState(raw);
+
+    migrateDropAssistantIdColumns(db);
+    expect(() => migrateDropAssistantIdColumns(db)).not.toThrow();
+  });
+});

package/src/memory/migrations/__tests__/209-strip-thinking-from-consolidated.test.ts

@@ -0,0 +1,224 @@
+/**
+ * Tests for migration 209 — stripping thinking / redacted_thinking blocks from
+ * persisted assistant messages.
+ *
+ * The migration rewrites content entirely inside SQLite (JSON1), dispatched a
+ * rowid window at a time through `runAsyncSqlite`. These tests drive the step
+ * directly against a real DB and assert the at-rest content, idempotency,
+ * scoping (assistant rows only), tolerance of malformed content, and that the
+ * persisted rowid watermark is honored for resume.
+ */
+import { describe, expect, test } from "bun:test";
+
+const { getDb, getSqlite } = await import("../../db-connection.js");
+const { initializeDb } = await import("../../db-init.js");
+const { migrateStripThinkingFromConsolidated } =
+  await import("../209-strip-thinking-from-consolidated.js");
+
+await initializeDb();
+
+const CONV = "conv-209";
+getSqlite()
+  .query(
+    `INSERT OR IGNORE INTO conversations (id, created_at, updated_at) VALUES (?, ?, ?)`,
+  )
+  .run(CONV, Date.now(), Date.now());
+
+let seq = 0;
+function insert(role: string, content: string): { id: string; rowid: number } {
+  const id = `m209-${seq++}`;
+  getSqlite()
+    .query(
+      `INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES (?, ?, ?, ?, ?)`,
+    )
+    .run(id, CONV, role, content, Date.now());
+  const rowid = (
+    getSqlite()
+      .query(`SELECT rowid AS r FROM messages WHERE id = ?`)
+      .get(id) as { r: number }
+  ).r;
+  return { id, rowid };
+}
+
+function content(id: string): string {
+  return (
+    getSqlite().query(`SELECT content FROM messages WHERE id = ?`).get(id) as {
+      content: string;
+    }
+  ).content;
+}
+
+function blocks(id: string): Array<Record<string, unknown>> {
+  return JSON.parse(content(id));
+}
+
+describe("migration 209 — strip thinking from consolidated assistant messages", () => {
+  test("strips thinking blocks but keeps text and tool_use, preserving order", async () => {
+    const { id } = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "secret", signature: "sig" },
+        { type: "text", text: "hello" },
+        { type: "tool_use", id: "t1", name: "x", input: { a: 1 } },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(blocks(id)).toEqual([
+      { type: "text", text: "hello" },
+      { type: "tool_use", id: "t1", name: "x", input: { a: 1 } },
+    ]);
+  });
+
+  test("strips redacted_thinking blocks", async () => {
+    const { id } = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "redacted_thinking", data: "blob" },
+        { type: "text", text: "world" },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(blocks(id)).toEqual([{ type: "text", text: "world" }]);
+  });
+
+  test("all-thinking message becomes the null-byte placeholder sentinel", async () => {
+    const { id } = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "a", signature: "s1" },
+        { type: "redacted_thinking", data: "b" },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    const result = blocks(id);
+    expect(result).toHaveLength(1);
+    expect(result[0].type).toBe("text");
+    expect(result[0].text).toBe("\x00__PLACEHOLDER__[internal blocks omitted]");
+    // The leading byte must be a literal NUL, produced by SQLite's char(0).
+    expect((result[0].text as string).charCodeAt(0)).toBe(0);
+  });
+
+  test("preserves blocks with a missing/null type", async () => {
+    const { id } = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "x", signature: "s" },
+        { foo: "bar" },
+        { type: "text", text: "kept" },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(blocks(id)).toEqual([
+      { foo: "bar" },
+      { type: "text", text: "kept" },
+    ]);
+  });
+
+  test("leaves thinking-free assistant messages untouched", async () => {
+    const original = JSON.stringify([
+      { type: "text", text: "I was thinking about lunch" },
+    ]);
+    const { id } = insert("assistant", original);
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    // Substring 'thinking' appears in the text but no block is of thinking type.
+    expect(content(id)).toBe(original);
+  });
+
+  test("does not touch non-assistant roles", async () => {
+    const original = JSON.stringify([
+      { type: "thinking", thinking: "x", signature: "s" },
+      { type: "text", text: "u" },
+    ]);
+    const { id } = insert("user", original);
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(content(id)).toBe(original);
+  });
+
+  test("tolerates non-array and invalid JSON content", async () => {
+    const obj = insert(
+      "assistant",
+      JSON.stringify({ type: "thinking", thinking: "x" }),
+    );
+    const invalid = insert("assistant", "{not json with thinking");
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(content(obj.id)).toBe(
+      JSON.stringify({ type: "thinking", thinking: "x" }),
+    );
+    expect(content(invalid.id)).toBe("{not json with thinking");
+  });
+
+  test("is idempotent across repeated runs", async () => {
+    const { id } = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "x", signature: "s" },
+        { type: "text", text: "stable" },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+    const once = content(id);
+    await migrateStripThinkingFromConsolidated(getDb());
+    const twice = content(id);
+
+    expect(twice).toBe(once);
+    expect(blocks(id)).toEqual([{ type: "text", text: "stable" }]);
+  });
+
+  test("honors the persisted rowid watermark and resumes above it", async () => {
+    const below = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "x", signature: "s" },
+        { type: "text", text: "below" },
+      ]),
+    );
+    const above = insert(
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "y", signature: "s" },
+        { type: "text", text: "above" },
+      ]),
+    );
+
+    // Pretend a prior run already swept through `below`'s rowid: the sweep must
+    // resume strictly above it, leaving `below` untouched and cleaning `above`.
+    getSqlite()
+      .query(
+        `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, ?, ?)`,
+      )
+      .run(
+        "migration_209_strip_thinking_watermark",
+        String(below.rowid),
+        Date.now(),
+      );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    expect(blocks(below.id)).toEqual([
+      { type: "thinking", thinking: "x", signature: "s" },
+      { type: "text", text: "below" },
+    ]);
+    expect(blocks(above.id)).toEqual([{ type: "text", text: "above" }]);
+
+    // The watermark is cleared once the sweep reaches the end of the table.
+    const wm = getSqlite()
+      .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
+      .get("migration_209_strip_thinking_watermark");
+    expect(wm).toBeNull();
+  });
+});

package/src/memory/migrations/__tests__/run-migrations.test.ts

@@ -54,7 +54,7 @@ describe("runMigrationSteps — checkpointing", () => {
 
   test("records step completions in the shared memory_checkpoints ledger", async () => {
     /**
-     * Step bookkeeping lives in the same memory_checkpoints table the registry
+     * Step bookkeeping lives in the same memory_checkpoints table the step runner
      * uses, under the `step:` namespace — one ledger for all applied state.
      */
 
@@ -110,7 +110,7 @@ describe("runMigrationSteps — checkpointing", () => {
      * before the loop so a migration interrupted mid-flight re-runs this boot.
      */
 
-    // GIVEN a database with a stalled registry checkpoint left by a crash
+    // GIVEN a database with a stalled step checkpoint left by a crash
     const db = createTestDb();
     const raw = getSqliteFrom(db);
     raw.run(

package/src/memory/migrations/run-migrations.ts

@@ -4,10 +4,45 @@ import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 
 const log = getLogger("db-init");
 
+/**
+ * Rollback metadata for a migration step that can be reversed.
+ *
+ * A single step may carry multiple rollback entries (e.g. migration 162 has
+ * two: one for the timestamp conversion and one for the table rebuild). Each
+ * entry has its own monotonic version number for rollback ordering and its
+ * own `down()` function.
+ */
+export interface RollbackEntry {
+  /** Monotonic version used for rollback ordering. */
+  version: number;
+  /** Human-readable description for diagnostics. */
+  description: string;
+  /** Reverse the migration. Must be idempotent — safe to re-run. */
+  down: (database: DrizzleDb) => void;
+}
+
+/**
+ * Object form of a migration step, carrying optional rollback metadata and
+ * dependency declarations inline — no separate registry needed.
+ */
+export interface MigrationStepObject {
+  /** Step name — used for `step:*` checkpoint key, logging, and skip logic. */
+  name: string;
+  /** Forward migration body. */
+  run: (database: DrizzleDb) => void | Promise<void>;
+  /** Step names that must complete before this step runs. */
+  dependsOn?: string[];
+  /** Rollback entries — present if this step has down() function(s). */
+  rollback?: RollbackEntry[];
+}
+
 /**
  * A single forward migration step, identified for checkpointing and logging by
  * its `.name`. Anonymous steps (empty `.name`) cannot be tracked and always run.
  *
+ * Bare function steps are identified by `Function.name`. Object-form steps
+ * carry an explicit `name` plus optional `rollback` and `dependsOn` metadata.
+ *
  * A step may be synchronous or return a promise. The runner awaits an async
  * step to completion before checkpointing it and moving on, so ordering is
  * preserved exactly as for sync steps: step N+1 never starts — and is never
@@ -16,7 +51,56 @@ const log = getLogger("db-init");
  * thread between batches while still guaranteeing later migrations observe its
  * completed result.
  */
-export type MigrationStep = (database: DrizzleDb) => void | Promise<void>;
+export type MigrationStep =
+  | MigrationStepObject
+  | ((database: DrizzleDb) => void | Promise<void>);
+
+/**
+ * Normalize a {@link MigrationStep} (which may be a bare function or an object)
+ * into its object form so callers don't need to type-narrow.
+ */
+export function normalizeStep(step: MigrationStep): MigrationStepObject {
+  if (typeof step === "function") {
+    return { name: step.name, run: step };
+  }
+  return step;
+}
+
+/**
+ * Extract the step name from either form of {@link MigrationStep}.
+ */
+export function getStepName(step: MigrationStep): string {
+  return typeof step === "function" ? step.name : step.name;
+}
+
+/**
+ * Get the maximum rollback version across all steps that carry rollback
+ * metadata. Returns 0 if no steps have rollback entries.
+ */
+export function getMaxRollbackVersion(steps: MigrationStep[]): number {
+  let max = 0;
+  for (const step of steps) {
+    if (typeof step === "function") continue;
+    if (!step.rollback) continue;
+    for (const entry of step.rollback) {
+      if (entry.version > max) max = entry.version;
+    }
+  }
+  return max;
+}
+
+/**
+ * Collect all step names that appear in the steps array (both bare-function
+ * and object-form). Used by {@link validateMigrationState} to detect
+ * checkpoints from a newer daemon version.
+ */
+export function getKnownStepNames(steps: MigrationStep[]): Set<string> {
+  const names = new Set<string>();
+  for (const step of steps) {
+    names.add(getStepName(step));
+  }
+  return names;
+}
 
 export interface MigrationRunResult {
   /** Steps that ran and completed successfully this boot. */
@@ -30,9 +114,8 @@ export interface MigrationRunResult {
 /**
  * Prefix under which forward-step completions are recorded in the shared
  * `memory_checkpoints` ledger. A distinct namespace keeps step bookkeeping from
- * colliding with registry checkpoint keys (`migration_*`, `backfill_*`,
- * `drop_*`) and is deliberately chosen so `validateMigrationState` does not
- * mistake a step record for an unknown registry migration.
+ * colliding with legacy checkpoint keys (`migration_*`, `backfill_*`,
+ * `drop_*`) left by older migration functions.
  */
 export const STEP_CHECKPOINT_PREFIX = "step:";
 
@@ -161,7 +244,8 @@ export async function runMigrationSteps(
   const ran: string[] = [];
 
   for (const step of steps) {
-    const name = step.name;
+    const obj = normalizeStep(step);
+    const name = obj.name;
     const checkpointable = name !== "";
 
     if (checkpointable && applied.has(name)) {
@@ -175,7 +259,7 @@ export async function runMigrationSteps(
         markStarted.run(`${STEP_CHECKPOINT_PREFIX}${name}`, Date.now());
       }
       log.info({ migration: name }, `Starting migration: ${name}`);
-      const result = step(database);
+      const result = obj.run(database);
       if (result instanceof Promise) {
         await result;
       }

package/src/memory/migrations/schema-introspection.ts

@@ -16,3 +16,17 @@ export function tableHasColumn(
 
   return columns.some((column) => column.name === columnName);
 }
+
+/**
+ * True when a base table with the given name exists. Useful for migrations
+ * that recreate indexes — `CREATE INDEX IF NOT EXISTS` guards against the
+ * index already existing, but still throws "no such table" if the underlying
+ * table was already dropped by an earlier migration.
+ */
+export function tableExists(database: DrizzleDb, tableName: string): boolean {
+  const raw = getSqliteFrom(database);
+  const row = raw
+    .query(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`)
+    .get(tableName);
+  return row != null;
+}