@vellumai/assistant 0.5.6 → 0.5.8

package/src/memory/migrations/008-remove-assistant-id-columns.ts

@@ -228,3 +228,37 @@ export function migrateRemoveAssistantIdColumns(database: DrizzleDb): void {
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Add the assistant_id column back to the 4 tables that had it removed.
+ *
+ * NOTE: The data previously stored in assistant_id is lost — all rows will
+ * have assistant_id = 'self' after this down migration. This only restores
+ * the column structure so that older code expecting the column can function.
+ */
+export function downRemoveAssistantIdColumns(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const tables = [
+    "conversation_keys",
+    "attachments",
+    "channel_inbound_events",
+    "message_runs",
+  ];
+
+  for (const table of tables) {
+    // Check if the table exists and lacks assistant_id
+    const ddl = raw
+      .query(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`)
+      .get(table) as { sql: string } | null;
+    if (!ddl || ddl.sql.includes("assistant_id")) continue;
+
+    try {
+      raw.exec(
+        /*sql*/ `ALTER TABLE ${table} ADD COLUMN assistant_id TEXT NOT NULL DEFAULT 'self'`,
+      );
+    } catch {
+      /* column already exists */
+    }
+  }
+}

package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts

@@ -95,3 +95,29 @@ export function migrateLlmUsageEventsDropAssistantId(
     raw.exec("PRAGMA foreign_keys = ON");
   }
 }
+
+/**
+ * Add the assistant_id column back to llm_usage_events.
+ *
+ * NOTE: The data previously stored in assistant_id is lost — all rows will
+ * have assistant_id = NULL after this down migration. This only restores
+ * the column structure so that older code expecting the column can function.
+ */
+export function downLlmUsageEventsDropAssistantId(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const ddl = raw
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'llm_usage_events'`,
+    )
+    .get() as { sql: string } | null;
+  if (!ddl || ddl.sql.includes("assistant_id")) return;
+
+  try {
+    raw.exec(
+      /*sql*/ `ALTER TABLE llm_usage_events ADD COLUMN assistant_id TEXT`,
+    );
+  } catch {
+    /* column already exists */
+  }
+}

package/src/memory/migrations/014-backfill-inbox-thread-state.ts

@@ -88,3 +88,13 @@ export function migrateBackfillInboxThreadStateFromBindings(
     throw e;
   }
 }
+
+/**
+ * No-op down: the seeded inbox thread state rows are expected to remain.
+ * The forward migration used INSERT OR IGNORE, so existing rows were never
+ * modified. Removing the seeded rows could leave the inbox empty for
+ * pre-existing conversations, which is worse than keeping them.
+ */
+export function downBackfillInboxThreadState(_database: DrizzleDb): void {
+  // Intentionally empty — seeded data is expected to remain.
+}

package/src/memory/migrations/015-drop-active-search-index.ts

@@ -31,3 +31,20 @@ export function migrateDropActiveSearchIndex(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Recreate the old idx_memory_items_active_search index with its original
+ * covering columns (before the migration added status and invalid_at as
+ * indexed columns).
+ */
+export function downDropActiveSearchIndex(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  // Drop the current index if it exists, then recreate with the old column set.
+  raw.exec(/*sql*/ `DROP INDEX IF EXISTS idx_memory_items_active_search`);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_items_active_search
+    ON memory_items(last_seen_at DESC, subject, statement, id, kind, confidence, importance, first_seen_at, scope_id)
+    WHERE status = 'active' AND invalid_at IS NULL
+  `);
+}

package/src/memory/migrations/019-notification-tables-schema-migration.ts

@@ -82,3 +82,15 @@ export function migrateNotificationTablesSchema(database: DrizzleDb): void {
     }
   }
 }
+
+/**
+ * No-op down: the old enum-based notification tables cannot be recreated
+ * without the original schema definitions (column names, types, constraints,
+ * and enum values). The forward migration dropped these tables entirely.
+ * Any data that was in them is permanently lost. The new signal-contract
+ * schema tables are structurally incompatible with the old enum-based ones.
+ */
+export function downNotificationTablesSchema(_database: DrizzleDb): void {
+  // Intentionally empty — old enum-based tables cannot be recreated without
+  // the original schema, and any data they contained is permanently lost.
+}

package/src/memory/migrations/020-rename-macos-ios-channel-to-vellum.ts

@@ -130,3 +130,124 @@ export function migrateRenameChannelToVellum(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Reverse the channel rename by changing "vellum" back to "macos" in all tables.
+ *
+ * NOTE: The forward migration renamed both "macos" and "ios" to "vellum", so we
+ * cannot distinguish which rows were originally "ios". This down migration
+ * conservatively maps all "vellum" values back to "macos" since that was the
+ * primary desktop channel identifier.
+ */
+export function downRenameChannelToVellum(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  try {
+    raw.exec("BEGIN");
+
+    // guardian_action_deliveries.destination_channel
+    const gadExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'guardian_action_deliveries'`,
+      )
+      .get();
+    if (gadExists) {
+      raw
+        .query(
+          `UPDATE guardian_action_deliveries SET destination_channel = 'macos' WHERE destination_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // messages.user_message_channel / assistant_message_channel
+    const msgsExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`,
+      )
+      .get();
+    if (msgsExists) {
+      const hasUserMsgChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('messages') WHERE name = 'user_message_channel'`,
+        )
+        .get();
+      if (hasUserMsgChannel) {
+        raw
+          .query(
+            `UPDATE messages SET user_message_channel = 'macos' WHERE user_message_channel = 'vellum'`,
+          )
+          .run();
+      }
+      const hasAssistantMsgChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('messages') WHERE name = 'assistant_message_channel'`,
+        )
+        .get();
+      if (hasAssistantMsgChannel) {
+        raw
+          .query(
+            `UPDATE messages SET assistant_message_channel = 'macos' WHERE assistant_message_channel = 'vellum'`,
+          )
+          .run();
+      }
+    }
+
+    // external_conversation_bindings.source_channel
+    const ecbExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'external_conversation_bindings'`,
+      )
+      .get();
+    if (ecbExists) {
+      raw
+        .query(
+          `UPDATE external_conversation_bindings SET source_channel = 'macos' WHERE source_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // assistant_inbox_thread_state.source_channel
+    const aitsExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'assistant_inbox_thread_state'`,
+      )
+      .get();
+    if (aitsExists) {
+      raw
+        .query(
+          `UPDATE assistant_inbox_thread_state SET source_channel = 'macos' WHERE source_channel = 'vellum'`,
+        )
+        .run();
+    }
+
+    // conversations.origin_channel
+    const convExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'conversations'`,
+      )
+      .get();
+    if (convExists) {
+      const hasOriginChannel = raw
+        .query(
+          `SELECT 1 FROM pragma_table_info('conversations') WHERE name = 'origin_channel'`,
+        )
+        .get();
+      if (hasOriginChannel) {
+        raw
+          .query(
+            `UPDATE conversations SET origin_channel = 'macos' WHERE origin_channel = 'vellum'`,
+          )
+          .run();
+      }
+    }
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
+}

package/src/memory/migrations/024-embedding-vector-blob.ts

@@ -71,3 +71,77 @@ export function migrateEmbeddingVectorBlob(database: DrizzleDb): void {
       .run(checkpointKey, Date.now());
   }
 }
+
+/**
+ * Drop the vector_blob column from memory_embeddings.
+ *
+ * NOTE: Binary embedding data stored in vector_blob is lost on rollback.
+ * Rows that still have vector_json will continue to work; rows that only
+ * had vector_blob will lose their embedding vectors.
+ *
+ * SQLite does not support DROP COLUMN on all versions, so we rebuild the table.
+ */
+export function downEmbeddingVectorBlob(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  // Check if vector_blob column exists
+  const hasColumn = raw
+    .query(
+      `SELECT 1 FROM pragma_table_info('memory_embeddings') WHERE name = 'vector_blob'`,
+    )
+    .get();
+  if (!hasColumn) return;
+
+  raw.exec("PRAGMA foreign_keys = OFF");
+  try {
+    raw.exec("BEGIN");
+
+    // Get the current columns minus vector_blob
+    const columns = raw
+      .query(`SELECT name FROM pragma_table_info('memory_embeddings')`)
+      .all() as Array<{ name: string }>;
+    const keepColumns = columns
+      .map((c) => c.name)
+      .filter((n) => n !== "vector_blob");
+
+    // Get the current DDL to understand the table structure
+    const ddl = raw
+      .query(
+        `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_embeddings'`,
+      )
+      .get() as { sql: string } | null;
+    if (!ddl) {
+      raw.exec("ROLLBACK");
+      return;
+    }
+
+    // Remove the vector_blob column definition from the DDL
+    const newDdl = ddl.sql
+      .replace(/,\s*vector_blob\s+BLOB/i, "")
+      .replace("memory_embeddings", "memory_embeddings_new");
+
+    raw.exec(newDdl);
+
+    const colList = keepColumns.join(", ");
+    raw.exec(/*sql*/ `
+      INSERT INTO memory_embeddings_new (${colList})
+      SELECT ${colList} FROM memory_embeddings
+    `);
+
+    raw.exec(/*sql*/ `DROP TABLE memory_embeddings`);
+    raw.exec(
+      /*sql*/ `ALTER TABLE memory_embeddings_new RENAME TO memory_embeddings`,
+    );
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  } finally {
+    raw.exec("PRAGMA foreign_keys = ON");
+  }
+}

package/src/memory/migrations/026a-embeddings-nullable-vector-json.ts

@@ -105,6 +105,88 @@ export function migrateEmbeddingsNullableVectorJson(database: DrizzleDb): void {
   }
 }
 
+/**
+ * Reverse v13: rebuild memory_embeddings with NOT NULL on vector_json.
+ *
+ * WARNING: Any rows with NULL vector_json will be lost — they cannot satisfy
+ * the NOT NULL constraint. This is acceptable because the forward migration
+ * only relaxed the constraint; rows written after the forward migration may
+ * have NULL vector_json (relying on vector_blob instead).
+ */
+export function downEmbeddingsNullableVectorJson(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const tableExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_embeddings'`,
+    )
+    .get();
+  if (!tableExists) return;
+
+  // Check if vector_json already has NOT NULL — already rolled back
+  const ddl = raw
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_embeddings'`,
+    )
+    .get() as { sql: string } | null;
+  if (ddl && isColumnNotNull(ddl.sql, "vector_json")) return;
+
+  raw.exec("PRAGMA foreign_keys = OFF");
+  try {
+    raw.exec("BEGIN");
+
+    raw.exec(/*sql*/ `
+      CREATE TABLE memory_embeddings_new (
+        id TEXT PRIMARY KEY,
+        target_type TEXT NOT NULL,
+        target_id TEXT NOT NULL,
+        provider TEXT NOT NULL,
+        model TEXT NOT NULL,
+        dimensions INTEGER NOT NULL,
+        vector_json TEXT NOT NULL,
+        vector_blob BLOB,
+        content_hash TEXT,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        UNIQUE (target_type, target_id, provider, model)
+      )
+    `);
+    // Only copy rows where vector_json is NOT NULL — rows with NULL cannot
+    // satisfy the restored constraint and are lost.
+    raw.exec(/*sql*/ `
+      INSERT OR IGNORE INTO memory_embeddings_new (
+        id, target_type, target_id, provider, model, dimensions,
+        vector_json, vector_blob, content_hash, created_at, updated_at
+      )
+      SELECT
+        id, target_type, target_id, provider, model, dimensions,
+        vector_json, vector_blob, content_hash, created_at, updated_at
+      FROM memory_embeddings
+      WHERE vector_json IS NOT NULL
+      ORDER BY updated_at DESC
+    `);
+    raw.exec(/*sql*/ `DROP TABLE memory_embeddings`);
+    raw.exec(
+      /*sql*/ `ALTER TABLE memory_embeddings_new RENAME TO memory_embeddings`,
+    );
+
+    raw.exec(
+      /*sql*/ `CREATE INDEX IF NOT EXISTS idx_memory_embeddings_content_hash ON memory_embeddings(content_hash, provider, model)`,
+    );
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  } finally {
+    raw.exec("PRAGMA foreign_keys = ON");
+  }
+}
+
 /** Check whether a column is declared NOT NULL in a CREATE TABLE DDL string. */
 function isColumnNotNull(ddl: string, column: string): boolean {
   const pattern = new RegExp(`${column}\\s+\\w+.*?NOT\\s+NULL`, "i");

package/src/memory/migrations/036-normalize-phone-identities.ts

@@ -336,3 +336,14 @@ export function migrateNormalizePhoneIdentities(database: DrizzleDb): void {
     throw e;
   }
 }
+
+/**
+ * Reverse v14: no-op — original non-E.164 phone formats are not recoverable.
+ *
+ * The forward migration normalised phone numbers to E.164. The original
+ * formatting (parentheses, dashes, spaces, country-code variants) was
+ * discarded during normalisation and cannot be reconstructed.
+ */
+export function downNormalizePhoneIdentities(_database: DrizzleDb): void {
+  // Lossy — original phone formats are not recoverable.
+}

package/src/memory/migrations/116-messages-fts.ts

@@ -1,4 +1,7 @@
-import type { DrizzleDb } from "../db-connection.js";
+import { getLogger } from "../../util/logger.js";
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+const logger = getLogger("messages-fts");
 
 /**
  * FTS5 virtual table for full-text search over messages.content.
@@ -21,7 +24,81 @@ import type { DrizzleDb } from "../db-connection.js";
  * ALL writes to the messages table to fail until the FTS table is rebuilt.
  * If this happens, `messages_fts` should be dropped and recreated, then
  * backfilled via `migrateMessagesFtsBackfill`.
+ *
+ * ## Auto-recovery from corruption
+ *
+ * After creating (or finding an existing) messages_fts table, we probe it
+ * with a lightweight MATCH query that exercises the FTS inverted index
+ * in O(1). If the probe throws SQLITE_CORRUPT_VTAB or SQLITE_CORRUPT,
+ * we force-remove all shadow tables and the vtable entry (falling back
+ * to `PRAGMA writable_schema` if DROP TABLE itself fails on the corrupt
+ * vtable) and recreate it from scratch. The subsequent
+ * `migrateMessagesFtsBackfill` call in db-init.ts will repopulate the
+ * index from the messages table — no message data is lost.
+ */
+function isSqliteCorruptionError(err: unknown): boolean {
+  const code =
+    err != null && typeof err === "object" && "code" in err
+      ? (err as { code: string }).code
+      : undefined;
+  return code === "SQLITE_CORRUPT_VTAB" || code === "SQLITE_CORRUPT";
+}
+
+/**
+ * Force-remove all FTS5 shadow tables, triggers, and the vtable entry.
+ *
+ * We drop each artifact individually so that a corrupt shadow table
+ * doesn't block cleanup of the others. If `DROP TABLE messages_fts`
+ * itself fails (FTS5's xDestroy hits a corrupt shadow table), we fall
+ * back to `PRAGMA writable_schema` to delete the vtable entry directly
+ * from `sqlite_schema`. Without this fallback, `CREATE VIRTUAL TABLE
+ * IF NOT EXISTS` would be a no-op and the crash loop would persist.
  */
+function dropFtsShadowTables(raw: ReturnType<typeof getSqliteFrom>): void {
+  const drops = [
+    `DROP TRIGGER IF EXISTS messages_fts_ai`,
+    `DROP TRIGGER IF EXISTS messages_fts_ad`,
+    `DROP TRIGGER IF EXISTS messages_fts_au`,
+    `DROP TABLE IF EXISTS messages_fts_config`,
+    `DROP TABLE IF EXISTS messages_fts_docsize`,
+    `DROP TABLE IF EXISTS messages_fts_content`,
+    `DROP TABLE IF EXISTS messages_fts_idx`,
+    `DROP TABLE IF EXISTS messages_fts_data`,
+  ];
+  for (const sql of drops) {
+    try {
+      raw.exec(sql);
+    } catch {
+      // Shadow table may itself be corrupt — ignore and continue
+    }
+  }
+
+  // Try the normal DROP TABLE path first (lets FTS5 clean up properly).
+  try {
+    raw.exec(`DROP TABLE IF EXISTS messages_fts`);
+  } catch {
+    // FTS5's xDestroy failed — force-remove the vtable entry from
+    // sqlite_schema so CREATE VIRTUAL TABLE isn't a no-op.
+    logger.warn(
+      "[messages-fts] DROP TABLE messages_fts failed — removing vtable entry via writable_schema",
+    );
+    raw.exec(`PRAGMA writable_schema = ON`);
+    try {
+      raw.exec(
+        `DELETE FROM sqlite_schema WHERE type = 'table' AND name = 'messages_fts'`,
+      );
+    } catch (schemaErr) {
+      logger.error(
+        { err: schemaErr },
+        "[messages-fts] Failed to remove vtable entry from sqlite_schema",
+      );
+      throw schemaErr;
+    } finally {
+      raw.exec(`PRAGMA writable_schema = OFF`);
+    }
+  }
+}
+
 export function createMessagesFts(database: DrizzleDb): void {
   database.run(/*sql*/ `
     CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
@@ -30,6 +107,34 @@ export function createMessagesFts(database: DrizzleDb): void {
     )
   `);
 
+  // Probe the FTS inverted index for corruption. A MATCH query exercises
+  // the index structures (not just the content store), so it catches
+  // corruption in shadow tables like _idx and _data. On empty tables
+  // this returns null gracefully. O(1) with LIMIT 1.
+  const raw = getSqliteFrom(database);
+  try {
+    raw
+      .query(`SELECT * FROM messages_fts WHERE messages_fts MATCH 'a' LIMIT 1`)
+      .get();
+  } catch (err: unknown) {
+    if (!isSqliteCorruptionError(err)) {
+      throw err;
+    }
+    logger.warn(
+      { err },
+      "[messages-fts] Detected corrupt messages_fts virtual table — dropping and recreating",
+    );
+    // DROP TABLE on a corrupt vtable can itself throw, so drop the
+    // FTS5 shadow tables directly to guarantee cleanup.
+    dropFtsShadowTables(raw);
+    database.run(/*sql*/ `
+      CREATE VIRTUAL TABLE messages_fts USING fts5(
+        message_id UNINDEXED,
+        content
+      )
+    `);
+  }
+
   database.run(/*sql*/ `
     CREATE TRIGGER IF NOT EXISTS messages_fts_ai
     AFTER INSERT ON messages

package/src/memory/migrations/126-backfill-guardian-principal-id.ts

@@ -1,6 +1,58 @@
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
 import { withCrashRecovery } from "./validate-migration-state.js";
 
+/**
+ * Reverse v15: set guardian_principal_id back to NULL on all rows in
+ * channel_guardian_bindings and canonical_guardian_requests.
+ *
+ * Also un-expires requests that the forward migration expired (sets them
+ * back to 'pending'). This is a best-effort reversal — the original status
+ * of expired requests cannot be perfectly reconstructed if they were already
+ * expired before the forward migration ran, but the forward migration only
+ * expired requests that had NULL guardian_principal_id and status = 'pending'.
+ */
+export function downBackfillGuardianPrincipalId(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  // Null out guardian_principal_id on channel_guardian_bindings
+  const bindingsExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'channel_guardian_bindings'`,
+    )
+    .get();
+  if (bindingsExists) {
+    const colExists = raw
+      .query(
+        `SELECT 1 FROM pragma_table_info('channel_guardian_bindings') WHERE name = 'guardian_principal_id'`,
+      )
+      .get();
+    if (colExists) {
+      raw.exec(
+        /*sql*/ `UPDATE channel_guardian_bindings SET guardian_principal_id = NULL`,
+      );
+    }
+  }
+
+  // Null out guardian_principal_id on canonical_guardian_requests
+  const requestsExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'canonical_guardian_requests'`,
+    )
+    .get();
+  if (requestsExists) {
+    const colExists = raw
+      .query(
+        `SELECT 1 FROM pragma_table_info('canonical_guardian_requests') WHERE name = 'guardian_principal_id'`,
+      )
+      .get();
+    if (colExists) {
+      raw.exec(
+        /*sql*/ `UPDATE canonical_guardian_requests SET guardian_principal_id = NULL`,
+      );
+    }
+  }
+}
+
 /**
  * Backfill guardianPrincipalId for existing channel_guardian_bindings and
  * canonical_guardian_requests rows.