@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/memory/migrations/209-strip-thinking-from-consolidated.ts

@@ -1,6 +1,85 @@
+import { getLogger } from "../../util/logger.js";
+import { getDbPath } from "../../util/platform.js";
+import { runAsyncSqlite } from "../db-async-query.js";
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
+
+const log = getLogger("db-init");
+
+/**
+ * Checkpoint key holding the highest `messages.rowid` already swept by this
+ * migration. Persisted after every window so an interrupted run resumes from
+ * where it left off instead of restarting the whole table scan. The value is a
+ * plain integer string — deliberately not `started`/`rolling_back`, so
+ * `recoverCrashedMigrations` never mistakes it for a stalled step and clears
+ * it, and not `step:`-prefixed, so `validateMigrationState` ignores it.
+ */
+const WATERMARK_KEY = "migration_209_strip_thinking_watermark";
+
+/**
+ * Number of `rowid` values swept per `runAsyncSqlite` dispatch. Each window is
+ * one off-thread subprocess transaction, so the size bounds both the WAL growth
+ * per statement and how long a single write lock is held, while keeping the
+ * number of subprocess spawns low on a large table.
+ */
+const ROWID_WINDOW = 100_000;
+
+/** SQL predicate: this `json_each` element is an internal reasoning block. */
+const IS_THINKING = `json_extract(value, '$.type') IN ('thinking', 'redacted_thinking')`;
+
+/**
+ * SQL predicate: this `json_each` element is kept. `IS NOT` (not `!=`) so a
+ * block with a NULL/absent `type` is preserved, matching the JS filter the
+ * original migration used (`b.type !== 'thinking'`).
+ */
+const IS_KEPT = `json_extract(value, '$.type') IS NOT 'thinking' AND json_extract(value, '$.type') IS NOT 'redacted_thinking'`;
+
+/**
+ * Build the two set-based UPDATEs that sweep one rowid window `(lo, hi]`.
+ *
+ *  - Statement A strips thinking/redacted_thinking blocks from rows that retain
+ *    at least one kept block, rebuilding the content array in place. `json(value)`
+ *    re-embeds each surviving element as JSON rather than a quoted string.
+ *  - Statement B replaces all-thinking rows (no kept block survives) with the
+ *    null-byte placeholder sentinel so the message isn't left empty. `char(0)`
+ *    is evaluated inside SQLite, so the literal NUL is produced in the database
+ *    and never written into the SQL piped to the sqlite3 CLI.
+ *
+ * Both are gated by a cheap `content LIKE '%thinking%'` substring prefilter
+ * (covers both `thinking` and `redacted_thinking`) so the expensive `json_each`
+ * work only runs for rows that could possibly contain a reasoning block. The
+ * two statements touch disjoint row sets (A requires a kept block, B requires
+ * none), so their order within the window is immaterial.
+ */
+function windowSql(lo: number, hi: number): string {
+  const scope = `role = 'assistant'
+       AND rowid > ${lo} AND rowid <= ${hi}
+       AND content LIKE '%thinking%'
+       AND json_valid(content)
+       AND json_type(content) = 'array'`;
+
+  const stripMixed = /*sql*/ `
+    UPDATE messages
+    SET content = (
+      SELECT json_group_array(json(value))
+      FROM json_each(messages.content)
+      WHERE ${IS_KEPT}
+    )
+    WHERE ${scope}
+      AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_THINKING})
+      AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_KEPT});`;
+
+  const placeholderOnly = /*sql*/ `
+    UPDATE messages
+    SET content = json_array(
+      json_object('type', 'text', 'text', char(0) || '__PLACEHOLDER__[internal blocks omitted]')
+    )
+    WHERE ${scope}
+      AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_THINKING})
+      AND NOT EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_KEPT});`;
+
+  return `${stripMixed}\n${placeholderOnly}`;
+}
 
 /**
  * Strip thinking and redacted_thinking blocks from all assistant messages.
@@ -12,74 +91,62 @@ import { withCrashRecovery } from "./validate-migration-state.js";
  * provider no longer needs to strip, enabling append-only conversation
  * history and stable prefix caching.
  *
- * Idempotent — safe to re-run.
+ * The rewrite runs entirely inside SQLite (via JSON1), dispatched through
+ * {@link runAsyncSqlite} a rowid window at a time. On a host with the `sqlite3`
+ * CLI each window executes in a subprocess, leaving the daemon's event loop
+ * free to answer `/healthz` while a multi-GB table is rewritten — the original
+ * synchronous in-process loop blocked the loop for minutes and tripped the
+ * startup probe into a restart loop. Progress is checkpointed per window, so an
+ * interrupted run resumes instead of rescanning from the start.
+ *
+ * Idempotent — safe to re-run. Already-cleaned rows no longer contain a
+ * thinking block, so the substring prefilter skips them.
  */
-export function migrateStripThinkingFromConsolidated(
+export async function migrateStripThinkingFromConsolidated(
   database: DrizzleDb,
-): void {
-  withCrashRecovery(
-    database,
-    "migration_strip_thinking_from_consolidated_v1",
-    () => {
-      const raw = getSqliteFrom(database);
-
-      const BATCH_SIZE = 100;
-      let lastRowid = 0;
-
-      for (;;) {
-        const rows = raw
-          .query(
-            `SELECT rowid, id, content FROM messages
-             WHERE role = 'assistant'
-               AND rowid > ?
-             ORDER BY rowid
-             LIMIT ?`,
-          )
-          .all(lastRowid, BATCH_SIZE) as Array<{
-          rowid: number;
-          id: string;
-          content: string;
-        }>;
-
-        if (rows.length === 0) break;
-
-        for (const row of rows) {
-          lastRowid = row.rowid;
-
-          let blocks: Array<{ type: string }>;
-          try {
-            const parsed = JSON.parse(row.content);
-            if (!Array.isArray(parsed)) continue;
-            blocks = parsed;
-          } catch {
-            continue;
-          }
-
-          const hasThinking = blocks.some(
-            (b) => b.type === "thinking" || b.type === "redacted_thinking",
-          );
-          if (!hasThinking) continue;
-
-          const stripped = blocks.filter(
-            (b) => b.type !== "thinking" && b.type !== "redacted_thinking",
-          );
-
-          // Preserve at least one block so the message isn't empty.
-          const finalContent =
-            stripped.length > 0
-              ? stripped
-              : [
-                  {
-                    type: "text" as const,
-                    text: "\x00__PLACEHOLDER__[internal blocks omitted]",
-                  },
-                ];
-
-          raw
-            .query(`UPDATE messages SET content = ? WHERE id = ?`)
-            .run(JSON.stringify(finalContent), row.id);
-        }
-      }
-    },
+): Promise<void> {
+  const raw = getSqliteFrom(database);
+  const dbPath = getDbPath();
+
+  const maxRow = (
+    raw.query(`SELECT MAX(rowid) AS m FROM messages`).get() as {
+      m: number | null;
+    }
+  ).m;
+  if (maxRow == null) return; // empty table — nothing to sweep
+
+  const watermarkRow = raw
+    .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
+    .get(WATERMARK_KEY) as { value: string } | undefined;
+  let lo = watermarkRow ? Number.parseInt(watermarkRow.value, 10) || 0 : 0;
+
+  const setWatermark = raw.query(
+    `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, ?, ?)`,
+  );
+
+  while (lo < maxRow) {
+    const hi = Math.min(lo + ROWID_WINDOW, maxRow);
+
+    const res = await runAsyncSqlite(windowSql(lo, hi), { dbPath });
+    if (!res.ok) {
+      // Leave the watermark at the last completed window; throwing reports the
+      // step failed so the runner retries it (from the watermark) next boot
+      // rather than checkpointing it as done.
+      throw new Error(
+        `strip-thinking window (${lo}, ${hi}] failed: ${res.error}`,
+      );
+    }
+
+    lo = hi;
+    setWatermark.run(WATERMARK_KEY, String(lo), Date.now());
+  }
+
+  // Bound WAL growth left by the windowed rewrites, then drop the watermark so
+  // a future re-run (e.g. after a rollback) starts clean.
+  await runAsyncSqlite(`PRAGMA wal_checkpoint(TRUNCATE);`, { dbPath });
+  raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(WATERMARK_KEY);
+
+  log.info(
+    "Migration 209: stripped thinking blocks from consolidated messages",
   );
 }

package/src/memory/migrations/211-memory-recall-logs-query-context.ts

@@ -1,21 +1,16 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
 import { tableHasColumn } from "./schema-introspection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_memory_recall_logs_query_context_v1";
 
 /**
  * Add query_context column to memory_recall_logs to persist the query text
  * that drove semantic search, enabling the inspector to show what was searched.
  */
 export function migrateMemoryRecallLogsQueryContext(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    if (!tableHasColumn(database, "memory_recall_logs", "query_context")) {
-      const raw = getSqliteFrom(database);
-      raw.exec(
-        `ALTER TABLE memory_recall_logs ADD COLUMN query_context TEXT`,
-      );
-    }
-  });
+  if (!tableHasColumn(database, "memory_recall_logs", "query_context")) {
+    const raw = getSqliteFrom(database);
+    raw.exec(
+      `ALTER TABLE memory_recall_logs ADD COLUMN query_context TEXT`,
+    );
+  }
 }

package/src/memory/migrations/212-llm-request-logs-created-at-index.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_llm_request_logs_created_at_index_v1";
 
 /**
  * Add an index on `llm_request_logs.created_at` so time-range deletes
@@ -10,10 +7,8 @@ const CHECKPOINT_KEY = "migration_llm_request_logs_created_at_index_v1";
  * table scan.
  */
 export function migrateLlmRequestLogsCreatedAtIndex(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
-    raw.exec(
-      `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
-    );
-  });
+  const raw = getSqliteFrom(database);
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
+  );
 }

package/src/memory/migrations/217-conversation-host-access.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_conversation_host_access_v1";
 
 /**
  * Add conversation-scoped host access state with a safe default of disabled.
@@ -10,23 +7,21 @@ const CHECKPOINT_KEY = "migration_conversation_host_access_v1";
  * Idempotent: ALTER TABLE is guarded and the backfill only touches NULL rows.
  */
 export function migrateConversationHostAccess(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-    try {
-      raw.exec(
-        `ALTER TABLE conversations ADD COLUMN host_access INTEGER NOT NULL DEFAULT 0`,
-      );
-    } catch {
-      // Column already exists.
-    }
+  try {
+    raw.exec(
+      `ALTER TABLE conversations ADD COLUMN host_access INTEGER NOT NULL DEFAULT 0`,
+    );
+  } catch {
+    // Column already exists.
+  }
 
-    raw.exec(`
-      UPDATE conversations
-      SET host_access = 0
-      WHERE host_access IS NULL
-    `);
-  });
+  raw.exec(`
+    UPDATE conversations
+    SET host_access = 0
+    WHERE host_access IS NULL
+  `);
 }
 
 /**

package/src/memory/migrations/220-normalize-user-file-by-principal.ts

@@ -1,6 +1,5 @@
 import { computeUserFileBaseSlug } from "../../contacts/contact-store.js";
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
 
 /**
  * Reverse is a no-op. This migration only consolidates `user_file` across
@@ -85,106 +84,100 @@ export function isAutoIncrementedUserFile(
 export function migrateNormalizeUserFileByPrincipal(
   database: DrizzleDb,
 ): void {
-  withCrashRecovery(
-    database,
-    "migration_normalize_user_file_by_principal_v1",
-    () => {
-      const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-      const tableExists = raw
-        .query(
-          `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'contacts'`,
-        )
-        .get();
-      if (!tableExists) return;
+  const tableExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'contacts'`,
+    )
+    .get();
+  if (!tableExists) return;
 
-      const userFileColExists = raw
-        .query(
-          `SELECT 1 FROM pragma_table_info('contacts') WHERE name = 'user_file'`,
-        )
-        .get();
-      const principalColExists = raw
-        .query(
-          `SELECT 1 FROM pragma_table_info('contacts') WHERE name = 'principal_id'`,
-        )
-        .get();
-      if (!userFileColExists || !principalColExists) return;
+  const userFileColExists = raw
+    .query(
+      `SELECT 1 FROM pragma_table_info('contacts') WHERE name = 'user_file'`,
+    )
+    .get();
+  const principalColExists = raw
+    .query(
+      `SELECT 1 FROM pragma_table_info('contacts') WHERE name = 'principal_id'`,
+    )
+    .get();
+  if (!userFileColExists || !principalColExists) return;
 
-      try {
-        raw.exec("BEGIN");
+  try {
+    raw.exec("BEGIN");
 
-        const principals = raw
-          .query(
-            /*sql*/ `
-            SELECT principal_id
-            FROM contacts
-            WHERE principal_id IS NOT NULL
-            GROUP BY principal_id
-            HAVING COUNT(DISTINCT COALESCE(user_file, '')) > 1
-          `,
-          )
-          .all() as Array<{ principal_id: string }>;
+    const principals = raw
+      .query(
+        /*sql*/ `
+        SELECT principal_id
+        FROM contacts
+        WHERE principal_id IS NOT NULL
+        GROUP BY principal_id
+        HAVING COUNT(DISTINCT COALESCE(user_file, '')) > 1
+      `,
+      )
+      .all() as Array<{ principal_id: string }>;
 
-        // Fetch all non-null candidates and rank in JS. The auto-increment
-        // classification is a regex that SQLite's GLOB can't express cleanly
-        // (unbounded digit count, date-pattern exclusion), and keeping the
-        // logic in one place avoids SQL/JS drift.
-        const selectCandidates = raw.prepare(
-          /*sql*/ `
-          SELECT user_file, display_name, created_at, id FROM contacts
-          WHERE principal_id = ? AND user_file IS NOT NULL
-          `,
-        );
+    // Fetch all non-null candidates and rank in JS. The auto-increment
+    // classification is a regex that SQLite's GLOB can't express cleanly
+    // (unbounded digit count, date-pattern exclusion), and keeping the
+    // logic in one place avoids SQL/JS drift.
+    const selectCandidates = raw.prepare(
+      /*sql*/ `
+      SELECT user_file, display_name, created_at, id FROM contacts
+      WHERE principal_id = ? AND user_file IS NOT NULL
+      `,
+    );
 
-        const updateSiblings = raw.prepare(
-          /*sql*/ `
-          UPDATE contacts
-          SET user_file = ?, updated_at = ?
-          WHERE principal_id = ?
-            AND (user_file IS NULL OR user_file != ?)
-          `,
-        );
+    const updateSiblings = raw.prepare(
+      /*sql*/ `
+      UPDATE contacts
+      SET user_file = ?, updated_at = ?
+      WHERE principal_id = ?
+        AND (user_file IS NULL OR user_file != ?)
+      `,
+    );
 
-        for (const { principal_id } of principals) {
-          const candidates = selectCandidates.all(principal_id) as Array<{
-            user_file: string;
-            display_name: string;
-            created_at: number;
-            id: string;
-          }>;
-          if (candidates.length === 0) continue;
+    for (const { principal_id } of principals) {
+      const candidates = selectCandidates.all(principal_id) as Array<{
+        user_file: string;
+        display_name: string;
+        created_at: number;
+        id: string;
+      }>;
+      if (candidates.length === 0) continue;
 
-          candidates.sort((a, b) => {
-            const aAuto = isAutoIncrementedUserFile(a.user_file, a.display_name)
-              ? 1
-              : 0;
-            const bAuto = isAutoIncrementedUserFile(b.user_file, b.display_name)
-              ? 1
-              : 0;
-            if (aAuto !== bAuto) return aAuto - bAuto;
-            if (a.created_at !== b.created_at)
-              return a.created_at - b.created_at;
-            return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
-          });
+      candidates.sort((a, b) => {
+        const aAuto = isAutoIncrementedUserFile(a.user_file, a.display_name)
+          ? 1
+          : 0;
+        const bAuto = isAutoIncrementedUserFile(b.user_file, b.display_name)
+          ? 1
+          : 0;
+        if (aAuto !== bAuto) return aAuto - bAuto;
+        if (a.created_at !== b.created_at)
+          return a.created_at - b.created_at;
+        return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+      });
 
-          const canonical = candidates[0]!.user_file;
-          updateSiblings.run(
-            canonical,
-            Date.now(),
-            principal_id,
-            canonical,
-          );
-        }
+      const canonical = candidates[0]!.user_file;
+      updateSiblings.run(
+        canonical,
+        Date.now(),
+        principal_id,
+        canonical,
+      );
+    }
 
-        raw.exec("COMMIT");
-      } catch (e) {
-        try {
-          raw.exec("ROLLBACK");
-        } catch {
-          /* no active transaction */
-        }
-        throw e;
-      }
-    },
-  );
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
 }

package/src/memory/migrations/222-strip-placeholder-sentinels-from-messages.ts

@@ -1,7 +1,6 @@
 import { isPlaceholderSentinelText } from "../../providers/placeholder-sentinels.js";
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
 
 /**
  * Strip provider placeholder sentinel text blocks from persisted assistant
@@ -25,59 +24,53 @@ import { withCrashRecovery } from "./validate-migration-state.js";
 export function migrateStripPlaceholderSentinelsFromMessages(
   database: DrizzleDb,
 ): void {
-  withCrashRecovery(
-    database,
-    "migration_strip_placeholder_sentinels_from_messages_v1",
-    () => {
-      const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-      const BATCH_SIZE = 100;
-      let lastRowid = 0;
+  const BATCH_SIZE = 100;
+  let lastRowid = 0;
 
-      for (;;) {
-        const rows = raw
-          .query(
-            `SELECT rowid, id, content FROM messages
-             WHERE role = 'assistant'
-               AND content LIKE '%__PLACEHOLDER__%'
-               AND rowid > ?
-             ORDER BY rowid
-             LIMIT ?`,
-          )
-          .all(lastRowid, BATCH_SIZE) as Array<{
-          rowid: number;
-          id: string;
-          content: string;
-        }>;
+  for (;;) {
+    const rows = raw
+      .query(
+        `SELECT rowid, id, content FROM messages
+         WHERE role = 'assistant'
+           AND content LIKE '%__PLACEHOLDER__%'
+           AND rowid > ?
+         ORDER BY rowid
+         LIMIT ?`,
+      )
+      .all(lastRowid, BATCH_SIZE) as Array<{
+      rowid: number;
+      id: string;
+      content: string;
+    }>;
 
-        if (rows.length === 0) break;
+    if (rows.length === 0) break;
 
-        for (const row of rows) {
-          lastRowid = row.rowid;
+    for (const row of rows) {
+      lastRowid = row.rowid;
 
-          let blocks: Array<Record<string, unknown>>;
-          try {
-            const parsed = JSON.parse(row.content);
-            if (!Array.isArray(parsed)) continue;
-            blocks = parsed;
-          } catch {
-            continue;
-          }
+      let blocks: Array<Record<string, unknown>>;
+      try {
+        const parsed = JSON.parse(row.content);
+        if (!Array.isArray(parsed)) continue;
+        blocks = parsed;
+      } catch {
+        continue;
+      }
 
-          const stripped = blocks.filter((b) => {
-            if (typeof b !== "object" || b === null) return false;
-            if (b.type !== "text") return true;
-            const text = typeof b.text === "string" ? b.text : "";
-            return !isPlaceholderSentinelText(text);
-          });
+      const stripped = blocks.filter((b) => {
+        if (typeof b !== "object" || b === null) return false;
+        if (b.type !== "text") return true;
+        const text = typeof b.text === "string" ? b.text : "";
+        return !isPlaceholderSentinelText(text);
+      });
 
-          if (stripped.length === blocks.length) continue;
+      if (stripped.length === blocks.length) continue;
 
-          raw
-            .query(`UPDATE messages SET content = ? WHERE id = ?`)
-            .run(JSON.stringify(stripped), row.id);
-        }
-      }
-    },
-  );
+      raw
+        .query(`UPDATE messages SET content = ? WHERE id = ?`)
+        .run(JSON.stringify(stripped), row.id);
+    }
+  }
 }