@martian-engineering/lossless-claw 0.7.0 → 0.8.0

package/src/db/migration.ts

@@ -2,6 +2,10 @@ import type { DatabaseSync } from "node:sqlite";
 import { getLcmDbFeatures } from "./features.js";
 import { parseUtcTimestampOrNull } from "../store/parse-utc-timestamp.js";
 
+type MigrationLogger = {
+  info?: (message: string) => void;
+};
+
 type SummaryColumnInfo = {
   name?: string;
 };
@@ -27,6 +31,18 @@ type SummaryParentEdgeRow = {
   parent_summary_id: string;
 };
 
+type TableNameRow = {
+  name?: string;
+};
+
+type FtsTableSpec = {
+  tableName: string;
+  createSql: string;
+  seedSql: string;
+  expectedColumns: string[];
+  staleSchemaPatterns?: string[];
+};
+
 function ensureSummaryDepthColumn(db: DatabaseSync): void {
   const summaryColumns = db.prepare(`PRAGMA table_info(summaries)`).all() as SummaryColumnInfo[];
   const hasDepth = summaryColumns.some((col) => col.name === "depth");
@@ -107,6 +123,29 @@ function ensureCompactionTelemetryColumns(db: DatabaseSync): void {
   }
 }
 
+function describeMigrationError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function runMigrationStep(
+  name: string,
+  log: MigrationLogger | undefined,
+  step: () => void,
+): void {
+  const startedAt = Date.now();
+  try {
+    step();
+    log?.info?.(
+      `[lcm] migration step complete: step=${name} durationMs=${Date.now() - startedAt}`,
+    );
+  } catch (error) {
+    log?.info?.(
+      `[lcm] migration step failed: step=${name} durationMs=${Date.now() - startedAt} error=${describeMigrationError(error)}`,
+    );
+    throw error;
+  }
+}
+
 function backfillSummaryDepths(db: DatabaseSync): void {
   // Leaves are always depth 0, even if legacy rows had malformed values.
   db.exec(`UPDATE summaries SET depth = 0 WHERE kind = 'leaf'`);
@@ -444,10 +483,89 @@ function backfillToolCallColumns(db: DatabaseSync): void {
   );
 }
 
+function getExistingTableNames(db: DatabaseSync, names: string[]): Set<string> {
+  if (names.length === 0) {
+    return new Set();
+  }
+  const placeholders = names.map(() => "?").join(", ");
+  const rows = db
+    .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders})`)
+    .all(...names) as TableNameRow[];
+  return new Set(
+    rows
+      .map((row) => row.name)
+      .filter((name): name is string => typeof name === "string" && name.length > 0),
+  );
+}
+
+function getFtsShadowTableNames(tableName: string): string[] {
+  return [
+    `${tableName}_data`,
+    `${tableName}_idx`,
+    `${tableName}_content`,
+    `${tableName}_docsize`,
+    `${tableName}_config`,
+  ];
+}
+
+function quoteSqlIdentifier(identifier: string): string {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
+    throw new Error(`Invalid SQL identifier: ${identifier}`);
+  }
+  return `"${identifier.replaceAll(`"`, `""`)}"`;
+}
+
+function shouldRecreateStandaloneFtsTable(db: DatabaseSync, spec: FtsTableSpec): boolean {
+  const shadowTables = getFtsShadowTableNames(spec.tableName);
+  const existingTables = getExistingTableNames(db, [spec.tableName, ...shadowTables]);
+  if (!existingTables.has(spec.tableName)) {
+    return true;
+  }
+  if (shadowTables.some((name) => !existingTables.has(name))) {
+    return true;
+  }
+
+  try {
+    const info = db
+      .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name = ?")
+      .get(spec.tableName) as { sql?: string } | undefined;
+    const sql = info?.sql ?? "";
+    if (spec.staleSchemaPatterns?.some((pattern) => sql.includes(pattern))) {
+      return true;
+    }
+
+    const columns = db
+      .prepare(`PRAGMA table_info(${quoteSqlIdentifier(spec.tableName)})`)
+      .all() as SummaryColumnInfo[];
+    const columnNames = new Set(
+      columns
+        .map((col) => col.name)
+        .filter((name): name is string => typeof name === "string" && name.length > 0),
+    );
+    return spec.expectedColumns.some((column) => !columnNames.has(column));
+  } catch {
+    return true;
+  }
+}
+
+function ensureStandaloneFtsTable(db: DatabaseSync, spec: FtsTableSpec): void {
+  if (!shouldRecreateStandaloneFtsTable(db, spec)) {
+    return;
+  }
+
+  db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(spec.tableName)}`);
+  for (const shadowTableName of getFtsShadowTableNames(spec.tableName)) {
+    db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(shadowTableName)}`);
+  }
+  db.exec(spec.createSql);
+  db.exec(spec.seedSql);
+}
+
 export function runLcmMigrations(
   db: DatabaseSync,
-  options?: { fts5Available?: boolean },
+  options?: { fts5Available?: boolean; log?: MigrationLogger },
 ): void {
+  const log = options?.log;
   db.exec(`
     CREATE TABLE IF NOT EXISTS conversations (
       conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -645,79 +763,80 @@ export function runLcmMigrations(
     ON conversations (session_key, active, created_at)
   `);
   db.exec(`DROP INDEX IF EXISTS conversations_session_key_idx`);
-  ensureSummaryDepthColumn(db);
-  ensureSummaryMetadataColumns(db);
-  ensureSummaryModelColumn(db);
-  ensureCompactionTelemetryColumns(db);
-  backfillSummaryDepths(db);
+  runMigrationStep("ensureSummaryDepthColumn", log, () => ensureSummaryDepthColumn(db));
+  runMigrationStep("ensureSummaryMetadataColumns", log, () =>
+    ensureSummaryMetadataColumns(db),
+  );
+  runMigrationStep("ensureSummaryModelColumn", log, () => ensureSummaryModelColumn(db));
+  runMigrationStep("ensureCompactionTelemetryColumns", log, () =>
+    ensureCompactionTelemetryColumns(db),
+  );
+  runMigrationStep("backfillSummaryDepths", log, () => backfillSummaryDepths(db));
   // Index on depth — created AFTER backfillSummaryDepths to avoid index
   // maintenance overhead during bulk depth updates on large existing DBs.
-  db.exec(`CREATE INDEX IF NOT EXISTS summaries_conv_depth_kind_idx ON summaries (conversation_id, depth, kind)`);
-  backfillSummaryMetadata(db);
-  backfillToolCallColumns(db);
+  runMigrationStep("createSummariesDepthIndex", log, () =>
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS summaries_conv_depth_kind_idx ON summaries (conversation_id, depth, kind)`,
+    ),
+  );
+  runMigrationStep("backfillSummaryMetadata", log, () => backfillSummaryMetadata(db));
+  runMigrationStep("backfillToolCallColumns", log, () => backfillToolCallColumns(db));
 
-  const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
+  const detectedFeatures = options?.fts5Available === false ? null : getLcmDbFeatures(db);
+  const fts5Available = options?.fts5Available ?? detectedFeatures?.fts5Available ?? false;
   if (!fts5Available) {
     return;
   }
 
+  const trigramTokenizerAvailable = detectedFeatures?.trigramTokenizerAvailable ?? false;
+  if (!trigramTokenizerAvailable) {
+    try {
+      db.exec(`DROP TABLE IF EXISTS summaries_fts_cjk`);
+    } catch {
+      // Best effort only. A stale virtual table should not block core migration.
+    }
+  }
+
   // FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
-  const hasFts = db
-    .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
-    .get();
-
-  if (hasFts) {
-    // Check for stale schema: external-content FTS tables with content_rowid cause errors.
-    // Drop and recreate as standalone FTS if the old schema is detected.
-    const ftsSchema = (
-      db
-        .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'")
-        .get() as { sql: string } | undefined
-    )?.sql;
-    if (ftsSchema && ftsSchema.includes("content_rowid")) {
-      db.exec("DROP TABLE messages_fts");
-      db.exec(`
+  runMigrationStep("ensureMessagesFts", log, () => {
+    ensureStandaloneFtsTable(db, {
+      tableName: "messages_fts",
+      createSql: `
         CREATE VIRTUAL TABLE messages_fts USING fts5(
           content,
           tokenize='porter unicode61'
-        );
-        INSERT INTO messages_fts(rowid, content) SELECT message_id, content FROM messages;
-      `);
-    }
-  } else {
-    db.exec(`
-      CREATE VIRTUAL TABLE messages_fts USING fts5(
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-  }
-
-  const summariesFtsInfo = db
-    .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='summaries_fts'")
-    .get() as { sql?: string } | undefined;
-  const summariesFtsSql = summariesFtsInfo?.sql ?? "";
-  const summariesFtsColumns = db.prepare(`PRAGMA table_info(summaries_fts)`).all() as Array<{
-    name?: string;
-  }>;
-  const hasSummaryIdColumn = summariesFtsColumns.some((col) => col.name === "summary_id");
-  const shouldRecreateSummariesFts =
-    !summariesFtsInfo ||
-    !hasSummaryIdColumn ||
-    summariesFtsSql.includes("content_rowid='summary_id'") ||
-    summariesFtsSql.includes('content_rowid="summary_id"');
-  if (shouldRecreateSummariesFts) {
-    db.exec(`
-      DROP TABLE IF EXISTS summaries_fts;
-      CREATE VIRTUAL TABLE summaries_fts USING fts5(
-        summary_id UNINDEXED,
-        content,
-        tokenize='porter unicode61'
-      );
-      INSERT INTO summaries_fts(summary_id, content)
-      SELECT summary_id, content FROM summaries;
-    `);
-  }
+        )
+      `,
+      seedSql: `
+        INSERT INTO messages_fts(rowid, content)
+        SELECT message_id, content FROM messages
+      `,
+      expectedColumns: ["content"],
+      staleSchemaPatterns: ["content_rowid"],
+    });
+  });
+
+  runMigrationStep("ensureSummariesFts", log, () => {
+    ensureStandaloneFtsTable(db, {
+      tableName: "summaries_fts",
+      createSql: `
+        CREATE VIRTUAL TABLE summaries_fts USING fts5(
+          summary_id UNINDEXED,
+          content,
+          tokenize='porter unicode61'
+        )
+      `,
+      seedSql: `
+        INSERT INTO summaries_fts(summary_id, content)
+        SELECT summary_id, content FROM summaries
+      `,
+      expectedColumns: ["summary_id", "content"],
+      staleSchemaPatterns: [
+        "content_rowid='summary_id'",
+        'content_rowid="summary_id"',
+      ],
+    });
+  });
 
   // ── CJK trigram FTS table ────────────────────────────────────────────────
   // FTS5 unicode61 (porter) tokenizer cannot segment CJK ideographs, so CJK
@@ -727,20 +846,23 @@ export function runLcmMigrations(
   //
   // A trigram-tokenized table indexes every 3-character substring, enabling
   // native CJK substring matching via FTS5 MATCH with OR semantics.
-  const cjkTableExists = db
-    .prepare(
-      "SELECT 1 FROM sqlite_master WHERE type='table' AND name='summaries_fts_cjk'",
-    )
-    .get();
-  if (!cjkTableExists) {
-    db.exec(`
-      CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
-        summary_id UNINDEXED,
-        content,
-        tokenize='trigram'
-      );
-      INSERT INTO summaries_fts_cjk(summary_id, content)
-      SELECT summary_id, content FROM summaries;
-    `);
-  }
+  runMigrationStep("ensureSummariesFtsCjk", log, () => {
+    if (trigramTokenizerAvailable) {
+      ensureStandaloneFtsTable(db, {
+        tableName: "summaries_fts_cjk",
+        createSql: `
+          CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
+            summary_id UNINDEXED,
+            content,
+            tokenize='trigram'
+          )
+        `,
+        seedSql: `
+          INSERT INTO summaries_fts_cjk(summary_id, content)
+          SELECT summary_id, content FROM summaries
+        `,
+        expectedColumns: ["summary_id", "content"],
+      });
+    }
+  });
 }