@martian-engineering/lossless-claw 0.6.3 → 0.8.0

package/src/db/connection.ts

@@ -8,30 +8,49 @@ const SQLITE_BUSY_TIMEOUT_MS = 5_000;
 const connectionsByPath = new Map<ConnectionKey, Set<DatabaseSync>>();
 const connectionIndex = new Map<DatabaseSync, ConnectionKey>();
 
-function isInMemoryPath(dbPath: string): boolean {
+export function isInMemoryPath(dbPath: string): boolean {
   const normalized = dbPath.trim();
   return normalized === ":memory:" || normalized.startsWith("file::memory:");
 }
 
-function normalizePath(dbPath: string): ConnectionKey {
-  if (isInMemoryPath(dbPath)) {
+export function getFileBackedDatabasePath(dbPath: string): string | null {
+  const trimmed = dbPath.trim();
+  if (!trimmed || isInMemoryPath(trimmed)) {
+    return null;
+  }
+  return resolve(trimmed);
+}
+
+export function normalizePath(dbPath: string): ConnectionKey {
+  const fileBackedDatabasePath = getFileBackedDatabasePath(dbPath);
+  if (!fileBackedDatabasePath) {
     const trimmed = dbPath.trim();
     return trimmed.length > 0 ? trimmed : ":memory:";
   }
-  return resolve(dbPath);
+  return fileBackedDatabasePath;
 }
 
 function ensureDbDirectory(dbPath: string): void {
-  if (isInMemoryPath(dbPath)) {
+  const fileBackedDatabasePath = getFileBackedDatabasePath(dbPath);
+  if (!fileBackedDatabasePath) {
     return;
   }
-  mkdirSync(dirname(dbPath), { recursive: true });
+  mkdirSync(dirname(fileBackedDatabasePath), { recursive: true });
 }
 
 function configureConnection(db: DatabaseSync): DatabaseSync {
   db.exec("PRAGMA journal_mode = WAL");
   db.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`);
   db.exec("PRAGMA foreign_keys = ON");
+  // 64MB page cache (default 2MB is severely undersized for multi-GB databases
+  // with concurrent agents). Memory is demand-allocated, released on close.
+  db.exec("PRAGMA cache_size = -65536");
+  // NORMAL is officially recommended for WAL mode — crash-safe for app crashes,
+  // only risks data loss on power failure (OS/kernel crash). The bootstrap
+  // process re-ingests any lost transactions from session files.
+  db.exec("PRAGMA synchronous = NORMAL");
+  // Keep temp tables/indexes in RAM (helps ordinal resequencing).
+  db.exec("PRAGMA temp_store = MEMORY");
   return db;
 }
 
@@ -66,6 +85,9 @@ function closeDatabase(db: DatabaseSync | undefined): void {
     return;
   }
   try {
+    // Update query planner statistics for tables that changed since last optimize.
+    // Separate try so a SQLITE_BUSY/SQLITE_READONLY from optimize doesn't skip close.
+    try { db.exec("PRAGMA optimize"); } catch { /* best-effort */ }
     db.close();
   } catch {
     // Ignore close failures; callers are shutting down anyway.
@@ -81,7 +103,13 @@ function closeDatabase(db: DatabaseSync | undefined): void {
  */
 export function createLcmDatabaseConnection(dbPath: string): DatabaseSync {
   ensureDbDirectory(dbPath);
-  const db = configureConnection(new DatabaseSync(dbPath));
+  const db = new DatabaseSync(dbPath);
+  try {
+    configureConnection(db);
+  } catch (err) {
+    try { db.close(); } catch { /* ignore cleanup failure */ }
+    throw err;
+  }
   trackConnection(dbPath, db);
   return db;
 }

package/src/db/features.ts

@@ -2,19 +2,20 @@ import type { DatabaseSync } from "node:sqlite";
 
 export type LcmDbFeatures = {
   fts5Available: boolean;
+  trigramTokenizerAvailable: boolean;
 };
 
 const featureCache = new WeakMap<DatabaseSync, LcmDbFeatures>();
 
-function probeFts5(db: DatabaseSync): boolean {
+function probeVirtualTable(db: DatabaseSync, sql: string): boolean {
   try {
-    db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
-    db.exec("CREATE VIRTUAL TABLE temp.__lcm_fts5_probe USING fts5(content)");
-    db.exec("DROP TABLE temp.__lcm_fts5_probe");
+    db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe");
+    db.exec(sql);
+    db.exec("DROP TABLE temp.__lcm_virtual_table_probe");
     return true;
   } catch {
     try {
-      db.exec("DROP TABLE IF EXISTS temp.__lcm_fts5_probe");
+      db.exec("DROP TABLE IF EXISTS temp.__lcm_virtual_table_probe");
     } catch {
       // Ignore cleanup failures after a failed probe.
     }
@@ -22,6 +23,20 @@ function probeFts5(db: DatabaseSync): boolean {
   }
 }
 
+function probeFts5(db: DatabaseSync): boolean {
+  return probeVirtualTable(
+    db,
+    "CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content)",
+  );
+}
+
+function probeTrigramTokenizer(db: DatabaseSync): boolean {
+  return probeVirtualTable(
+    db,
+    "CREATE VIRTUAL TABLE temp.__lcm_virtual_table_probe USING fts5(content, tokenize='trigram')",
+  );
+}
+
 /**
  * Detect SQLite features exposed by the current Node runtime.
  *
@@ -36,7 +51,11 @@ export function getLcmDbFeatures(db: DatabaseSync): LcmDbFeatures {
 
   const detected: LcmDbFeatures = {
     fts5Available: probeFts5(db),
+    trigramTokenizerAvailable: false,
   };
+  if (detected.fts5Available) {
+    detected.trigramTokenizerAvailable = probeTrigramTokenizer(db);
+  }
   featureCache.set(db, detected);
   return detected;
 }

package/src/db/migration.ts

@@ -2,6 +2,10 @@ import type { DatabaseSync } from "node:sqlite";
 import { getLcmDbFeatures } from "./features.js";
 import { parseUtcTimestampOrNull } from "../store/parse-utc-timestamp.js";
 
+type MigrationLogger = {
+  info?: (message: string) => void;
+};
+
 type SummaryColumnInfo = {
   name?: string;
 };
@@ -27,6 +31,18 @@ type SummaryParentEdgeRow = {
   parent_summary_id: string;
 };
 
+type TableNameRow = {
+  name?: string;
+};
+
+type FtsTableSpec = {
+  tableName: string;
+  createSql: string;
+  seedSql: string;
+  expectedColumns: string[];
+  staleSchemaPatterns?: string[];
+};
+
 function ensureSummaryDepthColumn(db: DatabaseSync): void {
   const summaryColumns = db.prepare(`PRAGMA table_info(summaries)`).all() as SummaryColumnInfo[];
   const hasDepth = summaryColumns.some((col) => col.name === "depth");
@@ -78,6 +94,58 @@ function ensureSummaryModelColumn(db: DatabaseSync): void {
   }
 }
 
+function ensureCompactionTelemetryColumns(db: DatabaseSync): void {
+  const telemetryColumns = db.prepare(`PRAGMA table_info(conversation_compaction_telemetry)`).all() as SummaryColumnInfo[];
+  const hasLastLeafCompactionAt = telemetryColumns.some((col) => col.name === "last_leaf_compaction_at");
+  const hasTurnsSinceLeafCompaction = telemetryColumns.some((col) => col.name === "turns_since_leaf_compaction");
+  const hasTokensAccumulatedSinceLeafCompaction = telemetryColumns.some(
+    (col) => col.name === "tokens_accumulated_since_leaf_compaction",
+  );
+  const hasLastActivityBand = telemetryColumns.some((col) => col.name === "last_activity_band");
+
+  if (!hasLastLeafCompactionAt) {
+    db.exec(`ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_leaf_compaction_at TEXT`);
+  }
+  if (!hasTurnsSinceLeafCompaction) {
+    db.exec(
+      `ALTER TABLE conversation_compaction_telemetry ADD COLUMN turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`,
+    );
+  }
+  if (!hasTokensAccumulatedSinceLeafCompaction) {
+    db.exec(
+      `ALTER TABLE conversation_compaction_telemetry ADD COLUMN tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0`,
+    );
+  }
+  if (!hasLastActivityBand) {
+    db.exec(
+      `ALTER TABLE conversation_compaction_telemetry ADD COLUMN last_activity_band TEXT NOT NULL DEFAULT 'low' CHECK (last_activity_band IN ('low', 'medium', 'high'))`,
+    );
+  }
+}
+
+function describeMigrationError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function runMigrationStep(
+  name: string,
+  log: MigrationLogger | undefined,
+  step: () => void,
+): void {
+  const startedAt = Date.now();
+  try {
+    step();
+    log?.info?.(
+      `[lcm] migration step complete: step=${name} durationMs=${Date.now() - startedAt}`,
+    );
+  } catch (error) {
+    log?.info?.(
+      `[lcm] migration step failed: step=${name} durationMs=${Date.now() - startedAt} error=${describeMigrationError(error)}`,
+    );
+    throw error;
+  }
+}
+
 function backfillSummaryDepths(db: DatabaseSync): void {
   // Leaves are always depth 0, even if legacy rows had malformed values.
   db.exec(`UPDATE summaries SET depth = 0 WHERE kind = 'leaf'`);
@@ -415,10 +483,89 @@ function backfillToolCallColumns(db: DatabaseSync): void {
   );
 }
 
+function getExistingTableNames(db: DatabaseSync, names: string[]): Set<string> {
+  if (names.length === 0) {
+    return new Set();
+  }
+  const placeholders = names.map(() => "?").join(", ");
+  const rows = db
+    .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name IN (${placeholders})`)
+    .all(...names) as TableNameRow[];
+  return new Set(
+    rows
+      .map((row) => row.name)
+      .filter((name): name is string => typeof name === "string" && name.length > 0),
+  );
+}
+
+function getFtsShadowTableNames(tableName: string): string[] {
+  return [
+    `${tableName}_data`,
+    `${tableName}_idx`,
+    `${tableName}_content`,
+    `${tableName}_docsize`,
+    `${tableName}_config`,
+  ];
+}
+
+function quoteSqlIdentifier(identifier: string): string {
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
+    throw new Error(`Invalid SQL identifier: ${identifier}`);
+  }
+  return `"${identifier.replaceAll(`"`, `""`)}"`;
+}
+
+function shouldRecreateStandaloneFtsTable(db: DatabaseSync, spec: FtsTableSpec): boolean {
+  const shadowTables = getFtsShadowTableNames(spec.tableName);
+  const existingTables = getExistingTableNames(db, [spec.tableName, ...shadowTables]);
+  if (!existingTables.has(spec.tableName)) {
+    return true;
+  }
+  if (shadowTables.some((name) => !existingTables.has(name))) {
+    return true;
+  }
+
+  try {
+    const info = db
+      .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name = ?")
+      .get(spec.tableName) as { sql?: string } | undefined;
+    const sql = info?.sql ?? "";
+    if (spec.staleSchemaPatterns?.some((pattern) => sql.includes(pattern))) {
+      return true;
+    }
+
+    const columns = db
+      .prepare(`PRAGMA table_info(${quoteSqlIdentifier(spec.tableName)})`)
+      .all() as SummaryColumnInfo[];
+    const columnNames = new Set(
+      columns
+        .map((col) => col.name)
+        .filter((name): name is string => typeof name === "string" && name.length > 0),
+    );
+    return spec.expectedColumns.some((column) => !columnNames.has(column));
+  } catch {
+    return true;
+  }
+}
+
+function ensureStandaloneFtsTable(db: DatabaseSync, spec: FtsTableSpec): void {
+  if (!shouldRecreateStandaloneFtsTable(db, spec)) {
+    return;
+  }
+
+  db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(spec.tableName)}`);
+  for (const shadowTableName of getFtsShadowTableNames(spec.tableName)) {
+    db.exec(`DROP TABLE IF EXISTS ${quoteSqlIdentifier(shadowTableName)}`);
+  }
+  db.exec(spec.createSql);
+  db.exec(spec.seedSql);
+}
+
 export function runLcmMigrations(
   db: DatabaseSync,
-  options?: { fts5Available?: boolean },
+  options?: { fts5Available?: boolean; log?: MigrationLogger },
 ): void {
+  const log = options?.log;
   db.exec(`
     CREATE TABLE IF NOT EXISTS conversations (
       conversation_id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -546,15 +693,39 @@ export function runLcmMigrations(
       updated_at TEXT NOT NULL DEFAULT (datetime('now'))
     );
 
+    CREATE TABLE IF NOT EXISTS conversation_compaction_telemetry (
+      conversation_id INTEGER PRIMARY KEY REFERENCES conversations(conversation_id) ON DELETE CASCADE,
+      last_observed_cache_read INTEGER,
+      last_observed_cache_write INTEGER,
+      last_observed_cache_hit_at TEXT,
+      last_observed_cache_break_at TEXT,
+      cache_state TEXT NOT NULL DEFAULT 'unknown'
+        CHECK (cache_state IN ('hot', 'cold', 'unknown')),
+      retention TEXT,
+      last_leaf_compaction_at TEXT,
+      turns_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
+      tokens_accumulated_since_leaf_compaction INTEGER NOT NULL DEFAULT 0,
+      last_activity_band TEXT NOT NULL DEFAULT 'low'
+        CHECK (last_activity_band IN ('low', 'medium', 'high')),
+      updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
     -- Indexes
     CREATE INDEX IF NOT EXISTS messages_conv_seq_idx ON messages (conversation_id, seq);
     CREATE INDEX IF NOT EXISTS summaries_conv_created_idx ON summaries (conversation_id, created_at);
+    CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
+    CREATE INDEX IF NOT EXISTS summary_parents_parent_summary_idx ON summary_parents (parent_summary_id);
     CREATE INDEX IF NOT EXISTS message_parts_message_idx ON message_parts (message_id);
     CREATE INDEX IF NOT EXISTS message_parts_type_idx ON message_parts (part_type);
     CREATE INDEX IF NOT EXISTS context_items_conv_idx ON context_items (conversation_id, ordinal);
     CREATE INDEX IF NOT EXISTS large_files_conv_idx ON large_files (conversation_id, created_at);
     CREATE INDEX IF NOT EXISTS bootstrap_state_path_idx
       ON conversation_bootstrap_state (session_file_path, updated_at);
+    CREATE INDEX IF NOT EXISTS compaction_telemetry_state_idx
+      ON conversation_compaction_telemetry (cache_state, updated_at);
+
+    -- Speed up summary_messages lookups by message_id (PK is summary_id,message_id)
+    CREATE INDEX IF NOT EXISTS summary_messages_message_idx ON summary_messages (message_id);
   `);
 
   // Forward-compatible conversations migration for existing DBs.
@@ -592,75 +763,80 @@ export function runLcmMigrations(
     ON conversations (session_key, active, created_at)
   `);
   db.exec(`DROP INDEX IF EXISTS conversations_session_key_idx`);
-  ensureSummaryDepthColumn(db);
-  ensureSummaryMetadataColumns(db);
-  ensureSummaryModelColumn(db);
-  backfillSummaryDepths(db);
-  backfillSummaryMetadata(db);
-  backfillToolCallColumns(db);
-
-  const fts5Available = options?.fts5Available ?? getLcmDbFeatures(db).fts5Available;
+  runMigrationStep("ensureSummaryDepthColumn", log, () => ensureSummaryDepthColumn(db));
+  runMigrationStep("ensureSummaryMetadataColumns", log, () =>
+    ensureSummaryMetadataColumns(db),
+  );
+  runMigrationStep("ensureSummaryModelColumn", log, () => ensureSummaryModelColumn(db));
+  runMigrationStep("ensureCompactionTelemetryColumns", log, () =>
+    ensureCompactionTelemetryColumns(db),
+  );
+  runMigrationStep("backfillSummaryDepths", log, () => backfillSummaryDepths(db));
+  // Index on depth — created AFTER backfillSummaryDepths to avoid index
+  // maintenance overhead during bulk depth updates on large existing DBs.
+  runMigrationStep("createSummariesDepthIndex", log, () =>
+    db.exec(
+      `CREATE INDEX IF NOT EXISTS summaries_conv_depth_kind_idx ON summaries (conversation_id, depth, kind)`,
+    ),
+  );
+  runMigrationStep("backfillSummaryMetadata", log, () => backfillSummaryMetadata(db));
+  runMigrationStep("backfillToolCallColumns", log, () => backfillToolCallColumns(db));
+
+  const detectedFeatures = options?.fts5Available === false ? null : getLcmDbFeatures(db);
+  const fts5Available = options?.fts5Available ?? detectedFeatures?.fts5Available ?? false;
   if (!fts5Available) {
     return;
   }
 
+  const trigramTokenizerAvailable = detectedFeatures?.trigramTokenizerAvailable ?? false;
+  if (!trigramTokenizerAvailable) {
+    try {
+      db.exec(`DROP TABLE IF EXISTS summaries_fts_cjk`);
+    } catch {
+      // Best effort only. A stale virtual table should not block core migration.
+    }
+  }
+
   // FTS5 virtual tables for full-text search (cannot use IF NOT EXISTS, so check manually)
-  const hasFts = db
-    .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'")
-    .get();
-
-  if (hasFts) {
-    // Check for stale schema: external-content FTS tables with content_rowid cause errors.
-    // Drop and recreate as standalone FTS if the old schema is detected.
-    const ftsSchema = (
-      db
-        .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='messages_fts'")
-        .get() as { sql: string } | undefined
-    )?.sql;
-    if (ftsSchema && ftsSchema.includes("content_rowid")) {
-      db.exec("DROP TABLE messages_fts");
-      db.exec(`
+  runMigrationStep("ensureMessagesFts", log, () => {
+    ensureStandaloneFtsTable(db, {
+      tableName: "messages_fts",
+      createSql: `
         CREATE VIRTUAL TABLE messages_fts USING fts5(
           content,
           tokenize='porter unicode61'
-        );
-        INSERT INTO messages_fts(rowid, content) SELECT message_id, content FROM messages;
-      `);
-    }
-  } else {
-    db.exec(`
-      CREATE VIRTUAL TABLE messages_fts USING fts5(
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-  }
-
-  const summariesFtsInfo = db
-    .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='summaries_fts'")
-    .get() as { sql?: string } | undefined;
-  const summariesFtsSql = summariesFtsInfo?.sql ?? "";
-  const summariesFtsColumns = db.prepare(`PRAGMA table_info(summaries_fts)`).all() as Array<{
-    name?: string;
-  }>;
-  const hasSummaryIdColumn = summariesFtsColumns.some((col) => col.name === "summary_id");
-  const shouldRecreateSummariesFts =
-    !summariesFtsInfo ||
-    !hasSummaryIdColumn ||
-    summariesFtsSql.includes("content_rowid='summary_id'") ||
-    summariesFtsSql.includes('content_rowid="summary_id"');
-  if (shouldRecreateSummariesFts) {
-    db.exec(`
-      DROP TABLE IF EXISTS summaries_fts;
-      CREATE VIRTUAL TABLE summaries_fts USING fts5(
-        summary_id UNINDEXED,
-        content,
-        tokenize='porter unicode61'
-      );
-      INSERT INTO summaries_fts(summary_id, content)
-      SELECT summary_id, content FROM summaries;
-    `);
-  }
+        )
+      `,
+      seedSql: `
+        INSERT INTO messages_fts(rowid, content)
+        SELECT message_id, content FROM messages
+      `,
+      expectedColumns: ["content"],
+      staleSchemaPatterns: ["content_rowid"],
+    });
+  });
+
+  runMigrationStep("ensureSummariesFts", log, () => {
+    ensureStandaloneFtsTable(db, {
+      tableName: "summaries_fts",
+      createSql: `
+        CREATE VIRTUAL TABLE summaries_fts USING fts5(
+          summary_id UNINDEXED,
+          content,
+          tokenize='porter unicode61'
+        )
+      `,
+      seedSql: `
+        INSERT INTO summaries_fts(summary_id, content)
+        SELECT summary_id, content FROM summaries
+      `,
+      expectedColumns: ["summary_id", "content"],
+      staleSchemaPatterns: [
+        "content_rowid='summary_id'",
+        'content_rowid="summary_id"',
+      ],
+    });
+  });
 
   // ── CJK trigram FTS table ────────────────────────────────────────────────
   // FTS5 unicode61 (porter) tokenizer cannot segment CJK ideographs, so CJK
@@ -670,20 +846,23 @@ export function runLcmMigrations(
   //
   // A trigram-tokenized table indexes every 3-character substring, enabling
   // native CJK substring matching via FTS5 MATCH with OR semantics.
-  const cjkTableExists = db
-    .prepare(
-      "SELECT 1 FROM sqlite_master WHERE type='table' AND name='summaries_fts_cjk'",
-    )
-    .get();
-  if (!cjkTableExists) {
-    db.exec(`
-      CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
-        summary_id UNINDEXED,
-        content,
-        tokenize='trigram'
-      );
-      INSERT INTO summaries_fts_cjk(summary_id, content)
-      SELECT summary_id, content FROM summaries;
-    `);
-  }
+  runMigrationStep("ensureSummariesFtsCjk", log, () => {
+    if (trigramTokenizerAvailable) {
+      ensureStandaloneFtsTable(db, {
+        tableName: "summaries_fts_cjk",
+        createSql: `
+          CREATE VIRTUAL TABLE summaries_fts_cjk USING fts5(
+            summary_id UNINDEXED,
+            content,
+            tokenize='trigram'
+          )
+        `,
+        seedSql: `
+          INSERT INTO summaries_fts_cjk(summary_id, content)
+          SELECT summary_id, content FROM summaries
+        `,
+        expectedColumns: ["summary_id", "content"],
+      });
+    }
+  });
 }