onadit 0.0.1 → 1.0.0

package/dist/src/storage/tables.js

@@ -0,0 +1,290 @@
+/**
+ * LanceDB table schemas and initialization.
+ *
+ * Four tables:
+ * - structured_docs: indexed documents from watched/scanned directories
+ * - conversations_hot: recent conversations (<=30 days)
+ * - conversations_cold: older conversations (>30 days)
+ * - digests: weekly summaries
+ *
+ * Tables are created idempotently: if they already exist, they are opened as-is.
+ * A seed row is inserted at creation time to establish the schema, then removed.
+ */
+import path from "node:path";
+import { getConnection, closeConnection, resetConnectionRef } from "./db.js";
+import { EMBEDDING_DIMENSIONS, LOCAL_EMBEDDING_DIMENSIONS, } from "../providers/types.js";
+// --- Table names ---
+/** Core tables that must exist after first boot. */
+export const CORE_TABLE_NAMES = [
+    "structured_docs",
+    "conversations_hot",
+    "conversations_cold",
+    "digests",
+];
+/** All tables including optional ones created by initTables. */
+export const TABLE_NAMES = [
+    ...CORE_TABLE_NAMES,
+    "structured_docs_local",
+];
+// --- Seed data (establishes schema, then deleted) ---
+export const SEED_ID = "__seed__";
+const zeroVec = () => new Array(EMBEDDING_DIMENSIONS).fill(0);
+const localZeroVec = () => new Array(LOCAL_EMBEDDING_DIMENSIONS).fill(0);
+function structuredDocSeed() {
+    return [{
+            id: SEED_ID,
+            source_path: "",
+            content: "",
+            embedding: zeroVec(),
+            source_type: "filesystem_watched",
+            source_scope: "project",
+            client_name: "",
+            indexed_at: "",
+            metadata: "{}",
+        }];
+}
+function structuredDocLocalSeed() {
+    return [{
+            id: SEED_ID,
+            source_path: "",
+            content: "",
+            embedding: localZeroVec(),
+            source_type: "filesystem_watched",
+            source_scope: "project",
+            client_name: "",
+            indexed_at: "",
+            metadata: "{}",
+        }];
+}
+function conversationSeed() {
+    return [{
+            id: SEED_ID,
+            content: "",
+            summary: "",
+            embedding: zeroVec(),
+            source: "claude_code",
+            domain: "tech",
+            intent: "build",
+            quality: "medium",
+            topics: "[]",
+            decisions: "[]",
+            key_outputs: "[]",
+            indexed_at: "",
+            idempotency_key: "",
+            conversation_fingerprint: "",
+            metadata_provider: "",
+            embedding_provider: "",
+            parent_id: "",
+            chunk_index: "",
+            chunk_count: "",
+        }];
+}
+function digestSeed() {
+    return [{
+            id: SEED_ID,
+            content: "",
+            summary: "",
+            embedding: zeroVec(),
+            period_start: "",
+            period_end: "",
+            digest_type: "",
+            created_at: "",
+        }];
+}
+// --- Table handle cache ---
+const tableHandles = new Map();
+/**
+ * Swap lock: set to true while commitAllSwaps is running.
+ * getTable() throws a retryable error during this window so readers
+ * get a clear signal instead of a cryptic "table is closed" error.
+ */
+let swapInProgress = false;
+export function setSwapInProgress(v) {
+    swapInProgress = v;
+}
+export function isSwapInProgress() {
+    return swapInProgress;
+}
+/**
+ * Initialize all four LanceDB tables. Idempotent — safe to call on every startup.
+ * Creates tables that don't exist; opens tables that already do.
+ */
+export async function initTables(dataDir) {
+    const dbPath = path.join(dataDir, "lancedb");
+    const conn = await getConnection(dbPath);
+    const existing = new Set(await conn.tableNames());
+    for (const name of TABLE_NAMES) {
+        if (existing.has(name)) {
+            const table = await conn.openTable(name);
+            await migrateConversationColumns(name, table);
+            tableHandles.set(name, table);
+        }
+        else {
+            const seed = seedForTable(name);
+            const table = await conn.createTable(name, seed);
+            await table.delete(`id = '${SEED_ID}'`);
+            tableHandles.set(name, table);
+        }
+    }
+}
+/**
+ * Open EXISTING tables into the handle cache WITHOUT creating or migrating any.
+ *
+ * The read-only counterpart to {@link initTables}, for recon/dry-run/diagnostic
+ * paths that must never mutate the live schema. `initTables` adds missing
+ * columns (`migrateConversationColumns`) and creates missing tables — both are
+ * schema writes. Running it from a chunk-aware tool against a dir served by a
+ * pre-chunk daemon is what added the chunk columns and broke that daemon's
+ * writes (the `chunk-backfill --dry-run` incident). A read-only consumer has no
+ * business doing either, so it opens tables as-is and leaves the schema and the
+ * on-disk version untouched.
+ *
+ * Opens every table in {@link TABLE_NAMES} that exists; silently skips ones that
+ * don't (it will NOT create them). Any name in `require` that is missing throws
+ * a clear error rather than being created — so a caller that needs a specific
+ * table fails loudly instead of operating on a table that a later
+ * schema-migrating open would have silently created.
+ */
+export async function openTablesReadOnly(dataDir, opts = {}) {
+    const dbPath = path.join(dataDir, "lancedb");
+    const conn = await getConnection(dbPath);
+    const existing = new Set(await conn.tableNames());
+    // Drop any handles cached from a PRIOR open (a different data dir, or an
+    // earlier boot of this one). Without this, a read-only open of dir B that
+    // skips a table missing from B would leave dir A's handle for that table in
+    // the cache, and getTable() would silently serve dir A's data. A read-only
+    // open must reflect exactly the tables that exist in `dataDir`, nothing else.
+    for (const table of tableHandles.values()) {
+        try {
+            table.close();
+        }
+        catch {
+            // Already closed / never opened — nothing to release.
+        }
+    }
+    tableHandles.clear();
+    for (const name of opts.require ?? []) {
+        if (!existing.has(name)) {
+            throw new Error(`Table "${name}" does not exist at ${dbPath}. Read-only open will ` +
+                `not create or migrate tables — boot the daemon (or run an explicit ` +
+                `write-mode command) to initialize storage first.`);
+        }
+    }
+    for (const name of TABLE_NAMES) {
+        if (!existing.has(name))
+            continue;
+        const table = await conn.openTable(name);
+        tableHandles.set(name, table);
+    }
+}
+/**
+ * Backfill string columns added to the conversation schema after a table
+ * was first created. Each missing column is added filled with an empty
+ * string for existing rows:
+ *   - `conversation_fingerprint`: ingest refresh-on-grow treats empty as
+ *     "cannot prove unchanged" and re-derives from content.
+ *   - `metadata_provider` / `embedding_provider`: empty means
+ *     "enriched / valid / provider-unknown" — i.e. a legacy row is never
+ *     mistaken for a no-AI (`rift-basic`) or no-key (`rift-none`) row, so
+ *     it is never targeted by a future enrichment/embedding backfill.
+ *   - `parent_id` / `chunk_index` / `chunk_count`: empty means
+ *     unchunked/legacy — `chunk_count` empty (or "1") is read as "not part of
+ *     a chunk set", so a legacy single-row conversation is never grouped.
+ *
+ * Idempotent: only columns absent from the live schema are added, so this
+ * is safe to call on every startup across mixed-vintage tables.
+ */
+async function migrateConversationColumns(name, table) {
+    if (name !== "conversations_hot" && name !== "conversations_cold")
+        return;
+    const schema = await table.schema();
+    const present = new Set(schema.fields.map((f) => f.name));
+    const missing = [
+        "conversation_fingerprint",
+        "metadata_provider",
+        "embedding_provider",
+        "parent_id",
+        "chunk_index",
+        "chunk_count",
+    ]
+        .filter((col) => !present.has(col))
+        .map((col) => ({ name: col, valueSql: "CAST('' AS STRING)" }));
+    if (missing.length === 0)
+        return;
+    await table.addColumns(missing);
+}
+/**
+ * Get a typed table handle. Throws if tables haven't been initialized
+ * or if a shadow swap is in progress.
+ */
+export function getTable(name) {
+    if (swapInProgress) {
+        throw new Error("Table swap in progress — retry shortly");
+    }
+    const table = tableHandles.get(name);
+    if (!table) {
+        throw new Error(`Table "${name}" not initialized. Call initTables() first.`);
+    }
+    return table;
+}
+/** Replace the cached handle for a table (used by shadow-swap after commit). */
+export function setTable(name, table) {
+    tableHandles.set(name, table);
+}
+/**
+ * Close all live table handles without checking the swap lock.
+ * Used internally by shadow-swap during commitAllSwaps.
+ */
+export function closeAllHandles() {
+    for (const table of tableHandles.values()) {
+        try {
+            table.close();
+        }
+        catch {
+            // Table might already be closed or uninitialized.
+        }
+    }
+    tableHandles.clear();
+}
+/**
+ * Refresh all cached table handles to see the latest data.
+ * Called after worker processes write to the same LanceDB files.
+ */
+export async function refreshTables() {
+    for (const [name, table] of tableHandles) {
+        try {
+            await table.checkoutLatest();
+        }
+        catch {
+            // If checkout fails (table closed/corrupt), skip — getTable()
+            // will throw on next access and the caller can handle it.
+        }
+    }
+}
+/** Close all table handles and the connection. */
+export function closeTables() {
+    for (const table of tableHandles.values()) {
+        table.close();
+    }
+    tableHandles.clear();
+    closeConnection();
+}
+/** Reset all state (for tests). */
+export function resetTables() {
+    tableHandles.clear();
+    resetConnectionRef();
+}
+export function seedForTable(name) {
+    switch (name) {
+        case "structured_docs":
+            return structuredDocSeed();
+        case "structured_docs_local":
+            return structuredDocLocalSeed();
+        case "conversations_hot":
+        case "conversations_cold":
+            return conversationSeed();
+        case "digests":
+            return digestSeed();
+    }
+}
+//# sourceMappingURL=tables.js.map

package/dist/src/storage/tables.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tables.js","sourceRoot":"","sources":["../../../src/storage/tables.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EACL,oBAAoB,EACpB,0BAA0B,GAO3B,MAAM,uBAAuB,CAAC;AAE/B,sBAAsB;AAEtB,oDAAoD;AACpD,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,iBAAiB;IACjB,mBAAmB;IACnB,oBAAoB;IACpB,SAAS;CACD,CAAC;AAEX,gEAAgE;AAChE,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,GAAG,gBAAgB;IACnB,uBAAuB;CACf,CAAC;AAiGX,uDAAuD;AAEvD,MAAM,CAAC,MAAM,OAAO,GAAG,UAAU,CAAC;AAClC,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,KAAK,CAAS,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACtE,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,IAAI,KAAK,CAAS,0BAA0B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAEjF,SAAS,iBAAiB;IACxB,OAAO,CAAC;YACN,EAAE,EAAE,OAAO;YACX,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,OAAO,EAAE;YACpB,WAAW,EAAE,oBAAoB;YACjC,YAAY,EAAE,SAAS;YACvB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB;IAC7B,OAAO,CAAC;YACN,EAAE,EAAE,OAAO;YACX,WAAW,EAAE,EAAE;YACf,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,YAAY,EAAE;YACzB,WAAW,EAAE,oBAAoB;YACjC,YAAY,EAAE,SAAS;YACvB,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO,CAAC;YACN,EAAE,EAAE,OAAO;YACX,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,OAAO,EAAE;YACpB,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,QAAQ;YACjB,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,WAAW,EAAE,IAAI;YACjB,UAAU,EAAE,EAAE;YACd,eAAe,EAAE,EAAE;YACnB,wBAAwB,EAAE,EAAE;YAC5B,iBAAiB,EAAE,EAAE;YACrB,kBAAkB,EAAE,EAAE;YACtB,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,EAAE;YACf,WAAW,EAAE,EAAE;SAChB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,CAAC;YACN,EAAE,EAAE,OAAO;YACX,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,EAAE;YACX,SAAS,EAAE,OAAO,EAAE;YACpB,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,EAAE;YACd,WAAW,EAAE,EAAE;YACf,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;AACL,CAAC;AAED,6BAA6B;AAE7B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEjD;;;;GAIG;AACH,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B,MAAM,UAAU,iBAAiB,CAAC,CAAU;IAC1C,cAAc,GAAG,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAe;IAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,0BAA0B,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC9C,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACjD,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,OAAO,GAAG,CAAC,CAAC;YACxC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,OAAe,EACf,OAA2C,EAAE;IAE7C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAElD,yEAAyE;IACzE,0EAA0E;IAC1E,4EAA4E;IAC5E,2EAA2E;IAC3E,8EAA8E;IAC9E,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;IACH,CAAC;IACD,YAAY,CAAC,KAAK,EAAE,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACb,UAAU,IAAI,uBAAuB,MAAM,wBAAwB;gBACjE,qEAAqE;gBACrE,kDAAkD,CACrD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS;QAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACzC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,KAAK,UAAU,0BAA0B,CACvC,IAAe,EACf,KAAY;IAEZ,IAAI,IAAI,KAAK,mBAAmB,IAAI,IAAI,KAAK,oBAAoB;QAAE,OAAO;IAC1E,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAmB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAI;QACf,0BAA0B;QAC1B,mBAAmB;QACnB,oBAAoB;QACpB,WAAW;QACX,aAAa;QACb,aAAa;KACJ;SACR,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAClC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACjC,MAAM,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAe;IACtC,IAAI,cAAc,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,6CAA6C,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CAAC,IAAe,EAAE,KAAY;IACpD,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe;IAC7B,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;IACH,CAAC;IACD,YAAY,CAAC,KAAK,EAAE,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,8DAA8D;YAC9D,0DAA0D;QAC5D,CAAC;IACH,CAAC;AACH,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,WAAW;IACzB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;IACD,YAAY,CAAC,KAAK,EAAE,CAAC;IACrB,eAAe,EAAE,CAAC;AACpB,CAAC;AAED,mCAAmC;AACnC,MAAM,UAAU,WAAW;IACzB,YAAY,CAAC,KAAK,EAAE,CAAC;IACrB,kBAAkB,EAAE,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAe;IAC1C,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,iBAAiB;YACpB,OAAO,iBAAiB,EAAE,CAAC;QAC7B,KAAK,uBAAuB;YAC1B,OAAO,sBAAsB,EAAE,CAAC;QAClC,KAAK,mBAAmB,CAAC;QACzB,KAAK,oBAAoB;YACvB,OAAO,gBAAgB,EAAE,CAAC;QAC5B,KAAK,SAAS;YACZ,OAAO,UAAU,EAAE,CAAC;IACxB,CAAC;AACH,CAAC"}