onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/storage/integrity.d.ts

@@ -0,0 +1,11 @@
+import { type TableName } from "./tables.js";
+export interface IntegrityResult {
+    healthy: boolean;
+    corruptTables: TableName[];
+}
+/**
+ * Check all four tables for readability.
+ * Returns healthy:true if all are readable, or the list of corrupt tables.
+ */
+export declare function checkIntegrity(dataDir: string): Promise<IntegrityResult>;
+//# sourceMappingURL=integrity.d.ts.map

package/dist/src/storage/integrity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"integrity.d.ts","sourceRoot":"","sources":["../../../src/storage/integrity.ts"],"names":[],"mappings":"AAeA,OAAO,EAAiC,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAE5E,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,aAAa,EAAE,SAAS,EAAE,CAAC;CAC5B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAiD9E"}

package/dist/src/storage/integrity.js

@@ -0,0 +1,66 @@
+/**
+ * Startup integrity check — Slice 15.
+ *
+ * Verifies LanceDB tables are readable and plausibly intact.
+ * If corruption is detected, returns a signal to trigger auto-rebuild.
+ *
+ * The check is designed to be fast enough for startup use:
+ * - Opens each table
+ * - Attempts a countRows() call
+ * - Verifies the count is a non-negative number
+ *
+ * Any table that fails to open or query is considered corrupt.
+ */
+import path from "node:path";
+import { getConnection } from "./db.js";
+import { TABLE_NAMES, CORE_TABLE_NAMES } from "./tables.js";
+/**
+ * Check all four tables for readability.
+ * Returns healthy:true if all are readable, or the list of corrupt tables.
+ */
+export async function checkIntegrity(dataDir) {
+    const dbPath = path.join(dataDir, "lancedb");
+    const corruptTables = [];
+    let conn;
+    try {
+        conn = await getConnection(dbPath);
+    }
+    catch {
+        // Can't even connect — all tables are suspect.
+        return { healthy: false, corruptTables: [...TABLE_NAMES] };
+    }
+    const existing = new Set(await conn.tableNames());
+    // Fresh install: no tables exist yet — nothing to be corrupt.
+    // initTables() will create them. A rebuild would be pointless since
+    // there is no raw data to rebuild from on first boot.
+    if (existing.size === 0) {
+        return { healthy: true, corruptTables: [] };
+    }
+    // Core tables must exist after first boot. Optional tables
+    // (structured_docs_local) are created by initTables if missing.
+    const coreSet = new Set(CORE_TABLE_NAMES);
+    for (const name of TABLE_NAMES) {
+        if (!existing.has(name)) {
+            if (coreSet.has(name)) {
+                corruptTables.push(name);
+            }
+            continue;
+        }
+        try {
+            const table = await conn.openTable(name);
+            const count = await table.countRows();
+            table.close();
+            if (typeof count !== "number" || count < 0) {
+                corruptTables.push(name);
+            }
+        }
+        catch {
+            corruptTables.push(name);
+        }
+    }
+    return {
+        healthy: corruptTables.length === 0,
+        corruptTables,
+    };
+}
+//# sourceMappingURL=integrity.js.map

package/dist/src/storage/integrity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"integrity.js","sourceRoot":"","sources":["../../../src/storage/integrity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAkB,MAAM,aAAa,CAAC;AAO5E;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAe;IAClD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7C,MAAM,aAAa,GAAgB,EAAE,CAAC;IAEtC,IAAI,IAAI,CAAC;IACT,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,+CAA+C;QAC/C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,GAAG,WAAW,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IAElD,8DAA8D;IAC9D,oEAAoE;IACpE,sDAAsD;IACtD,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IAC9C,CAAC;IAED,2DAA2D;IAC3D,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;YACtC,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBAC3C,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,aAAa,CAAC,MAAM,KAAK,CAAC;QACnC,aAAa;KACd,CAAC;AACJ,CAAC"}

package/dist/src/storage/maintenance.d.ts

@@ -0,0 +1,27 @@
+import { type TableName } from "./tables.js";
+/**
+ * Versions younger than this are kept for time-travel recovery. Matches
+ * LanceDB's own default. Readers in the daemon hold the version current at
+ * their last refresh; compaction runs within an hour of the writes that made
+ * it necessary, so that version is far younger than this window.
+ */
+export declare const VERSION_RETENTION_MS: number;
+/** Compact once a table has this many small fragments. */
+export declare const SMALL_FRAGMENT_THRESHOLD = 16;
+export interface TableMaintenanceResult {
+    table: TableName;
+    fragments_before: number;
+    fragments_after: number;
+    indexes_created: string[];
+    ms: number;
+}
+/**
+ * Why maintenance is due, one line per table that needs it. Empty = nothing
+ * to do. Cheap: one `stats()` per table plus index listing.
+ */
+export declare function maintenanceReasons(tables?: readonly TableName[]): Promise<string[]>;
+export declare function runTableMaintenance(opts?: {
+    now?: Date;
+    tables?: readonly TableName[];
+}): Promise<TableMaintenanceResult[]>;
+//# sourceMappingURL=maintenance.d.ts.map

package/dist/src/storage/maintenance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"maintenance.d.ts","sourceRoot":"","sources":["../../../src/storage/maintenance.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAyB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAGpE;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,QAA0B,CAAC;AAE5D,0DAA0D;AAC1D,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,EAAE,EAAE,MAAM,CAAC;CACZ;AAUD;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,GAAE,SAAS,SAAS,EAAgB,GACzC,OAAO,CAAC,MAAM,EAAE,CAAC,CAenB;AAED,wBAAsB,mBAAmB,CAAC,IAAI,GAAE;IAC9C,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,MAAM,CAAC,EAAE,SAAS,SAAS,EAAE,CAAC;CAC1B,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAuBzC"}

package/dist/src/storage/maintenance.js

@@ -0,0 +1,66 @@
+import { getTable, TABLE_NAMES } from "./tables.js";
+import { ensureFullTextIndexes, fullTextIndexGaps } from "./fts.js";
+/**
+ * Versions younger than this are kept for time-travel recovery. Matches
+ * LanceDB's own default. Readers in the daemon hold the version current at
+ * their last refresh; compaction runs within an hour of the writes that made
+ * it necessary, so that version is far younger than this window.
+ */
+export const VERSION_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+/** Compact once a table has this many small fragments. */
+export const SMALL_FRAGMENT_THRESHOLD = 16;
+function tryGetTable(name) {
+    try {
+        return getTable(name);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Why maintenance is due, one line per table that needs it. Empty = nothing
+ * to do. Cheap: one `stats()` per table plus index listing.
+ */
+export async function maintenanceReasons(tables = TABLE_NAMES) {
+    const reasons = [];
+    for (const name of tables) {
+        const table = tryGetTable(name);
+        if (!table)
+            continue;
+        const stats = await table.stats();
+        const small = stats.fragmentStats.numSmallFragments;
+        if (small > SMALL_FRAGMENT_THRESHOLD) {
+            reasons.push(`${name}: ${small} small fragments`);
+        }
+        for (const gap of await fullTextIndexGaps(name, table)) {
+            reasons.push(`${name}: ${gap}`);
+        }
+    }
+    return reasons;
+}
+export async function runTableMaintenance(opts = {}) {
+    const now = opts.now ?? new Date();
+    const cleanupOlderThan = new Date(now.getTime() - VERSION_RETENTION_MS);
+    const results = [];
+    for (const name of opts.tables ?? TABLE_NAMES) {
+        const table = tryGetTable(name);
+        if (!table)
+            continue;
+        const started = performance.now();
+        const before = (await table.stats()).fragmentStats.numFragments;
+        // Index creation first: a new index covers every row at once, and the
+        // optimize pass below then compacts data and index together.
+        const created = await ensureFullTextIndexes(name, table);
+        await table.optimize({ cleanupOlderThan });
+        const after = (await table.stats()).fragmentStats.numFragments;
+        results.push({
+            table: name,
+            fragments_before: before,
+            fragments_after: after,
+            indexes_created: created,
+            ms: Math.round(performance.now() - started),
+        });
+    }
+    return results;
+}
+//# sourceMappingURL=maintenance.js.map

package/dist/src/storage/maintenance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"maintenance.js","sourceRoot":"","sources":["../../../src/storage/maintenance.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAkB,MAAM,aAAa,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAEpE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5D,0DAA0D;AAC1D,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAU3C,SAAS,WAAW,CAAC,IAAe;IAClC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,SAA+B,WAAW;IAE1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC;QACpD,IAAI,KAAK,GAAG,wBAAwB,EAAE,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,KAAK,kBAAkB,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,MAAM,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAGtC,EAAE;IACJ,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACnC,MAAM,gBAAgB,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,oBAAoB,CAAC,CAAC;IACxE,MAAM,OAAO,GAA6B,EAAE,CAAC;IAC7C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC;QAChE,sEAAsE;QACtE,6DAA6D;QAC7D,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzD,MAAM,KAAK,CAAC,QAAQ,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,YAAY,CAAC;QAC/D,OAAO,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,IAAI;YACX,gBAAgB,EAAE,MAAM;YACxB,eAAe,EAAE,KAAK;YACtB,eAAe,EAAE,OAAO;YACxB,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/storage/rebuild.d.ts

@@ -0,0 +1,71 @@
+import type { Config } from "../config/schema.js";
+import type { EmbeddingProvider } from "../providers/types.js";
+import type { TableName } from "./tables.js";
+/**
+ * Shared duplicate-resolution policy for conversation raw files.
+ *
+ * Both repair paths — reconcile and full rebuild — must pick the SAME winner
+ * when several raw files compete for one row id or idempotency_key, otherwise
+ * the canonical conversation row could flip depending on which tool last ran,
+ * which is corrosive to trust and debugging. Policy: most-recent raw wins,
+ * ordered by filename (`<timestamp>_<id>.json`, ISO-8601-derived → lexicographic
+ * == chronological). Compare basenames so the ordering is stable regardless of
+ * which source folder a file lives in. Sorts most-recent FIRST (descending).
+ */
+export declare function compareRawRecencyDesc(aPath: string, bPath: string): number;
+export interface RebuildDeps {
+    config: Config;
+    embeddingProvider: EmbeddingProvider;
+    dataDir: string;
+    /** Ollama embedding for local-extraction sources (Slice 19). */
+    localEmbeddingProvider?: EmbeddingProvider;
+}
+export interface RebuildReport {
+    tables_rebuilt: TableName[];
+    rows_reindexed: number;
+    /**
+     * Conversation raw files skipped because a row with the same non-empty
+     * `idempotency_key` was already inserted into the rebuilt index under a
+     * different row id. Mirrors reconcile's `missing_skipped_duplicate_key`:
+     * without it, an orphan raw left on disk (e.g. one reconcile already
+     * skipped) would resurface as a duplicate every time the index is rebuilt
+     * from raw — rebuild groups by row id and would otherwise miss the dup.
+     */
+    conversations_skipped_duplicate_key: number;
+}
+/**
+ * Full rebuild from raw disk state via shadow-table swap.
+ * Creates shadow tables for all four tables, populates them from
+ * Source Recoverability Contract locations, then atomically swaps.
+ */
+export declare function fullRebuild(deps: RebuildDeps): Promise<RebuildReport>;
+export type ReindexScope = "all" | "conversations" | "documents" | "structured_docs";
+export interface ReembedOptions {
+    embeddingProvider: EmbeddingProvider;
+    scope: ReindexScope;
+    /** For per-row provider routing in structured_docs. */
+    config?: Config;
+    localEmbeddingProvider?: EmbeddingProvider;
+}
+export declare function reembed(embeddingProvider: EmbeddingProvider, scope: ReindexScope, dataDir: string, config?: Config, localEmbeddingProvider?: EmbeddingProvider): Promise<number>;
+/**
+ * Remove structured-doc rows whose source file no longer exists on disk.
+ * Scoped to filesystem-sourced rows (watched + scheduled_scan).
+ *
+ * Why: deletions between scheduled scans (or while a watcher is offline)
+ * leave ghost rows in the index, so search can return content that no
+ * longer exists. This pass closes that gap.
+ *
+ * Safety: skips rows whose owning source root is unreachable, so an
+ * unmounted drive cannot trigger mass deletion. If `config` is omitted
+ * the function still runs, but without the unmounted-root guard.
+ */
+export declare function removeStructuredDocOrphans(dataDir: string, config?: Config): Promise<number>;
+export declare function extractIdFromFilename(filename: string): string | undefined;
+/**
+ * Extract the ISO timestamp from a raw conversation filename.
+ * Format: `{timestamp}_{id}.json` where timestamp is ISO with colons/dots
+ * replaced by hyphens (e.g. `2024-01-01T00-00-00-000Z_conv-1.json`).
+ */
+export declare function extractTimestampFromFilename(filename: string): Date | undefined;
+//# sourceMappingURL=rebuild.d.ts.map

package/dist/src/storage/rebuild.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rebuild.d.ts","sourceRoot":"","sources":["../../../src/storage/rebuild.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAa/D,OAAO,KAAK,EAIV,SAAS,EACV,MAAM,aAAa,CAAC;AAWrB;;;;;;;;;;GAUG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,SAAS,EAAE,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB;;;;;;;OAOG;IACH,mCAAmC,EAAE,MAAM,CAAC;CAC7C;AAED;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAgD3E;AA2UD,MAAM,MAAM,YAAY,GACpB,KAAK,GACL,eAAe,GACf,WAAW,GACX,iBAAiB,CAAC;AAEtB,MAAM,WAAW,cAAc;IAC7B,iBAAiB,EAAE,iBAAiB,CAAC;IACrC,KAAK,EAAE,YAAY,CAAC;IACpB,uDAAuD;IACvD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sBAAsB,CAAC,EAAE,iBAAiB,CAAC;CAC5C;AAED,wBAAsB,OAAO,CAC3B,iBAAiB,EAAE,iBAAiB,EACpC,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,EACf,sBAAsB,CAAC,EAAE,iBAAiB,GACzC,OAAO,CAAC,MAAM,CAAC,CA2CjB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAC9C,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,CAoDjB;AAoQD,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG1E;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,QAAQ,EAAE,MAAM,GACf,IAAI,GAAG,SAAS,CAclB"}