docsgov 0.1.0

package/dist/dedup/indexdb/load.js

@@ -0,0 +1,110 @@
+/**
+ * loadAllSectionsWithEmbeddings loads every section row and its embedding.
+ *
+ * Returns { sections, embeddings } where embeddings maps section ID to the
+ * L2-normalized float32 vector. The returned slice is in DB scan order; callers
+ * that need a deterministic order must sort after this call.
+ */
+export function loadAllSectionsWithEmbeddings(db) {
+    const rows = db
+        .prepare(`
+      SELECT id, file_path, heading, heading_level, anchor, start_line, end_line,
+             content_hash, raw_content, embed_text, prose_word_count,
+             has_table, has_code, inbound_count, embedding
+      FROM sections
+    `)
+        .all();
+    const sections = [];
+    const embeddings = new Map();
+    for (const r of rows) {
+        const sec = {
+            id: r.id,
+            file_path: r.file_path,
+            heading: r.heading,
+            heading_level: r.heading_level,
+            anchor: r.anchor,
+            start_line: r.start_line,
+            end_line: r.end_line,
+            content_hash: r.content_hash,
+            raw_content: r.raw_content,
+            embed_text: r.embed_text,
+            prose_word_count: r.prose_word_count,
+            // Go scans INTEGER 0/1 into an int then compares != 0.
+            has_table: r.has_table !== 0,
+            has_code: r.has_code !== 0,
+            inbound_count: r.inbound_count,
+        };
+        sections.push(sec);
+        embeddings.set(sec.id, decodeVec(r.embedding));
+    }
+    return { sections, embeddings };
+}
+/**
+ * loadAllBlocksWithEmbeddings loads every block row.
+ *
+ * Returns { blocks, embeddings } where embeddings maps content_hash to the
+ * L2-normalized float32 vector. A NULL or empty embedding BLOB means the block
+ * has no vector (tables are exact-hash only); those content_hashes are absent
+ * from the map — no entry is inserted. When multiple block rows share a
+ * content_hash the map holds one entry (identical vectors).
+ */
+export function loadAllBlocksWithEmbeddings(db) {
+    const rows = db
+        .prepare(`
+      SELECT section_id, block_index, file_path, heading, kind,
+             start_line, end_line, content_hash, embedding
+      FROM blocks
+    `)
+        .all();
+    const blocks = [];
+    const embeddings = new Map();
+    for (const r of rows) {
+        const rec = {
+            SectionID: r.section_id,
+            Index: r.block_index,
+            FilePath: r.file_path,
+            Heading: r.heading,
+            Kind: r.kind,
+            StartLine: r.start_line,
+            EndLine: r.end_line,
+            ContentHash: r.content_hash,
+            // Text is not persisted; leave it empty on load.
+            Text: "",
+            // TableRows is not persisted; Go leaves the zero value on load.
+            TableRows: 0,
+        };
+        blocks.push(rec);
+        const vec = decodeVecOrNull(r.embedding);
+        if (vec !== null) {
+            embeddings.set(rec.ContentHash, vec);
+        }
+    }
+    return { blocks, embeddings };
+}
+/**
+ * decodeVec decodes a little-endian float32 BLOB to a Float32Array. An empty or
+ * NULL blob yields an empty Float32Array (Go returned nil; callers treat a
+ * zero-length vector the same way). Mirrors the indexer's encodeVec inverse and
+ * is duplicated here to keep indexdb a leaf package.
+ */
+export function decodeVec(buf) {
+    return decodeVecOrNull(buf) ?? new Float32Array(0);
+}
+/**
+ * decodeVecOrNull decodes a little-endian float32 BLOB, returning null for an
+ * empty/NULL blob. Used by block loading where "no vector" must stay absent from
+ * the embeddings map (NULL → absent, not a zero-length entry), matching Go's
+ * `if vec := decodeVec(blob); vec != nil`.
+ */
+function decodeVecOrNull(buf) {
+    if (buf === null || buf.length === 0) {
+        return null;
+    }
+    const v = new Float32Array(Math.floor(buf.length / 4));
+    // DataView reads at the BLOB's own byte offset; little-endian per Go.
+    const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+    for (let i = 0; i < v.length; i++) {
+        v[i] = view.getFloat32(i * 4, true);
+    }
+    return v;
+}

package/dist/dedup/indexdb/migrations.js

@@ -0,0 +1,58 @@
+// The schema-migration registry.
+//
+// node:sqlite has no transaction object: a migration runs on the open
+// connection while the caller holds an explicit BEGIN/COMMIT. So where Go's
+// `apply` took a *sql.Tx, the TS version takes the DatabaseSync directly and
+// must NOT issue BEGIN/COMMIT/ROLLBACK itself — applyMigration owns that.
+import { createBlocksSQL } from "./schema.js";
+/**
+ * createBlocksTableV2 is the v1->v2 additive migration: it creates the blocks
+ * table and its indexes on the passed connection, leaving sections/meta (and
+ * their embeddings) untouched. The blocks DDL is shared with createTableSQL via
+ * createBlocksSQL so it is defined exactly once.
+ */
+function createBlocksTableV2(db, _deps) {
+    db.exec(createBlocksSQL);
+}
+/**
+ * migrations is the ordered registry of available migrations.
+ *
+ * Usage: Open looks for a migration whose fromVersion matches the stored
+ * schema_version. If found, it applies the migration inside one exclusive tx
+ * and bumps the stored version to SchemaVersion. If not found, it falls back to
+ * delete-and-recreate (StatusSchemaRecreated).
+ *
+ * Exported (vs. Go's package-private slice) so the test helper registerTestMigration
+ * can append/restore it from the *.test.ts file; production code should not mutate it.
+ */
+export const migrations = [
+    // v1->v2: additive — add the blocks table + indexes; existing section rows
+    // and embeddings are preserved.
+    { fromVersion: "1", apply: createBlocksTableV2 },
+];
+/** findMigration returns the migration that can upgrade fromVersion, or undefined. */
+export function findMigration(fromVersion) {
+    return migrations.find((m) => m.fromVersion === fromVersion);
+}
+/**
+ * registerTestMigration appends a test-only migration to the registry and
+ * returns a cleanup function that removes it again. Mirrors Go's
+ * RegisterTestMigration(*testing.T, ...) which registered t.Cleanup; vitest has
+ * no per-test cleanup hook, so the caller invokes the returned function (e.g. in
+ * a finally block or afterEach) to keep the production registry unpolluted.
+ *
+ * The inner `apply` is wrapped to drop the MigrationDeps argument, matching the
+ * Go helper's narrower test signature.
+ */
+export function registerTestMigration(fromVersion, apply) {
+    migrations.push({
+        fromVersion,
+        apply: (db, _deps) => apply(db),
+    });
+    return () => {
+        // Remove the last migration (the one we just added).
+        if (migrations.length > 0) {
+            migrations.pop();
+        }
+    };
+}

package/dist/dedup/indexdb/schema.js

@@ -0,0 +1,83 @@
+// DDL and open parameters for the dedup index DB.
+//
+// The CREATE TABLE strings are the persisted on-disk format — they MUST match
+// the Go source byte-for-byte (column names, order, types, indexes) so a DB
+// written by the Go binary and one written here are interchangeable.
+/**
+ * SchemaVersion is the current schema version stored in the meta table.
+ * Bump this constant when making a schema change; pair the bump with a
+ * registered migration if the change is additive and embeddings should
+ * be preserved.
+ */
+export const SchemaVersion = "2";
+/**
+ * SQLiteOpenParams are the SQLite connection parameters, locked per the plan:
+ *   - WAL journaling: standard for single-writer, low-concurrency workloads
+ *   - busy_timeout=5000ms: gives concurrent readers from another process time
+ *     to bail rather than returning an immediate error
+ *   - txlock=immediate: prevents deadlocks when multiple statements run on the
+ *     same connection
+ *
+ * Go appended these to the sql.Open URI string for the mattn driver. node:sqlite
+ * takes a plain path, so openDB applies the equivalent via PRAGMAs (see indexdb.ts).
+ * The string is retained as documentation of the intended params.
+ */
+export const SQLiteOpenParams = "?_journal_mode=WAL&_busy_timeout=5000&_txlock=immediate";
+/**
+ * MaxOpenConns mirrors Go's pool cap of 1 (required there to avoid deadlocks
+ * with go-sqlite3 + exclusive transactions). node:sqlite is a single synchronous
+ * connection by construction, so this is documentation only — there is no pool.
+ */
+export const MaxOpenConns = 1;
+/**
+ * createBlocksSQL is the DDL for the v2 blocks table + its indexes. It is shared
+ * by createTableSQL (fresh create) and the v1->v2 migration (existing DBs) so the
+ * blocks schema is defined exactly once.
+ */
+export const createBlocksSQL = `
+CREATE TABLE IF NOT EXISTS blocks (
+  section_id    TEXT NOT NULL,
+  block_index   INTEGER NOT NULL,
+  file_path     TEXT NOT NULL,
+  heading       TEXT NOT NULL,
+  kind          TEXT NOT NULL,
+  start_line    INTEGER NOT NULL,
+  end_line      INTEGER NOT NULL,
+  content_hash  TEXT NOT NULL,
+  embedding     BLOB,
+  PRIMARY KEY (section_id, block_index)
+);
+CREATE INDEX IF NOT EXISTS idx_blocks_hash ON blocks(content_hash);
+CREATE INDEX IF NOT EXISTS idx_blocks_file ON blocks(file_path);
+`;
+/**
+ * createTableSQL is the DDL for the dedup index schema (sections + meta + blocks).
+ * Exported for tests that need to set up a DB with a specific state.
+ */
+export const createTableSQL = `
+CREATE TABLE IF NOT EXISTS sections (
+  id                  TEXT PRIMARY KEY,
+  file_path           TEXT NOT NULL,
+  heading             TEXT NOT NULL,
+  heading_level       INTEGER NOT NULL,
+  anchor              TEXT NOT NULL,
+  start_line          INTEGER NOT NULL,
+  end_line            INTEGER NOT NULL,
+  content_hash        TEXT NOT NULL,
+  raw_content         TEXT NOT NULL,
+  embed_text          TEXT NOT NULL,
+  prose_word_count    INTEGER NOT NULL,
+  has_table           INTEGER NOT NULL,
+  has_code            INTEGER NOT NULL,
+  inbound_count       INTEGER NOT NULL DEFAULT 0,
+  embedding           BLOB    NOT NULL,
+  updated_at          TEXT    NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_sections_file ON sections(file_path);
+CREATE INDEX IF NOT EXISTS idx_sections_hash ON sections(content_hash);
+
+CREATE TABLE IF NOT EXISTS meta (
+  key   TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+` + createBlocksSQL;

package/dist/dedup/indexer/index.js

@@ -0,0 +1,9 @@
+// Barrel for the indexer package: the two-pass dedup indexing pipeline.
+//
+// The production entry point is `run` (Go's Run). The remaining exports are the
+// pipeline primitives Go exposed via export_test.go for white-box tests
+// (blockEligible, blockEmbeddable, embedBlocks, collectSectionsAndBlocks) plus
+// parseLinks; they have no other callers but stay exported so the wiring-contract
+// tests can target them directly.
+export { run, collectSectionsAndBlocks, computeInboundCounts, embedBlocks, blockEligible, blockEmbeddable, encodeVec, decodeVec, } from "./indexer.js";
+export { parseLinks, isExternal, resolveLink } from "./links.js";