docsgov 0.1.0

package/dist/dedup/gitignore.js

@@ -0,0 +1,97 @@
+// Manages the docgov-owned block in .docgov/dedup/.gitignore that keeps the
+// local SQLite index cache out of git.
+//
+// Ported from internal/dedup/gitignore.go. Go's os.ReadFile/WriteFile become
+// node:fs/promises readFile/writeFile; a missing file (ENOENT) is treated as
+// empty existing content (Go's os.ErrNotExist branch). Byte-for-byte output is a
+// contract — the gitignore tests assert exact content — so the header string
+// (em-dash and backtick included) and the blank-line / trailing-newline rules
+// are preserved verbatim.
+import { readFile, writeFile } from "node:fs/promises";
+import * as path from "node:path";
+/** dedupGitignoreHeader labels the block docgov manages in .docgov/dedup/.gitignore. */
+export const dedupGitignoreHeader = "# docgov dedup index cache — local, rebuilt by `docgov dedup`.";
+/**
+ * dedupGitignoreRules are the patterns that keep the local SQLite index cache out
+ * of git: the DB plus its WAL-mode sidecars. config.yml and the .gitignore itself
+ * stay tracked.
+ */
+export const dedupGitignoreRules = ["index.db", "index.db-wal", "index.db-shm"];
+/** isNotExist reports whether err is a Node ENOENT (Go's os.ErrNotExist). */
+function isNotExist(err) {
+    return (typeof err === "object" &&
+        err !== null &&
+        err.code === "ENOENT");
+}
+/**
+ * ensureDedupGitignore makes sure .docgov/dedup/.gitignore excludes the index
+ * cache. It is a no-op when every rule is already present (matched as an exact,
+ * trimmed line — comments and blanks ignored); otherwise it creates the file or
+ * appends only the missing rules, leaving any existing content intact so a user's
+ * own entries survive. dedupDir must already exist.
+ */
+export async function ensureDedupGitignore(dedupDir) {
+    const filePath = path.join(dedupDir, ".gitignore");
+    let existing = "";
+    try {
+        existing = await readFile(filePath, "utf8");
+    }
+    catch (err) {
+        if (!isNotExist(err)) {
+            throw new Error(`read ${quote(filePath)}: ${String(err)}`, { cause: err });
+        }
+        // Missing file: existing stays "" — same as Go's empty-on-ErrNotExist read.
+    }
+    const present = gitignoreLineSet(existing);
+    const missing = [];
+    for (const rule of dedupGitignoreRules) {
+        if (!present.has(rule)) {
+            missing.push(rule);
+        }
+    }
+    if (missing.length === 0) {
+        return; // rules already covered — nothing to write
+    }
+    try {
+        await writeFile(filePath, appendGitignoreBlock(existing, missing));
+    }
+    catch (err) {
+        throw new Error(`write ${quote(filePath)}: ${String(err)}`, { cause: err });
+    }
+}
+/**
+ * gitignoreLineSet returns the set of trimmed, non-comment, non-blank lines in a
+ * .gitignore body — the rules already in effect.
+ */
+export function gitignoreLineSet(body) {
+    const set = new Set();
+    for (const raw of body.split("\n")) {
+        const line = raw.trim();
+        if (line === "" || line.startsWith("#")) {
+            continue;
+        }
+        set.add(line);
+    }
+    return set;
+}
+/**
+ * appendGitignoreBlock returns existing followed by the docgov header and the
+ * missing rules. When existing is empty the block stands alone; otherwise it is
+ * separated from prior content by a blank line, and a trailing newline is added
+ * first if needed so the header never glues onto a user's last line.
+ */
+export function appendGitignoreBlock(existing, missing) {
+    const block = dedupGitignoreHeader + "\n" + missing.join("\n") + "\n";
+    if (existing.trim().length === 0) {
+        return block;
+    }
+    let prefix = existing;
+    if (!prefix.endsWith("\n")) {
+        prefix += "\n";
+    }
+    return prefix + "\n" + block;
+}
+/** quote renders a path as a Go %q-style double-quoted string for error messages. */
+function quote(s) {
+    return JSON.stringify(s);
+}

package/dist/dedup/gitignore.test.js

@@ -0,0 +1,98 @@
+/**
+ * Behavior-encoding tests for the docgov-owned .docgov/dedup/.gitignore block,
+ * ported from internal/dedup/gitignore_test.go.
+ *
+ * WHY each case matters: this file decides what stays out of git. The exact
+ * bytes are a contract — a user-tracked config.yml must never be ignored, the
+ * user's own entries must survive, and rules must never duplicate — so the tests
+ * assert on literal content, not just rule membership.
+ *
+ * Go used t.TempDir + os.ReadFile/WriteFile; vitest has neither, so we mkdtemp
+ * under os.tmpdir, clean each dir in afterEach, and read/write with node:fs.
+ */
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { dedupGitignoreRules, ensureDedupGitignore, gitignoreLineSet, } from "./gitignore.js";
+const tmpDirs = [];
+afterEach(() => {
+    for (const d of tmpDirs.splice(0)) {
+        rmSync(d, { recursive: true, force: true });
+    }
+});
+function newDir() {
+    const dir = mkdtempSync(join(tmpdir(), "dedup-gitignore-"));
+    tmpDirs.push(dir);
+    return dir;
+}
+function readGitignore(dir) {
+    return readFileSync(join(dir, ".gitignore"), "utf8");
+}
+function writeGitignore(dir, content) {
+    writeFileSync(join(dir, ".gitignore"), content);
+}
+/** countLine counts trimmed lines in body that equal want (Go's countLine helper). */
+function countLine(body, want) {
+    let n = 0;
+    for (const raw of body.split("\n")) {
+        if (raw.trim() === want) {
+            n++;
+        }
+    }
+    return n;
+}
+describe("ensureDedupGitignore", () => {
+    // WHY: first run on a fresh folder must drop a .gitignore that excludes the
+    // index.db cache + its WAL/SHM sidecars — but never the tracked config.yml.
+    it("creates the file with exactly the cache rules when absent", async () => {
+        const dir = newDir();
+        await ensureDedupGitignore(dir);
+        const got = readGitignore(dir);
+        const want = "# docgov dedup index cache — local, rebuilt by `docgov dedup`.\n" +
+            "index.db\nindex.db-wal\nindex.db-shm\n";
+        expect(got).toBe(want);
+        const set = gitignoreLineSet(got);
+        for (const rule of dedupGitignoreRules) {
+            expect(set.has(rule)).toBe(true);
+        }
+        // config.yml must stay tracked.
+        expect(set.has("config.yml")).toBe(false);
+    });
+    // WHY: the check is on the rules, not on file existence — when every cache
+    // rule is already present (alongside the user's own entries), the file must be
+    // left byte-for-byte untouched so we never churn a user's file.
+    it("leaves the file untouched when all rules are already present", async () => {
+        const dir = newDir();
+        const custom = "# my notes\nscratch/\nindex.db\nindex.db-wal\nindex.db-shm\n";
+        writeGitignore(dir, custom);
+        await ensureDedupGitignore(dir);
+        expect(readGitignore(dir)).toBe(custom);
+    });
+    // WHY: an existing .gitignore missing some cache rules must keep its content
+    // and gain ONLY the missing rules — index.db already there must not duplicate.
+    it("appends only the missing rules and keeps existing content", async () => {
+        const dir = newDir();
+        const custom = "# my notes\nindex.db\n";
+        writeGitignore(dir, custom);
+        await ensureDedupGitignore(dir);
+        const got = readGitignore(dir);
+        expect(got).toContain(custom);
+        const set = gitignoreLineSet(got);
+        for (const rule of dedupGitignoreRules) {
+            expect(set.has(rule)).toBe(true);
+        }
+        // index.db was already present — must appear exactly once (no duplicate).
+        expect(countLine(got, "index.db")).toBe(1);
+    });
+    // WHY: a second call after an append must be a no-op — once all rules are
+    // present, nothing more is written (idempotency keeps repeated runs clean).
+    it("is idempotent: a second call after an append changes nothing", async () => {
+        const dir = newDir();
+        writeGitignore(dir, "# my notes\nindex.db\n");
+        await ensureDedupGitignore(dir);
+        const after = readGitignore(dir);
+        await ensureDedupGitignore(dir);
+        expect(readGitignore(dir)).toBe(after);
+    });
+});

package/dist/dedup/index.js

@@ -0,0 +1,11 @@
+// Barrel for the top-level dedup facade.
+//
+// Exports the two entry points the CLI calls (Index, Analyze) and the public
+// report types (so cmd and the dedup/report subpackage can import them from the
+// package root). Config types + Default and the config loader are re-exported
+// too, matching the Go facade's surface. The subpackages (analyzer, indexer,
+// indexdb, …) keep their own barrels and are not re-exported here.
+export { Index, Analyze } from "./dedup.js";
+export { Default } from "./config.js";
+export { Load, YAMLTypeError } from "./configload.js";
+export { ensureDedupGitignore, dedupGitignoreHeader, dedupGitignoreRules, } from "./gitignore.js";

package/dist/dedup/indexdb/errors.js

@@ -0,0 +1,48 @@
+// Sentinel error and the Open outcome enum for the dedup index DB.
+//
+// Go's `var ErrIndexMissing = errors.New(...)` becomes an Error subclass so
+// callers match with `instanceof` instead of errors.Is.
+/**
+ * ErrIndexMissing is thrown when a caller requests read-only mode but the index
+ * database does not exist. In v1, Open always creates the DB, so this sentinel
+ * is never thrown by v1 callers — it exists for the v2 read-only mode hook.
+ */
+export class ErrIndexMissing extends Error {
+    constructor(message = "indexdb: index database does not exist") {
+        super(message);
+        this.name = "ErrIndexMissing";
+    }
+}
+/**
+ * OpenStatus indicates what Open did when it opened (or created) the DB.
+ * It is data, not state — callers read the second element of Open's return;
+ * there is no Store.status() accessor.
+ *
+ * The numeric values match Go's iota so StatusOpened stays the zero value
+ * ("opened, nothing to report").
+ */
+export var OpenStatus;
+(function (OpenStatus) {
+    /**
+     * The DB existed and all meta matched — the common case. Zero value so a
+     * freshly-declared status reads as "opened, nothing to report."
+     */
+    OpenStatus[OpenStatus["StatusOpened"] = 0] = "StatusOpened";
+    /** The DB was created for the first time. */
+    OpenStatus[OpenStatus["StatusFresh"] = 1] = "StatusFresh";
+    /**
+     * The stored schema_version was lower than the binary's SchemaVersion and
+     * registered additive migrations covered the gap. Embeddings are preserved.
+     */
+    OpenStatus[OpenStatus["StatusSchemaMigrated"] = 2] = "StatusSchemaMigrated";
+    /**
+     * The stored schema_version mismatched and no migration path existed — the
+     * DB file was deleted and recreated.
+     */
+    OpenStatus[OpenStatus["StatusSchemaRecreated"] = 3] = "StatusSchemaRecreated";
+    /**
+     * The stored embedder_model or embedder_dim did not match the binary's
+     * constants — the sections table was purged and a full re-embed is needed.
+     */
+    OpenStatus[OpenStatus["StatusEmbedderPurged"] = 4] = "StatusEmbedderPurged";
+})(OpenStatus || (OpenStatus = {}));

package/dist/dedup/indexdb/index.js

@@ -0,0 +1,6 @@
+// Barrel for the indexdb package: the SQLite-backed dedup index cache.
+export { Store, open, openDB, } from "./indexdb.js";
+export { ErrIndexMissing, OpenStatus } from "./errors.js";
+export { SchemaVersion, SQLiteOpenParams, MaxOpenConns, createTableSQL, createBlocksSQL, } from "./schema.js";
+export { migrations, findMigration, registerTestMigration, } from "./migrations.js";
+export { loadAllSectionsWithEmbeddings, loadAllBlocksWithEmbeddings, decodeVec, } from "./load.js";

package/dist/dedup/indexdb/indexdb.js

@@ -0,0 +1,302 @@
+/**
+ * Manages the dedup section index stored in a SQLite database.
+ *
+ * The DB lives at <repo_root>/.docgov/dedup/index.db. Callers resolve the repo
+ * root and pass the absolute DB path to open — this package never walks for
+ * .docgov or touches the sentinel constant (that stays in internal/repo).
+ *
+ * Ported from internal/dedup/indexdb. node:sqlite is synchronous, so Go's
+ * context.Context / database/sql async plumbing is dropped: open/close/queries
+ * are plain synchronous calls. A *sql.Tx becomes explicit BEGIN/COMMIT/ROLLBACK
+ * statements on the single connection, and *sql.Rows (a lazy cursor) becomes an
+ * eagerly-materialized array (node:sqlite has no row cursor — stmt.all reads all
+ * rows at once).
+ */
+import { existsSync, rmSync } from "node:fs";
+import { DatabaseSync } from "node:sqlite";
+import { OpenStatus } from "./errors.js";
+import { loadAllBlocksWithEmbeddings, loadAllSectionsWithEmbeddings } from "./load.js";
+import { findMigration } from "./migrations.js";
+import { createTableSQL, SchemaVersion, SQLiteOpenParams } from "./schema.js";
+/** Store is an open dedup index database. */
+export class Store {
+    /** The underlying synchronous SQLite connection. */
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    /** Closes the underlying database connection. */
+    close() {
+        this.db.close();
+    }
+    /**
+     * loadAllSectionsWithEmbeddings loads every section row and its embedding.
+     * embeddings maps section ID to the decoded float32 vector. See load.ts.
+     */
+    loadAllSectionsWithEmbeddings() {
+        return loadAllSectionsWithEmbeddings(this.db);
+    }
+    /**
+     * loadAllBlocksWithEmbeddings loads every block row. embeddings maps
+     * content_hash to the decoded float32 vector; NULL-embedding (table) blocks
+     * are absent from the map. See load.ts.
+     */
+    loadAllBlocksWithEmbeddings() {
+        return loadAllBlocksWithEmbeddings(this.db);
+    }
+    /**
+     * QuerySections returns all rows from the sections table with the fields the
+     * indexer needs for diff-and-skip logic: id, content_hash, embedding.
+     *
+     * Go returned a lazy *sql.Rows the caller had to Close; node:sqlite has no
+     * cursor, so this materializes the full result set into an array.
+     */
+    querySections() {
+        const rows = this.db
+            .prepare(`SELECT id, content_hash, embedding FROM sections`)
+            .all();
+        return rows.map((r) => ({
+            id: r.id,
+            content_hash: r.content_hash,
+            embedding: r.embedding,
+        }));
+    }
+    /**
+     * QueryBlocks returns the four columns the indexer needs for diff/prune:
+     * section_id, block_index, content_hash, embedding.
+     *
+     * Like querySections, this materializes the whole result set (no *sql.Rows).
+     */
+    queryBlocks() {
+        const rows = this.db
+            .prepare(`SELECT section_id, block_index, content_hash, embedding FROM blocks`)
+            .all();
+        return rows.map((r) => ({
+            section_id: r.section_id,
+            block_index: r.block_index,
+            content_hash: r.content_hash,
+            embedding: r.embedding,
+        }));
+    }
+    /**
+     * ExecTx runs fn inside a single exclusive transaction. If fn throws, the
+     * transaction is rolled back and the error rethrown; otherwise it is committed.
+     *
+     * node:sqlite has no *sql.Tx object — fn receives the connection and must run
+     * its statements on it without issuing its own BEGIN/COMMIT/ROLLBACK.
+     */
+    execTx(fn) {
+        this.db.exec("BEGIN");
+        try {
+            fn(this.db);
+        }
+        catch (err) {
+            this.db.exec("ROLLBACK");
+            throw err;
+        }
+        this.db.exec("COMMIT");
+    }
+}
+/**
+ * Open opens (or creates) the dedup index DB at path.
+ *
+ * embedderModel and embedderDim are the expected embedder identity values. They
+ * are stored in the meta table on first open and compared on subsequent opens;
+ * a mismatch triggers a sections purge (StatusEmbedderPurged).
+ *
+ * Reconciliation logic (returns { store, status }):
+ *   - New file → create schema + write meta → StatusFresh
+ *   - Existing, all meta matches → StatusOpened (zero value)
+ *   - schema_version mismatch + registered migration → apply in tx → StatusSchemaMigrated
+ *   - schema_version mismatch + no migration → delete file + recreate → StatusSchemaRecreated
+ *   - embedder_model or embedder_dim mismatch → DELETE FROM sections → StatusEmbedderPurged
+ *
+ * path must be an absolute OS path.
+ */
+export function open(path, embedderModel, embedderDim) {
+    // Distinguish fresh vs existing by checking the file.
+    const isFresh = !existsSync(path);
+    let db = openDB(path);
+    const embedderDimStr = String(embedderDim);
+    if (isFresh) {
+        // First-time open: create schema and write meta.
+        try {
+            createSchema(db);
+            writeMetaValues(db, embedderModel, embedderDimStr);
+        }
+        catch (err) {
+            db.close();
+            throw new Error(`indexdb.open: initialize ${quote(path)}: ${String(err)}`, { cause: err });
+        }
+        return { store: new Store(db), status: OpenStatus.StatusFresh };
+    }
+    // Existing DB: read stored meta.
+    let stored;
+    try {
+        stored = readMeta(db);
+    }
+    catch (err) {
+        db.close();
+        // Meta table might not exist in a very old or corrupt DB — treat as recreate.
+        try {
+            return recreateDB(path, embedderModel, embedderDimStr);
+        }
+        catch (rerr) {
+            throw new Error(`indexdb.open: recreate after bad meta ${quote(path)}: ${String(rerr)}`, {
+                cause: rerr,
+            });
+        }
+    }
+    // Check schema_version.
+    if (stored.version !== SchemaVersion) {
+        db.close();
+        const mig = findMigration(stored.version);
+        if (mig !== undefined) {
+            db = openDB(path);
+            try {
+                applyMigration(db, mig);
+            }
+            catch (err) {
+                db.close();
+                throw new Error(`indexdb.open: apply migration ${quote(path)}: ${String(err)}`, {
+                    cause: err,
+                });
+            }
+            return { store: new Store(db), status: OpenStatus.StatusSchemaMigrated };
+        }
+        // No migration path — delete and recreate.
+        try {
+            return recreateDB(path, embedderModel, embedderDimStr);
+        }
+        catch (err) {
+            throw new Error(`indexdb.open: recreate ${quote(path)}: ${String(err)}`, { cause: err });
+        }
+    }
+    // Check embedder identity.
+    if (stored.model !== embedderModel || stored.dim !== embedderDimStr) {
+        try {
+            purgeEmbedderValues(db, embedderModel, embedderDimStr);
+        }
+        catch (err) {
+            db.close();
+            throw new Error(`indexdb.open: embedder purge ${quote(path)}: ${String(err)}`, { cause: err });
+        }
+        return { store: new Store(db), status: OpenStatus.StatusEmbedderPurged };
+    }
+    return { store: new Store(db), status: OpenStatus.StatusOpened };
+}
+/**
+ * openDB opens the SQLite database at path and applies the locked connection
+ * PRAGMAs (the database/sql URI params have no node:sqlite equivalent).
+ */
+export function openDB(path) {
+    const db = new DatabaseSync(path);
+    // Equivalent of SQLiteOpenParams (?_journal_mode=WAL&_busy_timeout=5000&_txlock=immediate).
+    // txlock=immediate has no PRAGMA form; ExecTx/applyMigration use plain BEGIN,
+    // and the single-connection model makes the lock-upgrade deadlock it guarded
+    // against impossible here.
+    db.exec("PRAGMA journal_mode = WAL;");
+    db.exec("PRAGMA busy_timeout = 5000;");
+    return db;
+}
+/** createSchema runs the DDL for sections + meta + blocks tables. */
+function createSchema(db) {
+    db.exec(createTableSQL);
+}
+/** writeMetaValues inserts schema_version, embedder_model, and embedder_dim into meta. */
+function writeMetaValues(db, embedderModel, embedderDimStr) {
+    const stmt = db.prepare(`INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)`);
+    for (const [k, v] of [
+        ["schema_version", SchemaVersion],
+        ["embedder_model", embedderModel],
+        ["embedder_dim", embedderDimStr],
+    ]) {
+        stmt.run(k, v);
+    }
+}
+/** readMeta returns the stored schema_version, embedder_model, embedder_dim. */
+function readMeta(db) {
+    const rows = db.prepare(`SELECT key, value FROM meta`).all();
+    let version = "";
+    let model = "";
+    let dim = "";
+    for (const { key, value } of rows) {
+        switch (key) {
+            case "schema_version":
+                version = value;
+                break;
+            case "embedder_model":
+                model = value;
+                break;
+            case "embedder_dim":
+                dim = value;
+                break;
+        }
+    }
+    return { version, model, dim };
+}
+/**
+ * recreateDB deletes the DB file at path, re-creates it, and returns
+ * StatusSchemaRecreated. The SQLite sidecar files (-wal, -shm) are removed too
+ * so a recreate does not resurrect stale WAL pages from the deleted DB.
+ */
+function recreateDB(path, embedderModel, embedderDimStr) {
+    rmSync(path, { force: true });
+    rmSync(`${path}-wal`, { force: true });
+    rmSync(`${path}-shm`, { force: true });
+    const db = openDB(path);
+    try {
+        createSchema(db);
+        writeMetaValues(db, embedderModel, embedderDimStr);
+    }
+    catch (err) {
+        db.close();
+        throw err;
+    }
+    return { store: new Store(db), status: OpenStatus.StatusSchemaRecreated };
+}
+/**
+ * applyMigration runs a migration inside one exclusive transaction and bumps the
+ * stored schema_version to SchemaVersion.
+ */
+function applyMigration(db, mig) {
+    db.exec("BEGIN");
+    try {
+        const deps = {};
+        mig.apply(db, deps);
+        db.prepare(`INSERT OR REPLACE INTO meta(key, value) VALUES('schema_version', ?)`).run(SchemaVersion);
+    }
+    catch (err) {
+        db.exec("ROLLBACK");
+        throw new Error(`migration from ${quote(mig.fromVersion)}: ${String(err)}`, { cause: err });
+    }
+    db.exec("COMMIT");
+}
+/**
+ * purgeEmbedderValues deletes all rows from the sections table and updates the
+ * embedder meta fields to the provided values, all in one transaction.
+ */
+function purgeEmbedderValues(db, embedderModel, embedderDimStr) {
+    db.exec("BEGIN");
+    try {
+        db.exec(`DELETE FROM sections`);
+        const stmt = db.prepare(`INSERT OR REPLACE INTO meta(key, value) VALUES(?, ?)`);
+        for (const [k, v] of [
+            ["embedder_model", embedderModel],
+            ["embedder_dim", embedderDimStr],
+        ]) {
+            stmt.run(k, v);
+        }
+    }
+    catch (err) {
+        db.exec("ROLLBACK");
+        throw err;
+    }
+    db.exec("COMMIT");
+}
+/** quote wraps a value in double quotes for error messages (Go's %q for paths). */
+function quote(s) {
+    return `"${s}"`;
+}
+// Re-export so SQLiteOpenParams stays reachable for tests that open a raw DB.
+export { SQLiteOpenParams };