docsgov 0.1.0

package/dist/dedup/dedup.js

@@ -0,0 +1,143 @@
+// Top-level facade for the docgov dedup subsystem: the Index and Analyze entry
+// points the CLI calls.
+//
+// Ported from internal/dedup/dedup.go. Index walks <repoRoot>/docs/, extracts
+// eligible sections, embeds new/changed ones, and persists the index to
+// .docgov/dedup/index.db. Analyze loads that index and runs the layered
+// duplicate-detection algorithm, returning a structured Report.
+//
+// Reconciling the mixed sync/async stack vs Go:
+//   - ctx is dropped throughout (the ported embedder/indexdb already dropped it);
+//   - indexdb.open is synchronous and THROWS on failure (Go returned an error),
+//     so the Go `defer store.Close()` becomes a try/finally;
+//   - embedder.newEmbedder is async (model load); the Go `defer emb.Close()` and
+//     its real close are wrapped in try/finally;
+//   - the analyzer wants Map<string, number[]>; the indexdb loaders return
+//     Map<string, Float32Array>, so vectors are converted before the call.
+import { existsSync, promises as fsp } from "node:fs";
+import * as path from "node:path";
+import { analyze } from "./analyzer/index.js";
+import { Dimension, Embedder, Model } from "./embedder/index.js";
+import { ErrIndexMissing, open } from "./indexdb/index.js";
+import { run } from "./indexer/index.js";
+import { Load } from "./configload.js";
+import { ensureDedupGitignore } from "./gitignore.js";
+/** dbPathOf returns the index DB path under repoRoot (Go inlines this twice). */
+function dbPathOf(repoRoot) {
+    return path.join(repoRoot, ".docgov", "dedup", "index.db");
+}
+/**
+ * Index walks <repoRoot>/docs/, extracts eligible sections, embeds new and
+ * changed ones, and persists the index to .docgov/dedup/index.db.
+ *
+ * progress receives one-line status messages; pass a no-op for silence (Go's
+ * io.Discard) or one that writes to stderr from the CLI.
+ */
+export async function Index(repoRoot, progress) {
+    let cfg;
+    try {
+        cfg = await Load(repoRoot);
+    }
+    catch (err) {
+        throw new Error(`dedup.Index: load config: ${String(err)}`, { cause: err });
+    }
+    const dbPath = dbPathOf(repoRoot);
+    try {
+        await fsp.mkdir(path.dirname(dbPath), { recursive: true });
+    }
+    catch (err) {
+        throw new Error(`dedup.Index: create db dir: ${String(err)}`, { cause: err });
+    }
+    try {
+        await ensureDedupGitignore(path.dirname(dbPath));
+    }
+    catch (err) {
+        throw new Error(`dedup.Index: write gitignore: ${String(err)}`, { cause: err });
+    }
+    let store;
+    try {
+        ({ store } = open(dbPath, Model, Dimension));
+    }
+    catch (err) {
+        throw new Error(`dedup.Index: open db: ${String(err)}`, { cause: err });
+    }
+    try {
+        let emb;
+        try {
+            emb = await Embedder.newEmbedder();
+        }
+        catch (err) {
+            throw new Error(`dedup.Index: new embedder: ${String(err)}`, { cause: err });
+        }
+        try {
+            let stats;
+            try {
+                stats = await run(store, emb, repoRoot, cfg, progress);
+            }
+            catch (err) {
+                throw new Error(`dedup.Index: run: ${String(err)}`, { cause: err });
+            }
+            return {
+                sections: stats.sections,
+                embedded: stats.embedded,
+                pruned: stats.pruned,
+            };
+        }
+        finally {
+            await emb.close();
+        }
+    }
+    finally {
+        store.close();
+    }
+}
+/**
+ * Analyze loads the index from .docgov/dedup/index.db, runs the layered
+ * duplicate-detection algorithm, and returns a structured Report.
+ *
+ * Throws ErrIndexMissing (wrapped) when the index DB does not exist.
+ */
+export async function Analyze(repoRoot) {
+    const dbPath = dbPathOf(repoRoot);
+    if (!existsSync(dbPath)) {
+        // Throw the sentinel directly so callers match it with `instanceof`
+        // (the TS analogue of Go's errors.Is against indexdb.ErrIndexMissing).
+        throw new ErrIndexMissing("dedup.Analyze: indexdb: index database does not exist");
+    }
+    let cfg;
+    try {
+        cfg = await Load(repoRoot);
+    }
+    catch (err) {
+        throw new Error(`dedup.Analyze: load config: ${String(err)}`, { cause: err });
+    }
+    let store;
+    try {
+        ({ store } = open(dbPath, Model, Dimension));
+    }
+    catch (err) {
+        throw new Error(`dedup.Analyze: open db: ${String(err)}`, { cause: err });
+    }
+    try {
+        const { sections, embeddings } = store.loadAllSectionsWithEmbeddings();
+        const { blocks, embeddings: blockEmb } = store.loadAllBlocksWithEmbeddings();
+        return analyze(sections, toNumberMap(embeddings), blocks, toNumberMap(blockEmb), cfg);
+    }
+    finally {
+        store.close();
+    }
+}
+/**
+ * toNumberMap converts the Float32Array-valued embedding maps returned by the
+ * indexdb loaders into the number[]-valued maps the analyzer consumes. The
+ * analyzer reads vectors by index and length only, so this is a faithful,
+ * value-preserving conversion (Go used []float32 throughout; the TS analyzer
+ * settled on number[]).
+ */
+function toNumberMap(m) {
+    const out = new Map();
+    for (const [k, v] of m) {
+        out.set(k, Array.from(v));
+    }
+    return out;
+}

package/dist/dedup/dedup.test.js

@@ -0,0 +1,212 @@
+/**
+ * Behavior-encoding tests for the dedup facade (Index + Analyze), ported from
+ * internal/dedup/dedup_test.go and extended with a fake-embedder integration
+ * test driving the facade's Analyze over a really-indexed inline corpus.
+ *
+ * WHY each case matters:
+ *   - Analyze on a missing index must throw ErrIndexMissing (wrapped), so the
+ *     CLI can tell "run `dedup index` first" apart from a real failure;
+ *   - Analyze on an empty index is a valid "no duplicates" state, NOT an error;
+ *   - Analyze must surface L5 PartialOverlaps from blocks loaded out of the DB —
+ *     the facade wiring (loading blocks + passing them to the analyzer) is the
+ *     contract a regression to nil-blocks would silently break.
+ *
+ * The fake-embedder integration indexes two inline docs that share a verbatim
+ * prose block via the indexer (no model download), then asserts dedup.Analyze
+ * returns exactly the expected exact-overlap cluster — exercising the whole
+ * facade Analyze path on real persisted state.
+ *
+ * The real-embedder Index path is covered by an env-gated smoke test (the only
+ * allowed skip: it would download the ~1GB model).
+ */
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { afterEach, describe, expect, it } from "vitest";
+import { Dimension, Model } from "./embedder/index.js";
+import { ErrIndexMissing, open } from "./indexdb/index.js";
+import { encodeVec, run } from "./indexer/index.js";
+import { Default } from "./config.js";
+import { Analyze, Index } from "./dedup.js";
+// --- fake embedder ---------------------------------------------------------
+/** unitVec produces a deterministic vector with a 1.0 at position (seed % dim). */
+function unitVec(seed, dim) {
+    const v = new Array(dim).fill(0);
+    v[seed % dim] = 1.0;
+    return v;
+}
+/**
+ * FakeEmbedder returns deterministic orthogonal unit vectors per batch position
+ * so distinct sections never collapse into one HIGH group — the shared block is
+ * then the only duplication signal, surfacing via L5-exact (which is hash-based,
+ * not embedding-based). No model download.
+ */
+class FakeEmbedder {
+    async embed(texts) {
+        return texts.map((_, i) => unitVec(i, Dimension));
+    }
+    name() {
+        return Model;
+    }
+    dimension() {
+        return Dimension;
+    }
+}
+// --- temp dir plumbing -----------------------------------------------------
+const tmpDirs = [];
+afterEach(() => {
+    for (const d of tmpDirs.splice(0)) {
+        rmSync(d, { recursive: true, force: true });
+    }
+});
+/** newRepoRoot returns a fresh empty temp repo root. */
+function newRepoRoot() {
+    const dir = mkdtempSync(join(tmpdir(), "dedup-facade-"));
+    tmpDirs.push(dir);
+    return dir;
+}
+/** writeFiles writes relPath→content under repoRoot, creating parents. */
+function writeFiles(repoRoot, files) {
+    for (const [rel, content] of Object.entries(files)) {
+        const full = join(repoRoot, rel);
+        mkdirSync(dirname(full), { recursive: true });
+        writeFileSync(full, content);
+    }
+}
+/** dbPathOf mirrors the facade's index DB location. */
+function dbPathOf(repoRoot) {
+    return join(repoRoot, ".docgov", "dedup", "index.db");
+}
+/** createEmptyIndexDB opens (and immediately closes) a valid empty index DB. */
+function createEmptyIndexDB(repoRoot) {
+    const dbPath = dbPathOf(repoRoot);
+    mkdirSync(dirname(dbPath), { recursive: true });
+    const { store } = open(dbPath, Model, Dimension);
+    store.close();
+}
+const noProgress = () => { };
+describe("dedup.Analyze", () => {
+    // WHY: a missing index must be a distinct, matchable error so the CLI can
+    // prompt the user to index first rather than reporting a generic failure.
+    it("throws ErrIndexMissing when the index DB does not exist", async () => {
+        const repoRoot = newRepoRoot(); // empty, no .docgov/dedup/index.db
+        await expect(Analyze(repoRoot)).rejects.toBeInstanceOf(ErrIndexMissing);
+    });
+    // WHY: an existing but empty index is a valid "no duplicates" state — Analyze
+    // must return an empty Report, NOT an error and NOT ErrIndexMissing.
+    it("returns an empty Report for an empty index", async () => {
+        const repoRoot = newRepoRoot();
+        createEmptyIndexDB(repoRoot);
+        const report = await Analyze(repoRoot);
+        expect(report.HighGroups).toHaveLength(0);
+        expect(report.MaybePairs).toHaveLength(0);
+        expect(report.PartialOverlaps).toHaveLength(0);
+    });
+    // WHY: the facade must load blocks from the DB and feed them to the analyzer so
+    // a verbatim block shared across two files surfaces as exactly one exact
+    // PartialOverlaps cluster. This pins the block-loading wiring (Go's P7.T1 fix);
+    // a regression to nil-blocks would silently return zero clusters here.
+    // Seeds the DB directly (mirrors Go's TestAnalyze_EndToEnd_PartialOverlap) to
+    // assert the wiring independently of the markdown extractor.
+    it("returns one exact PartialOverlaps cluster for a shared verbatim block", async () => {
+        const repoRoot = newRepoRoot();
+        const dbPath = dbPathOf(repoRoot);
+        mkdirSync(dirname(dbPath), { recursive: true });
+        const { store } = open(dbPath, Model, Dimension);
+        // Orthogonal section embeddings so the two sections do NOT form a HIGH group
+        // (cosine 0); the block embedding is shared (verbatim-identical block).
+        const secAEmb = encodeVec(new Float32Array([1.0, 0.0, 0.0]));
+        const secBEmb = encodeVec(new Float32Array([0.0, 1.0, 0.0]));
+        const blockEmb = encodeVec(new Float32Array([0.5, 0.5, 0.0]));
+        const sharedHash = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899";
+        store.execTx((db) => {
+            const insertSection = db.prepare(`INSERT OR REPLACE INTO sections
+          (id, file_path, heading, heading_level, anchor, start_line, end_line,
+           content_hash, raw_content, embed_text, prose_word_count, has_table,
+           has_code, inbound_count, embedding, updated_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+            insertSection.run("sec-a-intro", "docs/a.md", "Introduction", 2, "introduction", 1, 20, "hash-a", "## Introduction\n\nContent a.", "content a", 15, 0, 0, 0, secAEmb, "2024-01-01T00:00:00Z");
+            insertSection.run("sec-b-intro", "docs/b.md", "Introduction", 2, "introduction", 1, 20, "hash-b", "## Introduction\n\nContent b.", "content b", 15, 0, 0, 0, secBEmb, "2024-01-01T00:00:00Z");
+            const insertBlock = db.prepare(`INSERT OR REPLACE INTO blocks
+          (section_id, block_index, file_path, heading, kind,
+           start_line, end_line, content_hash, embedding)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`);
+            insertBlock.run("sec-a-intro", 0, "docs/a.md", "Introduction", "prose", 3, 8, sharedHash, blockEmb);
+            insertBlock.run("sec-b-intro", 0, "docs/b.md", "Introduction", "prose", 3, 8, sharedHash, blockEmb);
+        });
+        store.close();
+        const report = await Analyze(repoRoot);
+        expect(report.PartialOverlaps).toHaveLength(1);
+        const cl = report.PartialOverlaps[0];
+        expect(cl.Exact).toBe(true);
+        expect(cl.Kind).toBe("prose");
+        expect(cl.Locations).toHaveLength(2);
+    });
+});
+describe("dedup facade end-to-end (fake embedder)", () => {
+    // WHY: this exercises the whole facade Analyze path over a really-indexed
+    // corpus — the indexer extracts/persists sections+blocks from inline docs, and
+    // Analyze must then surface the shared verbatim prose block as an exact
+    // overlap. The embedder is faked so no model is downloaded; L5-exact is
+    // hash-based so the fake vectors don't affect the cluster.
+    it("indexes two inline docs sharing a verbatim block and Analyze reports it", async () => {
+        const repoRoot = newRepoRoot();
+        // The shared paragraph is >10 words so the block is eligible. It is byte-
+        // identical in both files → identical content_hash → one exact cluster.
+        //
+        // Each section also carries its own distinct prose so the two SECTIONS are
+        // NOT a whole-section exact duplicate (which the analyzer would report as a
+        // HIGH group and then suppress the block-level overlap). Only the shared
+        // paragraph overlaps, so it must surface as an L5 PartialOverlaps cluster.
+        const sharedPara = "This shared paragraph appears verbatim in both documents and is long enough to be an eligible duplicate block for detection.";
+        const aIntro = "Alpha covers the ingestion side of the system, including the upload queue, the validation gateway, and the retry scheduler for failed batches.";
+        const bIntro = "Beta documents the reporting surface, covering scheduled exports, the metrics rollup job, and the long-term archival of historical aggregates.";
+        writeFiles(repoRoot, {
+            "docs/a.md": `## Alpha Overview\n\n${aIntro}\n\n${sharedPara}\n`,
+            "docs/b.md": `## Beta Overview\n\n${bIntro}\n\n${sharedPara}\n`,
+        });
+        // Index via the indexer with a fake embedder (the facade's Index would
+        // download the real model). This populates the same DB the facade reads.
+        const dbPath = dbPathOf(repoRoot);
+        mkdirSync(dirname(dbPath), { recursive: true });
+        const { store } = open(dbPath, Model, Dimension);
+        try {
+            const stats = await run(store, new FakeEmbedder(), repoRoot, Default(), noProgress);
+            expect(stats.sections).toBeGreaterThanOrEqual(2);
+        }
+        finally {
+            store.close();
+        }
+        const report = await Analyze(repoRoot);
+        // The shared verbatim block must surface as exactly one exact cluster across
+        // the two files.
+        expect(report.PartialOverlaps).toHaveLength(1);
+        const cl = report.PartialOverlaps[0];
+        expect(cl.Exact).toBe(true);
+        expect(new Set(cl.Locations.map((l) => l.FilePath))).toEqual(new Set(["docs/a.md", "docs/b.md"]));
+    });
+});
+describe("dedup.Index (real embedder)", () => {
+    // WHY: the Index path itself (real embedder, real walk, real persist) is only
+    // meaningful with the model. Gated behind DOCGOV_E2E so CI never downloads
+    // ~1GB — the single allowed env-skip per the porting conventions.
+    const runE2E = process.env["DOCGOV_E2E"] === "1";
+    it.skipIf(!runE2E)("indexes a corpus and persists sections", async () => {
+        const repoRoot = newRepoRoot();
+        writeFiles(repoRoot, {
+            "docs/guide.md": "## Introduction\n\nThis is the introduction section of the guide. It covers the basic concepts and provides an overview of the system.\n",
+        });
+        const stats = await Index(repoRoot, noProgress);
+        expect(stats.sections).toBeGreaterThan(0);
+        // The DB now exists; Analyze must succeed (not ErrIndexMissing).
+        const db = new DatabaseSync(dbPathOf(repoRoot));
+        try {
+            const row = db.prepare("SELECT COUNT(*) AS n FROM sections").get();
+            expect(row.n).toBeGreaterThan(0);
+        }
+        finally {
+            db.close();
+        }
+    });
+});

package/dist/dedup/dedupcfg/config.js

@@ -0,0 +1,112 @@
+// Package dedupcfg defines the Config types for the dedup subsystem.
+// It is a leaf package (no imports from within the dedup tree) so that the
+// indexer can import it without creating an import cycle with the top-level
+// dedup facade.
+//
+// The YAML overlay (Default() merged with .docgov/dedup/config.yml) lives in
+// the parent dedup package; this leaf only declares the shapes and defaults.
+// Because that overlay deserializes raw YAML keys onto these shapes, every
+// property name here keeps its EXACT YAML serialized spelling (snake_case),
+// not the Go field's PascalCase identifier.
+// headingBlacklisted reports whether heading matches any entry in
+// heading_blacklist or heading_blacklist_extra (case-insensitive substring).
+//
+// It is the single matcher for the heading blacklist, shared by the
+// canonical-selection path and the block-embedding gate so the two cannot
+// drift apart. (Go: AnalyzerConfig.HeadingBlacklisted.)
+export function headingBlacklisted(cfg, heading) {
+    const h = heading.toLowerCase();
+    for (const tok of cfg.heading_blacklist) {
+        if (h.includes(tok.toLowerCase())) {
+            return true;
+        }
+    }
+    for (const tok of cfg.heading_blacklist_extra ?? []) {
+        if (h.includes(tok.toLowerCase())) {
+            return true;
+        }
+    }
+    return false;
+}
+// defaultConfig returns the v1 locked default configuration.
+// (Go: Default().) Named defaultConfig because `default` is a reserved word.
+export function defaultConfig() {
+    return {
+        Markdown: {
+            min_prose_words: 10,
+            heading_token_min_len: 3,
+            hidden_dir_prefix: ".",
+            ignored_dirs: [
+                ".git",
+                "node_modules",
+                "vendor",
+                "dist",
+                "build",
+                ".next",
+                ".cache",
+                ".docgov",
+                "dedup-poc",
+                ".venv",
+            ],
+        },
+        Indexer: {
+            embed_progress_threshold: 100,
+            max_workers: 0,
+            external_url_prefixes: ["http://", "https://", "mailto:"],
+        },
+        Analyzer: {
+            thresh_high: 0.93,
+            thresh_maybe: 0.86,
+            distinctive_abs_min: 3,
+            distinctive_pct_of_headings: 0.03,
+            universal_stopwords: [
+                "the",
+                "a",
+                "an",
+                "of",
+                "and",
+                "or",
+                "to",
+                "with",
+                "for",
+                "in",
+                "on",
+                "is",
+                "are",
+                "be",
+                "by",
+                "from",
+                "as",
+                "at",
+            ],
+            differentiators: [
+                ["calendar days", "business days"],
+                ["sync", "async"],
+                ["create", "cancel"],
+                ["source", "target"],
+                ["inbound", "outbound"],
+                ["success", "failure"],
+                ["old flow", "new flow"],
+                ["deprecated", "current"],
+            ],
+            path_priority: ["docs/concepts/", "docs/architecture/", "docs/design/"],
+            heading_blacklist: ["related", "template rendering", "template sample"],
+            path_blacklist: ["changelog", "migration", "deprecated", "old", "legacy", "temporary"],
+        },
+        Report: {
+            preview_chars: 280,
+            preview_word_ratio: 0.6,
+            wrap_cols: 72,
+            separator: "---",
+        },
+        Embedder: {
+            batch_size: 32,
+        },
+        Block: {
+            min_words: 10,
+            table_min_rows: 2,
+            cosine_threshold: 0.95,
+            multiplicity_cap: 5,
+        },
+    };
+}

package/dist/dedup/dedupcfg/config.test.js

@@ -0,0 +1,70 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig, headingBlacklisted } from "./index.js";
+// The locked v1 BlockConfig defaults drive the L5 block-level clustering layer;
+// the cosine_threshold and multiplicity_cap in particular gate which blocks are
+// reported as duplicates. WHY: silently shifting any of these changes clustering
+// behavior, so the defaults are pinned exactly.
+describe("defaultConfig Block", () => {
+    it("returns the pinned v1 block thresholds", () => {
+        expect(defaultConfig().Block).toEqual({
+            min_words: 10,
+            table_min_rows: 2,
+            cosine_threshold: 0.95,
+            multiplicity_cap: 5,
+        });
+    });
+});
+// The clustering thresholds must keep their exact pinned values: thresh_maybe <
+// thresh_high and both in (0,1) is the invariant the analyzer relies on.
+// WHY: a drifted threshold would re-bucket near-duplicates and change which
+// sections cluster together.
+describe("defaultConfig Analyzer thresholds", () => {
+    it("pins thresh_high and thresh_maybe", () => {
+        const a = defaultConfig().Analyzer;
+        expect(a.thresh_high).toBe(0.93);
+        expect(a.thresh_maybe).toBe(0.86);
+        expect(a.thresh_maybe).toBeLessThan(a.thresh_high);
+    });
+});
+// headingBlacklisted is the single matcher shared by the canonical-selection
+// path and the block-embedding gate. Matching is case-insensitive substring
+// against heading_blacklist + heading_blacklist_extra.
+//
+// WHY: blacklisted headings must be recognised identically wherever the list is
+// consulted, so a single matcher prevents the two callers from drifting apart.
+describe("headingBlacklisted", () => {
+    const cfg = {
+        // Only the fields the matcher reads are populated; the rest are irrelevant
+        // to this behavior and omitted for focus.
+        thresh_high: 0.93,
+        thresh_maybe: 0.86,
+        distinctive_abs_min: 3,
+        distinctive_pct_of_headings: 0.03,
+        universal_stopwords: [],
+        differentiators: [],
+        path_priority: [],
+        heading_blacklist: ["related", "steps"],
+        heading_blacklist_extra: ["examples"],
+        path_blacklist: [],
+    };
+    it.each([
+        ["Steps", true], // exact, case-insensitive
+        ["Next Steps", true], // substring match
+        ["Related", true], // from heading_blacklist
+        ["Examples", true], // from heading_blacklist_extra
+        ["Order Lifecycle", false], // no match
+        ["", false], // empty heading
+    ])("headingBlacklisted(%j) === %j", (heading, want) => {
+        expect(headingBlacklisted(cfg, heading)).toBe(want);
+    });
+    // The default config has no heading_blacklist_extra; the matcher must treat
+    // the absent (undefined) list as empty rather than throwing. WHY: the YAML
+    // overlay only appends an extra list when present, so the common default path
+    // leaves it undefined.
+    it("treats absent heading_blacklist_extra as empty", () => {
+        const a = defaultConfig().Analyzer;
+        expect(a.heading_blacklist_extra).toBeUndefined();
+        expect(headingBlacklisted(a, "Related")).toBe(true);
+        expect(headingBlacklisted(a, "Order Lifecycle")).toBe(false);
+    });
+});

package/dist/dedup/dedupcfg/index.js

@@ -0,0 +1 @@
+export { defaultConfig, headingBlacklisted } from "./config.js";

package/dist/dedup/deduptypes/index.js

@@ -0,0 +1 @@
+export {};

package/dist/dedup/deduptypes/types.js

@@ -0,0 +1,9 @@
+// Package deduptypes defines the pure-data types returned by the dedup
+// analyzer. It is a leaf package (no imports from within the dedup subtree) so
+// that the analyzer can import it without creating a cycle with the top-level
+// dedup facade, which imports the analyzer.
+//
+// These are pure-data shapes (Go structs with no serialization tags): the
+// field names below are the exact spellings used by indexdb when it persists
+// and loads them, so they are kept verbatim.
+export {};

package/dist/dedup/deduptypes/types.test.js

@@ -0,0 +1,34 @@
+import { describe, it, expect } from "vitest";
+// Mirrors Go's TestCluster_Shape. The dedup analyzer produces these structs and
+// indexdb persists/loads them by these exact field names, so the shape itself is
+// the contract: a drift in a field name or type would silently break round-trip
+// serialization. Constructing the value and reading every field back pins it.
+describe("Cluster shape (indexdb persistence contract)", () => {
+    it("exposes every BlockLocation and Cluster field with the expected names and types", () => {
+        const loc = {
+            FilePath: "a.md",
+            Heading: "H",
+            StartLine: 1,
+            EndLine: 3,
+        };
+        const c = {
+            Kind: "prose",
+            ContentHash: "h",
+            Similarity: 0.97,
+            Exact: true,
+            Informational: false,
+            Locations: [loc],
+        };
+        expect(c.Kind).toBe("prose");
+        expect(c.ContentHash).toBe("h");
+        expect(c.Similarity).toBe(0.97);
+        expect(c.Exact).toBe(true);
+        expect(c.Informational).toBe(false);
+        expect(c.Locations).toHaveLength(1);
+        const l = c.Locations[0];
+        expect(l.FilePath).toBe("a.md");
+        expect(l.Heading).toBe("H");
+        expect(l.StartLine).toBe(1);
+        expect(l.EndLine).toBe(3);
+    });
+});

package/dist/dedup/embedder/cache.js

@@ -0,0 +1,23 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { Model } from "./constants.js";
+/**
+ * cacheDir returns the resolved model cache directory in precedence order:
+ *  1. explicit (the WithCacheDir option equivalent)
+ *  2. DOCGOV_MODEL_CACHE env var
+ *  3. ~/.cache/docgov/models/<sanitized-model-name>/
+ *
+ * The returned path is not guaranteed to exist; the caller creates it.
+ */
+export function cacheDir(explicit) {
+    if (explicit !== "") {
+        return explicit;
+    }
+    const env = process.env.DOCGOV_MODEL_CACHE;
+    if (env !== undefined && env !== "") {
+        return env;
+    }
+    // Sanitize the model name: replace / with _ for filesystem safety.
+    const sanitized = Model.replaceAll("/", "_");
+    return join(homedir(), ".cache", "docgov", "models", sanitized);
+}

package/dist/dedup/embedder/cache.test.js

@@ -0,0 +1,50 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, expect, test } from "vitest";
+import { cacheDir } from "./cache.js";
+import { Model } from "./constants.js";
+// Snapshot/restore DOCGOV_MODEL_CACHE so tests don't leak into each other or
+// the host env.
+let savedEnv;
+beforeEach(() => {
+    savedEnv = process.env.DOCGOV_MODEL_CACHE;
+    delete process.env.DOCGOV_MODEL_CACHE;
+});
+afterEach(() => {
+    if (savedEnv === undefined) {
+        delete process.env.DOCGOV_MODEL_CACHE;
+    }
+    else {
+        process.env.DOCGOV_MODEL_CACHE = savedEnv;
+    }
+});
+// WHY: tests pass an explicit cache dir (a temp dir) so CI never races on the
+// shared host cache. The explicit option MUST win over both the env var and
+// the default, or that isolation guarantee breaks.
+test("explicit cache dir wins over env var and default", () => {
+    process.env.DOCGOV_MODEL_CACHE = "/from/env";
+    expect(cacheDir("/explicit/dir")).toBe("/explicit/dir");
+});
+// WHY: DOCGOV_MODEL_CACHE lets operators relocate the multi-hundred-MB model
+// off the home partition. It must be honored when no explicit dir is given.
+test("DOCGOV_MODEL_CACHE is used when no explicit dir is given", () => {
+    process.env.DOCGOV_MODEL_CACHE = "/from/env";
+    expect(cacheDir("")).toBe("/from/env");
+});
+// WHY: an empty env var must NOT shadow the default (Go checks `env != ""`).
+// A stray `DOCGOV_MODEL_CACHE=` in a shell would otherwise route the cache to
+// the empty path.
+test("empty DOCGOV_MODEL_CACHE falls through to the default", () => {
+    process.env.DOCGOV_MODEL_CACHE = "";
+    const want = join(homedir(), ".cache", "docgov", "models", Model.replaceAll("/", "_"));
+    expect(cacheDir("")).toBe(want);
+});
+// WHY: the default path sanitizes the model name by replacing "/" with "_".
+// Leaving the slash in would create a nested dir under models/ and break the
+// has-onnx-file lookup, re-triggering downloads every run.
+test("default path sanitizes the model name slash to underscore", () => {
+    const got = cacheDir("");
+    expect(got).toContain("paraphrase-multilingual-mpnet-base-v2");
+    expect(got).not.toContain("sentence-transformers/");
+    expect(got).toContain("sentence-transformers_paraphrase-multilingual-mpnet-base-v2");
+});