docsgov 0.1.0

package/dist/dedup/analyzer/canonical.test.js

@@ -0,0 +1,70 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { canonicalRank, isDisqualified, lessRank } from "./canonical.js";
+// makeSection builds a minimal Section (TS snake_case shape) for rank tests.
+function makeSection(id, filePath, heading, level, anchor, contentHash, rawContent, inbound) {
+    return {
+        id,
+        file_path: filePath,
+        heading,
+        heading_level: level,
+        anchor,
+        start_line: 1,
+        end_line: 10,
+        content_hash: contentHash,
+        raw_content: rawContent,
+        embed_text: heading,
+        prose_word_count: 15,
+        has_table: false,
+        has_code: false,
+        inbound_count: inbound,
+    };
+}
+// Disqualification is the FIRST canonical-rank field, so a disqualified section
+// can never win election even if it is otherwise the best. These tests pin both
+// blacklist axes (heading + path) and the non-match case so the disqualifier
+// neither over- nor under-fires.
+describe("isDisqualified", () => {
+    const cfg = defaultConfig().Analyzer;
+    it("disqualifies a blacklisted heading ('Related')", () => {
+        const sec = makeSection("id1", "docs/concepts/foo.md", "Related", 2, "related", "hash1", "## Related\nSome content.", 0);
+        expect(isDisqualified(sec, cfg)).toBe(true);
+    });
+    it("disqualifies a blacklisted path ('changelog')", () => {
+        const sec = makeSection("id1", "docs/changelog/foo.md", "Overview", 2, "overview", "hash1", "## Overview\nSome content.", 0);
+        expect(isDisqualified(sec, cfg)).toBe(true);
+    });
+    it("does not disqualify a clean section", () => {
+        const sec = makeSection("id1", "docs/concepts/foo.md", "Order Lifecycle", 2, "order-lifecycle", "hash1", "## Order Lifecycle\nSome content.", 0);
+        expect(isDisqualified(sec, cfg)).toBe(false);
+    });
+});
+// The canonical-rank tuple ordering decides WHICH section becomes the reference
+// every other duplicate points to. Each test pins one field's contribution so
+// the election cannot silently re-order. lessRank encodes the lower-is-better
+// comparison the orchestrator uses to pick the winner.
+describe("canonicalRank + lessRank", () => {
+    const cfg = defaultConfig().Analyzer;
+    it("ranks a better path (docs/concepts) lower than docs/other", () => {
+        const conceptSec = makeSection("id1", "docs/concepts/foo.md", "Lifecycle", 2, "lifecycle", "hash1", "content", 0);
+        const otherSec = makeSection("id2", "docs/other/foo.md", "Lifecycle", 2, "lifecycle", "hash2", "content", 0);
+        expect(lessRank(canonicalRank(conceptSec, cfg), canonicalRank(otherSec, cfg))).toBe(true);
+    });
+    it("ranks higher inbound count lower (more incoming links wins)", () => {
+        const highInbound = makeSection("id1", "docs/concepts/foo.md", "Lifecycle", 2, "lifecycle", "hash1", "content", 5);
+        const lowInbound = makeSection("id2", "docs/concepts/bar.md", "Lifecycle", 2, "lifecycle", "hash2", "content", 0);
+        expect(lessRank(canonicalRank(highInbound, cfg), canonicalRank(lowInbound, cfg))).toBe(true);
+    });
+    it("ranks a disqualified section above (worse than) a qualified one", () => {
+        const disq = makeSection("id1", "docs/concepts/foo.md", "Related", 2, "related", "hash1", "content", 0);
+        const notDisq = makeSection("id2", "docs/concepts/foo.md", "Overview", 2, "overview", "hash2", "content", 0);
+        expect(lessRank(canonicalRank(notDisq, cfg), canonicalRank(disq, cfg))).toBe(true);
+    });
+    it("breaks ties on longer raw_content (NegLen), measured in UTF-8 bytes", () => {
+        // WHY: NegLen is -byteLength so the longer section wins. Pin the byte-length
+        // semantics: a 1-char multibyte string must out-rank a shorter ASCII one.
+        const longer = makeSection("id1", "docs/concepts/a.md", "Lifecycle", 2, "lifecycle", "h1", "abcde", 0);
+        const shorter = makeSection("id2", "docs/concepts/b.md", "Lifecycle", 2, "lifecycle", "h2", "ab", 0);
+        expect(lessRank(canonicalRank(longer, cfg), canonicalRank(shorter, cfg))).toBe(true);
+    });
+});

package/dist/dedup/analyzer/cosine_clusters.js

@@ -0,0 +1,169 @@
+// Ported from internal/dedup/analyzer/cosine_clusters.go.
+//
+// L5-cosine clustering over prose blocks. The cosine math, the >=threshold gate,
+// the same-file-pair skip, and the min-qualifying-cosine-per-cluster rule are all
+// behavior-load-bearing: they decide which near-copy blocks cluster together and
+// what Similarity is reported, so they are reproduced exactly (including the
+// float32->float64 dot-product accumulation and the union-find component merge).
+import { cmpStr, cmpNum } from "./order.js";
+/**
+ * dotProduct computes the dot product of two float32 vectors as float64.
+ * For L2-normalized vectors this equals the cosine similarity.
+ *
+ * Go (analyzer.go:dotProduct) accumulates in float64 over float32 elements;
+ * JS numbers are float64, and the vectors here are plain number[] sourced from
+ * the same embeddings, so the accumulation matches.
+ */
+function dotProduct(a, b) {
+    const n = a.length;
+    if (n === 0 || b.length < n) {
+        return 0;
+    }
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+        sum += a[i] * b[i];
+    }
+    return sum;
+}
+/**
+ * cosineClusters performs L5-cosine clustering over prose blocks only.
+ *
+ * It considers only blocks whose ContentHash is in blockEmb (prose with a vector)
+ * AND NOT in excludeHashes (exact-cluster hashes already reported). Tables (absent
+ * from blockEmb) are naturally excluded.
+ *
+ * All-pairs over the considered blocks (i<j): same-FilePath pairs are skipped. If
+ * dotProduct(vi, vj) >= cfg.Block.cosine_threshold, i and j are unioned via
+ * union-find.
+ *
+ * For each resulting component of >=2 blocks a Cluster is emitted:
+ *   - Exact:false, Kind:"prose", ContentHash:"" (members differ)
+ *   - Similarity = min qualifying pairwise cosine within the cluster
+ *   - Informational:false
+ *   - Locations sorted by (FilePath, StartLine)
+ *
+ * Clusters are sorted by first location (FilePath, StartLine) for determinism.
+ */
+export function cosineClusters(blocks, blockEmb, excludeHashes, cfg) {
+    const threshold = cfg.Block.cosine_threshold;
+    const cands = [];
+    for (const b of blocks) {
+        if (excludeHashes.has(b.ContentHash)) {
+            continue;
+        }
+        const v = blockEmb.get(b.ContentHash);
+        if (v === undefined) {
+            continue;
+        }
+        cands.push({ block: b, vec: v });
+    }
+    const n = cands.length;
+    if (n < 2) {
+        return [];
+    }
+    // Union-find (path-compressed) over candidate indices.
+    const parent = [];
+    for (let i = 0; i < n; i++) {
+        parent[i] = i;
+    }
+    const find = (x) => {
+        while (parent[x] !== x) {
+            parent[x] = parent[parent[x]];
+            x = parent[x];
+        }
+        return x;
+    };
+    const union = (a, b) => {
+        const ra = find(a);
+        const rb = find(b);
+        if (ra !== rb) {
+            parent[ra] = rb;
+        }
+    };
+    const edges = [];
+    for (let i = 0; i < n; i++) {
+        for (let j = i + 1; j < n; j++) {
+            // Skip same-file pairs.
+            if (cands[i].block.FilePath === cands[j].block.FilePath) {
+                continue;
+            }
+            const cos = dotProduct(cands[i].vec, cands[j].vec);
+            if (cos >= threshold) {
+                union(i, j);
+                edges.push({ i, j, cos });
+            }
+        }
+    }
+    // Assign each candidate to its root component.
+    const compMembers = new Map();
+    for (let i = 0; i < n; i++) {
+        const root = find(i);
+        const list = compMembers.get(root);
+        if (list) {
+            list.push(i);
+        }
+        else {
+            compMembers.set(root, [i]);
+        }
+    }
+    // Compute minimum qualifying cosine per component from recorded edges.
+    // Initialize to a value above threshold so the first edge always wins.
+    const compMinCos = new Map();
+    for (const root of compMembers.keys()) {
+        compMinCos.set(root, 2.0);
+    }
+    for (const e of edges) {
+        const root = find(e.i); // both ends in same component after union
+        if (e.cos < compMinCos.get(root)) {
+            compMinCos.set(root, e.cos);
+        }
+    }
+    // Build clusters for components with >=2 members.
+    const clusters = [];
+    for (const [root, idxs] of compMembers) {
+        if (idxs.length < 2) {
+            continue;
+        }
+        const locs = [];
+        for (const idx of idxs) {
+            const b = cands[idx].block;
+            locs.push({
+                FilePath: b.FilePath,
+                Heading: b.Heading,
+                StartLine: b.StartLine,
+                EndLine: b.EndLine,
+            });
+        }
+        // Sort locations by (FilePath, StartLine).
+        locs.sort((a, b) => {
+            const c = cmpStr(a.FilePath, b.FilePath);
+            if (c !== 0)
+                return c;
+            return cmpNum(a.StartLine, b.StartLine);
+        });
+        let minCos = compMinCos.get(root);
+        if (minCos > 1.0) {
+            // No edge recorded (shouldn't happen if len>=2 and union was called) — use threshold.
+            minCos = threshold;
+        }
+        clusters.push({
+            Kind: "prose",
+            ContentHash: "",
+            Similarity: minCos,
+            Exact: false,
+            Informational: false,
+            Locations: locs,
+        });
+    }
+    // Sort clusters by first location (FilePath, StartLine) for determinism.
+    clusters.sort((a, b) => {
+        if (a.Locations.length === 0 || b.Locations.length === 0) {
+            return 0;
+        }
+        const c = cmpStr(a.Locations[0].FilePath, b.Locations[0].FilePath);
+        if (c !== 0)
+            return c;
+        return cmpNum(a.Locations[0].StartLine, b.Locations[0].StartLine);
+    });
+    return clusters;
+}

package/dist/dedup/analyzer/cosine_clusters.test.js

@@ -0,0 +1,131 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { cosineClusters } from "./cosine_clusters.js";
+// Minimal BlockRecord builder for the cosine pass (only path/kind/lines/hash
+// drive clustering).
+function block(filePath, heading, kind, startLine, endLine, contentHash) {
+    return {
+        SectionID: "",
+        FilePath: filePath,
+        Heading: heading,
+        Index: 0,
+        Kind: kind,
+        StartLine: startLine,
+        EndLine: endLine,
+        ContentHash: contentHash,
+        Text: "",
+        TableRows: 0,
+    };
+}
+function copyVec(v) {
+    return v.slice();
+}
+// cosineClusters is the L5-cosine pass. The exact set of rules it enforces is
+// the contract a single test pins, because each rule prevents a specific class
+// of false positive/negative in near-duplicate detection:
+//   (a) cross-file pairs at cosine >= threshold cluster (the whole point),
+//   (b) below-threshold pairs do NOT cluster (no spurious merges),
+//   (c) tables (no embedding) never cluster (cosine is prose-only),
+//   (d) same-file pairs are skipped (intra-file structure, not cross-doc dup),
+//   (e) excludeHashes (already an exact cluster) are not re-reported,
+//   (f) output order is deterministic (report/index consume it).
+describe("cosineClusters", () => {
+    const cfg = defaultConfig(); // Block.cosine_threshold = 0.95
+    // Build the fixture once; reused for determinism check.
+    function fixture() {
+        const vA = [1.0, 0.0];
+        const vB = [Math.cos(0.2257), Math.sin(0.2257)]; // cosine with vA ~ 0.975 (> 0.95)
+        const vC = [0.0, 1.0]; // cosine with vA = 0 (< threshold)
+        const vSameFile = [0.0, -1.0]; // perpendicular to all others
+        const vE = [Math.cos(0.15), Math.sin(0.15)];
+        const hashA = "prose-hash-a";
+        const hashBx = "prose-hash-bx";
+        const hashC = "prose-hash-c";
+        const hashTable = "table-hash";
+        const hashG1 = "prose-g1";
+        const hashG2 = "prose-g2";
+        const hashExcl = "prose-excl";
+        const hashE = "prose-hash-e";
+        const hashF = "prose-hash-f";
+        const blocks = [
+            // (a) cross-file near-copy pair
+            block("docs/a.md", "Intro", "prose", 1, 5, hashA),
+            block("docs/b.md", "Overview", "prose", 2, 6, hashBx),
+            // (b) below-threshold
+            block("docs/c.md", "Other", "prose", 1, 3, hashC),
+            // (c) table block (absent from blockEmb)
+            block("docs/a.md", "Table", "table", 6, 10, hashTable),
+            // (d) same-file pair in docs/g.md only
+            block("docs/g.md", "G1", "prose", 1, 5, hashG1),
+            block("docs/g.md", "G2", "prose", 6, 10, hashG2),
+            // (e) excluded hash
+            block("docs/d.md", "Dup", "prose", 1, 5, hashExcl),
+            // (f) second cross-file pair
+            block("docs/e.md", "Appendix", "prose", 1, 4, hashE),
+            block("docs/f.md", "Appendix", "prose", 1, 4, hashF),
+        ];
+        const blockEmb = new Map([
+            [hashA, vA],
+            [hashBx, vB],
+            [hashC, vC],
+            [hashG1, vSameFile],
+            [hashG2, copyVec(vSameFile)],
+            [hashExcl, vA], // would pair with hashA but is excluded
+            [hashE, vE],
+            [hashF, copyVec(vE)],
+            // hashTable intentionally absent
+        ]);
+        const excludeHashes = new Set([hashExcl]);
+        return { blocks, blockEmb, excludeHashes };
+    }
+    it("applies all six clustering rules (a-f)", () => {
+        const { blocks, blockEmb, excludeHashes } = fixture();
+        const clusters = cosineClusters(blocks, blockEmb, excludeHashes, cfg);
+        // (c) tables never cluster.
+        for (const cl of clusters) {
+            for (const loc of cl.Locations) {
+                expect(loc.StartLine === 6 && loc.FilePath === "docs/a.md" && loc.EndLine === 10).toBe(false);
+            }
+        }
+        // (d) same-file-only docs/g.md blocks never appear.
+        for (const cl of clusters) {
+            for (const loc of cl.Locations) {
+                expect(loc.FilePath).not.toBe("docs/g.md");
+            }
+        }
+        // (e) excluded hash (docs/d.md) never appears.
+        for (const cl of clusters) {
+            for (const loc of cl.Locations) {
+                expect(loc.FilePath).not.toBe("docs/d.md");
+            }
+        }
+        // (b) below-threshold block (docs/c.md) never appears.
+        for (const cl of clusters) {
+            for (const loc of cl.Locations) {
+                expect(loc.FilePath).not.toBe("docs/c.md");
+            }
+        }
+        // (a) a cluster with docs/a.md:1 and docs/b.md exists.
+        const abCluster = clusters.find((cl) => {
+            const hasA = cl.Locations.some((l) => l.FilePath === "docs/a.md" && l.StartLine === 1);
+            const hasB = cl.Locations.some((l) => l.FilePath === "docs/b.md");
+            return hasA && hasB;
+        });
+        expect(abCluster).toBeDefined();
+        expect(abCluster.Exact).toBe(false);
+        expect(abCluster.ContentHash).toBe("");
+        expect(abCluster.Kind).toBe("prose");
+        expect(abCluster.Similarity).toBeGreaterThanOrEqual(cfg.Block.cosine_threshold);
+    });
+    it("(f) is deterministic across repeated calls", () => {
+        const { blocks, blockEmb, excludeHashes } = fixture();
+        const c1 = cosineClusters(blocks, blockEmb, excludeHashes, cfg);
+        const c2 = cosineClusters(blocks, blockEmb, excludeHashes, cfg);
+        expect(c1.length).toBe(c2.length);
+        for (let i = 0; i < c1.length; i++) {
+            expect(c1[i].Kind).toBe(c2[i].Kind);
+            expect(c1[i].Similarity).toBe(c2[i].Similarity);
+            expect(c1[i].Locations).toEqual(c2[i].Locations);
+        }
+    });
+});

package/dist/dedup/analyzer/distinctive.js

@@ -0,0 +1,85 @@
+// Ported from internal/dedup/analyzer/distinctive.go.
+//
+// Distinctive-heading-token filtering for L2 promotion: a token is "distinctive"
+// when it is rare across the corpus of headings. Two headings sharing a
+// distinctive token get promoted from MAYBE to HIGH, so the frequency threshold
+// and tokenization are behavior-load-bearing and pinned to the Go output.
+/**
+ * Matches runs of ASCII letters and CJK characters (Unified Ideographs block).
+ * Mirrors the Go headingTokenRE = `[A-Za-z\x{4E00}-\x{9FFF}]+` (Python POC's
+ * _HEADING_TOKEN_RE = re.compile(r"[A-Za-z一-鿿]+")). The `u` flag makes the
+ * 一-鿿 range a single code-point class, matching Go's rune semantics.
+ */
+const headingTokenRE = /[A-Za-z一-鿿]+/gu;
+/**
+ * buildDistinctiveFilter computes the set of tokens that are considered
+ * "distinctive" in the given corpus of headings.
+ *
+ * A token is distinctive iff its frequency across distinct headings is
+ * <= max(distinctive_abs_min, trunc(distinctive_pct_of_headings * total_headings)).
+ * Universal stopwords and tokens shorter than minTokenLen are excluded.
+ *
+ * @param minTokenLen  minimum rune length (default 3)
+ */
+export function buildDistinctiveFilter(headings, cfg, minTokenLen = 3) {
+    const stopwords = new Set(cfg.universal_stopwords);
+    const freq = new Map();
+    let total = 0;
+    for (const h of headings) {
+        total++;
+        for (const t of headingTokensOf(h, minTokenLen, stopwords)) {
+            freq.set(t, (freq.get(t) ?? 0) + 1);
+        }
+    }
+    if (total === 0) {
+        return new Set();
+    }
+    let threshold = cfg.distinctive_abs_min;
+    // Go's int(float64(total) * pct) truncates toward zero.
+    const pctThreshold = Math.trunc(total * cfg.distinctive_pct_of_headings);
+    if (pctThreshold > threshold) {
+        threshold = pctThreshold;
+    }
+    const filter = new Set();
+    for (const [t, c] of freq) {
+        if (c <= threshold) {
+            filter.add(t);
+        }
+    }
+    return filter;
+}
+/**
+ * distinctiveTokensOf returns the set of distinctive tokens present in heading.
+ *
+ * @param minTokenLen  minimum rune length (default 3)
+ */
+export function distinctiveTokensOf(heading, distinctiveFilter, cfg, minTokenLen = 3) {
+    const stopwords = new Set(cfg.universal_stopwords);
+    const tokens = headingTokensOf(heading, minTokenLen, stopwords);
+    const result = new Set();
+    for (const t of tokens) {
+        if (distinctiveFilter.has(t)) {
+            result.add(t);
+        }
+    }
+    return result;
+}
+/**
+ * headingTokensOf tokenizes a heading: case-folds, keeps [A-Za-z一-鿿]+ runs,
+ * drops stopwords, drops tokens shorter than minLen (measured in runes).
+ */
+function headingTokensOf(heading, minLen, stopwords) {
+    const matches = heading.toLowerCase().match(headingTokenRE) ?? [];
+    const result = new Set();
+    for (const t of matches) {
+        // Go: len([]rune(t)) — rune count, not UTF-16 length.
+        if ([...t].length < minLen) {
+            continue;
+        }
+        if (stopwords.has(t)) {
+            continue;
+        }
+        result.add(t);
+    }
+    return result;
+}

package/dist/dedup/analyzer/distinctive.test.js

@@ -0,0 +1,49 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { buildDistinctiveFilter, distinctiveTokensOf } from "./distinctive.js";
+// Distinctive tokens drive L2 promotion (MAYBE -> HIGH when two headings share a
+// rare token). The frequency threshold, stopword drop, and min-length drop each
+// gate which tokens count, so a drift would re-promote or under-promote pairs.
+// These tests pin each gate by WHY it exists.
+describe("buildDistinctiveFilter", () => {
+    const cfg = defaultConfig().Analyzer;
+    it("returns rare tokens (freq <= threshold)", () => {
+        const headings = [
+            "Order Lifecycle",
+            "Order Management",
+            "Payment Processing",
+            "Payment Gateway",
+            "Authentication Flow",
+        ];
+        // 5 headings -> threshold = max(3, trunc(0.03*5)) = max(3, 0) = 3.
+        // "lifecycle" appears once -> distinctive.
+        const filter = buildDistinctiveFilter(headings, cfg);
+        expect(filter.size).toBeGreaterThan(0);
+        expect(filter.has("lifecycle")).toBe(true);
+    });
+    it("drops universal stopwords", () => {
+        // WHY: stopwords are common-by-design; counting them would falsely promote
+        // pairs that merely both say "the".
+        const headings = ["The Overview", "A Summary", "An Introduction"];
+        const filter = buildDistinctiveFilter(headings, cfg);
+        for (const stop of ["the", "a", "an"]) {
+            expect(filter.has(stop)).toBe(false);
+        }
+    });
+    it("drops tokens shorter than the min token length (3)", () => {
+        // WHY: 2-char tokens like "go" are too generic to be a reliable signal.
+        const headings = ["Go API", "Go CLI", "Go SDK"];
+        const filter = buildDistinctiveFilter(headings, cfg);
+        expect(filter.has("go")).toBe(false);
+    });
+});
+describe("distinctiveTokensOf", () => {
+    const cfg = defaultConfig().Analyzer;
+    it("returns only the heading tokens that are in the distinctive filter", () => {
+        const headings = ["Order Lifecycle", "Payment Processing", "Order Management"];
+        const filter = buildDistinctiveFilter(headings, cfg);
+        const tokens = distinctiveTokensOf("Order Lifecycle", filter, cfg);
+        // "lifecycle" is distinctive (freq=1 <= threshold=3) and present in heading.
+        expect(tokens.has("lifecycle")).toBe(true);
+    });
+});

package/dist/dedup/analyzer/exact_clusters.js

@@ -0,0 +1,63 @@
+// Ported from internal/dedup/analyzer/exact_clusters.go.
+//
+// L5-exact clustering: buckets blocks by ContentHash and emits a Cluster for
+// every hash that appears verbatim across >=2 distinct files. Blocks duplicated
+// only within one file are skipped (mirrors L3's same-file skip).
+import { cmpStr, cmpNum } from "./order.js";
+/**
+ * exactClusters buckets the given blocks by ContentHash and returns one Cluster
+ * per bucket that spans >=2 distinct FilePaths.
+ *
+ * Each emitted Cluster has:
+ *   - Exact: true, Similarity: 1.0, Informational: false
+ *   - Kind and ContentHash from the bucket
+ *   - Locations sorted by (FilePath, StartLine)
+ *
+ * The returned slice is sorted by ContentHash for deterministic output.
+ */
+export function exactClusters(blocks) {
+    const buckets = new Map();
+    for (const b of blocks) {
+        let bkt = buckets.get(b.ContentHash);
+        if (!bkt) {
+            bkt = { kind: b.Kind, locations: [] };
+            buckets.set(b.ContentHash, bkt);
+        }
+        bkt.locations.push({
+            FilePath: b.FilePath,
+            Heading: b.Heading,
+            StartLine: b.StartLine,
+            EndLine: b.EndLine,
+        });
+    }
+    const clusters = [];
+    for (const [hash, bkt] of buckets) {
+        // Check for >=2 distinct FilePaths.
+        const seen = new Set();
+        for (const loc of bkt.locations) {
+            seen.add(loc.FilePath);
+        }
+        if (seen.size < 2) {
+            continue;
+        }
+        // Sort locations by (FilePath, StartLine) for determinism.
+        const locs = bkt.locations.slice();
+        locs.sort((a, b) => {
+            const c = cmpStr(a.FilePath, b.FilePath);
+            if (c !== 0)
+                return c;
+            return cmpNum(a.StartLine, b.StartLine);
+        });
+        clusters.push({
+            Kind: bkt.kind,
+            ContentHash: hash,
+            Similarity: 1.0,
+            Exact: true,
+            Informational: false,
+            Locations: locs,
+        });
+    }
+    // Sort clusters by ContentHash for deterministic output.
+    clusters.sort((a, b) => cmpStr(a.ContentHash, b.ContentHash));
+    return clusters;
+}

package/dist/dedup/analyzer/exact_clusters.test.js

@@ -0,0 +1,81 @@
+import { describe, it, expect } from "vitest";
+import { exactClusters } from "./exact_clusters.js";
+// makeBlock builds a minimal BlockRecord for exactClusters tests.
+function makeBlock(filePath, heading, contentHash, kind, startLine, endLine) {
+    return {
+        SectionID: "",
+        FilePath: filePath,
+        Heading: heading,
+        Index: 0,
+        Kind: kind,
+        StartLine: startLine,
+        EndLine: endLine,
+        ContentHash: contentHash,
+        Text: "",
+        TableRows: 0,
+    };
+}
+// exactClusters is the L5-exact pass: it must surface a block that appears
+// VERBATIM across >=2 files, and must NOT surface one duplicated only within a
+// single file (that is intra-file structure, not a cross-doc duplication
+// problem). Determinism (sorted by ContentHash, locations sorted) is part of the
+// contract because the report and the dedup index consume the order.
+describe("exactClusters", () => {
+    it("clusters cross-file verbatim blocks, skips same-file-only duplicates, and sorts deterministically", () => {
+        const proseHash = "prose-hash-aaaa";
+        const tableHash = "table-hash-bbbb";
+        const sameFileHash = "same-file-hash-c";
+        const blocks = [
+            // (a) prose block shared across docs/a.md and docs/b.md
+            makeBlock("docs/a.md", "Overview", proseHash, "prose", 5, 10),
+            makeBlock("docs/b.md", "Summary", proseHash, "prose", 12, 17),
+            // (b) prose duplicated only within docs/a.md — must NOT cluster
+            makeBlock("docs/a.md", "Details", sameFileHash, "prose", 20, 25),
+            makeBlock("docs/a.md", "Appendix", sameFileHash, "prose", 30, 35),
+            // (c) table block shared across docs/a.md and docs/c.md
+            makeBlock("docs/a.md", "Data", tableHash, "table", 40, 50),
+            makeBlock("docs/c.md", "Data", tableHash, "table", 3, 13),
+        ];
+        const clusters = exactClusters(blocks);
+        // (b) same-file-only duplicate must NOT appear.
+        expect(clusters.some((cl) => cl.ContentHash === sameFileHash)).toBe(false);
+        // Exactly two clusters: prose + table.
+        expect(clusters).toHaveLength(2);
+        // (d) sorted by ContentHash: proseHash < tableHash lexicographically.
+        expect(clusters[0].ContentHash).toBe(proseHash);
+        expect(clusters[1].ContentHash).toBe(tableHash);
+        // (a) prose cluster properties.
+        const prose = clusters[0];
+        expect(prose.Exact).toBe(true);
+        expect(prose.Similarity).toBe(1.0);
+        expect(prose.Kind).toBe("prose");
+        expect(prose.Informational).toBe(false);
+        expect(prose.Locations).toHaveLength(2);
+        // Locations sorted by (FilePath, StartLine): a.md:5 before b.md:12.
+        expect(prose.Locations[0]).toMatchObject({ FilePath: "docs/a.md", StartLine: 5 });
+        expect(prose.Locations[1]).toMatchObject({ FilePath: "docs/b.md", StartLine: 12 });
+        // (c) table cluster properties.
+        const table = clusters[1];
+        expect(table.Exact).toBe(true);
+        expect(table.Kind).toBe("table");
+        expect(table.Locations).toHaveLength(2);
+        expect(table.Locations[0].FilePath).toBe("docs/a.md");
+        expect(table.Locations[1].FilePath).toBe("docs/c.md");
+    });
+    it("emits all locations of a {A,A,B} bucket (2 distinct files qualifies)", () => {
+        const hash = "shared-hash-xxxx";
+        const blocks = [
+            makeBlock("docs/a.md", "S1", hash, "prose", 1, 5),
+            makeBlock("docs/a.md", "S2", hash, "prose", 10, 14),
+            makeBlock("docs/b.md", "S3", hash, "prose", 2, 6),
+        ];
+        const clusters = exactClusters(blocks);
+        expect(clusters).toHaveLength(1);
+        const cl = clusters[0];
+        expect(cl.Locations).toHaveLength(3);
+        // Sorted: a.md:1, a.md:10, b.md:2.
+        expect(cl.Locations[0]).toMatchObject({ FilePath: "docs/a.md", StartLine: 1 });
+        expect(cl.Locations[1]).toMatchObject({ FilePath: "docs/a.md", StartLine: 10 });
+        expect(cl.Locations[2].FilePath).toBe("docs/b.md");
+    });
+});

package/dist/dedup/analyzer/index.js

@@ -0,0 +1,14 @@
+// Barrel for the analyzer package: the layered duplicate-detection algorithm.
+//
+// The public surface is the `analyze` orchestrator plus the canonical-selection,
+// distinctive-token, differentiator, and preview primitives it composes (all of
+// which Go exports for white-box tests and for reuse by the dedup facade).
+export { analyze } from "./analyzer.js";
+export { canonicalRank, isDisqualified, lessRank, pathPriorityRank, } from "./canonical.js";
+export { buildDistinctiveFilter, distinctiveTokensOf } from "./distinctive.js";
+export { findDifferentiators } from "./safety.js";
+export { computePreview } from "./preview.js";
+export { exactClusters } from "./exact_clusters.js";
+export { cosineClusters } from "./cosine_clusters.js";
+export { applyMultiplicity, suppressKnownGroups } from "./multiplicity.js";
+export { partialOverlaps } from "./partial_overlaps.js";