docsgov 0.1.0

package/dist/dedup/analyzer/analyzer.test.js

@@ -0,0 +1,530 @@
+// Ported from the TestAnalyze_* cases in internal/dedup/analyzer/analyzer_test.go.
+//
+// These are the end-to-end spec for the layered pipeline: they pin which layer
+// fires (L1 exact / L2 promotion / L3 cosine HIGH-vs-MAYBE / L4 differentiator /
+// L5 blocks), how confidence and recommended action are derived, when groups and
+// pairs are dropped, and that block-level overlaps are wired through. The
+// per-primitive behavior (clusters, canonical rank, preview, distinctive tokens,
+// differentiators, multiplicity, suppression) is covered by the sibling
+// *.test.ts files; this file exercises the orchestrator that composes them.
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { analyze } from "./analyzer.js";
+const cfg = defaultConfig();
+// makeSection builds a minimal Section (TS snake_case shape) mirroring the Go
+// makeSection helper: StartLine=1, EndLine=10, prose_word_count=15.
+function makeSection(id, filePath, heading, level, anchor, contentHash, rawContent, inbound) {
+    return {
+        id,
+        file_path: filePath,
+        heading,
+        heading_level: level,
+        anchor,
+        start_line: 1,
+        end_line: 10,
+        content_hash: contentHash,
+        raw_content: rawContent,
+        embed_text: heading,
+        prose_word_count: 15,
+        has_table: false,
+        has_code: false,
+        inbound_count: inbound,
+    };
+}
+// makeUnitVec2D returns the 2D unit vector in the direction (x, y); the analyzer
+// treats embeddings as L2-normalized so the dot product equals cosine.
+function makeUnitVec2D(x, y) {
+    const norm = Math.sqrt(x * x + y * y);
+    if (norm === 0) {
+        return [0, 0];
+    }
+    return [x / norm, y / norm];
+}
+// emb builds the section-id -> vector map the analyzer consumes.
+function emb(entries) {
+    return new Map(Object.entries(entries));
+}
+// L1 must union sections that share a content_hash (cross-file) into a HIGH
+// group and mark the duplicate ExactMatch, with no MAYBE pair — exact identity
+// is the strongest signal and short-circuits cosine.
+describe("analyze L1 exact grouping", () => {
+    it("groups equal-hash cross-file sections as HIGH with ExactMatch and no MAYBE pairs", () => {
+        const hash = "abcdef1234567890";
+        const raw = "## Overview\nSome content here for testing.";
+        const sA = makeSection("id1", "docs/a.md", "Overview", 2, "overview", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Overview", 2, "overview", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+        expect(report.MaybePairs).toHaveLength(0);
+        const grp = report.HighGroups[0];
+        expect(grp.Members.length).toBeGreaterThanOrEqual(1);
+        // L1 sets ExactMatch on the duplicate member.
+        expect(grp.Members.some((m) => m.ExactMatch)).toBe(true);
+    });
+    it("groups same-file equal-hash sections too (L1 does not skip same-file)", () => {
+        const hash = "samehash1234567890";
+        const sA = makeSection("id1", "docs/a.md", "Overview", 2, "overview", hash, "## Overview\nSome test content.", 0);
+        const sB = makeSection("id2", "docs/a.md", "Summary", 3, "summary", hash, "## Summary\nSome test content.", 0);
+        const v = makeUnitVec2D(1, 0);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+    });
+});
+// L3 turns a high-cosine cross-file pair into a HIGH group, but L3 SKIPS
+// same-file pairs entirely (neither HIGH nor MAYBE), and a mid-band cosine
+// without a shared distinctive heading token stays a MAYBE pair (never unioned).
+describe("analyze L3 cosine grouping", () => {
+    it("groups a cross-file cosine>=thresh_high pair as HIGH", () => {
+        const raw = "## Order Lifecycle\nThe order transitions through states.";
+        const sA = makeSection("id1", "docs/a.md", "Order Lifecycle", 2, "order-lifecycle", "hash1", raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Order Lifecycle", 2, "order-lifecycle", "hash2", raw, 0);
+        const v = makeUnitVec2D(0.6, 0.8); // identical directions -> cosine 1.0
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+    });
+    it("skips a same-file pair entirely (no HIGH group, no MAYBE pair)", () => {
+        const sA = makeSection("id1", "docs/a.md", "Order Lifecycle", 2, "order-lifecycle", "hash1", "## Order Lifecycle\nThe order transitions through states.", 0);
+        const sB = makeSection("id2", "docs/a.md", "Order States", 2, "order-states", "hash2", "## Order States\nThe order state machine details.", 0);
+        const v = makeUnitVec2D(0.6, 0.8);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups).toHaveLength(0);
+        expect(report.MaybePairs).toHaveLength(0);
+    });
+    it("reports a mid-band cosine without shared distinctive token as a MAYBE pair, not a group", () => {
+        // Headings "Uploading Documents" / "Downloading Reports" share no distinctive
+        // token, so no L2 promotion; cosine ~0.90 sits in [thresh_maybe, thresh_high).
+        const sA = makeSection("id1", "docs/a.md", "Uploading Documents", 2, "uploading-documents", "hash1", "## Uploading Documents\nContent about uploading various files.", 0);
+        const sB = makeSection("id2", "docs/b.md", "Downloading Reports", 2, "downloading-reports", "hash2", "## Downloading Reports\nContent about downloading various files.", 0);
+        const theta = 0.451; // arccos(0.90)
+        const v1 = [1.0, 0.0];
+        const v2 = [Math.cos(theta), Math.sin(theta)];
+        const report = analyze([sA, sB], emb({ id1: v1, id2: v2 }), null, null, cfg);
+        expect(report.MaybePairs.length).toBeGreaterThan(0);
+        expect(report.HighGroups).toHaveLength(0);
+    });
+});
+// L4 differentiators NEVER demote the confidence tier — they only force the
+// recommended action to manual_review. A cosine=1.0 sync/async pair must stay
+// confidence="high" but action="manual_review".
+describe("analyze L4 differentiator", () => {
+    it("keeps confidence high but forces action=manual_review when a differentiator is present", () => {
+        const sA = makeSection("id1", "docs/a.md", "Sync Processing", 2, "sync-processing", "hash1", "## Sync Processing\nThis section covers sync operations in detail.", 0);
+        const sB = makeSection("id2", "docs/b.md", "Async Processing", 2, "async-processing", "hash2", "## Async Processing\nThis section covers async operations in detail.", 0);
+        const v = makeUnitVec2D(0.6, 0.8);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+        const grp = report.HighGroups[0];
+        expect(grp.Confidence).toBe("high");
+        expect(grp.Action).toBe("manual_review");
+    });
+});
+// A group whose every member is disqualified (here both headings are the
+// blacklisted "Related") carries no actionable canonical, so it is dropped from
+// the report entirely.
+describe("analyze disqualified group filtering", () => {
+    it("drops a group where every member is disqualified", () => {
+        const hash = "disqhash12345678";
+        const raw = "## Related\nSee also other sections.";
+        const sA = makeSection("id1", "docs/a.md", "Related", 2, "related", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Related", 2, "related", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups).toHaveLength(0);
+    });
+});
+// Confidence/action mapping: an all-exact (or all-HIGH) group with no
+// differentiator yields confidence="high" and action="replace_with_reference".
+describe("analyze confidence mapping", () => {
+    it("maps an all-exact group to high + replace_with_reference", () => {
+        const hash = "highconfhash1234";
+        const raw = "## Lifecycle\nLong content here for canonical test.";
+        const sA = makeSection("id1", "docs/concepts/a.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const sB = makeSection("id2", "docs/design/b.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+        const grp = report.HighGroups[0];
+        expect(grp.Confidence).toBe("high");
+        expect(grp.Action).toBe("replace_with_reference");
+    });
+});
+// Member previews are pre-computed by the orchestrator (so the renderer never
+// re-derives them): the canonical's Preview is non-empty and has the heading
+// line stripped.
+describe("analyze member preview", () => {
+    it("pre-computes the canonical member preview with the heading line stripped", () => {
+        const hash = "prevhash12345678";
+        const raw = "## Lifecycle\nThis is the detailed content of this lifecycle section.";
+        const sA = makeSection("id1", "docs/a.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const report = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(report.HighGroups.length).toBeGreaterThan(0);
+        const grp = report.HighGroups[0];
+        expect(grp.Canonical.Preview).not.toBe("");
+        expect(grp.Canonical.Preview).not.toContain("## Lifecycle");
+    });
+});
+// L5 wiring: a verbatim cross-file block surfaces as ONE exact PartialOverlap
+// cluster and is NOT re-reported as a cosine cluster (exact-pass excludeHashes
+// suppresses the duplicate). Passing nil blocks yields an empty PartialOverlaps
+// without disturbing the L1-L4 output.
+describe("analyze L5 partial overlaps", () => {
+    it("populates PartialOverlaps from a verbatim block and dedups exact-vs-cosine", () => {
+        const sA = makeSection("secA", "docs/a.md", "Overview", 2, "overview", "secHash12345678", "## Overview\nSome unique content here.", 0);
+        const sB = makeSection("secB", "docs/b.md", "Details", 2, "details", "differentHash1", "## Details\nCompletely different content.", 0);
+        const v = makeUnitVec2D(1, 0);
+        const vB = makeUnitVec2D(0, 1); // orthogonal -> no L3 match
+        const sectionEmb = emb({ secA: v, secB: vB });
+        const verbatimHash = "verbatim-block-hash-xyz";
+        const blocks = [
+            mkBlock("secA", "docs/a.md", "Overview", "prose", 3, 6, verbatimHash),
+            mkBlock("secB", "docs/b.md", "Details", "prose", 4, 7, verbatimHash),
+        ];
+        // verbatimHash also has a block embedding: it would be a cosine candidate
+        // were it not added to excludeHashes by the exact pass.
+        const blockEmb = emb({ [verbatimHash]: makeUnitVec2D(0.5, 0.5) });
+        const rep = analyze([sA, sB], sectionEmb, blocks, blockEmb, cfg);
+        expect(rep.PartialOverlaps).toHaveLength(1);
+        const cl = rep.PartialOverlaps[0];
+        expect(cl.Exact).toBe(true);
+        expect(cl.Kind).toBe("prose");
+        expect(cl.ContentHash).toBe(verbatimHash);
+        expect(cl.Locations).toHaveLength(2);
+        // The verbatim block appears exactly once as exact, never as cosine.
+        const exactCount = rep.PartialOverlaps.filter((c) => c.ContentHash === verbatimHash && c.Exact).length;
+        const cosineCount = rep.PartialOverlaps.filter((c) => c.ContentHash === verbatimHash && !c.Exact).length;
+        expect(exactCount).toBe(1);
+        expect(cosineCount).toBe(0);
+    });
+    it("yields empty PartialOverlaps for nil blocks without disturbing L1-L4 output", () => {
+        const hash = "noBlockHash12345";
+        const raw = "## Lifecycle\nContent for lifecycle section testing.";
+        const sA = makeSection("id1", "docs/a.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const rep = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, cfg);
+        expect(rep.PartialOverlaps).toHaveLength(0);
+        expect(rep.HighGroups.length).toBeGreaterThan(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+    });
+});
+// Fewer than two sections cannot form a pair or group, so Analyze must
+// short-circuit to a fully-empty report before doing any embedding work. Pinned
+// because the rest of the pipeline assumes n>=2 (pair loops, union-find).
+describe("analyze trivial input", () => {
+    // WHY: a single section can never duplicate anything; returning early avoids
+    // wasted work and guarantees an empty, well-formed report.
+    it("returns an empty report for fewer than two sections", () => {
+        const s = makeSection("only", "docs/a.md", "Solo", 2, "solo", "h", "## Solo\nx", 0);
+        const rep = analyze([s], emb({ only: makeUnitVec2D(1, 0) }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+        expect(rep.PartialOverlaps).toHaveLength(0);
+    });
+    // WHY: zero sections is the degenerate boundary; it must not throw and must
+    // yield an empty report (same n<2 guard).
+    it("returns an empty report for zero sections", () => {
+        const rep = analyze([], new Map(), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+        expect(rep.PartialOverlaps).toHaveLength(0);
+    });
+});
+// L2 promotion is the layer that rescues a mid-band cosine pair from MAYBE to
+// HIGH when the headings share a distinctive token. Without it, near-duplicate
+// sections with telltale shared headings would be under-reported as mere pairs.
+describe("analyze L2 heading promotion", () => {
+    // WHY: a [thresh_maybe, thresh_high) cosine WITH a shared distinctive heading
+    // token must be PROMOTED to a HIGH group (not left a MAYBE pair). This pins the
+    // promotion branch that the heading signal triggers.
+    it("promotes a mid-band cosine pair with a shared distinctive heading token to HIGH", () => {
+        // Three shared distinctive tokens (zebra, payment, apple) so the reason
+        // builder must SORT them lexicographically (apple, payment, zebra),
+        // exercising both directions of the string comparator.
+        const sA = makeSection("id1", "docs/a.md", "Zebra Payment Apple", 2, "zebra-payment-apple", "h1", "## Zebra Payment Apple\nReconcile incoming payments daily here.", 0);
+        const sB = makeSection("id2", "docs/b.md", "Zebra Payment Apple", 2, "zebra-payment-apple", "h2", "## Zebra Payment Apple\nReconcile the incoming payments daily here.", 0);
+        // cosine ~0.90 sits inside [0.86, 0.93); the shared distinctive tokens
+        // (freq 2 <= distinctive threshold 3) drive the promotion.
+        const theta = Math.acos(0.9);
+        const v1 = [1, 0];
+        const v2 = [Math.cos(theta), Math.sin(theta)];
+        const rep = analyze([sA, sB], emb({ id1: v1, id2: v2 }), null, null, cfg);
+        expect(rep.HighGroups.length).toBeGreaterThan(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+        // The HIGH reason set records the shared distinctive heading tokens, sorted.
+        const grp = rep.HighGroups[0];
+        const allReasons = grp.Members.flatMap((m) => m.Reasons).join(" ");
+        expect(allReasons).toContain("shared distinctive heading tokens: apple, payment, zebra");
+    });
+});
+// A custom heading_token_min_len of 0 forces the analyzer to fall back to the
+// default minimum (3). Pinned because the fallback guards against a misconfigured
+// length that would otherwise admit one- and two-letter heading tokens.
+describe("analyze min-token-len fallback", () => {
+    // WHY: heading_token_min_len <= 0 must be coerced to 3, so the distinctive
+    // filter behaves identically to the default. We assert grouping still works,
+    // exercising the <=0 fallback path.
+    it("coerces a non-positive heading_token_min_len to the default", () => {
+        const custom = defaultConfig();
+        custom.Markdown.heading_token_min_len = 0;
+        const hash = "minlenhash123456";
+        const raw = "## Reconciliation\nSome content for the fallback test here.";
+        const sA = makeSection("id1", "docs/a.md", "Reconciliation", 2, "reconciliation", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Reconciliation", 2, "reconciliation", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const rep = analyze([sA, sB], emb({ id1: v, id2: v.slice() }), null, null, custom);
+        expect(rep.HighGroups.length).toBeGreaterThan(0);
+    });
+});
+// L1 must union three sections that share a content_hash into ONE group; the
+// third union sees an already-merged root, exercising the no-op union branch.
+describe("analyze L1 three-way exact", () => {
+    // WHY: three equal-hash cross-file sections collapse into a single HIGH group,
+    // not three pairwise groups. This pins union-find idempotence (the third
+    // union(ra===rb) is a no-op) and the all-exact -> high confidence mapping.
+    it("collapses three equal-hash sections into one HIGH group", () => {
+        const hash = "triplehash123456";
+        const raw = "## Shared\nThe very same content repeated across three docs.";
+        const sA = makeSection("id1", "docs/a.md", "Shared", 2, "shared", hash, raw, 0);
+        const sB = makeSection("id2", "docs/b.md", "Shared", 2, "shared", hash, raw, 0);
+        const sC = makeSection("id3", "docs/c.md", "Shared", 2, "shared", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        const rep = analyze([sA, sB, sC], emb({ id1: v, id2: v.slice(), id3: v.slice() }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(1);
+        // canonical + 2 duplicate members = 3 sections in one group.
+        expect(rep.HighGroups[0].Members).toHaveLength(2);
+        expect(rep.HighGroups[0].Confidence).toBe("high");
+    });
+});
+// A transitive HIGH group is built when A~B and B~C cross the high bar but A~C is
+// below thresh_maybe (no recorded pair). The canonical's link to the far member
+// must be reconstructed via the best transitive pair — this is the contract that
+// keeps a distant group member's similarity/reason non-empty.
+describe("analyze transitive group", () => {
+    // WHY: when the canonical has no DIRECT recorded pair with a transitively
+    // grouped member, buildGroup must fall back to the best pair among earlier
+    // members (findBestPairInfo) rather than emitting a 0-similarity bare member.
+    // A regression that dropped this fallback would mislabel real duplicates.
+    it("reconstructs a transitive member's similarity from the best intermediate pair", () => {
+        // A is the best canonical (docs/concepts/ has top path priority). B ranks
+        // before C (longer raw_content -> better NegLen) so sorted = [A, B, C].
+        // A·B = 0.95 (HIGH, unioned). B·C = 0.94 (HIGH, unioned). A·C = 0.80
+        // (< thresh_maybe, NOT recorded) -> C is only transitively in the group, with
+        // no direct (A,C) pair. Headings share no token (no L2 promotion noise).
+        const sA = makeSection("idA", "docs/concepts/a.md", "Alpha", 2, "alpha", "hA", "## Alpha\nAlpha content here.", 0);
+        const sB = makeSection("idB", "docs/guides/b.md", "Bravo", 2, "bravo", "hB", "## Bravo\nBravo content here padded longer to win NegLen tiebreak.", 0);
+        const sC = makeSection("idC", "docs/guides/c.md", "Charlie", 2, "charlie", "hC", "## Charlie\nShort.", 0);
+        const vA = [1, 0, 0];
+        const vB = [0.95, Math.sqrt(1 - 0.95 * 0.95), 0];
+        const vC = [0.8, 0.5765, 0.166]; // |vC|~=1; A·C=0.80, B·C~=0.94
+        const rep = analyze([sA, sB, sC], emb({ idA: vA, idB: vB, idC: vC }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(1);
+        const grp = rep.HighGroups[0];
+        // A wins canonical via path priority.
+        expect(grp.Canonical.SectionID).toBe("idA");
+        expect(grp.Members).toHaveLength(2);
+        // The transitive member (idC) has no direct (A,C) pair; buildGroup falls back
+        // to the best pair among earlier members and finds (B,C), so idC carries that
+        // pair's similarity (not the 0.0 placeholder).
+        const cMember = grp.Members.find((m) => m.SectionID === "idC");
+        expect(cMember.Similarity).toBeGreaterThan(0.9);
+    });
+    // WHY: when ALL three pairwise links are recorded and one is a plain MAYBE
+    // (mid-band, no shared token) yet the trio is still grouped via promotion/high,
+    // the group confidence must drop to "medium" (its weakest internal pair is not
+    // high/promoted). This pins the medium-tier mapping the renderer keys on.
+    it("downgrades a group with a non-high internal pair to medium confidence", () => {
+        // A·B = 0.94 (HIGH). B·C = 0.90 + shared "settlement" -> promoted (HIGH).
+        // A·C = 0.90, NO shared distinctive token -> plain MAYBE (recorded, internal).
+        const sA = makeSection("idA", "docs/a.md", "Payment Reconciliation", 2, "payment-reconciliation", "hA", "## Payment Reconciliation\nReconcile payments across the ledger.", 0);
+        const sB = makeSection("idB", "docs/b.md", "Payment Settlement", 2, "payment-settlement", "hB", "## Payment Settlement\nSettle payments across the ledger.", 0);
+        const sC = makeSection("idC", "docs/c.md", "Settlement Workflow", 2, "settlement-workflow", "hC", "## Settlement Workflow\nWorkflow that drives settlement.", 0);
+        // vA=(1,0,0); A·B=0.94; A·C=0.90; B·C=0.90.
+        const vA = [1, 0, 0];
+        const vB = [0.94, Math.sqrt(1 - 0.94 * 0.94), 0]; // (0.94, 0.34117..., 0)
+        // Solve vC: c1=0.90; B·C=0.94*0.90 + 0.34117*c2 = 0.90 -> c2=0.15828;
+        // c3=sqrt(1 - 0.90^2 - 0.15828^2).
+        const c1 = 0.9;
+        const c2 = (0.9 - 0.94 * 0.9) / Math.sqrt(1 - 0.94 * 0.94);
+        const c3 = Math.sqrt(1 - c1 * c1 - c2 * c2);
+        const vC = [c1, c2, c3];
+        const rep = analyze([sA, sB, sC], emb({ idA: vA, idB: vB, idC: vC }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(1);
+        expect(rep.HighGroups[0].Confidence).toBe("medium");
+        expect(rep.HighGroups[0].Action).toBe("manual_review");
+        // The plain-MAYBE (A,C) pair is internal to the group, so it is NOT also
+        // emitted as a standalone MAYBE pair.
+        expect(rep.MaybePairs).toHaveLength(0);
+    });
+});
+// Possible (MAYBE) pairs are filtered: a pair whose BOTH endpoints are
+// disqualified carries no actionable canonical and must be dropped before it
+// reaches the report. Pinned because surfacing an un-actionable pair wastes
+// reviewer time.
+describe("analyze MAYBE pair filtering", () => {
+    // WHY: both sections sit on blacklisted (deprecated) paths, so even a genuine
+    // mid-band cosine pair has no canonical worth keeping -> dropped.
+    it("drops a MAYBE pair when both endpoints are disqualified", () => {
+        const sA = makeSection("id1", "docs/deprecated/a.md", "Uploading Documents", 2, "uploading-documents", "h1", "## Uploading Documents\nContent about uploading various files here.", 0);
+        const sB = makeSection("id2", "docs/deprecated/b.md", "Downloading Reports", 2, "downloading-reports", "h2", "## Downloading Reports\nContent about downloading various files here.", 0);
+        const theta = 0.451; // arccos(0.90) -> mid-band, no shared distinctive token
+        const v1 = [1, 0];
+        const v2 = [Math.cos(theta), Math.sin(theta)];
+        const rep = analyze([sA, sB], emb({ id1: v1, id2: v2 }), null, null, cfg);
+        expect(rep.MaybePairs).toHaveLength(0);
+        expect(rep.HighGroups).toHaveLength(0);
+    });
+    // WHY: when a surviving MAYBE pair's second endpoint (index j) ranks BETTER
+    // than the first (index i), the canonical must be j, not i. This pins the
+    // canonical-by-rank else branch in buildPair; choosing the wrong canonical
+    // would point reviewers at the worse section to keep.
+    it("elects the higher-priority endpoint as the pair canonical even when it is index j", () => {
+        // Index 0 (worse: plain path, deep heading); index 1 (better: docs/concepts/).
+        const sWorse = makeSection("id1", "docs/misc/a.md", "Uploading Documents", 4, "uploading-documents", "h1", "## Uploading Documents\nContent about uploading various files here.", 0);
+        const sBetter = makeSection("id2", "docs/concepts/b.md", "Downloading Reports", 2, "downloading-reports", "h2", "## Downloading Reports\nContent about downloading various files here.", 0);
+        const theta = 0.451;
+        const v1 = [1, 0];
+        const v2 = [Math.cos(theta), Math.sin(theta)];
+        const rep = analyze([sWorse, sBetter], emb({ id1: v1, id2: v2 }), null, null, cfg);
+        expect(rep.MaybePairs).toHaveLength(1);
+        // The better-ranked section (index j = id2) is elected canonical.
+        expect(rep.MaybePairs[0].Canonical.SectionID).toBe("id2");
+        expect(rep.MaybePairs[0].Candidate.SectionID).toBe("id1");
+    });
+});
+// Output ordering is part of the report contract: HIGH groups are sorted by
+// (-size, -best_similarity) so the biggest/strongest duplicates lead, and MAYBE
+// pairs are sorted by descending similarity. Pinned because a renderer that
+// trusted input order would present duplicates in a meaningless sequence.
+describe("analyze output ordering", () => {
+    // WHY: two HIGH groups of different sizes must be ordered largest-first
+    // (sortGroups via groupScore's -size term). A regression that lost the size
+    // ordering would bury the most impactful duplicate group below a smaller one.
+    it("orders HIGH groups largest-first", () => {
+        // Group X: 2 exact sections (hashX). Group Y: 3 exact sections (hashY).
+        const rawX = "## Topic X\nContent X repeated verbatim across two files here.";
+        const rawY = "## Topic Y\nContent Y repeated verbatim across three files here.";
+        const x1 = makeSection("x1", "docs/x1.md", "Topic X", 2, "topic-x", "hashX", rawX, 0);
+        const x2 = makeSection("x2", "docs/x2.md", "Topic X", 2, "topic-x", "hashX", rawX, 0);
+        const y1 = makeSection("y1", "docs/y1.md", "Topic Y", 2, "topic-y", "hashY", rawY, 0);
+        const y2 = makeSection("y2", "docs/y2.md", "Topic Y", 2, "topic-y", "hashY", rawY, 0);
+        const y3 = makeSection("y3", "docs/y3.md", "Topic Y", 2, "topic-y", "hashY", rawY, 0);
+        // X sections embed along dim 0, Y sections along dim 1 (orthogonal) so the
+        // two exact groups never cross-union via L3 cosine.
+        const vx = [1, 0];
+        const vy = [0, 1];
+        const rep = analyze([x1, x2, y1, y2, y3], emb({ x1: vx, x2: vx.slice(), y1: vy, y2: vy.slice(), y3: vy.slice() }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(2);
+        // The 3-member group (Y) sorts before the 2-member group (X).
+        expect(rep.HighGroups[0].Members).toHaveLength(2); // canonical + 2 dups = 3
+        expect(rep.HighGroups[1].Members).toHaveLength(1); // canonical + 1 dup = 2
+        expect(rep.HighGroups[0].Canonical.Heading).toBe("Topic Y");
+    });
+    // WHY: two surviving MAYBE pairs with different similarities must be ordered
+    // most-similar-first (sortBySimDesc). A reviewer scans the top of the list, so
+    // the strongest candidate must lead.
+    it("orders MAYBE pairs by descending similarity", () => {
+        // Pair 1: cosine ~0.91. Pair 2: cosine ~0.88. Neither shares a distinctive
+        // heading token, so both stay MAYBE (no promotion, no union).
+        const p1a = makeSection("p1a", "docs/p1a.md", "Uploading Documents", 2, "uploading-documents", "h1a", "## Uploading Documents\nContent about uploading various files here.", 0);
+        const p1b = makeSection("p1b", "docs/p1b.md", "Downloading Reports", 2, "downloading-reports", "h1b", "## Downloading Reports\nContent about downloading various files here.", 0);
+        const p2a = makeSection("p2a", "docs/p2a.md", "Sending Emails", 2, "sending-emails", "h2a", "## Sending Emails\nContent about sending various messages here.", 0);
+        const p2b = makeSection("p2b", "docs/p2b.md", "Receiving Faxes", 2, "receiving-faxes", "h2b", "## Receiving Faxes\nContent about receiving various messages here.", 0);
+        // Each pair lives in its own orthogonal 4D subspace (pair1 in dims 0-1, pair2
+        // in dims 2-3) so the only mid-band cosines are WITHIN each pair; cross-pair
+        // dot products are 0 and never form spurious pairs.
+        const high = Math.acos(0.91);
+        const low = Math.acos(0.88);
+        const rep = analyze([p1a, p1b, p2a, p2b], emb({
+            p1a: [1, 0, 0, 0],
+            p1b: [Math.cos(high), Math.sin(high), 0, 0],
+            p2a: [0, 0, 1, 0],
+            p2b: [0, 0, Math.cos(low), Math.sin(low)],
+        }), null, null, cfg);
+        expect(rep.MaybePairs).toHaveLength(2);
+        // Descending similarity: the 0.91 pair leads the 0.88 pair.
+        expect(rep.MaybePairs[0].Similarity).toBeGreaterThan(rep.MaybePairs[1].Similarity);
+    });
+    // WHY: when the FIRST endpoint (index i) is the better canonical, buildPair
+    // must keep i as canonical (the then-branch). Combined with the j-better case
+    // above, this pins both arms of the canonical-by-rank choice for pairs.
+    it("keeps index i as the pair canonical when it ranks better", () => {
+        // index 0 sits in docs/concepts/ (top priority) -> i is the better canonical.
+        const sBetter = makeSection("id1", "docs/concepts/a.md", "Uploading Documents", 2, "uploading-documents", "h1", "## Uploading Documents\nContent about uploading various files here.", 0);
+        const sWorse = makeSection("id2", "docs/misc/b.md", "Downloading Reports", 4, "downloading-reports", "h2", "## Downloading Reports\nContent about downloading various files here.", 0);
+        const theta = 0.451;
+        const rep = analyze([sBetter, sWorse], emb({ id1: [1, 0], id2: [Math.cos(theta), Math.sin(theta)] }), null, null, cfg);
+        expect(rep.MaybePairs).toHaveLength(1);
+        expect(rep.MaybePairs[0].Canonical.SectionID).toBe("id1");
+        expect(rep.MaybePairs[0].Candidate.SectionID).toBe("id2");
+    });
+});
+// dotProduct must treat a missing or short embedding as cosine 0 (never NaN/throw)
+// so a section without a usable vector simply forms no pairs. Pinned because a
+// missing embedding must degrade gracefully, not corrupt the whole report.
+describe("analyze missing/short embeddings", () => {
+    // WHY: a section with NO embedding entry yields dotProduct 0 (< thresh_maybe),
+    // producing no pair — the analyzer must not crash on the undefined vector.
+    it("treats a section with no embedding as cosine 0 (no pair)", () => {
+        const sA = makeSection("id1", "docs/a.md", "Alpha", 2, "alpha", "h1", "## Alpha\nContent for the missing-embedding test here.", 0);
+        const sB = makeSection("id2", "docs/b.md", "Beta", 2, "beta", "h2", "## Beta\nContent for the missing-embedding test here.", 0);
+        // id2 deliberately omitted from the embedding map -> vecs[1] is undefined.
+        const rep = analyze([sA, sB], emb({ id1: makeUnitVec2D(1, 0) }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+    });
+    // WHY: an embedding shorter than its counterpart (length mismatch) also yields
+    // cosine 0 via the length guard — defends against ragged vectors producing a
+    // spurious high score from a truncated dot product.
+    it("treats a length-mismatched embedding as cosine 0 (no pair)", () => {
+        const sA = makeSection("id1", "docs/a.md", "Alpha", 2, "alpha", "h1", "## Alpha\nContent for the short-embedding test here.", 0);
+        const sB = makeSection("id2", "docs/b.md", "Beta", 2, "beta", "h2", "## Beta\nContent for the short-embedding test here.", 0);
+        // id2 has an empty vector; b.length (0) < n (2) -> dotProduct returns 0.
+        const rep = analyze([sA, sB], emb({ id1: [1, 0], id2: [] }), null, null, cfg);
+        expect(rep.HighGroups).toHaveLength(0);
+        expect(rep.MaybePairs).toHaveLength(0);
+    });
+});
+// L5 secGroup wiring: when a HIGH group exists AND blocks are supplied, the
+// section->group map must be populated so suppressKnownGroups can drop block
+// overlaps already covered by that group. Pinned because an unpopulated secGroup
+// would let L5 double-report blocks already captured by L1-L4.
+describe("analyze L5 secGroup population", () => {
+    // WHY: two grouped sections each contribute a verbatim block; because both
+    // blocks' sections are in the SAME HIGH group, the block cluster is suppressed
+    // (already known) rather than re-reported by L5.
+    it("suppresses a block cluster whose sections are all in one HIGH group", () => {
+        const hash = "l5grouphash12345";
+        const raw = "## Lifecycle\nThe shared lifecycle content used in both docs.";
+        const sA = makeSection("secA", "docs/a.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const sB = makeSection("secB", "docs/b.md", "Lifecycle", 2, "lifecycle", hash, raw, 0);
+        const v = makeUnitVec2D(1, 0);
+        // A verbatim block shared by both grouped sections.
+        const blockHash = "covered-block-hash";
+        const blocks = [
+            mkBlock("secA", "docs/a.md", "Lifecycle", "prose", 3, 6, blockHash),
+            mkBlock("secB", "docs/b.md", "Lifecycle", "prose", 3, 6, blockHash),
+        ];
+        const rep = analyze([sA, sB], emb({ secA: v, secB: v.slice() }), blocks, null, cfg);
+        expect(rep.HighGroups.length).toBeGreaterThan(0);
+        // Both block locations resolve to the same HIGH group -> suppressed.
+        expect(rep.PartialOverlaps).toHaveLength(0);
+    });
+});
+// mkBlock builds a minimal BlockRecord for the L5 wiring tests.
+function mkBlock(sectionID, filePath, heading, kind, startLine, endLine, contentHash) {
+    return {
+        SectionID: sectionID,
+        FilePath: filePath,
+        Heading: heading,
+        Index: 0,
+        Kind: kind,
+        StartLine: startLine,
+        EndLine: endLine,
+        ContentHash: contentHash,
+        Text: "",
+        TableRows: 0,
+    };
+}

package/dist/dedup/analyzer/canonical.js

@@ -0,0 +1,74 @@
+// Ported from internal/dedup/analyzer/canonical.go.
+//
+// Canonical-section selection: each duplicate group elects the section with the
+// lowest rank tuple as its canonical reference. The tuple ordering is the
+// behavior contract — drifting any field's comparison re-elects a different
+// canonical and changes the recommended edits, so the order is pinned exactly.
+import { headingBlacklisted } from "../dedupcfg/index.js";
+/**
+ * isDisqualified returns true if the section matches the heading or path
+ * blacklist (case-insensitive substring on the path, shared heading matcher).
+ */
+export function isDisqualified(s, cfg) {
+    if (headingBlacklisted(cfg, s.heading)) {
+        return true;
+    }
+    const path = s.file_path.toLowerCase();
+    for (const tok of cfg.path_blacklist) {
+        if (path.includes(tok.toLowerCase())) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * canonicalRank computes the rank tuple for a section.
+ * Lower tuple (compared via lessRank) = better canonical.
+ *
+ * NegLen mirrors Go's -len(raw_content), which is the UTF-8 BYTE length, not the
+ * rune/UTF-16 count — kept exact so the longest-content tiebreak matches Go.
+ */
+export function canonicalRank(s, cfg) {
+    const disq = isDisqualified(s, cfg) ? 1 : 0;
+    return {
+        Disqualified: disq,
+        PathPriority: pathPriorityRank(s.file_path, cfg.path_priority),
+        NegInbound: -s.inbound_count,
+        HeadingLevel: s.heading_level,
+        NegLen: -Buffer.byteLength(s.raw_content, "utf8"),
+        SectionID: s.id,
+    };
+}
+/** lessRank returns true if a is strictly better (lower) than b. */
+export function lessRank(a, b) {
+    if (a.Disqualified !== b.Disqualified) {
+        return a.Disqualified < b.Disqualified;
+    }
+    if (a.PathPriority !== b.PathPriority) {
+        return a.PathPriority < b.PathPriority;
+    }
+    if (a.NegInbound !== b.NegInbound) {
+        return a.NegInbound < b.NegInbound;
+    }
+    if (a.HeadingLevel !== b.HeadingLevel) {
+        return a.HeadingLevel < b.HeadingLevel;
+    }
+    if (a.NegLen !== b.NegLen) {
+        return a.NegLen < b.NegLen;
+    }
+    return a.SectionID < b.SectionID;
+}
+/**
+ * pathPriorityRank returns the path priority index for the given file path.
+ * Lower index = higher priority. Path is case-folded before comparison.
+ */
+export function pathPriorityRank(filePath, pathPriority) {
+    const p = filePath.toLowerCase();
+    for (let i = 0; i < pathPriority.length; i++) {
+        const prefix = pathPriority[i];
+        if (p.startsWith(prefix.toLowerCase())) {
+            return i;
+        }
+    }
+    return pathPriority.length;
+}