docsgov 0.1.0

package/dist/dedup/analyzer/multiplicity.js

@@ -0,0 +1,110 @@
+// Ported from internal/dedup/analyzer/multiplicity.go.
+//
+// Two post-processing passes over L5 clusters:
+//   - applyMultiplicity: downgrade boilerplate (a block in too many files) to
+//     Informational so it is reported but not flagged as accidental duplication.
+//   - suppressKnownGroups: drop clusters already fully captured by one L1-L4 HIGH
+//     group so L5 only surfaces NOVEL overlaps.
+// The cap comparison and the "all-in-one-real-group" suppression predicate are
+// behavior-load-bearing and reproduced exactly.
+/**
+ * applyMultiplicity iterates over clusters and, for each, counts the number of
+ * DISTINCT FilePaths across its Locations. If that count EXCEEDS
+ * cfg.Block.multiplicity_cap, the cluster is downgraded to Informational=true
+ * (boilerplate heuristic). Otherwise Informational is left as-is.
+ *
+ * Returns a NEW slice of NEW Cluster values; input clusters are never mutated.
+ */
+export function applyMultiplicity(clusters, cfg) {
+    const multiplicityCap = cfg.Block.multiplicity_cap;
+    const out = [];
+    for (const cl of clusters) {
+        const seen = new Set();
+        for (const loc of cl.Locations) {
+            seen.add(loc.FilePath);
+        }
+        const newCl = { ...cl }; // copy all fields
+        if (seen.size > multiplicityCap) {
+            newCl.Informational = true;
+        }
+        out.push(newCl);
+    }
+    return out;
+}
+/**
+ * suppressKnownGroups drops clusters that are already fully captured by a single
+ * L1-L4 HIGH group, so L5 only surfaces novel overlaps.
+ *
+ * secGroup maps section_id -> HIGH-group id; only sections that are members of a
+ * multi-member HIGH group appear. Sections absent from the map are "ungrouped".
+ *
+ * Because BlockLocation does not carry a section_id, this builds a join index
+ * from blocks: (FilePath, StartLine) -> SectionID. StartLine is unique within a
+ * file, so the key is sufficient.
+ *
+ * SUPPRESSION RULE: for each cluster, resolve every location to its section's
+ * group key — secGroup[sectionID] if present, else a unique sentinel per location
+ * (ungrouped). DROP the cluster IFF every location resolves to the SAME single
+ * real HIGH group. Otherwise KEEP the cluster unchanged.
+ */
+export function suppressKnownGroups(clusters, blocks, secGroup) {
+    // Build join index: (FilePath, StartLine) -> SectionID.
+    // A NUL-joined composite key cannot collide for distinct (path, line) pairs.
+    const joinIdx = new Map();
+    for (const b of blocks) {
+        joinIdx.set(locKey(b.FilePath, b.StartLine), b.SectionID);
+    }
+    // uniqueSentinel generates a unique NEGATIVE integer sentinel so ungrouped
+    // locations never collide with a real group id (all real group ids are >=0)
+    // and never collide with each other.
+    let sentinelNext = -1;
+    const nextSentinel = () => {
+        const v = sentinelNext;
+        sentinelNext--;
+        return v;
+    };
+    const out = [];
+    for (const cl of clusters) {
+        // Resolve each location to its group id or a unique sentinel.
+        const groupIDs = [];
+        for (const loc of cl.Locations) {
+            // Go: missing join entry yields the zero value "" for sectionID, which
+            // then misses secGroup too — preserved by the ?? "" default below.
+            const sectionID = joinIdx.get(locKey(loc.FilePath, loc.StartLine)) ?? "";
+            const gid = secGroup.get(sectionID);
+            if (gid !== undefined) {
+                groupIDs.push(gid);
+            }
+            else {
+                // Ungrouped: unique sentinel so it cannot equal any real group.
+                groupIDs.push(nextSentinel());
+            }
+        }
+        // Check if all locations resolve to the SAME single real group id.
+        // A real group id is >=0 (sentinels are negative).
+        let allSameGroup = true;
+        const first = groupIDs[0];
+        if (first < 0) {
+            // First location is ungrouped -> cannot be "all same real group".
+            allSameGroup = false;
+        }
+        else {
+            for (let k = 1; k < groupIDs.length; k++) {
+                if (groupIDs[k] !== first) {
+                    allSameGroup = false;
+                    break;
+                }
+            }
+        }
+        if (allSameGroup) {
+            // Entire cluster is inside one known HIGH group -> suppress.
+            continue;
+        }
+        out.push(cl);
+    }
+    return out;
+}
+/** Composite key for the (FilePath, StartLine) join index. */
+function locKey(filePath, startLine) {
+    return `${filePath}\x00${startLine}`;
+}

package/dist/dedup/analyzer/multiplicity.test.js

@@ -0,0 +1,123 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { applyMultiplicity, suppressKnownGroups } from "./multiplicity.js";
+function block(filePath, startLine, sectionID) {
+    return {
+        SectionID: sectionID,
+        FilePath: filePath,
+        Heading: "",
+        Index: 0,
+        Kind: "prose",
+        StartLine: startLine,
+        EndLine: startLine + 1,
+        ContentHash: "",
+        Text: "",
+        TableRows: 0,
+    };
+}
+// applyMultiplicity downgrades a block that recurs in MORE than the cap of
+// distinct files to Informational: such a block is almost certainly intentional
+// boilerplate, not an accidental duplication a reviewer should act on. The
+// strict ">" boundary (cap itself stays actionable) and input immutability are
+// the contract — a drift would either spam reviewers or hide real duplicates.
+describe("applyMultiplicity", () => {
+    it("flags over-cap clusters Informational, leaves at/under-cap actionable, and never mutates input", () => {
+        const cfg = defaultConfig();
+        cfg.Block.multiplicity_cap = 3; // >3 distinct files -> informational
+        const overCap = {
+            Kind: "prose",
+            ContentHash: "over-cap-hash",
+            Exact: true,
+            Informational: false,
+            Similarity: 1.0,
+            Locations: [
+                { FilePath: "docs/a.md", Heading: "H", StartLine: 1, EndLine: 5 },
+                { FilePath: "docs/b.md", Heading: "H", StartLine: 1, EndLine: 5 },
+                { FilePath: "docs/c.md", Heading: "H", StartLine: 1, EndLine: 5 },
+                { FilePath: "docs/d.md", Heading: "H", StartLine: 1, EndLine: 5 },
+            ],
+        };
+        const atCap = {
+            Kind: "prose",
+            ContentHash: "at-cap-hash",
+            Exact: true,
+            Informational: false,
+            Similarity: 1.0,
+            Locations: [
+                { FilePath: "docs/a.md", Heading: "H", StartLine: 10, EndLine: 15 },
+                { FilePath: "docs/b.md", Heading: "H", StartLine: 10, EndLine: 15 },
+                { FilePath: "docs/c.md", Heading: "H", StartLine: 10, EndLine: 15 },
+            ],
+        };
+        const underCap = {
+            Kind: "prose",
+            ContentHash: "under-cap-hash",
+            Exact: true,
+            Informational: false,
+            Similarity: 1.0,
+            Locations: [
+                { FilePath: "docs/a.md", Heading: "S1", StartLine: 20, EndLine: 25 },
+                { FilePath: "docs/a.md", Heading: "S2", StartLine: 30, EndLine: 35 },
+            ],
+        };
+        const input = [overCap, atCap, underCap];
+        const got = applyMultiplicity(input, cfg);
+        expect(got).toHaveLength(3);
+        expect(got[0].Informational).toBe(true); // > cap
+        expect(got[1].Informational).toBe(false); // exactly cap
+        expect(got[2].Informational).toBe(false); // 1 distinct file
+        // Immutability: input must not be modified.
+        expect(input[0].Informational).toBe(false);
+    });
+});
+// suppressKnownGroups removes L5 clusters that an L1-L4 HIGH group already fully
+// covers, so L5 only surfaces NOVEL overlaps. The predicate "drop iff every
+// location resolves to the SAME single real group" is the contract: an ungrouped
+// or cross-group member means there is something new to report, so it is kept.
+describe("suppressKnownGroups", () => {
+    it("drops fully-covered clusters and keeps ungrouped / cross-group / all-ungrouped ones", () => {
+        // secA,secB share group 1; secC,secD share group 2; secE,secF ungrouped.
+        const secGroup = new Map([
+            ["secA", 1],
+            ["secB", 1],
+            ["secC", 2],
+            ["secD", 2],
+        ]);
+        const blocks = [
+            block("docs/a.md", 1, "secA"),
+            block("docs/b.md", 1, "secB"),
+            block("docs/c.md", 1, "secC"),
+            block("docs/d.md", 1, "secD"),
+            block("docs/e.md", 1, "secE"), // ungrouped
+            block("docs/f.md", 1, "secF"), // ungrouped
+        ];
+        const mkCluster = (hash, locs) => ({
+            Kind: "prose",
+            ContentHash: hash,
+            Exact: true,
+            Informational: false,
+            Similarity: 1.0,
+            Locations: locs.map(([fp, ln]) => ({ FilePath: fp, Heading: "", StartLine: ln, EndLine: ln + 1 })),
+        });
+        // (a) both in group 1 -> dropped.
+        const caseA = mkCluster("hash-a", [["docs/a.md", 1], ["docs/b.md", 1]]);
+        // (b) group 1, group 1, ungrouped -> kept (novel via secE).
+        const caseB = mkCluster("hash-b", [["docs/a.md", 1], ["docs/b.md", 1], ["docs/e.md", 1]]);
+        // (c) group 1 + group 2 -> kept (two groups).
+        const caseC = mkCluster("hash-c", [["docs/a.md", 1], ["docs/c.md", 1]]);
+        // (d) all-ungrouped -> kept.
+        const caseD = mkCluster("hash-d", [["docs/e.md", 1], ["docs/f.md", 1]]);
+        const got = suppressKnownGroups([caseA, caseB, caseC, caseD], blocks, secGroup);
+        // (a) hash-a dropped.
+        expect(got.some((cl) => cl.ContentHash === "hash-a")).toBe(false);
+        // remaining: b, c, d.
+        expect(got).toHaveLength(3);
+        // (b) hash-b kept with all 3 locations.
+        const bCluster = got.find((cl) => cl.ContentHash === "hash-b");
+        expect(bCluster).toBeDefined();
+        expect(bCluster.Locations).toHaveLength(3);
+        // (c) and (d) kept.
+        expect(got.some((cl) => cl.ContentHash === "hash-c")).toBe(true);
+        expect(got.some((cl) => cl.ContentHash === "hash-d")).toBe(true);
+    });
+});

package/dist/dedup/analyzer/order.js

@@ -0,0 +1,22 @@
+// Comparison helpers mirroring Go's cmp.Compare, used by the cluster sorters.
+//
+// Go's cmp.Compare returns -1 / 0 / +1 and, for strings, orders byte-by-byte
+// (lexicographic). For the ASCII content hashes and docs/ file paths these
+// sorters see, UTF-16 code-unit order equals Go's byte order, so a plain
+// comparison reproduces the deterministic cluster ordering exactly.
+/** cmpNum reproduces cmp.Compare for numbers: -1 if a<b, +1 if a>b, else 0. */
+export function cmpNum(a, b) {
+    if (a < b)
+        return -1;
+    if (a > b)
+        return 1;
+    return 0;
+}
+/** cmpStr reproduces cmp.Compare for strings: lexicographic, -1 / 0 / +1. */
+export function cmpStr(a, b) {
+    if (a < b)
+        return -1;
+    if (a > b)
+        return 1;
+    return 0;
+}

package/dist/dedup/analyzer/partial_overlaps.js

@@ -0,0 +1,65 @@
+// Ported from internal/dedup/analyzer/partial_overlaps.go.
+//
+// Combines the L5-exact and L5-cosine passes, applies the multiplicity cap,
+// suppresses location-pairs already covered by an L1-L4 HIGH group, and returns
+// the result in a deterministic total order. The exact pass feeds excludeHashes
+// to the cosine pass so a verbatim block is never reported twice.
+import { cosineClusters } from "./cosine_clusters.js";
+import { exactClusters } from "./exact_clusters.js";
+import { applyMultiplicity, suppressKnownGroups } from "./multiplicity.js";
+import { cmpStr, cmpNum } from "./order.js";
+/**
+ * partialOverlaps runs the full L5 pipeline.
+ *
+ * secGroup maps section_id -> HIGH group index (from the final groups slice in
+ * Analyze). Only sections that are members of a multi-member HIGH group appear;
+ * other sections are absent (ungrouped).
+ *
+ * The returned slice is sorted by
+ * (Locations[0].FilePath, Locations[0].StartLine, ContentHash) for stable
+ * interleaving of exact and cosine clusters.
+ */
+export function partialOverlaps(blocks, blockEmb, secGroup, cfg) {
+    // Step 1: exact clusters.
+    const exact = exactClusters(blocks);
+    // Step 2: build excludeHashes from exact clusters so verbatim pairs are not
+    // re-reported by the cosine pass.
+    const excludeHashes = new Set();
+    for (const cl of exact) {
+        if (cl.ContentHash !== "") {
+            excludeHashes.add(cl.ContentHash);
+        }
+    }
+    // Step 3: cosine clusters (tables excluded, same-file pairs skipped).
+    const cosine = cosineClusters(blocks, blockEmb, excludeHashes, cfg);
+    // Step 4: combine into a fresh slice.
+    let all = [...exact, ...cosine];
+    // Step 5: apply multiplicity cap.
+    all = applyMultiplicity(all, cfg);
+    // Step 6: suppress clusters fully covered by a single L1-L4 HIGH group.
+    all = suppressKnownGroups(all, blocks, secGroup);
+    // Step 7: deterministic total order — sort by (first location FilePath,
+    // first location StartLine, ContentHash).
+    all.sort((a, b) => {
+        let aFile = "";
+        let bFile = "";
+        let aLine = 0;
+        let bLine = 0;
+        if (a.Locations.length > 0) {
+            aFile = a.Locations[0].FilePath;
+            aLine = a.Locations[0].StartLine;
+        }
+        if (b.Locations.length > 0) {
+            bFile = b.Locations[0].FilePath;
+            bLine = b.Locations[0].StartLine;
+        }
+        let c = cmpStr(aFile, bFile);
+        if (c !== 0)
+            return c;
+        c = cmpNum(aLine, bLine);
+        if (c !== 0)
+            return c;
+        return cmpStr(a.ContentHash, b.ContentHash);
+    });
+    return all;
+}

package/dist/dedup/analyzer/partial_overlaps.test.js

@@ -0,0 +1,161 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { partialOverlaps } from "./partial_overlaps.js";
+function block(filePath, sectionID, heading, startLine, endLine, contentHash) {
+    return {
+        SectionID: sectionID,
+        FilePath: filePath,
+        Heading: heading,
+        Index: 0,
+        Kind: "prose",
+        StartLine: startLine,
+        EndLine: endLine,
+        ContentHash: contentHash,
+        Text: "",
+        TableRows: 0,
+    };
+}
+// partialOverlaps wires the L5 pipeline together. The load-bearing behavior is
+// the exact->cosine handoff: a verbatim cross-file block is reported ONCE as an
+// exact cluster and is NOT re-reported as a cosine near-copy, because the exact
+// pass feeds its hash into excludeHashes. Mirrors Go's
+// TestAnalyze_PopulatesPartialOverlaps (here exercised directly on the primitive
+// the orchestrator calls). Pinned because double-reporting the same block would
+// inflate the dedup report and mislead reviewers.
+describe("partialOverlaps", () => {
+    it("reports a verbatim cross-file block once (exact), not also as a cosine cluster", () => {
+        const cfg = defaultConfig();
+        const verbatimHash = "verbatim-block-hash-xyz";
+        const blocks = [
+            block("docs/a.md", "secA", "Overview", 3, 6, verbatimHash),
+            block("docs/b.md", "secB", "Details", 4, 7, verbatimHash),
+        ];
+        // blockEmb HAS an entry for verbatimHash — it would be a cosine candidate too
+        // if the exact pass did not exclude it.
+        const blockEmb = new Map([[verbatimHash, [0.5, 0.5]]]);
+        // No HIGH groups, so nothing is suppressed.
+        const secGroup = new Map();
+        const got = partialOverlaps(blocks, blockEmb, secGroup, cfg);
+        expect(got).toHaveLength(1);
+        const cl = got[0];
+        expect(cl.Exact).toBe(true);
+        expect(cl.Kind).toBe("prose");
+        expect(cl.ContentHash).toBe(verbatimHash);
+        expect(cl.Locations).toHaveLength(2);
+        // Exact-vs-cosine dedup: exactly one exact, zero cosine for this hash.
+        const forHash = got.filter((c) => c.ContentHash === verbatimHash);
+        expect(forHash.filter((c) => c.Exact)).toHaveLength(1);
+        expect(forHash.filter((c) => !c.Exact)).toHaveLength(0);
+    });
+    // WHY: the final total order is the contract the report renderer relies on to
+    // interleave exact and cosine clusters reproducibly. With two clusters whose
+    // first locations differ by FilePath, the FilePath comparator (cmpStr) must
+    // decide the order — a regression that ignored FilePath would scramble the
+    // report between runs.
+    it("orders clusters by first-location FilePath", () => {
+        const cfg = defaultConfig();
+        // hashZ spans files starting with z..., hashA spans files starting with a...
+        const blocks = [
+            block("docs/za.md", "s1", "H", 3, 6, "hashZ"),
+            block("docs/zb.md", "s2", "H", 3, 6, "hashZ"),
+            block("docs/aa.md", "s3", "H", 3, 6, "hashA"),
+            block("docs/ab.md", "s4", "H", 3, 6, "hashA"),
+        ];
+        const got = partialOverlaps(blocks, new Map(), new Map(), cfg);
+        expect(got).toHaveLength(2);
+        // hashA cluster (first loc docs/aa.md) sorts before hashZ (docs/za.md).
+        expect(got[0].Locations[0].FilePath).toBe("docs/aa.md");
+        expect(got[1].Locations[0].FilePath).toBe("docs/za.md");
+    });
+    // WHY: when two clusters share the same first-location FilePath, ordering must
+    // fall through to StartLine (cmpNum). This pins the second comparator key so a
+    // stable ordering survives same-file clusters at different lines.
+    it("breaks a FilePath tie by first-location StartLine", () => {
+        const cfg = defaultConfig();
+        // Both clusters' first location is docs/a.md; the second files differ so the
+        // cross-file requirement holds, but the first-location StartLine differs.
+        const blocks = [
+            block("docs/a.md", "s1", "H", 20, 22, "hashLate"),
+            block("docs/b.md", "s2", "H", 5, 7, "hashLate"),
+            block("docs/a.md", "s3", "H", 4, 6, "hashEarly"),
+            block("docs/c.md", "s4", "H", 9, 11, "hashEarly"),
+        ];
+        const got = partialOverlaps(blocks, new Map(), new Map(), cfg);
+        expect(got).toHaveLength(2);
+        // hashEarly's first loc is docs/a.md:4 (StartLine 4 < 20) so it sorts first.
+        expect(got[0].ContentHash).toBe("hashEarly");
+        expect(got[0].Locations[0].StartLine).toBe(4);
+        expect(got[1].ContentHash).toBe("hashLate");
+        expect(got[1].Locations[0].StartLine).toBe(20);
+    });
+    // WHY: when first FilePath AND StartLine are equal, the final tiebreak is
+    // ContentHash (cmpStr). Two distinct verbatim blocks that happen to start at
+    // the same file+line must still be totally ordered, deterministically, by hash.
+    it("breaks a FilePath+StartLine tie by ContentHash", () => {
+        const cfg = defaultConfig();
+        // Construct two exact clusters whose sorted-first location is identical
+        // (docs/a.md, line 3). hashB's locations are a.md:3 + b.md:3; hashA's are
+        // a.md:3 + c.md:3. After per-cluster location sort, both lead with a.md:3.
+        const blocks = [
+            block("docs/a.md", "s1", "H", 3, 6, "hashB"),
+            block("docs/b.md", "s2", "H", 3, 6, "hashB"),
+            block("docs/a.md", "s3", "H", 3, 6, "hashA"),
+            block("docs/c.md", "s4", "H", 3, 6, "hashA"),
+        ];
+        const got = partialOverlaps(blocks, new Map(), new Map(), cfg);
+        expect(got).toHaveLength(2);
+        expect(got[0].Locations[0].FilePath).toBe("docs/a.md");
+        expect(got[1].Locations[0].FilePath).toBe("docs/a.md");
+        expect(got[0].Locations[0].StartLine).toBe(3);
+        expect(got[1].Locations[0].StartLine).toBe(3);
+        // Tie resolved by ContentHash: "hashA" < "hashB".
+        expect(got[0].ContentHash).toBe("hashA");
+        expect(got[1].ContentHash).toBe("hashB");
+    });
+    // WHY: an exact cluster can carry an empty ContentHash (when blocks share the
+    // empty-string hash across files). The excludeHashes builder must NOT add ""
+    // to the exclude set, otherwise every cosine candidate lacking a hash would be
+    // wrongly suppressed. This pins the `ContentHash !== ""` guard.
+    it("does not add an empty ContentHash to the cosine exclude set", () => {
+        const cfg = defaultConfig();
+        // Two cross-file blocks with empty ContentHash -> one exact cluster, hash "".
+        // A separate cross-file prose pair has DISTINCT hashes but near-1.0 cosine
+        // embeddings -> it must still surface as a cosine cluster (proving "" did not
+        // poison the exclude set / suppress unrelated cosine candidates).
+        const blocks = [
+            block("docs/a.md", "s1", "H", 1, 2, ""),
+            block("docs/b.md", "s2", "H", 1, 2, ""),
+            block("docs/c.md", "s3", "H", 10, 12, "cosHashC"),
+            block("docs/d.md", "s4", "H", 10, 12, "cosHashD"),
+        ];
+        const blockEmb = new Map([
+            ["cosHashC", [1, 0]],
+            ["cosHashD", [1, 0]],
+        ]);
+        const got = partialOverlaps(blocks, blockEmb, new Map(), cfg);
+        const emptyExact = got.filter((c) => c.Exact && c.ContentHash === "");
+        expect(emptyExact).toHaveLength(1);
+        // The real-hash cosine cluster survived (its hash was never excluded).
+        const cosine = got.filter((c) => !c.Exact);
+        expect(cosine).toHaveLength(1);
+        expect(cosine[0].Kind).toBe("prose");
+    });
+    // WHY: suppression must drop a cluster whose every location lives in the same
+    // L1-L4 HIGH group, so L5 only surfaces NOVEL overlaps. Without this, a block
+    // already reported via its section group would double-report at the block level.
+    it("suppresses a cluster fully covered by one HIGH group", () => {
+        const cfg = defaultConfig();
+        const h = "covered-hash";
+        const blocks = [
+            block("docs/a.md", "secA", "H", 3, 6, h),
+            block("docs/b.md", "secB", "H", 3, 6, h),
+        ];
+        // Both sections are members of the same HIGH group (id 0) -> fully covered.
+        const secGroup = new Map([
+            ["secA", 0],
+            ["secB", 0],
+        ]);
+        const got = partialOverlaps(blocks, new Map(), secGroup, cfg);
+        expect(got).toHaveLength(0);
+    });
+});

package/dist/dedup/analyzer/preview.js

@@ -0,0 +1,84 @@
+// Ported from internal/dedup/analyzer/preview.go.
+//
+// Prose-only preview extraction for the report renderer. The regex set is
+// parity-load-bearing (plan §791): they are package-level constants, not Config,
+// and the cleanup order matters. Go's RE2 `\s` is the ASCII set [\t\n\f\r ]
+// (NOT \v, NOT Unicode spaces), so every `\s` is spelled out explicitly here —
+// JS `\s` would match more and drift the output.
+/** (?s)```.*?``` — fenced code blocks (dotAll). */
+const codeFenceRE = /```[\s\S]*?```/g;
+/** (?m)(?:^[ \t]{4,}.*\n?)+ — indented code blocks. */
+const indentedCodeRE = /(?:^[ \t]{4,}.*\n?)+/gm;
+/** ![alt](url) — keep alt text. */
+const imageRE = /!\[([^\]]*)\]\([^)]+\)/g;
+/** [label](url) — keep label text. */
+const linkRE = /\[([^\]]+)\]\([^)]+\)/g;
+/** `code` — keep inner text. */
+const inlineCodeRE = /`([^`]+)`/g;
+/** (?m)^\s*(?:[-+*]|\d+\.|>+)\s+ — leading list / blockquote markers. */
+const leadMarkerRE = /^[\t\n\f\r ]*(?:[-+*]|\d+\.|>+)[\t\n\f\r ]+/gm;
+/** \*{1,3}([^*\n]+)\*{1,3} — bold/italic with asterisks. */
+const boldItalAstRE = /\*{1,3}([^*\n]+)\*{1,3}/g;
+/** _{1,3}([^_\n]+)_{1,3} — bold/italic with underscores. */
+const boldItalUndRE = /_{1,3}([^_\n]+)_{1,3}/g;
+/** \s+ — whitespace collapse (Go ASCII \s). */
+const whitespaceRE = /[\t\n\f\r ]+/g;
+/**
+ * computePreview extracts a clean prose-only preview from a section's raw markdown.
+ *
+ * It drops the section's heading line (if the first line starts with #), strips
+ * fenced and indented code blocks, image syntax, link URLs (keeping label text),
+ * list/blockquote markers, and emphasis markers. Truncates at maxChars on a word
+ * boundary, appending "…" if truncated. Wrapping is the renderer's concern.
+ */
+export function computePreview(rawContent, maxChars, wordRatio) {
+    let text = rawContent;
+    // Drop the entire first line if it's a heading line (## …).
+    const idx = text.indexOf("\n");
+    if (idx >= 0) {
+        // Go: strings.TrimLeft(text[:idx], " \t") — left-trim only space/tab.
+        const first = trimLeftSpaceTab(text.slice(0, idx));
+        if (first.startsWith("#")) {
+            text = text.slice(idx + 1);
+        }
+    }
+    // Strip code blocks before anything else.
+    text = text.replace(codeFenceRE, " ");
+    text = text.replace(indentedCodeRE, " ");
+    // Markdown link/image cleanup.
+    text = text.replace(imageRE, "$1");
+    text = text.replace(linkRE, "$1");
+    text = text.replace(inlineCodeRE, "$1");
+    // Strip leading list and blockquote markers per line.
+    text = text.replace(leadMarkerRE, "");
+    // Strip bold/italic emphasis.
+    text = text.replace(boldItalAstRE, "$1");
+    text = text.replace(boldItalUndRE, "$1");
+    // Collapse whitespace.
+    text = text.replace(whitespaceRE, " ").trim();
+    if (text === "") {
+        return "(no prose content)";
+    }
+    // Go: utf8.RuneCountInString — rune count, not UTF-16 length.
+    const runes = [...text];
+    const runeLen = runes.length;
+    if (runeLen <= maxChars) {
+        return text;
+    }
+    // Truncate at word boundary.
+    let cut = runes.slice(0, maxChars).join("");
+    const lastSpace = cut.lastIndexOf(" ");
+    // Go: int(float64(maxChars)*wordRatio) — truncate toward zero.
+    if (lastSpace > Math.trunc(maxChars * wordRatio)) {
+        cut = cut.slice(0, lastSpace);
+    }
+    return cut + "…";
+}
+/** Left-trims only ASCII space and tab (Go strings.TrimLeft(s, " \t")). */
+function trimLeftSpaceTab(s) {
+    let i = 0;
+    while (i < s.length && (s[i] === " " || s[i] === "\t")) {
+        i++;
+    }
+    return s.slice(i);
+}

package/dist/dedup/analyzer/preview.test.js

@@ -0,0 +1,46 @@
+import { describe, it, expect } from "vitest";
+import { computePreview } from "./preview.js";
+// The preview is the human-facing snippet a reviewer reads to decide whether two
+// sections are truly duplicates. It must show PROSE only — heading lines and code
+// fences would make unrelated sections look alike (or alike sections look
+// different), so each strip rule is pinned by WHY it matters, plus the
+// word-boundary truncation contract.
+describe("computePreview", () => {
+    it("strips the leading heading line", () => {
+        const raw = "## My Heading\n\nThis is the prose content of the section.";
+        const got = computePreview(raw, 280, 0.6);
+        expect(got).not.toContain("## My Heading");
+        expect(got).toContain("This is the prose content");
+    });
+    it("strips fenced code blocks (code is noise for prose comparison)", () => {
+        const raw = "## Heading\n\nSome prose.\n\n```go\nfunc main() {}\n```\n\nMore prose.";
+        const got = computePreview(raw, 280, 0.6);
+        expect(got).not.toContain("func main");
+        expect(got).toContain("Some prose");
+    });
+    it("truncates at a word boundary and appends an ellipsis", () => {
+        // WHY: cutting mid-word would corrupt the snippet; the ellipsis signals
+        // truncation. Build > 280 chars of repeated whole words.
+        let body = "";
+        while (body.length < 400) {
+            body += "documentation ";
+        }
+        const raw = `## Heading\n\n${body}`;
+        const got = computePreview(raw, 280, 0.6);
+        // A little slack for the ellipsis character.
+        expect([...got].length).toBeLessThanOrEqual(285);
+        expect(got.endsWith("…")).toBe(true);
+    });
+    it("does not truncate short content", () => {
+        const raw = "## Heading\n\nShort content.";
+        const got = computePreview(raw, 280, 0.6);
+        expect(got.endsWith("…")).toBe(false);
+    });
+    it("returns a placeholder when no prose remains after stripping", () => {
+        // WHY: a code-only section has no comparable prose; the placeholder keeps the
+        // report readable instead of showing an empty snippet.
+        const raw = "## Heading\n\n```go\nfunc main() {}\n```";
+        const got = computePreview(raw, 280, 0.6);
+        expect(got).toBe("(no prose content)");
+    });
+});

package/dist/dedup/analyzer/safety.js

@@ -0,0 +1,27 @@
+// Ported from internal/dedup/analyzer/safety.go.
+//
+// Differentiator detection for the L4 safety net: differentiators NEVER block
+// group membership; they force recommended_action = manual_review and annotate
+// the group with a reason tag.
+/**
+ * findDifferentiators checks for differentiator word-pair presence in two
+ * section texts. If section A contains one side of a pair and section B contains
+ * the other (case-insensitive substring match), a reason tag is added to the
+ * returned slice.
+ *
+ * @param textA  raw text of the first section
+ * @param textB  raw text of the second section
+ * @param differentiators  list of [left, right] word pairs (Go [2]string)
+ */
+export function findDifferentiators(textA, textB, differentiators) {
+    const a = textA.toLowerCase();
+    const b = textB.toLowerCase();
+    const hits = [];
+    for (const [left, right] of differentiators) {
+        if ((a.includes(left) && b.includes(right)) ||
+            (a.includes(right) && b.includes(left))) {
+            hits.push(`differentiator detected: '${left}' vs '${right}'`);
+        }
+    }
+    return hits;
+}