docsgov 0.1.0

package/dist/dedup/mdsection/section.test.js

@@ -0,0 +1,221 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { createHash } from "node:crypto";
+import { describe, expect, it } from "vitest";
+import { extract, extractFromFile, extractFromFileWithBlocks, } from "./index.js";
+const here = fileURLToPath(new URL(".", import.meta.url));
+const fixtureFile = `${here}testdata/fixture.md`;
+function loadFixture() {
+    return readFileSync(fixtureFile, "utf8");
+}
+function extractFixture() {
+    return extract(fixtureFile, loadFixture());
+}
+describe("extract — section splitting", () => {
+    // The exclusive-ownership invariant is what lets clustering attribute every
+    // duplicated line to exactly one section; overlapping ranges would double-count.
+    it("produces non-overlapping [start_line, end_line) ranges", () => {
+        const sections = extractFixture();
+        for (let i = 0; i < sections.length; i++) {
+            for (let j = i + 1; j < sections.length; j++) {
+                const a = sections[i];
+                const b = sections[j];
+                if (a.file_path !== b.file_path)
+                    continue;
+                const overlap = a.start_line < b.end_line && b.start_line < a.end_line;
+                expect(overlap, `sections ${i} and ${j} overlap`).toBe(false);
+            }
+        }
+    });
+    // All-eligible fixture: the union of section spans must exactly cover the file
+    // from its first heading to EOF — the contiguity half of exclusive ownership.
+    it("covers every line from the first heading to EOF (all-eligible fixture)", () => {
+        const allEligibleFile = `${here}testdata/all_eligible.md`;
+        const data = readFileSync(allEligibleFile, "utf8");
+        const sections = extract(allEligibleFile, data);
+        const lines = data.replace(/\n+$/, "").split("\n");
+        const totalLines = lines.length;
+        let firstHeadingLine = 0;
+        for (let i = 0; i < lines.length; i++) {
+            if (lines[i].startsWith("#")) {
+                firstHeadingLine = i + 1;
+                break;
+            }
+        }
+        const expected = totalLines - firstHeadingLine + 1;
+        let got = 0;
+        for (const s of sections)
+            got += s.end_line - s.start_line;
+        expect(got).toBe(expected);
+    });
+    // The eligibility gate is the dedup pool's admission filter: a code-only
+    // section carries no prose to embed, so admitting it would pollute clusters.
+    it("excludes sections below the prose-word threshold (Code Only)", () => {
+        const sections = extractFixture();
+        expect(sections.find((s) => s.heading === "Code Only")).toBeUndefined();
+    });
+    // Heading levels drive nothing downstream here but a wrong level would mean we
+    // mis-parsed the heading; the fixture is all H2, so every section must be 2.
+    it("records the heading level", () => {
+        for (const s of extractFixture()) {
+            expect(s.heading_level, s.heading).toBe(2);
+        }
+    });
+    // Line numbers are 1-indexed and a section always spans at least its heading,
+    // so start_line>0 and end_line>start_line are structural invariants.
+    it("uses 1-indexed lines with end_line > start_line", () => {
+        for (const s of extractFixture()) {
+            expect(s.start_line, s.heading).toBeGreaterThan(0);
+            expect(s.end_line, s.heading).toBeGreaterThan(s.start_line);
+        }
+    });
+    // raw_content must begin with the heading line so the on-disk section text can
+    // be located and re-displayed verbatim from the stored record.
+    it("starts raw_content with the heading line", () => {
+        for (const s of extractFixture()) {
+            expect(s.raw_content.startsWith(`## ${s.heading}`), s.heading).toBe(true);
+        }
+    });
+});
+describe("extract — anchors and IDs", () => {
+    // GitHub-style collision suffixes ("installation", "installation-1") must be
+    // assigned per-file in document order; this is what keeps two same-named
+    // headings addressable as distinct sections.
+    it("suffixes duplicate-heading anchors in order", () => {
+        const anchors = extractFixture()
+            .filter((s) => s.heading === "Installation")
+            .map((s) => s.anchor);
+        expect(anchors).toEqual(["installation", "installation-1"]);
+    });
+    // Because the anchor is part of the sectionid hash input, collision-suffix
+    // anchors MUST yield distinct IDs — otherwise two sections would key alike.
+    it("derives distinct IDs from collision-suffixed anchors", () => {
+        const ids = extractFixture()
+            .filter((s) => s.heading === "Installation")
+            .map((s) => s.id);
+        expect(ids).toHaveLength(2);
+        expect(ids[0]).not.toBe(ids[1]);
+    });
+    // A CJK heading is letters under \p{L}, so it survives slugification intact and
+    // must still produce a valid 16-char ID and pass the prose gate. Guards that the
+    // Unicode strip rule isn't ASCII-only.
+    it("handles a CJK heading (non-empty anchor, 16-char ID, eligible)", () => {
+        const s = extractFixture().find((x) => x.heading === "補貨單流程");
+        expect(s, "CJK section").toBeDefined();
+        expect(s.anchor).not.toBe("");
+        expect(s.id).toHaveLength(16);
+        expect(s.prose_word_count).toBeGreaterThanOrEqual(10);
+    });
+});
+describe("extract — derived fields", () => {
+    // The table linearization format ("h=v, h=v; …") is the locked embed
+    // representation; clustering compares these strings, so the exact shape matters.
+    it("linearizes a table into embed_text and sets has_table", () => {
+        const s = extractFixture().find((x) => x.heading === "Table Section");
+        expect(s, "Table Section").toBeDefined();
+        expect(s.has_table).toBe(true);
+        expect(s.embed_text).toContain("header a=val1a");
+    });
+    // has_code must stay false for prose-only sections; a stray true would mean we
+    // misclassified an inline span as a code block and skewed eligibility.
+    it("leaves has_code false for a prose-only section", () => {
+        const s = extractFixture().find((x) => x.heading === "Installation");
+        expect(s.has_code).toBe(false);
+    });
+    // prose_word_count gates eligibility, so it must reflect real word counts; a
+    // section known to exceed the threshold must report >= 10.
+    it("counts prose words above the gate for the first Installation", () => {
+        const s = extractFixture().find((x) => x.heading === "Installation" && x.anchor === "installation");
+        expect(s.prose_word_count).toBeGreaterThanOrEqual(10);
+    });
+    // List items are flattened to prose and counted; otherwise a list-only section
+    // would be wrongly excluded from the dedup pool.
+    it("flattens list items into counted prose", () => {
+        const s = extractFixture().find((x) => x.heading === "List Section");
+        expect(s, "List Section").toBeDefined();
+        expect(s.prose_word_count).toBeGreaterThanOrEqual(10);
+    });
+    // content_hash is sha256(embed_text); pinning the formula catches any drift in
+    // how the hash is computed, which would silently re-key every stored section.
+    it("sets content_hash = sha256(embed_text)", () => {
+        for (const s of extractFixture()) {
+            expect(s.content_hash).not.toBe("");
+            const want = createHash("sha256").update(s.embed_text).digest("hex");
+            expect(s.content_hash, s.heading).toBe(want);
+        }
+    });
+});
+describe("extract — file path handling", () => {
+    // file_path is stored verbatim and feeds the ID hash, so it must equal exactly
+    // what the caller passed (relative or absolute), never normalized.
+    it("stores file_path verbatim", () => {
+        for (const s of extractFixture()) {
+            expect(s.file_path).toBe(fixtureFile);
+        }
+    });
+    // extractFromFile must be equivalent to reading then extract — it is the same
+    // parse, just sourced from disk.
+    it("extractFromFile equals extract on the same content", () => {
+        const a = extractFromFile(fixtureFile);
+        const b = extract(fixtureFile, loadFixture());
+        expect(a.length).toBe(b.length);
+        expect(a.map((s) => s.id)).toEqual(b.map((s) => s.id));
+    });
+});
+describe("extract — golden output", () => {
+    // The golden pins the FULL extraction (IDs, hashes, line ranges, embed text)
+    // byte-for-byte against the Go implementation's output. Any change to section
+    // boundaries, slugs, or normalization that diverges from Go fails here — these
+    // values persist in the index, so divergence is a hard regression.
+    it("matches the Go golden JSON exactly", () => {
+        const sections = extractFixture();
+        // The golden was generated with file_path = "testdata/fixture.md"; rewrite
+        // our absolute path back to that relative form so IDs and the path field line
+        // up. (IDs in the golden hash the relative path.)
+        const relSections = extract("testdata/fixture.md", loadFixture());
+        const want = JSON.parse(readFileSync(`${here}testdata/sections.golden.json`, "utf8"));
+        expect(relSections).toEqual(want);
+        // Sanity: the absolute-path run yields the same count and headings, only the
+        // path-derived fields differ.
+        expect(sections.map((s) => s.heading)).toEqual(want.map((s) => s.heading));
+    });
+});
+describe("extractFromFileWithBlocks", () => {
+    // Extract and the blocks API share one parse; the sections half MUST be
+    // behavior-identical to Extract or two callers would disagree on the index.
+    it("returns sections identical to extract", () => {
+        const want = extract(fixtureFile, loadFixture());
+        const { sections: got } = extractFromFileWithBlocks(fixtureFile);
+        expect(got.length).toBe(want.length);
+        for (let i = 0; i < want.length; i++) {
+            expect(got[i]).toEqual(want[i]);
+        }
+    });
+    // Block records exist for ALL headings pre-eligibility: a table-only section
+    // (0 prose words, section-ineligible) still yields a "table" record so partial-
+    // duplication detection sees its rows. Code/HTML-only sections yield none.
+    it("emits blocks for ineligible sections and skips code/html", () => {
+        const fixture = `${here}testdata/allsections.md`;
+        const { blocks } = extractFromFileWithBlocks(fixture);
+        const byKey = new Map();
+        for (const br of blocks) {
+            const k = `${br.Heading}\u0000${br.Kind}`;
+            const arr = byKey.get(k) ?? [];
+            arr.push(br);
+            byKey.set(k, arr);
+        }
+        const tableBlocks = byKey.get("Table Only Section\u0000table") ?? [];
+        expect(tableBlocks.length, "table-only section table record").toBeGreaterThan(0);
+        expect(tableBlocks[0].SectionID).not.toBe("");
+        expect(tableBlocks[0].FilePath).toBe(fixture);
+        expect(tableBlocks[0].Heading).toBe("Table Only Section");
+        const proseBlocks = byKey.get("Prose Section\u0000prose") ?? [];
+        expect(proseBlocks.length, "prose section prose record").toBeGreaterThan(0);
+        for (const br of proseBlocks)
+            expect(br.SectionID).not.toBe("");
+        // Code-only section yields no records at all.
+        expect(blocks.some((br) => br.Heading === "Code Only Section")).toBe(false);
+        // List inside an eligible section folds to "prose".
+        expect((byKey.get("Eligible With List\u0000prose") ?? []).length).toBeGreaterThan(0);
+    });
+});

package/dist/dedup/report/floatfmt.js

@@ -0,0 +1,71 @@
+// formatFloat2 reproduces Go's fmt "%.2f" / strconv.FormatFloat(v, 'f', 2, 64)
+// byte-for-byte: the correctly-rounded 2-decimal form of the EXACT float64
+// value, using round-half-to-even on ties.
+//
+// WHY a custom formatter (not Number.prototype.toFixed): JS toFixed uses a
+// different (round-half-away-ish, float-buggy) algorithm. e.g. Go renders
+// (0.125).toFixed-equivalent as "0.12" (the exact double 0.125 lands on a
+// half and ties to even), while JS toFixed yields "0.13". The dedup similarity
+// scores are printed with %.2f and checked byte-for-byte by the golden oracle,
+// so the rounding must match Go exactly.
+//
+// Algorithm: recover the exact rational value (num/den) of the double from its
+// IEEE-754 bits via BigInt, compute round(value * 100) with round-half-to-even,
+// then place the decimal point. Verified against Go's %.2f across 5000+ random
+// values in [0,1) plus tie edge cases (all match).
+/** formatFloat2 returns x formatted like Go's "%.2f". */
+export function formatFloat2(x) {
+    if (!Number.isFinite(x)) {
+        // Go prints NaN/+Inf/-Inf for these; similarity scores are always finite,
+        // but mirror Go's tokens defensively rather than emitting "Infinity".
+        if (Number.isNaN(x)) {
+            return "NaN";
+        }
+        return x > 0 ? "+Inf" : "-Inf";
+    }
+    const neg = x < 0 || Object.is(x, -0);
+    const { num, den } = exactRational(Math.abs(x));
+    // q = round(num/den * 100) in hundredths, half-to-even.
+    const scaledNum = num * 100n;
+    let q = scaledNum / den;
+    const r = scaledNum % den;
+    const twice = r * 2n;
+    if (twice > den) {
+        q += 1n;
+    }
+    else if (twice === den && q % 2n === 1n) {
+        q += 1n; // tie -> round to even
+    }
+    let s = q.toString();
+    while (s.length < 3) {
+        s = "0" + s;
+    }
+    const intPart = s.slice(0, s.length - 2);
+    const frac = s.slice(s.length - 2);
+    return (neg ? "-" : "") + intPart + "." + frac;
+}
+// exactRational returns the exact value of a non-negative finite double as
+// num/den (both BigInt, den a power of two), decoded from its IEEE-754 bits.
+function exactRational(x) {
+    const buf = new DataView(new ArrayBuffer(8));
+    buf.setFloat64(0, x);
+    const hi = buf.getUint32(0);
+    const lo = buf.getUint32(4);
+    const exp = (hi >>> 20) & 0x7ff;
+    const mantHi = BigInt(hi & 0xfffff);
+    let mant = (mantHi << 32n) | BigInt(lo >>> 0);
+    let e;
+    if (exp === 0) {
+        // Subnormal.
+        e = -1074;
+    }
+    else {
+        mant |= 1n << 52n; // implicit leading bit
+        e = exp - 1075;
+    }
+    // value = mant * 2^e
+    if (e >= 0) {
+        return { num: mant << BigInt(e), den: 1n };
+    }
+    return { num: mant, den: 1n << BigInt(-e) };
+}

package/dist/dedup/report/floatfmt.test.js

@@ -0,0 +1,42 @@
+// Pins formatFloat2 to Go's fmt "%.2f" semantics.
+//
+// WHY this matters and is NOT covered by the renderer tests: the dedup
+// similarity scores are printed with %.2f and the golden oracle checks them
+// byte-for-byte. JS Number.prototype.toFixed rounds differently from Go on
+// half-way binary values (e.g. 0.125 -> "0.13" in JS but "0.12" in Go, because
+// the exact double 0.125 is a tie and Go rounds half-to-even). The renderer
+// fixtures only use values that happen to round the same either way (0.89,
+// 0.95, 0.96), so a naive toFixed would pass them yet still corrupt a real
+// cosine score that lands on a tie. These cases lock the Go behavior directly.
+import { describe, it, expect } from "vitest";
+import { formatFloat2 } from "./floatfmt.js";
+describe("formatFloat2 matches Go %.2f", () => {
+    it("formats the dedup similarity values used in fixtures", () => {
+        expect(formatFloat2(0.89)).toBe("0.89");
+        expect(formatFloat2(0.95)).toBe("0.95");
+        expect(formatFloat2(0.96)).toBe("0.96");
+        expect(formatFloat2(1.0)).toBe("1.00");
+        expect(formatFloat2(0)).toBe("0.00");
+    });
+    it("rounds half-to-even on exact-tie doubles (diverges from JS toFixed)", () => {
+        // 0.125 is exactly representable and is a tie at the 2nd decimal; Go rounds
+        // to the even digit (0.12). JS (0.125).toFixed(2) === "0.13" — the bug this
+        // formatter exists to avoid.
+        expect(formatFloat2(0.125)).toBe("0.12");
+        // 0.375 is exact and ties up to even (0.38).
+        expect(formatFloat2(0.375)).toBe("0.38");
+        // 2.675 is NOT exactly 2.675 in binary (slightly less), so it rounds DOWN to
+        // 2.67 — matching Go and demonstrating exact-value (not decimal) rounding.
+        expect(formatFloat2(2.675)).toBe("2.67");
+    });
+    it("rounds non-tie values to nearest", () => {
+        expect(formatFloat2(0.135)).toBe("0.14");
+        expect(formatFloat2(0.025)).toBe("0.03");
+        expect(formatFloat2(0.005000001)).toBe("0.01");
+        expect(formatFloat2(0.004999999)).toBe("0.00");
+    });
+    it("pads to two fraction digits and a leading integer digit", () => {
+        expect(formatFloat2(0.5)).toBe("0.50");
+        expect(formatFloat2(3)).toBe("3.00");
+    });
+});

package/dist/dedup/report/index.js

@@ -0,0 +1,8 @@
+// Barrel for the dedup report renderer.
+//
+// Port of internal/dedup/report. Exports Render (the plain-text report
+// renderer) and QuoteHeading (the Python-repr-style heading quoter, exported in
+// Go for the renderer's own use and for tests). formatFloat2 and wordWrap stay
+// internal — they are implementation details with no Go-exported counterpart.
+export { Render } from "./text.js";
+export { QuoteHeading } from "./quote.js";

package/dist/dedup/report/quote.js

@@ -0,0 +1,77 @@
+// QuoteHeading mimics Python's repr() for the string subset that appears in
+// markdown headings. Ported from internal/dedup/report/quote.go.
+//
+//   - Wrap in single quotes.
+//   - Backslash-escape '\', "'", control chars < 0x20, and DEL (0x7f).
+//   - Leave non-ASCII (UTF-8) bytes untouched — CJK characters render
+//     literally, NOT as \uXXXX.
+//
+// This is the locked rule from plan §366. The CJK case is tested explicitly:
+// input "補貨單流程" must produce "'補貨單流程'" (NOT escape sequences).
+//
+// Byte-for-byte note: Go iterates over the string's BYTES and passes any byte
+// >= 0x80 through verbatim. In TS we iterate over UTF-16 code units; every char
+// with codepoint >= 0x80 (including each surrogate half of an astral char) is
+// passed through verbatim, which reassembles to the identical bytes when the
+// result is UTF-8 encoded. All escaped cases are ASCII (< 0x80), so the
+// code-unit walk and the byte walk produce identical output.
+/** QuoteHeading returns a Python-repr-style single-quoted form of s. */
+export function QuoteHeading(s) {
+    let b = "'";
+    for (const ch of charUnits(s)) {
+        const c = ch.codePointAt(0);
+        // Non-ASCII: pass through verbatim (covers UTF-8 multi-byte sequences / CJK).
+        if (c >= 0x80) {
+            b += ch;
+            continue;
+        }
+        switch (ch) {
+            case "\\":
+                b += "\\\\";
+                break;
+            case "'":
+                b += "\\'";
+                break;
+            case "\t":
+                b += "\\t";
+                break;
+            case "\n":
+                b += "\\n";
+                break;
+            case "\r":
+                b += "\\r";
+                break;
+            default:
+                if (c < 0x20) {
+                    // Other control chars — use \xNN.
+                    b += "\\x" + hexDigit(c >> 4) + hexDigit(c & 0xf);
+                }
+                else if (c === 0x7f) {
+                    b += "\\x7f";
+                }
+                else {
+                    b += ch;
+                }
+        }
+    }
+    b += "'";
+    return b;
+}
+// charUnits yields one entry per UTF-16 code unit. We deliberately do NOT use a
+// codepoint iterator (for…of over the string) for ASCII handling, but since all
+// ASCII chars are single code units and astral chars (>= 0x80) pass through
+// verbatim either way, iterating by code unit is equivalent and avoids any
+// surrogate-pair edge case. We iterate by code unit so that a lone surrogate (if
+// any) is still passed through as a single >= 0x80 unit, matching Go's
+// byte-level pass-through of every non-ASCII byte.
+function* charUnits(s) {
+    for (let i = 0; i < s.length; i++) {
+        yield s.charAt(i);
+    }
+}
+function hexDigit(n) {
+    if (n < 10) {
+        return String.fromCharCode(0x30 + n); // '0'..'9'
+    }
+    return String.fromCharCode(0x61 + n - 10); // 'a'..'f'
+}

package/dist/dedup/report/quote.test.js

@@ -0,0 +1,67 @@
+// Pins QuoteHeading to Go's Python-repr-style heading quoting (quote.go).
+//
+// WHY this matters and is NOT covered elsewhere: the dedup text report prints
+// each group's heading via QuoteHeading and the golden oracle checks those
+// bytes against the Go binary. Two locked rules must hold exactly or the oracle
+// diverges: (1) non-ASCII bytes (CJK) pass through VERBATIM, never as \uXXXX
+// escapes; (2) control chars, quote, and backslash are backslash-escaped in the
+// precise Go spelling (\t \n \r, \xNN for other controls, \x7f for DEL). A
+// regression in either rule silently corrupts every report heading.
+import { describe, it, expect } from "vitest";
+import { QuoteHeading } from "./quote.js";
+describe("QuoteHeading", () => {
+    // WHY: the common case — a plain ASCII heading must be wrapped in single
+    // quotes with no escaping, the baseline the renderer relies on.
+    it("wraps a plain ASCII heading in single quotes", () => {
+        expect(QuoteHeading("Overview")).toBe("'Overview'");
+    });
+    // WHY: the locked CJK case from plan §366 — non-ASCII must render literally,
+    // NOT as \uXXXX. Escaping CJK here would make every Chinese heading mismatch
+    // the Go oracle byte-for-byte.
+    it("passes CJK characters through verbatim (no \\u escapes)", () => {
+        expect(QuoteHeading("補貨單流程")).toBe("'補貨單流程'");
+    });
+    // WHY: an embedded single quote must be backslash-escaped (\') so the quoted
+    // form stays a single Python-repr token; an unescaped quote would terminate
+    // the string early and corrupt the report.
+    it("escapes an embedded single quote", () => {
+        expect(QuoteHeading("it's")).toBe("'it\\'s'");
+    });
+    // WHY: a literal backslash must be doubled (\\) — Go escapes '\' first; a
+    // single backslash would be read as an escape lead-in downstream.
+    it("escapes a backslash", () => {
+        expect(QuoteHeading("a\\b")).toBe("'a\\\\b'");
+    });
+    // WHY: the three named control chars have dedicated short escapes in Go
+    // (\t \n \r); using \xNN for these would not match Go's output.
+    it("escapes tab, newline, and carriage return with named escapes", () => {
+        expect(QuoteHeading("a\tb\nc\rd")).toBe("'a\\tb\\nc\\rd'");
+    });
+    // WHY: other control chars (< 0x20) use the \xNN form with two lowercase hex
+    // digits; the hexDigit helper's 0-9 branch is exercised here (0x01 -> "01").
+    it("escapes a low control char as \\xNN with lowercase hex", () => {
+        expect(QuoteHeading("\x01")).toBe("'\\x01'");
+    });
+    // WHY: DEL (0x7f) is ASCII but non-printable; Go emits the literal "\x7f".
+    // This also drives hexDigit's a-f branch (0xf -> "f", 0x7 -> "7").
+    it("escapes DEL (0x7f) as \\x7f", () => {
+        expect(QuoteHeading("\x7f")).toBe("'\\x7f'");
+    });
+    // WHY: a printable ASCII char at the 0x20 boundary (space) must NOT be
+    // escaped — it falls through to the verbatim default, pinning the c >= 0x20
+    // branch that separates printable from control.
+    it("leaves a space and other printable ASCII unescaped", () => {
+        expect(QuoteHeading("a b")).toBe("'a b'");
+    });
+    // WHY: the empty heading must still produce the two surrounding quotes — the
+    // renderer wraps every heading and an empty one must not collapse to "".
+    it("returns just the quotes for an empty string", () => {
+        expect(QuoteHeading("")).toBe("''");
+    });
+    // WHY: an astral (>= 0x10000) char is two UTF-16 code units, each >= 0x80, so
+    // both pass through verbatim and reassemble to the original — covers the
+    // code-unit pass-through that the byte-equality note depends on.
+    it("passes an astral (surrogate-pair) char through unchanged", () => {
+        expect(QuoteHeading("a\u{1F600}b")).toBe("'a\u{1F600}b'");
+    });
+});