docsgov 0.1.0

package/dist/dedup/mdsection/anchor.test.js

@@ -0,0 +1,39 @@
+import { describe, expect, it } from "vitest";
+import { AnchorTracker, makeAnchor } from "./index.js";
+describe("makeAnchor — locked slugification", () => {
+    // Spaces become single hyphens and text is lowercased; this is the GitHub-style
+    // base slug that every section anchor (and thus its ID) is built from.
+    it("lowercases and hyphenates spaces", () => {
+        expect(makeAnchor("Table Section")).toBe("table-section");
+    });
+    // Punctuation outside [letter, digit, hyphen, space, underscore] is stripped
+    // BEFORE hyphenation, matching the locked strip regex; a wrong class would shift
+    // every anchor and re-key sections.
+    it("strips punctuation but keeps hyphens and underscores", () => {
+        expect(makeAnchor("Hello, World!")).toBe("hello-world");
+        expect(makeAnchor("a_b-c")).toBe("a_b-c");
+    });
+    // A run of whitespace collapses to ONE hyphen (not one per space), per Go's
+    // single-separator emission.
+    it("collapses whitespace runs to a single hyphen", () => {
+        expect(makeAnchor("foo   bar")).toBe("foo-bar");
+    });
+    // CJK characters are Unicode letters (\p{L}) and must pass through unchanged —
+    // the strip rule is Unicode-aware, not ASCII-only.
+    it("preserves CJK letters", () => {
+        expect(makeAnchor("補貨單流程")).toBe("補貨單流程");
+    });
+});
+describe("AnchorTracker — per-file collision suffixes", () => {
+    // The first occurrence is unsuffixed and each subsequent duplicate gets the
+    // next integer suffix, in assignment order. This is what keeps repeated
+    // headings addressable as distinct sections within one file.
+    it("suffixes duplicates -1, -2, … in order", () => {
+        const t = new AnchorTracker();
+        expect(t.assign("Installation")).toBe("installation");
+        expect(t.assign("Installation")).toBe("installation-1");
+        expect(t.assign("Installation")).toBe("installation-2");
+        // A different heading starts its own counter.
+        expect(t.assign("Usage")).toBe("usage");
+    });
+});

package/dist/dedup/mdsection/blocks.js

@@ -0,0 +1,409 @@
+/**
+ * Block classification, text extraction, table linearization, and BlockRecord
+ * production. Ported from internal/dedup/mdsection/blocks.go.
+ *
+ * The goldmark→mdast mapping that matters here:
+ *   - goldmark Text node with SoftLineBreak()/HardLineBreak() appends a space.
+ *     In mdast a soft break is an embedded "\n" inside one text node's value;
+ *     a hard break is a separate `break` node. We reproduce Go's "append a space"
+ *     behavior by turning each embedded "\n" into a space and emitting a space for
+ *     every `break` node.
+ *   - goldmark's table has a separate header node; mdast's `table` has the header
+ *     as its first `tableRow`. linearizeTable / countTableDataRows account for this.
+ *   - goldmark autolinks reach collectText as a childless leaf; mdast represents
+ *     `<https://…>` as a `link` node with a child text whose value is the URL, so
+ *     normal recursion already yields the visible URL text.
+ *   - inline raw HTML (`<br>`) is an mdast `html` node with no children; Go skips
+ *     it (only Text/AutoLink/String are emitted), so we skip `html`/`break`-value.
+ */
+import { createHash } from "node:crypto";
+/**
+ * Applies the same normalization pipeline as buildEmbedText:
+ * toLowerCase → collapseWhitespace → trim. This ensures a block hashed in
+ * isolation equals that same text as it appears inside a section's embed_text.
+ */
+export function normalizeBlockText(s) {
+    s = s.toLowerCase();
+    s = collapseWhitespace(s);
+    return s.trim();
+}
+/** Returns the hex-encoded SHA-256 of the normalized block text. */
+export function blockContentHash(s) {
+    const normalized = normalizeBlockText(s);
+    return createHash("sha256").update(normalized).digest("hex");
+}
+/**
+ * Splits prose on whitespace for word counting.
+ *
+ * Go uses `regexp.MustCompile("\\s+")`, whose `\s` in Go's RE2 is the ASCII set
+ * `[\t\n\f\r ]` (NOT vertical tab, NOT Unicode spaces). JS `\s` differs (it adds
+ * \v and Unicode spaces), so the class is spelled out explicitly to match Go.
+ */
+const wsRE = /[\t\n\f\r ]+/g;
+/**
+ * Classifies an mdast block node and returns a Block.
+ */
+export function classifyNode(n) {
+    switch (n.type) {
+        case "code":
+            // mdast `code` covers both fenced and indented code blocks
+            // (goldmark KindFencedCodeBlock + KindCodeBlock).
+            return { class: 2 /* BlockClass.Code */, text: "" };
+        case "html":
+            return { class: 3 /* BlockClass.HTML */, text: "" };
+        case "thematicBreak":
+            return { class: 3 /* BlockClass.HTML */, text: "" };
+        case "table":
+            return { class: 1 /* BlockClass.Table */, text: linearizeTable(n) };
+        case "list":
+            return classifyList(n);
+        case "paragraph":
+            if (isImageOnlyParagraph(n)) {
+                return { class: 3 /* BlockClass.HTML */, text: "" };
+            }
+            return { class: 0 /* BlockClass.Prose */, text: extractInlineText(n) };
+        case "blockquote":
+            return { class: 0 /* BlockClass.Prose */, text: extractBlockText(n) };
+        default: {
+            // For any unknown block type, extract as prose.
+            const text = extractBlockText(n);
+            if (text !== "") {
+                return { class: 0 /* BlockClass.Prose */, text };
+            }
+            return { class: 3 /* BlockClass.HTML */, text: "" };
+        }
+    }
+}
+/**
+ * Processes a list node, producing one chunk per list item joined by newlines.
+ * Returns a single Prose block.
+ */
+function classifyList(list) {
+    const parts = [];
+    for (const item of list.children) {
+        if (item.type !== "listItem") {
+            continue;
+        }
+        const text = listItemText(item);
+        if (text !== "") {
+            parts.push(text);
+        }
+    }
+    return { class: 0 /* BlockClass.Prose */, text: parts.join("\n") };
+}
+/**
+ * Extracts the prose text from a list item, joining all prose children with a
+ * single space (locked: list_item flattening).
+ */
+function listItemText(item) {
+    const chunks = [];
+    for (const child of item.children) {
+        let text;
+        switch (child.type) {
+            case "paragraph":
+                text = extractInlineText(child);
+                break;
+            default:
+                // mdast has no "textBlock"; any block child with children is descended.
+                if (hasChildren(child)) {
+                    text = extractBlockText(child);
+                }
+                else {
+                    text = "";
+                }
+        }
+        const t = text.trim();
+        if (t !== "") {
+            chunks.push(t);
+        }
+    }
+    return chunks.join(" ");
+}
+/** Returns true if the paragraph contains only image nodes. */
+function isImageOnlyParagraph(n) {
+    let hasChild = false;
+    for (const c of n.children) {
+        hasChild = true;
+        if (c.type !== "image") {
+            return false;
+        }
+    }
+    return hasChild;
+}
+/** Extracts plain text from a node's inline children. */
+export function extractInlineText(n) {
+    const out = [];
+    collectText(n, out);
+    return out.join("");
+}
+/**
+ * Recursively collects text from a node's children, mirroring goldmark's
+ * collectText: emit Text values, append a space for each soft/hard line break,
+ * skip markup-only inline leaves (raw inline HTML).
+ *
+ * mdast specifics:
+ *   - a `text` node value may contain "\n" (soft break) — each is turned into a
+ *     space, matching goldmark appending a space after a soft-broken Text node.
+ *   - a `break` node (hard break) emits a single space.
+ *   - `inlineCode`/`emphasis`/`strong`/`link`/`delete` recurse (their visible
+ *     text lives in child text nodes, except inlineCode which carries its own
+ *     `value`).
+ *   - `html` (inline raw HTML) and image-only leaves are skipped.
+ */
+function collectText(n, out) {
+    const children = getChildren(n);
+    if (children === undefined) {
+        return;
+    }
+    for (const c of children) {
+        switch (c.type) {
+            case "text":
+                // A soft line break appears as an embedded "\n"; goldmark appends a
+                // space after each soft-broken Text node, so map "\n" → " ".
+                out.push(c.value.replace(/\n/g, " "));
+                break;
+            case "break":
+                // Hard line break: goldmark appends a space.
+                out.push(" ");
+                break;
+            case "inlineCode":
+                // goldmark CodeSpan holds its text in child Text nodes; mdast carries it
+                // as `value`. Emit it directly (its visible text).
+                out.push(c.value);
+                break;
+            case "html":
+                // Inline raw HTML and markup-only leaves are skipped, consistent with
+                // block-level HTML being stripped from embed text.
+                break;
+            case "image":
+            case "imageReference":
+            case "footnoteReference":
+                // Markup-only / non-text inline leaves: skip.
+                break;
+            default:
+                if (hasChildren(c)) {
+                    collectText(c, out);
+                }
+        }
+    }
+}
+/** Extracts all text from a block node recursively. */
+export function extractBlockText(n) {
+    const out = [];
+    const children = getChildren(n);
+    if (children !== undefined) {
+        for (const c of children) {
+            switch (c.type) {
+                case "paragraph":
+                    out.push(extractInlineText(c));
+                    out.push("\n");
+                    break;
+                default:
+                    if (hasChildren(c)) {
+                        out.push(extractBlockText(c));
+                    }
+            }
+        }
+    }
+    return out.join("").trim();
+}
+/**
+ * Converts an mdast Table node to the locked linearized format:
+ *
+ *   "h1=v1, h2=v2; h1=v1b, h2=v2b"
+ *
+ * Semicolons separate rows; commas separate cells within a row; "=" separates
+ * header from cell value.
+ *
+ * mdast does not have a distinct header node: the first `tableRow` is the
+ * header and the remaining `tableRow`s are data rows.
+ */
+export function linearizeTable(table) {
+    const rows = table.children.filter((c) => c.type === "tableRow");
+    if (rows.length === 0) {
+        return "";
+    }
+    const headerRow = rows[0];
+    const headers = [];
+    for (const cell of headerRow.children) {
+        if (cell.type === "tableCell") {
+            headers.push(extractInlineText(cell).trim());
+        }
+    }
+    const dataRows = [];
+    for (let i = 1; i < rows.length; i++) {
+        const cells = [];
+        for (const cell of rows[i].children) {
+            if (cell.type === "tableCell") {
+                cells.push(extractInlineText(cell).trim());
+            }
+        }
+        dataRows.push(cells);
+    }
+    const rowStrs = [];
+    for (const row of dataRows) {
+        const cellStrs = [];
+        for (let i = 0; i < row.length; i++) {
+            const header = i < headers.length ? headers[i] : "";
+            cellStrs.push(`${header}=${row[i]}`);
+        }
+        rowStrs.push(cellStrs.join(", "));
+    }
+    return rowStrs.join("; ");
+}
+/**
+ * Returns the number of data rows in an mdast Table node — the count of
+ * `tableRow` children minus the single header row. This is the unambiguous
+ * source for the table eligibility gate; counting ";" in the linearized text
+ * over-counts when a cell value itself contains a ";".
+ */
+export function countTableDataRows(table) {
+    const total = table.children.reduce((n, c) => (c.type === "tableRow" ? n + 1 : n), 0);
+    return total > 0 ? total - 1 : 0;
+}
+/**
+ * Returns the 1-indexed start line of the first content of node n.
+ *
+ * Go's firstSegStart returns the byte offset of n's first line segment, or, when
+ * the node carries no segments (e.g. a list), descends to the first descendant
+ * that does. mdast attaches a position to every node — for a list this position
+ * starts at the first list item's content — so n.position.start.line already
+ * equals what Go computes for the locked fixtures. Returns -1 if no position.
+ */
+export function firstStartLine(n) {
+    if (n.position) {
+        return n.position.start.line;
+    }
+    return -1;
+}
+/**
+ * Produces BlockRecords for the given body nodes of a section. Each block's
+ * [StartLine, EndLine) is computed from node positions: StartLine =
+ * firstStartLine(n), EndLine = next block's StartLine, or sectionEndLine for the
+ * last block. Code and HTML blocks are excluded. Lists are folded into "prose".
+ *
+ * SectionID/FilePath/Heading are NOT stamped here — the caller fills them.
+ */
+export function collectBlockRecords(bodyNodes, sectionEndLine) {
+    const candidates = [];
+    for (const n of bodyNodes) {
+        const b = classifyNode(n);
+        if (b.class === 2 /* BlockClass.Code */ || b.class === 3 /* BlockClass.HTML */) {
+            continue;
+        }
+        const start = firstStartLine(n);
+        if (start < 0) {
+            continue;
+        }
+        candidates.push({ node: n, b, startLine: start });
+    }
+    if (candidates.length === 0) {
+        return [];
+    }
+    const records = [];
+    for (let i = 0; i < candidates.length; i++) {
+        const c = candidates[i];
+        const endLine = i + 1 < candidates.length ? candidates[i + 1].startLine : sectionEndLine;
+        let kind;
+        let tableRows = 0;
+        if (c.b.class === 1 /* BlockClass.Table */) {
+            kind = "table";
+            tableRows = countTableDataRows(c.node);
+        }
+        else {
+            kind = "prose";
+        }
+        records.push({
+            SectionID: "",
+            FilePath: "",
+            Heading: "",
+            Index: i,
+            Kind: kind,
+            StartLine: c.startLine,
+            EndLine: endLine,
+            ContentHash: blockContentHash(c.b.text),
+            Text: normalizeBlockText(c.b.text),
+            TableRows: tableRows,
+        });
+    }
+    return records;
+}
+/**
+ * Splits text on whitespace and returns the number of non-empty tokens.
+ * Matches the locked rule: split on Go-RE2 `\s+` (ASCII), filter empty splits.
+ */
+export function countWords(text) {
+    const parts = text.trim().split(wsRE);
+    let count = 0;
+    for (const p of parts) {
+        if (p !== "") {
+            count++;
+        }
+    }
+    return count;
+}
+/**
+ * Produces the lowercased, whitespace-collapsed embed text for a section from
+ * its classified blocks. Prose and table blocks are included; code and HTML are
+ * stripped.
+ */
+export function buildEmbedText(blocks) {
+    const parts = [];
+    for (const b of blocks) {
+        if (b.class === 0 /* BlockClass.Prose */ || b.class === 1 /* BlockClass.Table */) {
+            const t = b.text.trim();
+            if (t !== "") {
+                parts.push(t);
+            }
+        }
+    }
+    if (parts.length === 0) {
+        return "";
+    }
+    let combined = parts.join(" ");
+    combined = combined.toLowerCase();
+    combined = collapseWhitespace(combined);
+    return combined.trim();
+}
+/**
+ * Replaces runs of whitespace with a single space.
+ *
+ * Go uses `unicode.IsSpace`, whose rune set is the Unicode White_Space property;
+ * `\p{White_Space}` is the faithful equivalent (note this differs from countWords'
+ * ASCII-only split, matching Go's distinction between unicode.IsSpace and RE2 `\s`).
+ */
+export function collapseWhitespace(s) {
+    let out = "";
+    let inSpace = false;
+    for (const r of s) {
+        if (isUnicodeSpace(r)) {
+            if (!inSpace) {
+                out += " ";
+                inSpace = true;
+            }
+        }
+        else {
+            inSpace = false;
+            out += r;
+        }
+    }
+    return out;
+}
+const spaceRE = /^\p{White_Space}$/u;
+/** Mirrors Go's unicode.IsSpace for a single code point. */
+function isUnicodeSpace(r) {
+    return spaceRE.test(r);
+}
+/** True if a node has at least one child. */
+function hasChildren(n) {
+    const children = getChildren(n);
+    return children !== undefined && children.length > 0;
+}
+/** Returns a node's children array, or undefined for leaf nodes. */
+function getChildren(n) {
+    const c = n.children;
+    if (Array.isArray(c)) {
+        return c;
+    }
+    return undefined;
+}