docsgov 0.1.0

package/dist/dedup/indexer/indexer.test.js

@@ -0,0 +1,510 @@
+/**
+ * Behavior-encoding tests for the dedup indexing pipeline, ported from
+ * internal/dedup/indexer/indexer_test.go.
+ *
+ * Each test states WHY the behavior matters. The indexer is the incremental
+ * cache: it must (a) persist eligible sections/blocks, (b) re-embed ONLY the
+ * sections/blocks whose content hash changed — the cache's whole reason to exist,
+ * (c) populate inbound_count from intra-corpus links, (d) prune rows for deleted
+ * files, and (e) roll the whole run back on any failure so a crash never leaves a
+ * half-written index.
+ *
+ * The embedder is injected as a deterministic fake so tests never download the
+ * ~1GB model — this mirrors Go, which mocks the embedder at the indexer level.
+ * The DB is a real node:sqlite index opened at a temp path (the same engine the
+ * package uses), and raw queries read it back to assert on persisted state.
+ */
+import { mkdtempSync, rmSync, writeFileSync, mkdirSync, unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { afterEach, describe, expect, it } from "vitest";
+import { Dimension, Model } from "../embedder/index.js";
+import { open } from "../indexdb/index.js";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { blockEligible, blockEmbeddable, collectSectionsAndBlocks, embedBlocks, parseLinks, run, } from "./index.js";
+// --- fake embedder ---------------------------------------------------------
+/**
+ * fakeEmbedder returns deterministic unit vectors and records how many embed
+ * calls were made and what texts were embedded. errOn forces the next embed call
+ * to throw (for the rollback test).
+ */
+class FakeEmbedder {
+    calls = 0;
+    batches = [];
+    errOn = null;
+    async embed(texts) {
+        if (this.errOn !== null) {
+            const err = this.errOn;
+            this.errOn = null;
+            throw err;
+        }
+        this.calls++;
+        this.batches.push([...texts]);
+        return texts.map((_, i) => unitVec(i, Dimension));
+    }
+    name() {
+        return Model;
+    }
+    dimension() {
+        return Dimension;
+    }
+    totalTexts() {
+        return this.batches.reduce((n, b) => n + b.length, 0);
+    }
+}
+/** unitVec produces a deterministic vector with a 1.0 at position (seed % dim). */
+function unitVec(seed, dim) {
+    const v = new Array(dim).fill(0);
+    v[seed % dim] = 1.0;
+    return v;
+}
+// --- temp dir / DB plumbing ------------------------------------------------
+const tmpDirs = [];
+const openStores = [];
+afterEach(() => {
+    for (const s of openStores.splice(0)) {
+        try {
+            s.close();
+        }
+        catch {
+            /* already closed */
+        }
+    }
+    for (const d of tmpDirs.splice(0)) {
+        rmSync(d, { recursive: true, force: true });
+    }
+});
+/** openTestDB opens a fresh indexdb at a temp path; returns the store + path. */
+function openTestDB() {
+    const dir = mkdtempSync(join(tmpdir(), "indexer-db-"));
+    tmpDirs.push(dir);
+    const dbPath = join(dir, "index.db");
+    const { store } = open(dbPath, Model, Dimension);
+    openStores.push(store);
+    return { store, dbPath };
+}
+/** makeTestRepo writes the given files under a fresh temp root; returns repoRoot. */
+function makeTestRepo(files) {
+    const root = mkdtempSync(join(tmpdir(), "indexer-repo-"));
+    tmpDirs.push(root);
+    for (const [relPath, content] of Object.entries(files)) {
+        const full = join(root, relPath);
+        mkdirSync(dirname(full), { recursive: true });
+        writeFileSync(full, content);
+    }
+    return root;
+}
+const noProgress = () => { };
+// --- raw-DB read helpers (assert on persisted state) -----------------------
+function rawDB(dbPath) {
+    return new DatabaseSync(dbPath);
+}
+function countSectionsInDB(dbPath) {
+    const db = rawDB(dbPath);
+    try {
+        const row = db.prepare(`SELECT COUNT(*) AS n FROM sections`).get();
+        return row.n;
+    }
+    finally {
+        db.close();
+    }
+}
+function sectionHashesFromDB(dbPath) {
+    const db = rawDB(dbPath);
+    try {
+        const rows = db
+            .prepare(`SELECT content_hash FROM sections ORDER BY file_path, start_line`)
+            .all();
+        return rows.map((r) => r.content_hash);
+    }
+    finally {
+        db.close();
+    }
+}
+function countBlocksInDB(dbPath) {
+    const db = rawDB(dbPath);
+    try {
+        const row = db.prepare(`SELECT COUNT(*) AS n FROM blocks`).get();
+        return row.n;
+    }
+    finally {
+        db.close();
+    }
+}
+function blockFilePathsFromDB(dbPath) {
+    const db = rawDB(dbPath);
+    try {
+        const rows = db
+            .prepare(`SELECT DISTINCT file_path FROM blocks ORDER BY file_path`)
+            .all();
+        return rows.map((r) => r.file_path);
+    }
+    finally {
+        db.close();
+    }
+}
+function blockHashesFromDB(dbPath, filePath) {
+    const db = rawDB(dbPath);
+    try {
+        const rows = db
+            .prepare(`SELECT section_id, block_index, content_hash FROM blocks WHERE file_path=?`)
+            .all(filePath);
+        const out = new Map();
+        for (const r of rows) {
+            out.set(`${r.section_id}|${r.block_index}`, r.content_hash);
+        }
+        return out;
+    }
+    finally {
+        db.close();
+    }
+}
+function blockEmbedNullByHeadingFromDB(dbPath) {
+    const db = rawDB(dbPath);
+    try {
+        const rows = db
+            .prepare(`SELECT heading, (embedding IS NULL) AS is_null FROM blocks`)
+            .all();
+        const out = new Map();
+        for (const r of rows) {
+            out.set(r.heading, r.is_null !== 0);
+        }
+        return out;
+    }
+    finally {
+        db.close();
+    }
+}
+// --- tests -----------------------------------------------------------------
+describe("indexer.run", () => {
+    // WHY: the index is useless if a basic corpus produces no persisted sections.
+    it("indexes a simple corpus and persists sections", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/guide.md": "## Introduction\n\nThis is the introduction section of the guide. It covers the basic concepts and provides an overview of the system. Users should read this first.\n",
+        });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        const stats = await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        expect(stats.sections).toBeGreaterThan(0);
+        expect(countSectionsInDB(dbPath)).toBeGreaterThan(0);
+    });
+    // WHY: incremental caching is the package's reason to exist — re-indexing an
+    // unchanged corpus must do zero embedder work.
+    it("is idempotent: a second run on an unchanged corpus makes 0 new embed calls", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/guide.md": "## Introduction\n\nThis is the introduction section of the guide. It covers the basic concepts and provides an overview of the system. Users should read this first.\n\n## Setup\n\nThe setup section explains installation and configuration. Follow the steps below to get started with the tool. Make sure you have all prerequisites installed.\n",
+        });
+        const { store } = openTestDB();
+        const emb = new FakeEmbedder();
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const callsAfterFirst = emb.calls;
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        expect(emb.calls).toBe(callsAfterFirst);
+    });
+    // WHY: only the changed section (and its changed block) may be re-embedded;
+    // unchanged sections must reuse their cached vectors. A changed section
+    // re-embeds the section itself AND its one prose block = 2 texts; the unchanged
+    // section contributes none.
+    it("re-embeds only the changed section and its block", async () => {
+        const sectionA = "## Introduction\n\nThis is the introduction section of the guide. It covers the basic concepts and provides an overview of the system. Users should read this first.\n\n";
+        const sectionB = "## Setup\n\nThe setup section explains installation and configuration. Follow the steps below to get started with the tool. Make sure you have all prerequisites installed.\n";
+        const repoRoot = makeTestRepo({ "docs/guide.md": sectionA + sectionB });
+        const { store } = openTestDB();
+        const emb = new FakeEmbedder();
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const textsAfterFirst = emb.totalTexts();
+        const modified = "## Introduction\n\nThis introduction has been updated with new content. It covers advanced concepts. Users should read this thoroughly.\n\n" +
+            sectionB;
+        writeFileSync(join(repoRoot, "docs/guide.md"), modified);
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const newTexts = emb.totalTexts() - textsAfterFirst;
+        expect(newTexts).toBe(2);
+    });
+    // WHY: parity with the Python POC requires sections written in (file_path,
+    // start_line) order regardless of filesystem walk order.
+    it("writes sections in deterministic (file_path, start_line) order", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/z_last.md": "## Zebra Section\n\nThis section comes last in lexicographic order. It has sufficient prose words to be eligible for indexing. Multiple sentences ensure the minimum word count is met.\n",
+            "docs/a_first.md": "## Apple Section\n\nThis section comes first in lexicographic order. It has sufficient prose words to be eligible for indexing. Multiple sentences ensure the minimum word count is met.\n",
+        });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const db = rawDB(dbPath);
+        let paths;
+        try {
+            paths = db.prepare(`SELECT file_path FROM sections ORDER BY file_path, start_line`).all().map((r) => r.file_path);
+        }
+        finally {
+            db.close();
+        }
+        const sorted = [...paths].sort();
+        expect(paths).toEqual(sorted);
+    });
+    // WHY: a section whose file is deleted must be removed from the DB, or the
+    // index would accumulate dead rows that pollute clustering.
+    it("prunes sections whose file was removed", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/guide.md": "## Introduction\n\nThis is the introduction section. It explains the basic concepts and overview of the system. This is important to read first.\n",
+            "docs/extra.md": "## Extra Content\n\nThis extra file will be removed in the next run. It contains important information about the additional features. Please read it carefully.\n",
+        });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const countAfterFirst = countSectionsInDB(dbPath);
+        expect(countAfterFirst).toBeGreaterThanOrEqual(2);
+        unlinkSync(join(repoRoot, "docs/extra.md"));
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const countAfterSecond = countSectionsInDB(dbPath);
+        expect(countAfterSecond).toBeLessThan(countAfterFirst);
+    });
+    // WHY: the whole run is one transaction — an embedder failure must leave the
+    // on-disk DB exactly as it was, never half-written. We first index a corpus
+    // successfully, then make the SECOND run (after a content edit) fail at embed
+    // time; the persisted hashes must be byte-identical to the pre-failure state,
+    // proving the edited row was never committed.
+    it("rolls back the transaction when the embedder fails", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/guide.md": "## Introduction\n\nThis is the original introduction with plenty of words to be eligible for indexing and embedding into the dedup pool.\n",
+        });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        // First run succeeds and persists state.
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const hashBefore = sectionHashesFromDB(dbPath);
+        expect(hashBefore.length).toBeGreaterThan(0);
+        // Edit the content so the second run must re-embed, then force the embed to fail.
+        writeFileSync(join(repoRoot, "docs/guide.md"), "## Introduction\n\nThis introduction was edited and must trigger a fresh embedding call which is forced to fail for the rollback test.\n");
+        emb.errOn = new Error("fake embedder: forced error for rollback test");
+        await expect(run(store, emb, repoRoot, defaultConfig(), noProgress)).rejects.toThrow();
+        // DB unchanged: same hashes as before the failed run (edit was rolled back).
+        const hashAfter = sectionHashesFromDB(dbPath);
+        expect(hashAfter).toEqual(hashBefore);
+    });
+    // WHY: inbound_count drives canonical selection in the analyzer; Pass 2 must
+    // resolve an intra-corpus link to the target section and increment its count.
+    it("populates inbound_count from intra-corpus links", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/target.md": "## Target Section\n\nThis section is linked to by another document. It provides important information about a key concept. Many other sections reference this one.\n",
+            "docs/source.md": "## Source Section\n\nThis section links to the [Target Section](target.md#target-section). It references important information from the target document. Please see the link for details.\n",
+        });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        await run(store, emb, repoRoot, defaultConfig(), noProgress);
+        const db = rawDB(dbPath);
+        let maxInbound;
+        try {
+            const row = db.prepare(`SELECT MAX(inbound_count) AS m FROM sections`).get();
+            maxInbound = row.m ?? 0;
+        }
+        finally {
+            db.close();
+        }
+        expect(maxInbound).toBeGreaterThan(0);
+    });
+});
+describe("indexer.parseLinks", () => {
+    // WHY: cfg.Indexer.external_url_prefixes must actually drive external-link
+    // filtering — observable only when the caller passes a non-default prefix list.
+    it("filters external links using the supplied prefix list, not a hardcoded one", () => {
+        const raw = "See [the docs](http://example.com/target.md) for more info.";
+        const srcFile = "docs/source.md";
+        // Empty prefix list → http:// is treated as internal (not filtered).
+        const withEmpty = parseLinks(raw, srcFile, []);
+        expect(withEmpty.length).toBeGreaterThan(0);
+        // Default prefix list → http:// is external → filtered out.
+        const withHTTP = parseLinks(raw, srcFile, ["http://", "https://", "mailto:"]);
+        expect(withHTTP.length).toBe(0);
+    });
+});
+describe("indexer block gates and reuse", () => {
+    // WHY: L5 partial-duplication must (a) exclude sub-floor blocks from embedding
+    // and (b) embed each unique ContentHash at most once per run, reusing vectors
+    // for identical blocks so verbatim copies share one stored vector with no extra
+    // embedder calls.
+    it("gates ineligible blocks and dedups embeds by ContentHash", async () => {
+        const cfg = defaultConfig(); // Block.min_words == 10, table_min_rows == 2.
+        const mk = (over) => ({
+            SectionID: "",
+            FilePath: "",
+            Heading: "",
+            Index: 0,
+            Kind: "prose",
+            StartLine: 0,
+            EndLine: 0,
+            ContentHash: "",
+            Text: "",
+            TableRows: 0,
+            ...over,
+        });
+        const subFloor = mk({ Kind: "prose", ContentHash: "sub-floor-hash", Text: "one two three" });
+        const oneRowTable = mk({
+            Kind: "table",
+            ContentHash: "one-row-table-hash",
+            Text: "col=val",
+            TableRows: 1,
+        });
+        const eligibleProse = mk({
+            Kind: "prose",
+            ContentHash: "eligible-hash-A",
+            Text: "the quick brown fox jumps over the lazy dog again",
+        });
+        const sharedHash = "shared-hash-XY";
+        const sharedText = "this is a long enough prose block that qualifies for embedding";
+        const identicalA = mk({ Kind: "prose", ContentHash: sharedHash, Text: sharedText });
+        const identicalB = mk({ Kind: "prose", ContentHash: sharedHash, Text: sharedText });
+        // --- blockEligible gate ---
+        expect(blockEligible(subFloor, cfg)).toBe(false);
+        expect(blockEligible(oneRowTable, cfg)).toBe(false);
+        expect(blockEligible(eligibleProse, cfg)).toBe(true);
+        expect(blockEligible(identicalA, cfg)).toBe(true);
+        expect(blockEligible(identicalB, cfg)).toBe(true);
+        const all = [subFloor, oneRowTable, eligibleProse, identicalA, identicalB];
+        const eligible = all.filter((b) => blockEligible(b, cfg));
+        expect(eligible.length).toBe(3);
+        // --- embedBlocks: hash-based dedup + reuse ---
+        const existingVec = Float32Array.from([0.1, 0.2, 0.3]);
+        const existing = new Map([[eligibleProse.ContentHash, existingVec]]);
+        const emb = new FakeEmbedder();
+        const result = await embedBlocks(emb, eligible, existing, cfg);
+        // Existing hash present (reused, not re-embedded).
+        expect(result.has(eligibleProse.ContentHash)).toBe(true);
+        // Only one unique text embedded (sharedHash, shared by identicalA/B).
+        expect(emb.totalTexts()).toBe(1);
+        // The shared hash is present.
+        expect(result.has(sharedHash)).toBe(true);
+        // Table hash must NOT be in the result (tables are not embedded).
+        expect(result.has(oneRowTable.ContentHash)).toBe(false);
+        // The reused existing vector is preserved exactly.
+        expect(Array.from(result.get(eligibleProse.ContentHash))).toEqual(Array.from(existingVec));
+    });
+    // WHY: blockEmbeddable is the gate that decides whether a block gets a stored
+    // vector. A heading-blacklisted prose block must be excluded from L5-cosine
+    // clustering (NULL embedding); tables are never embeddable.
+    it("blockEmbeddable excludes blacklisted-heading prose and all tables", () => {
+        const cfg = defaultConfig();
+        cfg.Analyzer.heading_blacklist = [...cfg.Analyzer.heading_blacklist, "steps"];
+        const blacklisted = {
+            SectionID: "",
+            FilePath: "",
+            Heading: "Steps",
+            Index: 0,
+            Kind: "prose",
+            StartLine: 0,
+            EndLine: 0,
+            ContentHash: "h1",
+            Text: "",
+            TableRows: 0,
+        };
+        const normal = { ...blacklisted, Heading: "Overview", ContentHash: "h2" };
+        const table = { ...blacklisted, Heading: "Overview", Kind: "table", ContentHash: "h3" };
+        expect(blockEmbeddable(blacklisted, cfg)).toBe(false);
+        expect(blockEmbeddable(normal, cfg)).toBe(true);
+        expect(blockEmbeddable(table, cfg)).toBe(false);
+    });
+});
+describe("indexer block persistence", () => {
+    // WHY: applyChanges must persist eligible blocks in the same tx as sections and
+    // prune any (section_id, block_index) absent from the live eligible set. A
+    // changed block triggers one new embed; an unchanged block triggers none.
+    it("writes, prunes, and incrementally re-embeds blocks", async () => {
+        const proseA1 = "This is the first paragraph of file alpha with plenty of words to qualify.";
+        const proseA2 = "Here is another paragraph in file alpha containing more than ten words easily.";
+        const proseB1 = "This is the primary content of file beta with sufficient words to be eligible.";
+        const fileAlpha = "## Alpha Section\n\n" + proseA1 + "\n\n" + proseA2 + "\n";
+        const fileBeta = "## Beta Section\n\n" + proseB1 + "\n";
+        const repoRoot = makeTestRepo({ "docs/alpha.md": fileAlpha, "docs/beta.md": fileBeta });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        const cfg = defaultConfig();
+        // (a) Initial index: blocks persisted for both files.
+        await run(store, emb, repoRoot, cfg, noProgress);
+        const blocksAfterFirst = countBlocksInDB(dbPath);
+        expect(blocksAfterFirst).toBeGreaterThan(0);
+        expect(blockFilePathsFromDB(dbPath).length).toBe(2);
+        // (b) Delete beta.md: its blocks pruned; alpha's remain.
+        unlinkSync(join(repoRoot, "docs/beta.md"));
+        await run(store, emb, repoRoot, cfg, noProgress);
+        const filesAfterDelete = blockFilePathsFromDB(dbPath);
+        expect(filesAfterDelete).toEqual(["docs/alpha.md"]);
+        expect(countBlocksInDB(dbPath)).toBeLessThan(blocksAfterFirst);
+        // (c) Edit one block in alpha.md: changed block gets a new content_hash; the
+        // unchanged block keeps its hash.
+        const hashesBefore = blockHashesFromDB(dbPath, "docs/alpha.md");
+        expect(hashesBefore.size).toBeGreaterThanOrEqual(2);
+        const proseA1edited = "This paragraph has been edited so its content hash now differs from the original.";
+        const fileAlphaEdited = "## Alpha Section\n\n" + proseA1edited + "\n\n" + proseA2 + "\n";
+        writeFileSync(join(repoRoot, "docs/alpha.md"), fileAlphaEdited);
+        await run(store, emb, repoRoot, cfg, noProgress);
+        const hashesAfter = blockHashesFromDB(dbPath, "docs/alpha.md");
+        let unchanged = 0;
+        let changed = 0;
+        for (const [key, before] of hashesBefore) {
+            const after = hashesAfter.get(key);
+            if (after !== undefined) {
+                if (before === after) {
+                    unchanged++;
+                }
+                else {
+                    changed++;
+                }
+            }
+        }
+        expect(unchanged).toBeGreaterThanOrEqual(1);
+        expect(changed).toBeGreaterThanOrEqual(1);
+        // (d) Idempotent: re-index with no changes — zero embed calls.
+        emb.calls = 0;
+        emb.batches = [];
+        await run(store, emb, repoRoot, cfg, noProgress);
+        expect(emb.calls).toBe(0);
+    });
+    // WHY: the heading blacklist must exclude blocks from L5-cosine clustering even
+    // on re-index — a block embedded before its heading was blacklisted must lose
+    // its stored vector on the next run (stays stored for exact-hash, NULL vector).
+    it("nulls a block's embedding when its heading becomes blacklisted on re-index", async () => {
+        const overviewProse = "This overview paragraph has well over ten words so it clears the block word floor.";
+        const stepsProse = "These steps describe the procedure in more than ten words to clear the floor.";
+        const guide = "## Overview\n\n" + overviewProse + "\n\n## Steps\n\n" + stepsProse + "\n";
+        const repoRoot = makeTestRepo({ "docs/guide.md": guide });
+        const { store, dbPath } = openTestDB();
+        const emb = new FakeEmbedder();
+        // Run 1: default config — both blocks get a vector.
+        const cfg = defaultConfig();
+        await run(store, emb, repoRoot, cfg, noProgress);
+        let nulls = blockEmbedNullByHeadingFromDB(dbPath);
+        expect(nulls.get("Steps")).toBe(false);
+        expect(nulls.get("Overview")).toBe(false);
+        // Run 2: blacklist "steps" and re-index — Steps loses its vector (NULL) but
+        // remains stored; Overview intact.
+        cfg.Analyzer.heading_blacklist = [...cfg.Analyzer.heading_blacklist, "steps"];
+        await run(store, emb, repoRoot, cfg, noProgress);
+        nulls = blockEmbedNullByHeadingFromDB(dbPath);
+        expect(nulls.has("Steps")).toBe(true);
+        expect(nulls.get("Steps")).toBe(true);
+        expect(nulls.get("Overview")).toBe(false);
+    });
+});
+describe("indexer.collectSectionsAndBlocks", () => {
+    // WHY: L5 needs blocks from ALL sections — including table-only
+    // (section-ineligible) sections — and every block's FilePath must be the
+    // repo-relative slash-normalized path.
+    it("collects blocks from every file, including table-only sections, with repo-relative slash paths", async () => {
+        const repoRoot = makeTestRepo({
+            "docs/prose.md": "## Introduction\n\nThis section has enough prose words to be eligible for indexing. It covers several important topics and provides good context.\n",
+            "docs/table_only.md": "## Comparison\n\n| Name | Value |\n|------|-------|\n| A    | 1     |\n| B    | 2     |\n",
+        });
+        const cfg = defaultConfig();
+        const { blocks } = await collectSectionsAndBlocks(repoRoot, cfg);
+        expect(blocks.length).toBeGreaterThan(0);
+        for (const b of blocks) {
+            // Repo-relative slash path: no drive/absolute prefix, no backslash, docs/ prefix.
+            expect(b.FilePath.startsWith("/")).toBe(false);
+            expect(b.FilePath.includes("\\")).toBe(false);
+            expect(b.FilePath.startsWith("docs/")).toBe(true);
+        }
+        // The table block from the table-only (section-ineligible) file must be present.
+        const tableFound = blocks.some((b) => b.FilePath === "docs/table_only.md" && b.Kind === "table");
+        expect(tableFound).toBe(true);
+    });
+});

package/dist/dedup/indexer/links.js

@@ -0,0 +1,89 @@
+/**
+ * Inbound-link resolution for the indexer's Pass 2.
+ *
+ * Ported from internal/dedup/indexer/links.go. parseLinks extracts internal
+ * markdown link targets from a section's raw_content; the indexer resolves each
+ * target to a section ID and increments that section's inbound_count.
+ *
+ * Go used path.Dir/path.Clean/path.Join (the slash-only "path" package, not
+ * filepath) so resolution is OS-independent; the Node "node:path/posix" module is
+ * the exact analogue and keeps forward-slash semantics on every platform.
+ */
+import * as posix from "node:path/posix";
+/**
+ * linkRe matches markdown links: [text](url), excluding image links (!).
+ * Locked regex per plan §Phase2 "Link resolution".
+ *
+ * Go's regexp.FindAllStringSubmatch finds all non-overlapping matches. The
+ * leading `(?:^|[^!])` consumes one character before the `[`, which can cause
+ * adjacent matches to be skipped in Go too (the consumed char is not re-scanned);
+ * the global JS regex below has the identical non-overlapping behaviour because
+ * lastIndex advances past the whole match. The `g` flag is required for
+ * matchAll.
+ */
+const linkRe = /(?:^|[^!])\[(?<text>[^\]]+)\]\((?<url>[^)\s]+)\)/g;
+/**
+ * parseLinks extracts all internal link targets from rawContent.
+ * filePath is the file that contains the content (for relative resolution).
+ * externalPrefixes is the list of URL scheme prefixes treated as external;
+ * links matching any prefix are excluded from the result.
+ * Each returned link is a pair of (resolvedFilePath, anchor).
+ * Self-links (target == source file + same anchor) are not filtered here —
+ * the caller handles self-link exclusion.
+ */
+export function parseLinks(rawContent, filePath, externalPrefixes) {
+    const targets = [];
+    for (const m of rawContent.matchAll(linkRe)) {
+        const url = m.groups?.url;
+        if (url === undefined) {
+            continue;
+        }
+        if (isExternal(url, externalPrefixes)) {
+            continue;
+        }
+        const { filePath: resolved, anchor } = resolveLink(url, filePath);
+        targets.push({ filePath: resolved, anchor });
+    }
+    return targets;
+}
+/**
+ * isExternal returns true if url starts with any of the supplied external
+ * prefixes (from cfg.Indexer.external_url_prefixes).
+ */
+export function isExternal(url, externalPrefixes) {
+    for (const prefix of externalPrefixes) {
+        if (url.startsWith(prefix)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * resolveLink resolves a relative markdown URL against the source file path.
+ * Returns { filePath, anchor }.
+ *
+ * Examples:
+ *
+ *   resolveLink("../concepts/overview.md#intro", "docs/guide/setup.md")
+ *     → { filePath: "docs/concepts/overview.md", anchor: "intro" }
+ *
+ *   resolveLink("target.md", "docs/source.md")
+ *     → { filePath: "docs/target.md", anchor: "" }
+ */
+export function resolveLink(url, sourceFile) {
+    let anchor = "";
+    // Split anchor (Go: strings.LastIndex(url, "#")).
+    const idx = url.lastIndexOf("#");
+    if (idx >= 0) {
+        anchor = url.slice(idx + 1);
+        url = url.slice(0, idx);
+    }
+    // If url is empty (anchor-only link), it points to the same file.
+    if (url === "") {
+        return { filePath: sourceFile, anchor };
+    }
+    // Resolve relative to source file directory (slash-only, like Go's "path").
+    const sourceDir = posix.dirname(sourceFile);
+    const resolved = posix.normalize(posix.join(sourceDir, url));
+    return { filePath: resolved, anchor };
+}

package/dist/dedup/mdsection/anchor.js

@@ -0,0 +1,60 @@
+/**
+ * GitHub-style anchor slug generation and per-file collision tracking.
+ *
+ * Ported from internal/dedup/mdsection/anchor.go. The slugification rules are
+ * LOCKED: these anchors feed sectionid.derive and persist in the dedup index,
+ * so they must match the Go implementation character-for-character.
+ */
+/**
+ * Strips characters that are not Unicode letters, Unicode digits, hyphens,
+ * spaces, or underscores from heading text to form a GitHub-style anchor.
+ *
+ * Go source: `regexp.MustCompile("[^\\p{L}\\p{N}\\- _]+")`.
+ * The `u` flag makes `\p{L}`/`\p{N}` match the same Unicode classes Go's RE2
+ * matches for `\p{L}`/`\p{N}`.
+ */
+const anchorStripRE = /[^\p{L}\p{N}\- _]+/gu;
+/**
+ * Matches a run of Unicode whitespace. Go's makeAnchor uses `unicode.IsSpace`,
+ * whose rune set is exactly the Unicode White_Space property, so `\p{White_Space}`
+ * is the faithful equivalent. (Note: only ASCII space survives the strip step,
+ * since underscore is not whitespace and other separators are stripped — but the
+ * Unicode class is used to mirror Go exactly.)
+ */
+const wsRunRE = /\p{White_Space}+/gu;
+/**
+ * Returns the GitHub-style anchor slug for a heading text, applying the locked
+ * transformation:
+ *   1. Strip characters matching anchorStripRE.
+ *   2. Lowercase.
+ *   3. Replace whitespace runs with a single "-".
+ */
+export function makeAnchor(heading) {
+    // Remove non-(letter, digit, hyphen, space, underscore) characters.
+    let s = heading.replace(anchorStripRE, "");
+    // Lowercase.
+    s = s.toLowerCase();
+    // Replace whitespace runs (space treated as separator) with "-".
+    // Underscores are preserved (not whitespace), matching Go.
+    return s.replace(wsRunRE, "-");
+}
+/**
+ * Tracks per-file anchor collision counts and returns collision-suffixed
+ * anchors. Construct one per Extract call (per file).
+ */
+export class AnchorTracker {
+    seen = new Map();
+    /**
+     * Returns the anchor for the given heading text, appending "-1", "-2", … for
+     * duplicate headings within the same file. The first occurrence is unsuffixed.
+     */
+    assign(heading) {
+        const base = makeAnchor(heading);
+        const count = this.seen.get(base) ?? 0;
+        this.seen.set(base, count + 1);
+        if (count === 0) {
+            return base;
+        }
+        return `${base}-${count}`;
+    }
+}