docsgov 0.1.0

package/dist/dedup/analyzer/safety.test.js

@@ -0,0 +1,39 @@
+import { describe, it, expect } from "vitest";
+import { defaultConfig } from "../dedupcfg/index.js";
+import { findDifferentiators } from "./safety.js";
+// Differentiators are the L4 safety net: when two near-duplicate sections
+// disagree on a load-bearing axis (sync vs async, create vs cancel, ...) the
+// dedup tool must NOT auto-merge them. These tests pin WHY each branch of the
+// cross-side substring contract fires or stays silent — a drift here would let
+// the tool recommend merging genuinely-different sections.
+describe("findDifferentiators", () => {
+    const differentiators = defaultConfig().Analyzer.differentiators;
+    it("detects a pair when A has the left term and B has the right term", () => {
+        const textA = "This section explains sync operations in detail.";
+        const textB = "This section explains async operations in detail.";
+        const hits = findDifferentiators(textA, textB, differentiators);
+        expect(hits.length).toBeGreaterThan(0);
+        // The reason tag must name both sides so reviewers know what differs.
+        expect(hits.some((h) => h.includes("sync") && h.includes("async"))).toBe(true);
+    });
+    it("fires on a cross-side match (A=sync, B=async)", () => {
+        const textA = "Both sync and async operations.";
+        const textB = "Configuration of async jobs.";
+        const hits = findDifferentiators(textA, textB, differentiators);
+        expect(hits.length).toBeGreaterThan(0);
+    });
+    it("does NOT fire when both sides mention only the same term", () => {
+        // WHY: the rule needs A on one side AND B on the OTHER side. Two sections
+        // both about "sync" do not meet the pair contract, so no false positive.
+        const textA = "This section explains sync operations.";
+        const textB = "Configuration of sync workers.";
+        const hits = findDifferentiators(textA, textB, differentiators);
+        expect(hits).toHaveLength(0);
+    });
+    it("does NOT fire on unrelated text with no differentiator terms", () => {
+        const textA = "Overview of request handling middleware.";
+        const textB = "Configuration and request lifecycle overview.";
+        const hits = findDifferentiators(textA, textB, differentiators);
+        expect(hits).toHaveLength(0);
+    });
+});

package/dist/dedup/config.js

@@ -0,0 +1,18 @@
+// Top-level config facade for the dedup subsystem.
+//
+// Ported from internal/dedup/config.go. The Go facade re-exported the Config
+// types (as aliases) and Default() from the leaf dedupcfg package to break the
+// import cycle: internal/dedup/indexer imports dedupcfg directly, while the
+// facade re-exports for ergonomic callers. The same re-export holds here.
+//
+// SectionID (Go's re-export of sectionid.Derive) is intentionally NOT re-exported
+// here: it has no caller in the TS tree and the sectionid leaf is imported
+// directly where needed, so re-adding it would be speculative.
+import { defaultConfig } from "./dedupcfg/index.js";
+/**
+ * Default returns the v1 locked default configuration (Go: Default()).
+ * Delegates to dedupcfg.defaultConfig().
+ */
+export function Default() {
+    return defaultConfig();
+}

package/dist/dedup/configload.js

@@ -0,0 +1,299 @@
+// Loads and validates the dedup config: Default() overlaid with any
+// .docgov/dedup/config.yml present at repoRoot.
+//
+// Ported from internal/dedup/configload.go. Go used yaml.Unmarshal(data, &cfg),
+// which sets only the fields present in the YAML (scalars override, list fields
+// replace) and returns *yaml.TypeError on a type mismatch. The `yaml` package
+// here parses into plain JS values without that struct-aware overlay, so the
+// overlay + per-field type checking is done explicitly below. A type mismatch
+// throws YAMLTypeError so callers can match it (Go's errors.As against
+// *yaml.TypeError → instanceof YAMLTypeError here, per the Go-sentinel→subclass
+// porting rule).
+//
+// Two divergences from Go that the *_test cases pin:
+//   - list fields present in YAML REPLACE the default list entirely;
+//   - *_extra fields APPEND to the (possibly replaced) base list.
+//
+// ctx threading is dropped: there is no cancellation context in the TS tree (the
+// embedder/indexdb already dropped ctx), so Load takes only repoRoot.
+import { readFile } from "node:fs/promises";
+import * as path from "node:path";
+import { parse as parseYaml } from "yaml";
+import { Default } from "./config.js";
+/**
+ * YAMLTypeError is thrown when a config.yml value has the wrong type for its
+ * field (Go's *yaml.TypeError). Callers can match it with `instanceof` — the TS
+ * analogue of Go's errors.As(err, &typeErr).
+ */
+export class YAMLTypeError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "YAMLTypeError";
+    }
+}
+/** isNotExist reports whether err is a Node ENOENT (Go's os.IsNotExist). */
+function isNotExist(err) {
+    return (typeof err === "object" &&
+        err !== null &&
+        err.code === "ENOENT");
+}
+/** quote renders a path as a Go %q-style double-quoted string for error messages. */
+function quote(s) {
+    return JSON.stringify(s);
+}
+/**
+ * Load returns Default() overlaid with any .docgov/dedup/config.yml present at
+ * repoRoot. A missing file is silently skipped. A malformed YAML file throws a
+ * wrapped error (hard error — never falls back to defaults on parse failure).
+ * Validation runs last so user overlays are fully applied before checking.
+ */
+export async function Load(repoRoot) {
+    const cfg = Default();
+    const dedupDir = path.join(repoRoot, ".docgov", "dedup");
+    // Step 1: overlay config.yml onto defaults.
+    const configPath = path.join(dedupDir, "config.yml");
+    let data;
+    try {
+        data = await readFile(configPath, "utf8");
+    }
+    catch (err) {
+        if (!isNotExist(err)) {
+            throw new Error(`configload.Load: read ${quote(configPath)}: ${String(err)}`, { cause: err });
+        }
+        // Missing file: silently skip — cfg stays at defaults.
+    }
+    if (data !== undefined) {
+        let parsed;
+        try {
+            parsed = parseYaml(data);
+        }
+        catch (err) {
+            throw new Error(`configload.Load: parse ${quote(configPath)}: ${String(err)}`, { cause: err });
+        }
+        // An empty/null document is a no-op overlay (cfg stays at defaults).
+        if (parsed !== null && parsed !== undefined) {
+            if (typeof parsed !== "object" || Array.isArray(parsed)) {
+                throw new YAMLTypeError(`configload.Load: parse ${quote(configPath)}: expected a mapping`);
+            }
+            overlay(cfg, parsed, configPath);
+        }
+        // Apply *_extra fields: append extra entries to the (possibly overridden) base list.
+        cfg.Markdown.ignored_dirs = cfg.Markdown.ignored_dirs.concat(cfg.Markdown.ignored_dirs_extra ?? []);
+        cfg.Analyzer.heading_blacklist = cfg.Analyzer.heading_blacklist.concat(cfg.Analyzer.heading_blacklist_extra ?? []);
+    }
+    // Step 2: validate. Runs last so all overlays are applied before checking.
+    validate(cfg);
+    return cfg;
+}
+// --- overlay helpers -------------------------------------------------------
+/** typeErr builds a YAMLTypeError for a field that got the wrong YAML type. */
+function typeErr(configPath, key, want, got) {
+    return new YAMLTypeError(`configload.Load: parse ${quote(configPath)}: field ${key}: cannot unmarshal ${describe(got)} into ${want}`);
+}
+function describe(v) {
+    if (v === null)
+        return "null";
+    if (Array.isArray(v))
+        return "sequence";
+    return typeof v;
+}
+function asNumber(v, configPath, key) {
+    // YAML booleans/strings must NOT coerce to a number (yaml.v3 rejects them).
+    if (typeof v !== "number") {
+        throw typeErr(configPath, key, "number", v);
+    }
+    return v;
+}
+function asString(v, configPath, key) {
+    if (typeof v !== "string") {
+        throw typeErr(configPath, key, "string", v);
+    }
+    return v;
+}
+function asStringList(v, configPath, key) {
+    if (!Array.isArray(v)) {
+        throw typeErr(configPath, key, "sequence", v);
+    }
+    return v.map((item, i) => {
+        if (typeof item !== "string") {
+            throw typeErr(configPath, `${key}[${i}]`, "string", item);
+        }
+        return item;
+    });
+}
+function asDifferentiators(v, configPath, key) {
+    if (!Array.isArray(v)) {
+        throw typeErr(configPath, key, "sequence", v);
+    }
+    return v.map((pair, i) => {
+        if (!Array.isArray(pair) || pair.length !== 2) {
+            throw typeErr(configPath, `${key}[${i}]`, "2-element sequence", pair);
+        }
+        const a = pair[0];
+        const b = pair[1];
+        if (typeof a !== "string" || typeof b !== "string") {
+            throw typeErr(configPath, `${key}[${i}]`, "string pair", pair);
+        }
+        return [a, b];
+    });
+}
+/** has reports whether the YAML mapping carries key (present, even if null). */
+function has(rec, key) {
+    return Object.prototype.hasOwnProperty.call(rec, key);
+}
+/** group returns the sub-mapping under a top-level key, or null if absent/empty. */
+function group(rec, key, configPath) {
+    if (!has(rec, key)) {
+        return null;
+    }
+    const v = rec[key];
+    if (v === null || v === undefined) {
+        return null; // present-but-empty section → no overlay
+    }
+    if (typeof v !== "object" || Array.isArray(v)) {
+        throw typeErr(configPath, key, "mapping", v);
+    }
+    return v;
+}
+/**
+ * overlay applies the parsed YAML mapping onto cfg in place, mirroring
+ * yaml.Unmarshal(data, &cfg): only keys present in the YAML are set; list fields
+ * replace; *_extra fields are stored for later append in Load. Unknown keys are
+ * ignored (yaml.v3 defaults — no KnownFields strict mode).
+ */
+function overlay(cfg, root, p) {
+    const md = group(root, "markdown", p);
+    if (md !== null) {
+        if (has(md, "min_prose_words"))
+            cfg.Markdown.min_prose_words = asNumber(md["min_prose_words"], p, "markdown.min_prose_words");
+        if (has(md, "heading_token_min_len"))
+            cfg.Markdown.heading_token_min_len = asNumber(md["heading_token_min_len"], p, "markdown.heading_token_min_len");
+        if (has(md, "hidden_dir_prefix"))
+            cfg.Markdown.hidden_dir_prefix = asString(md["hidden_dir_prefix"], p, "markdown.hidden_dir_prefix");
+        if (has(md, "ignored_dirs"))
+            cfg.Markdown.ignored_dirs = asStringList(md["ignored_dirs"], p, "markdown.ignored_dirs");
+        if (has(md, "ignored_dirs_extra"))
+            cfg.Markdown.ignored_dirs_extra = asStringList(md["ignored_dirs_extra"], p, "markdown.ignored_dirs_extra");
+    }
+    const idx = group(root, "indexer", p);
+    if (idx !== null) {
+        if (has(idx, "embed_progress_threshold"))
+            cfg.Indexer.embed_progress_threshold = asNumber(idx["embed_progress_threshold"], p, "indexer.embed_progress_threshold");
+        if (has(idx, "max_workers"))
+            cfg.Indexer.max_workers = asNumber(idx["max_workers"], p, "indexer.max_workers");
+        if (has(idx, "external_url_prefixes"))
+            cfg.Indexer.external_url_prefixes = asStringList(idx["external_url_prefixes"], p, "indexer.external_url_prefixes");
+    }
+    const an = group(root, "analyzer", p);
+    if (an !== null) {
+        if (has(an, "thresh_high"))
+            cfg.Analyzer.thresh_high = asNumber(an["thresh_high"], p, "analyzer.thresh_high");
+        if (has(an, "thresh_maybe"))
+            cfg.Analyzer.thresh_maybe = asNumber(an["thresh_maybe"], p, "analyzer.thresh_maybe");
+        if (has(an, "distinctive_abs_min"))
+            cfg.Analyzer.distinctive_abs_min = asNumber(an["distinctive_abs_min"], p, "analyzer.distinctive_abs_min");
+        if (has(an, "distinctive_pct_of_headings"))
+            cfg.Analyzer.distinctive_pct_of_headings = asNumber(an["distinctive_pct_of_headings"], p, "analyzer.distinctive_pct_of_headings");
+        if (has(an, "universal_stopwords"))
+            cfg.Analyzer.universal_stopwords = asStringList(an["universal_stopwords"], p, "analyzer.universal_stopwords");
+        if (has(an, "differentiators"))
+            cfg.Analyzer.differentiators = asDifferentiators(an["differentiators"], p, "analyzer.differentiators");
+        if (has(an, "path_priority"))
+            cfg.Analyzer.path_priority = asStringList(an["path_priority"], p, "analyzer.path_priority");
+        if (has(an, "heading_blacklist"))
+            cfg.Analyzer.heading_blacklist = asStringList(an["heading_blacklist"], p, "analyzer.heading_blacklist");
+        if (has(an, "heading_blacklist_extra"))
+            cfg.Analyzer.heading_blacklist_extra = asStringList(an["heading_blacklist_extra"], p, "analyzer.heading_blacklist_extra");
+        if (has(an, "path_blacklist"))
+            cfg.Analyzer.path_blacklist = asStringList(an["path_blacklist"], p, "analyzer.path_blacklist");
+    }
+    const rep = group(root, "report", p);
+    if (rep !== null) {
+        if (has(rep, "preview_chars"))
+            cfg.Report.preview_chars = asNumber(rep["preview_chars"], p, "report.preview_chars");
+        if (has(rep, "preview_word_ratio"))
+            cfg.Report.preview_word_ratio = asNumber(rep["preview_word_ratio"], p, "report.preview_word_ratio");
+        if (has(rep, "wrap_cols"))
+            cfg.Report.wrap_cols = asNumber(rep["wrap_cols"], p, "report.wrap_cols");
+        if (has(rep, "separator"))
+            cfg.Report.separator = asString(rep["separator"], p, "report.separator");
+    }
+    const em = group(root, "embedder", p);
+    if (em !== null) {
+        if (has(em, "batch_size"))
+            cfg.Embedder.batch_size = asNumber(em["batch_size"], p, "embedder.batch_size");
+    }
+    const blk = group(root, "block", p);
+    if (blk !== null) {
+        if (has(blk, "min_words"))
+            cfg.Block.min_words = asNumber(blk["min_words"], p, "block.min_words");
+        if (has(blk, "table_min_rows"))
+            cfg.Block.table_min_rows = asNumber(blk["table_min_rows"], p, "block.table_min_rows");
+        if (has(blk, "cosine_threshold"))
+            cfg.Block.cosine_threshold = asNumber(blk["cosine_threshold"], p, "block.cosine_threshold");
+        if (has(blk, "multiplicity_cap"))
+            cfg.Block.multiplicity_cap = asNumber(blk["multiplicity_cap"], p, "block.multiplicity_cap");
+    }
+}
+// --- validation ------------------------------------------------------------
+/**
+ * validate checks that all scalar tunables are within their allowed ranges.
+ * Rules per the Go validate():
+ *   - 0 < ThreshMaybe < ThreshHigh < 1
+ *   - MinProseWords >= 1
+ *   - HeadingTokenMinLen >= 1
+ *   - DistinctiveAbsMin >= 1
+ *   - 0 < DistinctivePctOfHeadings <= 1
+ *   - PreviewChars >= 1
+ *   - WrapCols >= 1
+ *   - BatchSize >= 1
+ *   - Block.MinWords >= 1
+ *   - Block.TableMinRows >= 1
+ *   - 0 < Block.CosineThreshold < 1
+ *   - Block.MultiplicityCap >= 1
+ */
+function validate(cfg) {
+    const a = cfg.Analyzer;
+    if (a.thresh_maybe <= 0 || a.thresh_maybe >= 1) {
+        throw new Error(`configload.Load: thresh_maybe must be in (0, 1), got ${a.thresh_maybe}`);
+    }
+    if (a.thresh_high <= 0 || a.thresh_high >= 1) {
+        throw new Error(`configload.Load: thresh_high must be in (0, 1), got ${a.thresh_high}`);
+    }
+    if (a.thresh_maybe >= a.thresh_high) {
+        throw new Error(`configload.Load: thresh_maybe (${a.thresh_maybe}) must be < thresh_high (${a.thresh_high})`);
+    }
+    if (cfg.Markdown.min_prose_words < 1) {
+        throw new Error(`configload.Load: min_prose_words must be >= 1, got ${cfg.Markdown.min_prose_words}`);
+    }
+    if (cfg.Markdown.heading_token_min_len < 1) {
+        throw new Error(`configload.Load: heading_token_min_len must be >= 1, got ${cfg.Markdown.heading_token_min_len}`);
+    }
+    if (a.distinctive_abs_min < 1) {
+        throw new Error(`configload.Load: distinctive_abs_min must be >= 1, got ${a.distinctive_abs_min}`);
+    }
+    if (a.distinctive_pct_of_headings <= 0 || a.distinctive_pct_of_headings > 1) {
+        throw new Error(`configload.Load: distinctive_pct_of_headings must be in (0, 1], got ${a.distinctive_pct_of_headings}`);
+    }
+    if (cfg.Report.preview_chars < 1) {
+        throw new Error(`configload.Load: preview_chars must be >= 1, got ${cfg.Report.preview_chars}`);
+    }
+    if (cfg.Report.wrap_cols < 1) {
+        throw new Error(`configload.Load: wrap_cols must be >= 1, got ${cfg.Report.wrap_cols}`);
+    }
+    if (cfg.Embedder.batch_size < 1) {
+        throw new Error(`configload.Load: batch_size must be >= 1, got ${cfg.Embedder.batch_size}`);
+    }
+    if (cfg.Block.min_words < 1) {
+        throw new Error(`configload.Load: block.min_words must be >= 1, got ${cfg.Block.min_words}`);
+    }
+    if (cfg.Block.table_min_rows < 1) {
+        throw new Error(`configload.Load: block.table_min_rows must be >= 1, got ${cfg.Block.table_min_rows}`);
+    }
+    if (cfg.Block.cosine_threshold <= 0 || cfg.Block.cosine_threshold >= 1) {
+        throw new Error(`configload.Load: block.cosine_threshold must be in (0, 1), got ${cfg.Block.cosine_threshold}`);
+    }
+    if (cfg.Block.multiplicity_cap < 1) {
+        throw new Error(`configload.Load: block.multiplicity_cap must be >= 1, got ${cfg.Block.multiplicity_cap}`);
+    }
+}