@mailwoman/corpus 5.9.0 → 5.10.0

package/out/src/tools/golden-promote.js

@@ -0,0 +1,223 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
+ *   human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
+ *
+ *   ## What it does
+ *
+ *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
+ *   2. Reads the previous-version golden dir for forward-copy + dedup base
+ *   3. Applies filters that drop candidates unlikely to be human-typed:
+ *
+ *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
+ *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
+ *                 before locality only)
+ *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
+ *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
+ *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
+ *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
+ *   7. Writes MANIFEST.json with sha256 of each output file
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus golden promote \
+ *   --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
+ *   --bump-to v0.1.1 \
+ *   --prior v0.1.0
+ *   ```
+ */
+import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { readJSONL, sha256File, writeJSONL } from "@mailwoman/core/utils";
+// ── Filters ───────────────────────────────────────────────────────────────
+function normalize(s) {
+    return s.toLowerCase().replace(/\s+/g, " ").trim();
+}
+/**
+ * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
+ * together rather than human-typed.
+ */
+function isComponentsGlued(entry) {
+    const componentCount = Object.keys(entry.components).length;
+    if (componentCount < 5)
+        return false;
+    const separators = (entry.raw.match(/[,\n;]/g) ?? []).length;
+    return separators < 2;
+}
+/**
+ * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
+ * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
+ * often precedes locality there).
+ */
+function isPostcodeBadlyLeading(entry) {
+    if (Object.keys(entry.components).length < 4)
+        return false;
+    if (entry.country === "FR" || entry.country === "France")
+        return false;
+    const postcode = entry.components.postcode;
+    if (!postcode)
+        return false;
+    const postcodeIdx = entry.raw.indexOf(postcode);
+    if (postcodeIdx === -1)
+        return false;
+    return postcodeIdx < entry.raw.length / 3;
+}
+/**
+ * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
+ * suggests the LLM emitted markup instead of an address.
+ */
+function isSuspicious(entry) {
+    const raw = entry.raw;
+    // eslint-disable-next-line no-control-regex
+    if (/[\x00-\x1f\x7f]/.test(raw))
+        return true;
+    const openBrackets = (raw.match(/[[({<]/g) ?? []).length;
+    const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length;
+    if (openBrackets !== closeBrackets)
+        return true;
+    if (raw.split('"').length > 3)
+        return true;
+    // too many quote marks
+    return false;
+}
+// ── IO ─────────────────────────────────────────────────────────────────────
+/** Read a JSONL, tolerating a missing file (returns `[]`) — prior golden dirs may lack a bucket. */
+function readJSONLIfPresent(path) {
+    if (!existsSync(path))
+        return [];
+    return readJSONL(path);
+}
+// ── Main ───────────────────────────────────────────────────────────────────
+export async function promoteGolden(options, report) {
+    const prior = options.prior ?? "v0.1.0";
+    const goldenRoot = options.goldenRoot ?? "data/eval/golden";
+    const applyFilters = !options.noFilters;
+    const dryRun = options.dryRun ?? false;
+    report?.(`reading candidates: ${options.input}`);
+    const candidates = readJSONLIfPresent(options.input);
+    report?.(`  ${candidates.length} candidates loaded`);
+    // Forward-copy base: existing entries from the prior golden version go forward verbatim,
+    // and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
+    const priorDir = join(goldenRoot, prior);
+    const priorEntries = [];
+    const seenNormalized = new Set();
+    if (existsSync(priorDir)) {
+        for (const f of readdirSync(priorDir).filter((n) => n.endsWith(".jsonl"))) {
+            const country = f.replace(".jsonl", "").toUpperCase();
+            const entries = readJSONLIfPresent(join(priorDir, f));
+            priorEntries.push({ country, entries });
+            for (const e of entries) {
+                seenNormalized.add(normalize(e.raw));
+            }
+            report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`);
+        }
+    }
+    else {
+        report?.(`  ⚠ prior dir ${priorDir} not found — starting fresh`);
+    }
+    // Filter pass
+    const stats = {
+        candidatesIn: candidates.length,
+        filteredOut: { glued: 0, postcodeLeading: 0, suspicious: 0, duplicate: 0, forwardDup: 0 },
+        kept: 0,
+        perCountry: {},
+    };
+    const accepted = [];
+    const seenInBatch = new Set();
+    for (const cand of candidates) {
+        const norm = normalize(cand.raw);
+        // Dedup pass 1: against prior versioned golden
+        if (seenNormalized.has(norm)) {
+            stats.filteredOut.forwardDup++;
+            continue;
+        }
+        // Dedup pass 2: against this batch
+        if (seenInBatch.has(norm)) {
+            stats.filteredOut.duplicate++;
+            continue;
+        }
+        if (applyFilters) {
+            if (isComponentsGlued(cand)) {
+                stats.filteredOut.glued++;
+                continue;
+            }
+            if (isPostcodeBadlyLeading(cand)) {
+                stats.filteredOut.postcodeLeading++;
+                continue;
+            }
+            if (isSuspicious(cand)) {
+                stats.filteredOut.suspicious++;
+                continue;
+            }
+        }
+        accepted.push(cand);
+        seenInBatch.add(norm);
+        stats.kept++;
+        const country = cand.country || "OTHER";
+        stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1;
+    }
+    // Bucket per country, including forward-copied entries
+    const buckets = new Map();
+    for (const { country, entries } of priorEntries) {
+        // Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
+        buckets.set(country, [...entries]);
+    }
+    for (const cand of accepted) {
+        const key = (cand.country || "OTHER").toUpperCase();
+        if (!buckets.has(key)) {
+            buckets.set(key, []);
+        }
+        buckets.get(key).push(cand);
+    }
+    // Output
+    const outDir = join(goldenRoot, options.bumpTo);
+    report?.(`=== plan ===`);
+    report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`);
+    for (const [key, entries] of buckets) {
+        report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`);
+    }
+    report?.(`=== filter stats ===`);
+    report?.(`candidates in:        ${stats.candidatesIn}`);
+    report?.(`  filtered (glued):           ${stats.filteredOut.glued}`);
+    report?.(`  filtered (postcode-lead):   ${stats.filteredOut.postcodeLeading}`);
+    report?.(`  filtered (suspicious):      ${stats.filteredOut.suspicious}`);
+    report?.(`  filtered (dup-in-batch):    ${stats.filteredOut.duplicate}`);
+    report?.(`  filtered (dup-vs-prior):    ${stats.filteredOut.forwardDup}`);
+    report?.(`  kept:                       ${stats.kept}`);
+    report?.(`per-country (kept):`);
+    for (const [c, n] of Object.entries(stats.perCountry).sort((a, b) => b[1] - a[1])) {
+        report?.(`  ${c}: ${n}`);
+    }
+    if (dryRun) {
+        report?.(`(dry-run: no files written)`);
+        return stats;
+    }
+    mkdirSync(outDir, { recursive: true });
+    const manifest = {
+        promoted_at: new Date().toISOString(),
+        from: options.input,
+        from_sha256: await sha256File(options.input),
+        files: {},
+    };
+    for (const [key, entries] of buckets) {
+        const filename = `${key.toLowerCase()}.jsonl`;
+        const path = join(outDir, filename);
+        writeJSONL(path, entries);
+        manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) };
+    }
+    // Forward-copy non-.jsonl files (README.md, etc.) from prior
+    if (existsSync(priorDir)) {
+        for (const f of readdirSync(priorDir).filter((n) => !n.endsWith(".jsonl"))) {
+            copyFileSync(join(priorDir, f), join(outDir, f));
+            report?.(`  forward-copied: ${f}`);
+        }
+    }
+    writeFileSync(join(outDir, "MANIFEST.json"), JSON.stringify(manifest, null, 2) + "\n");
+    report?.(`✓ promoted to ${outDir}`);
+    return stats;
+}
+//# sourceMappingURL=golden-promote.js.map

package/out/src/tools/golden-promote.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-promote.js","sourceRoot":"","sources":["../../../src/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACzF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAsCzE,6EAA6E;AAE7E,SAAS,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACnD,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,KAAkB;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;IAE3D,IAAI,cAAc,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IACpC,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE5D,OAAO,UAAU,GAAG,CAAC,CAAA;AACtB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,KAAkB;IACjD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAE1D,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAA;IAE1C,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE/C,IAAI,WAAW,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAEpC,OAAO,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAkB;IACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAA;IAErB,4CAA4C;IAC5C,IAAI,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC5C,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IACxD,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE1D,IAAI,YAAY,KAAK,aAAa;QAAE,OAAO,IAAI,CAAA;IAE/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,uBAAuB;IACvB,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAE9E,oGAAoG;AACpG,SAAS,kBAAkB,CAAI,IAAY;IAC1C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAA;IAEhC,OAAO,SAAS,CAAI,IAAI,CAAC,CAAA;AAC1B,CAAC;AAED,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAA;IACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAA;IAC3D,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,SAAS,CAAA;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,EAAE,CAAC,uBAAuB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,kBAAkB,CAAc,OAAO,CAAC,KAAK,CAAC,CAAA;IACjE,MAAM,EAAE,CAAC,KAAK,UAAU,CAAC,MAAM,oBAAoB,CAAC,CAAA;IAEpD,yFAAyF;IACzF,oFAAoF;IACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;IACxC,MAAM,YAAY,GAAkD,EAAE,CAAA;IACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IAExC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC3E,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;YACrD,MAAM,OAAO,GAAG,kBAAkB,CAAc,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAA;YAClE,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;YAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzB,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,CAAC;YACD,MAAM,EAAE,CAAC,WAAW,OAAO,KAAK,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAC9E,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,iBAAiB,QAAQ,6BAA6B,CAAC,CAAA;IACjE,CAAC;IAED,cAAc;IACd,MAAM,KAAK,GAAiB;QAC3B,YAAY,EAAE,UAAU,CAAC,MAAM;QAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QACzF,IAAI,EAAE,CAAC;QACP,UAAU,EAAE,EAAE;KACd,CAAA;IACD,MAAM,QAAQ,GAAkB,EAAE,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IAErC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEhC,+CAA+C;QAC/C,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;YAC9B,SAAQ;QACT,CAAC;QAED,mCAAmC;QACnC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAA;YAC7B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;gBACzB,SAAQ;YACT,CAAC;YAED,IAAI,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAA;gBACnC,SAAQ;YACT,CAAC;YAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;gBAC9B,SAAQ;YACT,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnB,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACrB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,OAAO,CAAA;QACvC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,uDAAuD;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAA;IAEhD,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,YAAY,EAAE,CAAC;QACjD,8FAA8F;QAC9F,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;QAEnD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QACrB,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,SAAS;IACT,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACxB,MAAM,EAAE,CAAC,eAAe,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE9D,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,KAAK,GAAG,CAAC,WAAW,EAAE,WAAW,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IACpE,CAAC;IACD,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yBAAyB,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;IACpE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC,CAAA;IAC9E,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,CAAA;IACxE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,qBAAqB,CAAC,CAAA;IAE/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,6BAA6B,CAAC,CAAA;QAEvC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACtC,MAAM,QAAQ,GAKV;QACH,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,IAAI,EAAE,OAAO,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAC5C,KAAK,EAAE,EAAE;KACT,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACnC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QACzB,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAA;IACvF,CAAC;IAED,6DAA6D;IAC7D,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC5E,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC;IACF,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IACtF,MAAM,EAAE,CAAC,iBAAiB,MAAM,EAAE,CAAC,CAAA;IAEnC,OAAO,KAAK,CAAA;AACb,CAAC"}

package/out/src/tools/index.d.ts

@@ -0,0 +1,24 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
+ *   no `process.exit`: commands own parsing, rendering, and exit codes (see the 2026-07-09
+ *   scripts→Pastel spec).
+ */
+export * from "./align-shard.ts";
+export * from "./audit.ts";
+export * from "./corpus-stats.ts";
+export * from "./ingest-csv.ts";
+export * from "./overlay-manifest.ts";
+export * from "./fetch/download.ts";
+export * from "./fetch/index.ts";
+export * from "./golden-expand.ts";
+export * from "./golden-promote.ts";
+export * from "./jsonl-to-parquet.ts";
+export * from "./lint-shard.ts";
+export * from "./lint-shard-vocab.ts";
+export * from "./shard-kryptonite.ts";
+export * from "./shard-translit.ts";
+//# sourceMappingURL=index.d.ts.map

package/out/src/tools/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,uBAAuB,CAAA;AACrC,cAAc,qBAAqB,CAAA;AACnC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,qBAAqB,CAAA;AACnC,cAAc,uBAAuB,CAAA;AACrC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,qBAAqB,CAAA"}

package/out/src/tools/index.js

@@ -0,0 +1,24 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
+ *   no `process.exit`: commands own parsing, rendering, and exit codes (see the 2026-07-09
+ *   scripts→Pastel spec).
+ */
+export * from "./align-shard.js";
+export * from "./audit.js";
+export * from "./corpus-stats.js";
+export * from "./ingest-csv.js";
+export * from "./overlay-manifest.js";
+export * from "./fetch/download.js";
+export * from "./fetch/index.js";
+export * from "./golden-expand.js";
+export * from "./golden-promote.js";
+export * from "./jsonl-to-parquet.js";
+export * from "./lint-shard.js";
+export * from "./lint-shard-vocab.js";
+export * from "./shard-kryptonite.js";
+export * from "./shard-translit.js";
+//# sourceMappingURL=index.js.map

package/out/src/tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/tools/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,cAAc,kBAAkB,CAAA;AAChC,cAAc,YAAY,CAAA;AAC1B,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,uBAAuB,CAAA;AACrC,cAAc,qBAAqB,CAAA;AACnC,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,qBAAqB,CAAA;AACnC,cAAc,uBAAuB,CAAA;AACrC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,uBAAuB,CAAA;AACrC,cAAc,uBAAuB,CAAA;AACrC,cAAc,qBAAqB,CAAA"}

package/out/src/tools/ingest-csv.d.ts

@@ -0,0 +1,45 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ *
+ *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
+ *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus ingest-csv \
+ *   --input /data/corpus/sources/usgov-nppes/npidata_pfile.csv \
+ *   --table nppes_providers \
+ *   --output /data/corpus/sources/usgov-nppes/nppes.db
+ * ```
+ *
+ *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
+ *   derived from input filename) --output <path> SQLite database path (default: input dir /
+ *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
+ *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
+ *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
+ *   CREATE TABLE, but don't import
+ */
+/** Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted. */
+export interface IngestCSVOptions {
+    input: string;
+    table?: string;
+    output?: string;
+    sample?: number;
+    separator?: string;
+    skip?: number;
+    noHeader?: boolean;
+    dryRun?: boolean;
+}
+/**
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
+ * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
+ * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
+ * it.
+ */
+export declare function ingestCSV(options: IngestCSVOptions): Promise<void>;
+//# sourceMappingURL=ingest-csv.d.ts.map

package/out/src/tools/ingest-csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest-csv.d.ts","sourceRoot":"","sources":["../../../src/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAwWH,yGAAyG;AACzG,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAED;;;;;GAKG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAgBxE"}

package/out/src/tools/ingest-csv.js

@@ -0,0 +1,309 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ *
+ *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
+ *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus ingest-csv \
+ *   --input /data/corpus/sources/usgov-nppes/npidata_pfile.csv \
+ *   --table nppes_providers \
+ *   --output /data/corpus/sources/usgov-nppes/nppes.db
+ * ```
+ *
+ *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
+ *   derived from input filename) --output <path> SQLite database path (default: input dir /
+ *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
+ *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
+ *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
+ *   CREATE TABLE, but don't import
+ */
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { basename, dirname, extname, join } from "node:path";
+import { TextSpliterator } from "spliterator";
+// ---------------------------------------------------------------------------
+// Core: quote-aware CSV field splitting
+// ---------------------------------------------------------------------------
+const COMMA = 44;
+const DOUBLE_QUOTE = 34;
+// These stay hand-rolled on purpose — spliterator ≥ 3.2.0's CSVSpliterator does quote handling
+// correctly now, but it can't express this tool's CLI contract: `--skip N` drops N preamble lines
+// BEFORE the header (CSVSpliterator consumes row 1 as the header immediately, with no skip-first-N
+// hook), `--no-header` auto-generates `col_0…col_N` names from the first data row's width, and the
+// separator is any single caller-supplied byte. If a future edit removes those knobs, revisit;
+// until then, don't "finish the job" by swapping in CSVSpliterator.
+function splitCSVLine(line, separator = COMMA) {
+    const fields = [];
+    let start = 0;
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+        const ch = line.charCodeAt(i);
+        if (ch === DOUBLE_QUOTE) {
+            inQuotes = !inQuotes;
+        }
+        else if (ch === separator && !inQuotes) {
+            fields.push(line.slice(start, i));
+            start = i + 1;
+        }
+    }
+    fields.push(line.slice(start));
+    return fields;
+}
+function stripQuotes(field) {
+    const trimmed = field.trim();
+    if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
+        return trimmed.slice(1, -1).replace(/""/g, '"');
+    }
+    return trimmed;
+}
+// ---------------------------------------------------------------------------
+// Column name normalization
+// ---------------------------------------------------------------------------
+function normalizeColumnName(raw) {
+    return (raw
+        .trim()
+        .replace(/^"+|"+$/g, "")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "_")
+        .replace(/^_|_$/g, "") || "unnamed");
+}
+function dedupColumns(names) {
+    const seen = new Map();
+    return names.map((name) => {
+        const count = seen.get(name) ?? 0;
+        seen.set(name, count + 1);
+        return count === 0 ? name : `${name}_${count + 1}`;
+    });
+}
+function normalizeField(raw) {
+    let s = raw.trim();
+    if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
+        s = s.slice(1, -1).replace(/""/g, '"');
+    }
+    // Normalize common null-like values
+    if (!s || s === "null" || s === "NULL" || s === "N/A" || s === "n/a" || s === "-" || s === "<UNAVAIL>") {
+        return null;
+    }
+    return s;
+}
+function inferColumnType(samples) {
+    let nullCount = 0;
+    let intCount = 0;
+    let realCount = 0;
+    let textCount = 0;
+    for (const s of samples) {
+        if (s === null) {
+            nullCount++;
+            continue;
+        }
+        if (/^-?\d+$/.test(s)) {
+            intCount++;
+        }
+        else if (/^-?\d+\.?\d+$/.test(s)) {
+            realCount++;
+        }
+        else {
+            textCount++;
+        }
+    }
+    const total = samples.length;
+    const type = realCount / total >= 0.5 ? "REAL" : intCount / total >= 0.5 ? "INTEGER" : "TEXT";
+    return { name: "", type, nullable: nullCount / total >= 0.5 };
+}
+async function runIngest(opts) {
+    const sep = opts.separator.charCodeAt(0);
+    // --- Pass 1: read header + sample rows for type inference ---
+    process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`);
+    // CRLF-safe by construction: every field below flows through stripQuotes/normalizeField, both of
+    // which `.trim()`, so a trailing CR on the last column of a CRLF file is stripped. TextSpliterator's
+    // default skipEmpty matches readline row-for-row on files with a trailing newline (the common case);
+    // it drops interior blank lines that readline would have turned into all-null rows. The early `break`
+    // closes the file descriptor.
+    let headerLine = null;
+    const sampleRows = [];
+    let lineNum = 0;
+    for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+        lineNum++;
+        // Skip lines before header
+        if (lineNum <= opts.skipLines)
+            continue;
+        if (!headerLine && opts.hasHeader) {
+            headerLine = line;
+            continue;
+        }
+        if (sampleRows.length < opts.sampleSize) {
+            sampleRows.push(splitCSVLine(line, sep).map(stripQuotes));
+        }
+        else {
+            break;
+        }
+    }
+    if (!headerLine && opts.hasHeader) {
+        throw new Error("No header line found in CSV");
+    }
+    // --- Determine column names ---
+    let rawHeaders;
+    if (opts.hasHeader && headerLine) {
+        rawHeaders = splitCSVLine(headerLine, sep).map(stripQuotes);
+    }
+    else {
+        // Auto-generate column names: col_0, col_1, ...
+        const numCols = sampleRows[0]?.length ?? 0;
+        rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`);
+    }
+    const colNames = dedupColumns(rawHeaders.map(normalizeColumnName));
+    // --- Infer column types ---
+    const columns = colNames.map((name, i) => {
+        const samples = sampleRows.map((row) => {
+            const raw = row[i] ?? "";
+            return normalizeField(raw);
+        });
+        const info = inferColumnType(samples);
+        info.name = name;
+        return info;
+    });
+    // --- Generate SQL ---
+    const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ");
+    // Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
+    // so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+    const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`;
+    const tempCols = columns.map((c) => `"${c.name}"`).join(", ");
+    const insertSQL = `INSERT INTO "${opts.tableName}" (${tempCols})\nSELECT ${tempCols} FROM temp."${opts.tableName}_source";`;
+    process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`);
+    process.stderr.write(`${createTableSQL}\n\n`);
+    if (opts.dryRun) {
+        process.stderr.write("--dry-run: stopping before import\n");
+        return;
+    }
+    // --- Create database + import ---
+    const { DatabaseSync } = await import("node:sqlite");
+    mkdirSync(dirname(opts.outputPath), { recursive: true });
+    const db = new DatabaseSync(opts.outputPath);
+    db.exec("PRAGMA journal_mode = OFF"); // faster for bulk import
+    db.exec("PRAGMA synchronous = OFF");
+    db.exec(createTableSQL);
+    // Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
+    // NULL normalization and type coercion.
+    const csvBasename = basename(opts.inputPath);
+    const importSQL = [
+        `CREATE TEMP TABLE "${opts.tableName}_source" (${colDefs});`,
+        `.mode csv`,
+        `.separator "${opts.separator}"`,
+        `.import "${csvBasename}" --skip ${opts.skipLines + (opts.hasHeader ? 1 : 0)} --schema temp ${opts.tableName}_source`,
+        insertSQL,
+        `DROP TABLE temp."${opts.tableName}_source";`,
+    ];
+    // better-sqlite3 doesn't support .import natively, so we use a different approach:
+    // Read the CSV line-by-line and INSERT in a transaction.
+    process.stderr.write(`Importing rows...\n`);
+    const insertStmt = db.prepare(`INSERT INTO "${opts.tableName}" (${tempCols}) VALUES (${columns.map(() => "?").join(", ")})`);
+    let imported = 0;
+    let headerSkipped = false;
+    // node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+    const doInsert = () => {
+        db.exec("BEGIN");
+        try {
+            for (const row of batch) {
+                insertStmt.run(...row);
+            }
+            db.exec("COMMIT");
+        }
+        catch (err) {
+            db.exec("ROLLBACK");
+            throw err;
+        }
+    };
+    const batch = [];
+    const BATCH_SIZE = 10000;
+    for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+        lineNum++;
+        if (lineNum <= opts.skipLines)
+            continue;
+        if (opts.hasHeader && !headerSkipped) {
+            headerSkipped = true;
+            continue;
+        }
+        const fields = splitCSVLine(line, sep).map(stripQuotes);
+        const values = fields.map((f, i) => {
+            const v = normalizeField(f);
+            if (v === null)
+                return null;
+            const col = columns[i];
+            if (col?.type === "INTEGER" && /^-?\d+$/.test(v))
+                return parseInt(v, 10);
+            if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v))
+                return parseFloat(v);
+            return v;
+        });
+        // Pad or truncate to column count
+        while (values.length < columns.length) {
+            values.push(null);
+        }
+        values.length = columns.length;
+        batch.push(values);
+        if (batch.length >= BATCH_SIZE) {
+            doInsert();
+            imported += batch.length;
+            batch.length = 0;
+            if (imported % 100000 === 0) {
+                process.stderr.write(`  ${(imported / 1_000_000).toFixed(1)}M rows...\n`);
+            }
+        }
+    }
+    // Flush remaining
+    if (batch.length > 0) {
+        doInsert();
+        imported += batch.length;
+    }
+    process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`);
+    // Build a basic index on the first TEXT column (likely the primary key)
+    const firstTextCol = columns.find((c) => c.type === "TEXT");
+    if (firstTextCol) {
+        process.stderr.write(`Building index on "${firstTextCol.name}"...\n`);
+        db.exec(`CREATE INDEX IF NOT EXISTS idx_${opts.tableName}_${firstTextCol.name} ON "${opts.tableName}"("${firstTextCol.name}");`);
+    }
+    db.close();
+    // Write MANIFEST
+    const fileSize = (await import("node:fs/promises")).stat;
+    const stat = await (await import("node:fs/promises")).stat(opts.outputPath);
+    const manifest = {
+        ingested_at: new Date().toISOString(),
+        source_csv: basename(opts.inputPath),
+        table_name: opts.tableName,
+        columns: columns.map((c) => ({ name: c.name, type: c.type, nullable: c.nullable })),
+        row_count: imported,
+        db_bytes: stat.size,
+    };
+    const manifestPath = opts.outputPath.replace(/\.db$/, ".manifest.json");
+    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
+    process.stderr.write(`Done. ${imported.toLocaleString()} rows → ${opts.outputPath} (${(stat.size / 1024 / 1024).toFixed(0)} MB)\n`);
+}
+/**
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
+ * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
+ * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
+ * it.
+ */
+export async function ingestCSV(options) {
+    if (!existsSync(options.input)) {
+        throw new Error(`File not found: ${options.input}`);
+    }
+    const csvName = basename(options.input, extname(options.input));
+    await runIngest({
+        inputPath: options.input,
+        tableName: options.table ?? csvName.replace(/[^a-zA-Z0-9_]/g, "_"),
+        outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+        sampleSize: options.sample ?? 100,
+        separator: options.separator ?? ",",
+        skipLines: options.skip ?? 0,
+        hasHeader: !options.noHeader,
+        dryRun: options.dryRun ?? false,
+    });
+}
+//# sourceMappingURL=ingest-csv.js.map