@mailwoman/corpus 9.2.0 → 9.4.0

package/out/tools/audit.js

@@ -0,0 +1,282 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman corpus audit` — per-source slice-count vs source_weight diagnostic.
+ *
+ *   Reads a corpus dir's MANIFEST.json (or scans slices directly), counts slices per source,
+ *   optionally loads a training config to pair the counts with the configured source_weights, and
+ *   reports the estimated sampled-row distribution at training time.
+ *
+ *   Would have caught v0.3.0's "NAD = 411/674 train slices × 2.0 weight = ~75% of sampled mix"
+ *   finding before the v0.3.0 retrospective surfaced it.
+ *
+ *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
+ */
+import { pathExists, readDirectory, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { basename, join } from "path-ts";
+import { TextSpliterator } from "spliterator";
+/**
+ * Share of a slice one source may hold before the mix is flagged as dominated by it.
+ */
+const DOMINANT_SOURCE_SHARE = 0.4;
+/**
+ * How far the leading source may outweigh the runner-up before the mix is flagged. Catches the case where no single
+ * source clears {@link DOMINANT_SOURCE_SHARE} but the distribution is still lopsided.
+ */
+const MAX_TOP_TO_RUNNER_UP_RATIO = 1.5;
+/**
+ * Try parsing a training YAML's source_weights as a minimal regex-based extract. We don't pull in a YAML lib for this
+ * script — the syntax is so small that a regex over the source_weights block is sufficient + keeps the script
+ * dep-free.
+ */
+async function parseConfig(configPath) {
+    if (!(await pathExists(configPath)))
+        return null;
+    const weights = {};
+    let inBlock = false;
+    let blockIndent = -1;
+    for (const raw of TextSpliterator.from(await readLocalBuffer(configPath))) {
+        const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/);
+        if (sourceWeightsMatch) {
+            inBlock = true;
+            blockIndent = sourceWeightsMatch[1].length;
+            continue;
+        }
+        if (!inBlock)
+            continue;
+        // Skip blank lines and comments.
+        if (/^[\t ]*(#|$)/.test(raw))
+            continue;
+        // Lines indented MORE than `source_weights:` are entries; lines with ≤ indent end the block.
+        const indent = raw.match(/^[\t ]*/)[0].length;
+        if (indent <= blockIndent) {
+            inBlock = false;
+            continue;
+        }
+        const m = raw.match(/^[\t ]+([\w-]+):\s*([\d.]+)/);
+        if (m) {
+            weights[m[1]] = Number.parseFloat(m[2]);
+        }
+    }
+    return { sourceWeights: weights };
+}
+/**
+ * Scan a corpus directory's slices (typically under <corpus_dir>/train, /val, /test) and count slices per source per
+ * split.
+ */
+async function scanSlices(corpusDir, sampleCount) {
+    const stats = { bySplit: {}, totalSlices: 0, totalFiles: 0 };
+    for (const split of ["train", "val", "test"]) {
+        const splitDir = join(corpusDir, split);
+        if (!(await pathExists(splitDir)))
+            continue;
+        const files = (await readDirectory(splitDir)).filter((f) => f.endsWith(".parquet")).toSorted();
+        stats.totalFiles += files.length;
+        const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount));
+        const sampled = files.filter((_, i) => i % sampleEvery === 0).slice(0, sampleCount);
+        const splitMap = {};
+        // We can't read parquet without a dep, so we infer source from filenames where possible.
+        // The corpus build typically writes deterministically by source — fall back to "<unknown>"
+        // when filename gives no hint. For accurate per-source counts on real corpora, the
+        // MANIFEST.json route below is preferred.
+        for (const f of sampled) {
+            const inferred = inferSourceFromFilename(f);
+            splitMap[inferred] = (splitMap[inferred] ?? 0) + 1;
+        }
+        // Scale to estimated full-slice counts.
+        const scale = files.length / Math.max(sampled.length, 1);
+        for (const k of Object.keys(splitMap)) {
+            splitMap[k] = Math.round(splitMap[k] * scale);
+        }
+        stats.bySplit[split] = splitMap;
+        stats.totalSlices += files.length;
+    }
+    return stats;
+}
+function inferSourceFromFilename(filename) {
+    // Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet. The latter (current
+    // build at corpus-v0.3.0) gives no source signal in the filename — see manifestScan() for the
+    // authoritative path. Return "<unknown>" so the caller flags this case.
+    const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/);
+    if (m && m[1] !== undefined)
+        return m[1];
+    return "<unknown>";
+}
+/**
+ * Known source name prefixes. Corpus-v0.3.0 uses these as `source_id` prefixes; matching against the longest prefix
+ * that fits a given `first_source_id` recovers the canonical source name.
+ *
+ * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches `usgov-nad` rather than `usgov`. Sorted
+ * descending by length at use site.
+ */
+const KNOWN_SOURCE_PREFIXES = [
+    "wof-admin",
+    "wof-postalcode",
+    "ban",
+    "tiger",
+    "usgov-nad",
+    "usgov-nppes",
+    "usgov-hrsa-fqhc",
+    "usgov-imls-pls",
+    "state-ia-contractors",
+    "state-tx-notaries",
+    "state-ny-notaries",
+    "openaddresses",
+    // Synthetic adversarial sources (corpus-v0.4.0+, Thread B).
+    "deepseek-kryptonite",
+    "deepseek-translit-cyrl",
+    "deepseek-translit-jpan",
+    "deepseek-translit-hans",
+    "deepseek-translit-hang",
+    "deepseek-translit-armn",
+];
+/**
+ * Extract the source-name prefix from a `first_source_id` value.
+ */
+function sourceFromID(sourceID, knownPrefixes) {
+    // Sort longest-first so usgov-nad beats usgov, wof-admin beats wof.
+    const sorted = [...knownPrefixes].toSorted((a, b) => b.length - a.length);
+    for (const prefix of sorted) {
+        if (sourceID.startsWith(prefix + "-") || sourceID === prefix)
+            return prefix;
+    }
+    return "<unknown>";
+}
+/**
+ * Prefer reading MANIFEST.json when present — uses each slice's `first_source_id` + prefix matching to recover the
+ * source name. Falls back to scanSlices when MANIFEST is absent.
+ *
+ * NOTE: corpus-v0.3.0 slices can mix sources (see `last_source_id` differing from `first_source_id`). The first-row
+ * source is an approximation; reading the parquet's full source column would be authoritative but requires a parquet
+ * dep. For audit purposes the first-row approximation is accurate within ~5% for the corpus-v0.3.0 shape (most slices
+ * are >95% one source).
+ */
+async function manifestScan(corpusDir, knownPrefixes) {
+    const manifestPath = join(corpusDir, "MANIFEST.json");
+    if (!(await pathExists(manifestPath)))
+        return null;
+    const manifest = await readLocalJSONFile(manifestPath);
+    if (!Array.isArray(manifest.slices))
+        return null;
+    const bySplit = {};
+    for (const slice of manifest.slices) {
+        const split = slice.split;
+        const src = slice.source ?? sourceFromID(slice.first_source_id ?? "", knownPrefixes);
+        bySplit[split] ??= {};
+        bySplit[split][src] = (bySplit[split][src] ?? 0) + 1;
+    }
+    const total = Object.values(bySplit).reduce((sum, m) => sum + Object.values(m).reduce((a, b) => a + b, 0), 0);
+    return { bySplit, totalSlices: total, totalFiles: total };
+}
+function buildAuditRows(stats, weights) {
+    const totalSlices = Object.values(stats).reduce((a, b) => a + b, 0);
+    const allSources = new Set([...Object.keys(stats), ...Object.keys(weights)]);
+    const rows = [];
+    // Compute effective sample weight: slice_count × source_weight. Sources with no weight get the
+    // "—" marker (loader skips them).
+    const sampleWeights = [];
+    for (const src of allSources) {
+        const slices = stats[src] ?? 0;
+        const weight = weights[src];
+        const effective = weight !== undefined ? slices * weight : 0;
+        sampleWeights.push([src, effective]);
+    }
+    const totalSampleWeight = sampleWeights.reduce((a, [, w]) => a + w, 0);
+    for (const src of allSources) {
+        const slices = stats[src] ?? 0;
+        const weight = weights[src] ?? "—";
+        const effective = typeof weight === "number" ? (slices * weight) / Math.max(totalSampleWeight, 1) : "—";
+        rows.push({
+            source: src,
+            slices,
+            slicePct: totalSlices > 0 ? slices / totalSlices : 0,
+            weight,
+            effectiveSamplePct: typeof effective === "number" ? effective : "—",
+        });
+    }
+    // Flag the dominator: empirically calibrated against the v0.3.0 → v0.4.0 retrospective.
+    // v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space dilution
+    // was responsible for the coarse-F1 regression. So flag a source as "concentration warning"
+    // when it's above 40% effective sample OR more than 1.5× the next-highest.
+    const numeric = rows.filter((r) => typeof r.effectiveSamplePct === "number");
+    numeric.sort((a, b) => b.effectiveSamplePct - a.effectiveSamplePct);
+    if (numeric.length) {
+        const top = numeric[0];
+        const next = numeric[1]?.effectiveSamplePct ?? 0;
+        if (top.effectiveSamplePct > DOMINANT_SOURCE_SHARE ||
+            (next > 0 && top.effectiveSamplePct / next > MAX_TOP_TO_RUNNER_UP_RATIO)) {
+            top.overweightFactor = next > 0 ? top.effectiveSamplePct / next : Infinity;
+        }
+    }
+    rows.sort((a, b) => b.slices - a.slices);
+    return rows;
+}
+function formatPct(v) {
+    if (v === "—")
+        return "—";
+    return `${(v * 100).toFixed(1)}%`;
+}
+function printReport(corpusDir, configPath, stats, rows) {
+    console.log(`\nCorpus audit — ${corpusDir}`);
+    if (configPath) {
+        console.log(`Config:        ${configPath}`);
+    }
+    console.log(`Total slices:  ${stats.totalSlices}${stats.totalFiles !== stats.totalSlices ? ` (${stats.totalFiles} files on disk)` : ""}`);
+    console.log("");
+    const trainStats = stats.bySplit["train"];
+    if (trainStats) {
+        const total = Object.values(trainStats).reduce((a, b) => a + b, 0);
+        console.log(`Train split: ${total} slices`);
+        console.log("");
+        const headers = ["source", "slices", "slice %", "weight", "eff. sample %"];
+        const widths = [22, 8, 10, 8, 14];
+        const fmtRow = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join("  ");
+        console.log(fmtRow(headers));
+        console.log(fmtRow(widths.map((w) => "─".repeat(w))));
+        for (const row of rows) {
+            console.log(fmtRow([
+                row.source,
+                String(row.slices),
+                formatPct(row.slicePct),
+                typeof row.weight === "number" ? row.weight.toFixed(2) : "—",
+                formatPct(row.effectiveSamplePct),
+            ]));
+        }
+        console.log("");
+        const dominator = rows.find((r) => r.overweightFactor !== undefined);
+        if (dominator) {
+            const factor = dominator.overweightFactor;
+            const factorStr = factor === Infinity ? "∞" : factor?.toFixed(1);
+            console.error(`⚠ Concentration: ${dominator.source} would sample ${formatPct(dominator.effectiveSamplePct)} ` +
+                `of training rows (${factorStr}× the next-highest). ` +
+                `Past lesson: v0.3.0's NAD at ~52% caused the 21-label coarse regression. ` +
+                `Consider lowering this source's weight or boosting others.`);
+        }
+        else {
+            console.log("✓ No single-source concentration (top source < 40% effective sample AND < 1.5× next).");
+        }
+        const missingWeights = rows.filter((r) => r.weight === "—" && r.slices > 0);
+        if (missingWeights.length && configPath) {
+            console.error(`⚠ Sources present in corpus but absent from config.source_weights ` +
+                `(loader will skip them): ${missingWeights.map((r) => r.source).join(", ")}`);
+        }
+        const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.slices === 0);
+        if (orphanWeights.length) {
+            console.error(`⚠ Sources weighted in config but no slices found in corpus ` +
+                `(no-op weights): ${orphanWeights.map((r) => r.source).join(", ")}`);
+        }
+    }
+}
+export async function audit(opts) {
+    const config = opts.configPath ? await parseConfig(opts.configPath) : null;
+    // Compose the known-prefix list from both the hardcoded set and any extra names in the config
+    // (forward-compat for future adapters added before this file is updated).
+    const prefixes = [...new Set([...KNOWN_SOURCE_PREFIXES, ...Object.keys(config?.sourceWeights ?? {})])];
+    const stats = (await manifestScan(opts.corpusDir, prefixes)) ?? (await scanSlices(opts.corpusDir, opts.sampleSliceCount ?? 100));
+    const trainStats = stats.bySplit["train"] ?? {};
+    const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {});
+    printReport(opts.corpusDir, opts.configPath, stats, rows);
+}
+//# sourceMappingURL=audit.js.map

package/out/tools/audit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../lib/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1G,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAgCtC;;;;GAIG;AACH,KAAK,UAAU,WAAW,CAAC,UAAkB;IAC5C,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAA;YACd,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YAE3C,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QACtC,6FAA6F;QAC7F,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE9C,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAElD,IAAI,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,UAAU,CAAC,SAA0B,EAAE,WAAmB;IACxE,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAExE,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAEvC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAE,SAAQ;QAE3C,MAAM,KAAK,GAAG,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE9F,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,2FAA2F;QAC3F,mFAAmF;QACnF,0CAA0C;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;IAClC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,8FAA8F;IAC9F,8FAA8F;IAC9F,wEAAwE;IACxE,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,YAAY,CAAC,SAA0B,EAAE,aAAgC;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAErD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAElD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAErC,YAAY,CAAC,CAAA;IAEhB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QACpF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC1D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+FAA+F;IAC/F,kCAAkC;IAClC,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEvG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,MAAM;YACN,QAAQ,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,8FAA8F;IAC9F,4FAA4F;IAC5F,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAExC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CACnB,SAA0B,EAC1B,UAA8B,EAC9B,KAAiB,EACjB,IAAgB;IAEhB,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACvB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE3E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;QAExF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,6DAA6D;gBAC5D,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAe;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1E,8FAA8F;IAC9F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEtG,MAAM,KAAK,GACV,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAC,CAAA;IAEnH,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}

package/out/tools/corpus-stats.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
+ *
+ *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
+ *   as the baseline against which a new slice is compared.
+ *
+ *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
+ *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
+ *   mainline slice added, a source-pool re-weighted, etc.).
+ *
+ *   Output schema:
+ *
+ *   ```ts
+ *   interface CorpusStats {
+ *     row_count: number
+ *     slice_paths: string[]
+ *     tokens: { [token: string]: { [label: string]: number } }
+ *     bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
+ *     // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
+ *     // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
+ *   }
+ * ```
+ *
+ *   Usage: node scripts/build-corpus-stats.ts\
+ *   --slices <glob-pattern-or-dir>\
+ *   --output <stats.json>
+ *
+ *   For a quick local-corpus baseline (limited but useful for linter testing): node
+ *   scripts/build-corpus-stats.ts\
+ *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --output /tmp/corpus-stats-local.json
+ */
+export interface CorpusStatsOptions {
+    slicesArg: string;
+    outputPath: string;
+    limitPerSlice?: number;
+}
+export declare function buildCorpusStats(args: CorpusStatsOptions): Promise<void>;
+//# sourceMappingURL=corpus-stats.d.ts.map

package/out/tools/corpus-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAWH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAeD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqE9E"}

package/out/tools/corpus-stats.js

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
+ *
+ *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
+ *   as the baseline against which a new slice is compared.
+ *
+ *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
+ *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
+ *   mainline slice added, a source-pool re-weighted, etc.).
+ *
+ *   Output schema:
+ *
+ *   ```ts
+ *   interface CorpusStats {
+ *     row_count: number
+ *     slice_paths: string[]
+ *     tokens: { [token: string]: { [label: string]: number } }
+ *     bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
+ *     // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
+ *     // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
+ *   }
+ * ```
+ *
+ *   Usage: node scripts/build-corpus-stats.ts\
+ *   --slices <glob-pattern-or-dir>\
+ *   --output <stats.json>
+ *
+ *   For a quick local-corpus baseline (limited but useful for linter testing): node
+ *   scripts/build-corpus-stats.ts\
+ *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --output /tmp/corpus-stats-local.json
+ */
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { readDirectory, statPath } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { join } from "path-ts";
+import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats";
+const MIN_BIGRAM_COUNT = 2;
+async function discoverSlices(slicesArg) {
+    const stat = await statPath(slicesArg);
+    if (stat.isDirectory()) {
+        return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f));
+    }
+    if (stat.isFile() && slicesArg.endsWith(".parquet"))
+        return [slicesArg];
+    // Otherwise treat as a literal path list (one per line if it's stdin-friendly).
+    return [slicesArg];
+}
+export async function buildCorpusStats(args) {
+    const slicePaths = await discoverSlices(args.slicesArg);
+    console.error(`Discovered ${slicePaths.length} parquet slice(s)`);
+    const stats = createCooccurrenceStats();
+    const { tokens: tokenStats, bigrams: bigramStats } = stats;
+    let totalRows = 0;
+    for (const path of slicePaths) {
+        console.error(`Reading ${path}...`);
+        const before = totalRows;
+        for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerSlice)) {
+            totalRows++;
+            if (tokens.length !== labels.length)
+                continue; // skip malformed
+            accumulateCooccurrences(stats, tokens, labels);
+        }
+        console.error(`  ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`);
+    }
+    // Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
+    // stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
+    // detection.
+    let prunedBigrams = 0;
+    for (const [k, labelMap] of bigramStats) {
+        let total = 0;
+        for (const v of labelMap.values()) {
+            total += v;
+        }
+        if (total < MIN_BIGRAM_COUNT) {
+            bigramStats.delete(k);
+            prunedBigrams++;
+        }
+    }
+    console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`);
+    const out = {
+        row_count: totalRows,
+        slice_paths: slicePaths,
+        tokens: {},
+        bigrams: {},
+    };
+    for (const [tk, labelMap] of tokenStats) {
+        out.tokens[tk] = Object.fromEntries(labelMap);
+    }
+    for (const [k, labelMap] of bigramStats) {
+        out.bigrams[k] = Object.fromEntries(labelMap);
+    }
+    await writeLocalJSONFile(out, args.outputPath);
+    const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(JSON.stringify(out)));
+    console.error(`Wrote ${args.outputPath} (${formattedSize}) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`);
+}
+//# sourceMappingURL=corpus-stats.js.map

package/out/tools/corpus-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-stats.js","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAE3G,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAQ1B,KAAK,UAAU,cAAc,CAAC,SAAiB;IAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;IAEtC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;IAC7G,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,SAAS,CAAC,CAAA;IAEvE,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAA;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAwB;IAC9D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAEvD,OAAO,CAAC,KAAK,CAAC,cAAc,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAA;IAEjE,MAAM,KAAK,GAAG,uBAAuB,EAAE,CAAA;IACvC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,KAAK,CAAA;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,CAAA;QAEnC,MAAM,MAAM,GAAG,SAAS,CAAA;QAExB,IAAI,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvF,SAAS,EAAE,CAAA;YAEX,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,SAAQ,CAAC,iBAAiB;YAE/D,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/C,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,KAAK,SAAS,GAAG,MAAM,0BAA0B,UAAU,CAAC,IAAI,mBAAmB,WAAW,CAAC,IAAI,iBAAiB,CACpH,CAAA;IACF,CAAC;IAED,sFAAsF;IACtF,uFAAuF;IACvF,aAAa;IACb,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAA;QACX,CAAC;QAED,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YAC9B,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAErB,aAAa,EAAE,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,aAAa,uBAAuB,WAAW,CAAC,IAAI,SAAS,CAAC,CAAA;IAEtF,MAAM,GAAG,GAAG;QACX,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,EAA4C;QACpD,OAAO,EAAE,EAA4C;KACrD,CAAA;IAED,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IAC9C,MAAM,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAErF,OAAO,CAAC,KAAK,CACZ,SAAS,IAAI,CAAC,UAAU,KAAK,aAAa,OAAO,SAAS,UAAU,UAAU,CAAC,IAAI,YAAY,WAAW,CAAC,IAAI,UAAU,CACzH,CAAA;AACF,CAAC"}

package/out/tools/fetch/acra-sg.d.ts

@@ -0,0 +1,23 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
+ *   letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
+ *   `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
+ *   so the corpus adapter renders the string and the spans fall out by construction. A Singapore
+ *   postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
+ *
+ *   License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
+ *   adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
+ *
+ *   data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
+ *   `poll-download` answers it once the export is ready.
+ *
+ *   Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchACRASGOptions = BaseFetchOptions;
+export declare function fetchACRASG(options: FetchACRASGOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=acra-sg.d.ts.map

package/out/tools/fetch/acra-sg.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"acra-sg.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/acra-sg.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4C,MAAM,uBAAuB,CAAA;AAerH,MAAM,MAAM,kBAAkB,GAAG,gBAAgB,CAAA;AA4CjD,wBAAsB,WAAW,CAAC,OAAO,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAuFrH"}

package/out/tools/fetch/acra-sg.js

@@ -0,0 +1,126 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
+ *   letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
+ *   `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
+ *   so the corpus adapter renders the string and the spans fall out by construction. A Singapore
+ *   postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
+ *
+ *   License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
+ *   adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
+ *
+ *   data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
+ *   `poll-download` answers it once the export is ready.
+ *
+ *   Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
+ */
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters";
+import { pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { sleep } from "@mailwoman/core/utils/sleep";
+import { join } from "path-ts";
+import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
+const SLUG = "acra-sg";
+const COLLECTION_URL = "https://api-production.data.gov.sg/v2/public/api/collections/2/metadata";
+const DATASET_API = "https://api-production.data.gov.sg/v2/public/api/datasets";
+const DOWNLOAD_API = "https://api-open.data.gov.sg/v1/public/api/datasets";
+const LICENSE = "Singapore Open Data Licence version 1.0 — https://data.gov.sg/open-data-licence";
+const ATTRIBUTION = "Contains information from ACRA Information on Corporate Entities accessed on <date> from data.gov.sg which is made available under the terms of the Singapore Open Data Licence version 1.0 https://data.gov.sg/open-data-licence";
+const POLL_INTERVAL_MS = 3000;
+const POLL_ATTEMPTS = 40;
+async function readJSON(url) {
+    const res = await fetch(url, { headers: { accept: "application/json" } });
+    if (!res.ok)
+        throw new Error(`acra-sg: ${url} answered HTTP ${res.status}`);
+    return (await res.json());
+}
+/**
+ * Ask the portal to prepare the CSV export, then poll until it names the signed URL.
+ */
+async function signedURLFor(datasetID) {
+    await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`);
+    for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
+        await sleep(POLL_INTERVAL_MS);
+        const poll = await readJSON(`${DOWNLOAD_API}/${datasetID}/poll-download`);
+        if (poll.data?.status === "DOWNLOAD_SUCCESS" && poll.data.url)
+            return poll.data.url;
+    }
+    return undefined;
+}
+function filenameFor(name, datasetID) {
+    const letter = /\('([^']+)'\)/.exec(name)?.[1];
+    return `acra-entities-${(letter ?? datasetID).toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")}.csv`;
+}
+export async function fetchACRASG(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const collection = await readJSON(COLLECTION_URL);
+    const datasetIDs = collection.data?.collectionMetadata?.childDatasets ?? [];
+    report?.(`=== ${SLUG}: ${datasetIDs.length} datasets in collection 2`);
+    const previous = await loadCollectionFiles(manifestPath);
+    const files = new Map();
+    let fetched = 0;
+    let skipped = 0;
+    const failedCodes = [];
+    for (const datasetID of datasetIDs) {
+        const metadata = await readJSON(`${DATASET_API}/${datasetID}/metadata`);
+        const name = metadata.data?.name ?? datasetID;
+        const filename = filenameFor(name, datasetID);
+        const dest = join(destDir, filename);
+        const before = previous.get(filename);
+        if (before && (await pathExists(dest))) {
+            files.set(filename, before);
+            skipped++;
+            continue;
+        }
+        report?.(`--- ${name} (${datasetID})`);
+        let bytes;
+        try {
+            bytes = await withRetries(async () => {
+                const url = await signedURLFor(datasetID);
+                if (!url)
+                    throw new Error(`the export never reached DOWNLOAD_SUCCESS for ${datasetID}`);
+                const res = await fetch(url, { signal: AbortSignal.timeout(1_800_000) });
+                if (!res.ok)
+                    throw new Error(`HTTP ${res.status} on the signed URL for ${datasetID}`);
+                return streamBodyToFile(res, dest);
+            }, { report, label: name });
+        }
+        catch (error) {
+            report?.(`  ✗ ${error.message}`);
+            failedCodes.push(datasetID);
+            continue;
+        }
+        if (bytes < BYTES_PER_KIB) {
+            report?.(`  ✗ ${bytes} bytes — an error page, not the register`);
+            failedCodes.push(datasetID);
+            continue;
+        }
+        const sha = await sha256File(dest);
+        files.set(filename, {
+            source_url: `https://data.gov.sg/datasets/${datasetID}/view`,
+            downloaded_at: new Date().toISOString(),
+            filename,
+            sha256: sha,
+            bytes,
+        });
+        fetched++;
+        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+        const manifest = {
+            source: SLUG,
+            source_url: "https://data.gov.sg/collections/2/view",
+            license: LICENSE,
+            attribution: ATTRIBUTION,
+            downloaded_at: new Date().toISOString(),
+            files: [...files.values()],
+        };
+        await writeManifest(manifestPath, manifest);
+    }
+    return { fetched, skipped, failed: failedCodes.length, failedCodes };
+}
+//# sourceMappingURL=acra-sg.js.map

package/out/tools/fetch/acra-sg.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"acra-sg.js","sourceRoot":"","sources":["../../../lib/tools/fetch/acra-sg.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEzG,MAAM,IAAI,GAAG,SAAS,CAAA;AACtB,MAAM,cAAc,GAAG,yEAAyE,CAAA;AAChG,MAAM,WAAW,GAAG,2DAA2D,CAAA;AAC/E,MAAM,YAAY,GAAG,qDAAqD,CAAA;AAC1E,MAAM,OAAO,GAAG,iFAAiF,CAAA;AAEjG,MAAM,WAAW,GAChB,mOAAmO,CAAA;AAEpO,MAAM,gBAAgB,GAAG,IAAI,CAAA;AAC7B,MAAM,aAAa,GAAG,EAAE,CAAA;AAgBxB,KAAK,UAAU,QAAQ,CAAI,GAAW;IACrC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAA;IAEzE,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,YAAY,GAAG,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAE3E,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAM,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC5C,MAAM,QAAQ,CAAC,GAAG,YAAY,IAAI,SAAS,oBAAoB,CAAC,CAAA;IAEhE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,aAAa,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,MAAM,KAAK,CAAC,gBAAgB,CAAC,CAAA;QAC7B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAe,GAAG,YAAY,IAAI,SAAS,gBAAgB,CAAC,CAAA;QAEvF,IAAI,IAAI,CAAC,IAAI,EAAE,MAAM,KAAK,kBAAkB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;IACpF,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,SAAiB;IACnD,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAE9C,OAAO,iBAAiB,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,CAAA;AACjG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B,EAAE,MAA+B;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAqB,cAAc,CAAC,CAAA;IACrE,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,kBAAkB,EAAE,aAAa,IAAI,EAAE,CAAA;IAC3E,MAAM,EAAE,CAAC,OAAO,IAAI,KAAK,UAAU,CAAC,MAAM,2BAA2B,CAAC,CAAA;IAEtE,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;IAC/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAkB,GAAG,WAAW,IAAI,SAAS,WAAW,CAAC,CAAA;QACxF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,IAAI,SAAS,CAAA;QAC7C,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACpC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAErC,IAAI,MAAM,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAE3B,OAAO,EAAE,CAAA;YAET,SAAQ;QACT,CAAC;QAED,MAAM,EAAE,CAAC,OAAO,IAAI,KAAK,SAAS,GAAG,CAAC,CAAA;QACtC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,WAAW,CACxB,KAAK,IAAI,EAAE;gBACV,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAA;gBAEzC,IAAI,CAAC,GAAG;oBAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,SAAS,EAAE,CAAC,CAAA;gBACvF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;gBAExE,IAAI,CAAC,GAAG,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,0BAA0B,SAAS,EAAE,CAAC,CAAA;gBAErF,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACnC,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CACvB,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3C,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAE3B,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,OAAO,KAAK,0CAA0C,CAAC,CAAA;YAChE,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAE3B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE;YACnB,UAAU,EAAE,gCAAgC,SAAS,OAAO;YAC5D,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QAEF,OAAO,EAAE,CAAA;QACT,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,MAAM,QAAQ,GAA6B;YAC1C,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,wCAAwC;YACpD,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,WAAW;YACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;SAC1B,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACrE,CAAC"}