@mailwoman/corpus 9.2.0 → 9.3.0

package/out/src/tools/lint-shard.js

@@ -1,354 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Corpus linter. Compares a new shard against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
- *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new shard, compare the shard's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the shard's majority differs AND both have
- *        non-trivial counts (shard ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in shard while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of shard.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't gate). Per the design, the MANIFEST entry for a
- *   flagged shard should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-shard\
- *   --shard <new-shard.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
- */
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
-import { fileURLToPath } from "node:url";
-import { parseJSONStrict } from "@mailwoman/core/objects";
-import { ParquetReader } from "../parquet-wrapper/index.js";
-/**
- * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
- */
-const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
-/**
- * Examples printed per finding before the list is truncated.
- */
-const MAX_LISTED_EXAMPLES = 20;
-const SEP = "";
-/**
- * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
- * numbers miss.
- */
-const CORPUS_CONFIDENCE_FLOOR = 0.66;
-const SHARD_MIN_COUNT = 50;
-const CORPUS_MIN_COUNT = 200;
-const VACUUM_SHARD_MIN_COUNT = 20;
-const VACUUM_CORPUS_MIN_COUNT = 100;
-const BIGRAM_MIN_COUNT = 10;
-const ALL_O_RATIO_CEILING = 0.9;
-/**
- * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
- * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
- * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
- * the common path.
- */
-function defaultRulesPath() {
-    const sibling = new URL("./lint-rules.json", import.meta.url);
-    if (existsSync(sibling))
-        return fileURLToPath(sibling);
-    return fileURLToPath(new URL("../../../src/tools/lint-rules.json", import.meta.url));
-}
-/**
- * Stream a shard's `tokens`/`labels` columns.
- *
- * Projected rather than read whole: parquet is columnar, so the twelve unused columns are never touched. The rows feed
- * straight into {@link statsFromShard} and are never all resident.
- */
-async function* streamShard(shardPath) {
-    await using reader = await ParquetReader.openFile(shardPath);
-    yield* reader.project("tokens", "labels");
-}
-async function statsFromShard(rows) {
-    const out = {
-        rowCount: 0,
-        tokens: new Map(),
-        bigrams: new Map(),
-        truncatedRows: 0,
-        allORows: 0,
-    };
-    for await (const row of rows) {
-        out.rowCount++;
-        if (row.tokens.length !== row.labels.length) {
-            out.truncatedRows++;
-            continue;
-        }
-        if (row.labels.every((l) => l === "O")) {
-            out.allORows++;
-        }
-        for (let i = 0; i < row.tokens.length; i++) {
-            const tk = row.tokens[i];
-            const lb = row.labels[i];
-            let labelMap = out.tokens.get(tk);
-            if (!labelMap) {
-                labelMap = new Map();
-                out.tokens.set(tk, labelMap);
-            }
-            labelMap.set(lb, (labelMap.get(lb) ?? 0) + 1);
-            if (i + 1 < row.tokens.length) {
-                const bigramKey = tk + SEP + row.tokens[i + 1];
-                const bigramLabel = lb + SEP + row.labels[i + 1];
-                let bMap = out.bigrams.get(bigramKey);
-                if (!bMap) {
-                    bMap = new Map();
-                    out.bigrams.set(bigramKey, bMap);
-                }
-                bMap.set(bigramLabel, (bMap.get(bigramLabel) ?? 0) + 1);
-            }
-        }
-    }
-    return out;
-}
-function majorityLabel(distribution) {
-    const entries = distribution instanceof Map ? [...distribution.entries()] : Object.entries(distribution);
-    let bestLabel = "";
-    let bestCount = 0;
-    let total = 0;
-    for (const [label, count] of entries) {
-        total += count;
-        if (count > bestCount) {
-            bestCount = count;
-            bestLabel = label;
-        }
-    }
-    return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total };
-}
-function checkDistributionOutliers(shard, corpus) {
-    const flags = [];
-    for (const [token, shardLabelMap] of shard.tokens) {
-        const corpusLabelMap = corpus.tokens[token];
-        if (!corpusLabelMap)
-            continue;
-        const shardMaj = majorityLabel(shardLabelMap);
-        const corpusMaj = majorityLabel(corpusLabelMap);
-        if (corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
-            shardMaj.label !== corpusMaj.label &&
-            shardMaj.count >= SHARD_MIN_COUNT &&
-            corpusMaj.total >= CORPUS_MIN_COUNT) {
-            flags.push({
-                check: "distribution-outlier",
-                severity: "error",
-                token,
-                shardLabel: shardMaj.label,
-                corpusLabel: corpusMaj.label,
-                shardCount: shardMaj.count,
-                corpusCount: corpusMaj.count,
-                detail: `Token "${token}": shard majority is ${shardMaj.label} (${shardMaj.count}/${shardMaj.total}, ${(shardMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
-            });
-        }
-    }
-    return flags;
-}
-function checkLabelVacuum(shard, corpus) {
-    const flags = [];
-    for (const [token, shardLabelMap] of shard.tokens) {
-        const corpusLabelMap = corpus.tokens[token];
-        if (!corpusLabelMap)
-            continue;
-        const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0);
-        if (corpusTotal < VACUUM_CORPUS_MIN_COUNT)
-            continue;
-        for (const [label, shardCount] of shardLabelMap) {
-            if (shardCount < VACUUM_SHARD_MIN_COUNT)
-                continue;
-            if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
-                flags.push({
-                    check: "label-vacuum",
-                    severity: "error",
-                    token,
-                    shardLabel: label,
-                    shardCount,
-                    corpusCount: corpusTotal,
-                    detail: `Token "${token}": shard labels it ${label} ${shardCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
-                });
-            }
-        }
-    }
-    return flags;
-}
-function checkBigramCollisions(shard, corpus) {
-    const flags = [];
-    for (const [bigram, shardLabelMap] of shard.bigrams) {
-        const corpusLabelMap = corpus.bigrams[bigram];
-        if (!corpusLabelMap)
-            continue;
-        const shardMaj = majorityLabel(shardLabelMap);
-        const corpusMaj = majorityLabel(corpusLabelMap);
-        if (shardMaj.label !== corpusMaj.label &&
-            shardMaj.count >= BIGRAM_MIN_COUNT &&
-            corpusMaj.count >= BIGRAM_MIN_COUNT) {
-            const renderBigram = bigram.split(SEP).join(" ");
-            const renderShardLabel = shardMaj.label.split(SEP).join(" → ");
-            const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ");
-            flags.push({
-                check: "bigram-collision",
-                severity: "error",
-                bigram: renderBigram,
-                shardLabel: renderShardLabel,
-                corpusLabel: renderCorpusLabel,
-                shardCount: shardMaj.count,
-                corpusCount: corpusMaj.count,
-                detail: `Bigram "${renderBigram}": shard label-bigram is [${renderShardLabel}] (${shardMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
-            });
-        }
-    }
-    return flags;
-}
-function checkRules(shard, rulesFile) {
-    const flags = [];
-    const compiled = rulesFile.rules.map((r) => ({
-        rule: r,
-        regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
-    }));
-    for (const [token, labelMap] of shard.tokens) {
-        for (const { rule, regex } of compiled) {
-            if (!regex.test(token))
-                continue;
-            for (const [label, count] of labelMap) {
-                if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
-                    flags.push({
-                        check: "anti-pattern-rule",
-                        severity: rule.severity,
-                        ruleID: rule.id,
-                        token,
-                        shardLabel: label,
-                        shardCount: count,
-                        detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
-                    });
-                }
-            }
-        }
-    }
-    return flags;
-}
-function checkSanity(shard) {
-    const flags = [];
-    if (shard.truncatedRows > 0) {
-        flags.push({
-            check: "truncated-rows",
-            severity: "error",
-            detail: `${shard.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
-        });
-    }
-    const allORatio = shard.allORows / Math.max(1, shard.rowCount);
-    if (allORatio >= ALL_O_RATIO_CEILING) {
-        flags.push({
-            check: "all-O-shard",
-            severity: "warn",
-            detail: `${shard.allORows}/${shard.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Shard contributes no signal.`,
-        });
-    }
-    return flags;
-}
-function renderReport(opts, shard, flags) {
-    const errors = flags.filter((f) => f.severity === "error");
-    const warns = flags.filter((f) => f.severity === "warn");
-    const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠";
-    const lines = [
-        `# Corpus Lint: ${verdict}`,
-        "",
-        `- **Shard:** \`${opts.shardPath}\``,
-        `- **Corpus stats:** \`${opts.statsPath}\``,
-        `- **Rules:** \`${opts.rulesPath}\``,
-        `- **Shard rows:** ${shard.rowCount}`,
-        `- **Unique tokens:** ${shard.tokens.size}`,
-        `- **Unique bigrams:** ${shard.bigrams.size}`,
-        "",
-        `**Errors:** ${errors.length} (gates the shard's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
-        `**Warnings:** ${warns.length} (advisory)`,
-        "",
-    ];
-    if (!flags.length) {
-        lines.push("No anomalies detected.");
-        return lines.join("\n");
-    }
-    const byCheck = new Map();
-    for (const f of flags) {
-        const arr = byCheck.get(f.check) ?? [];
-        arr.push(f);
-        byCheck.set(f.check, arr);
-    }
-    for (const [check, list] of byCheck) {
-        lines.push(`## ${check} (${list.length})`);
-        lines.push("");
-        // Sort by shardCount desc — highest-volume issues first
-        list.sort((a, b) => (b.shardCount ?? 0) - (a.shardCount ?? 0));
-        for (const f of list.slice(0, 20)) {
-            lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`);
-        }
-        if (list.length > MAX_LISTED_EXAMPLES) {
-            lines.push(`- ... and ${list.length - 20} more`);
-        }
-        lines.push("");
-    }
-    return lines.join("\n");
-}
-/**
- * Lint a shard against corpus stats + the anti-pattern rules; print the markdown report to stdout.
- */
-export async function lintCorpusShard(options, report) {
-    const rulesPath = options.rulesPath ?? defaultRulesPath();
-    report?.(`Reading corpus stats from ${options.statsPath}...`);
-    const corpus = parseJSONStrict(readFileSync(options.statsPath, "utf8"));
-    report?.(`  ${corpus.row_count} rows from ${corpus.shard_paths.length} shard(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
-    report?.(`Reading shard from ${options.shardPath}...`);
-    const shard = await statsFromShard(streamShard(options.shardPath));
-    report?.(`  ${shard.rowCount} rows`);
-    report?.(`Loading rules from ${rulesPath}...`);
-    const rulesFile = parseJSONStrict(readFileSync(rulesPath, "utf8"));
-    report?.(`Running checks...`);
-    const flags = [
-        ...checkDistributionOutliers(shard, corpus),
-        ...checkLabelVacuum(shard, corpus),
-        ...checkBigramCollisions(shard, corpus),
-        ...checkRules(shard, rulesFile),
-        ...checkSanity(shard),
-    ];
-    const rendered = renderReport({ shardPath: options.shardPath, statsPath: options.statsPath, rulesPath }, shard, flags);
-    console.log(rendered);
-    if (options.outMd) {
-        writeFileSync(options.outMd, rendered);
-    }
-    if (options.outJSON) {
-        writeFileSync(options.outJSON, JSON.stringify({
-            shard: options.shardPath,
-            stats: options.statsPath,
-            flags,
-            summary: {
-                errors: flags.filter((f) => f.severity === "error").length,
-                warnings: flags.filter((f) => f.severity === "warn").length,
-            },
-        }, null, 2));
-    }
-    const errorCount = flags.filter((f) => f.severity === "error").length;
-    const warningCount = flags.filter((f) => f.severity === "warn").length;
-    if (errorCount > 0) {
-        report?.(`LINT FAILED: ${errorCount} error(s).`);
-    }
-    else {
-        report?.("LINT PASSED.");
-    }
-    return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered };
-}
-//# sourceMappingURL=lint-shard.js.map

package/out/src/tools/lint-shard.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard.js","sourceRoot":"","sources":["../../../src/tools/lint-shard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAExC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AAEzD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAE3D;;GAEG;AACH,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B,MAAM,GAAG,GAAG,EAAE,CAAA;AAEd;;;GAGG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AACpC,MAAM,eAAe,GAAG,EAAE,CAAA;AAC1B,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAC5B,MAAM,sBAAsB,GAAG,EAAE,CAAA;AACjC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,mBAAmB,GAAG,GAAG,CAAA;AAE/B;;;;;GAKG;AACH,SAAS,gBAAgB;IACxB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAE7D,IAAI,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,aAAa,CAAC,OAAO,CAAC,CAAA;IAEtD,OAAO,aAAa,CAAC,IAAI,GAAG,CAAC,oCAAoC,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACrF,CAAC;AAwDD;;;;;GAKG;AACH,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,SAAiB;IAC5C,YAAY,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAW,SAAS,CAAC,CAAA;IAEtE,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;AAC1C,CAAC;AAUD,KAAK,UAAU,cAAc,CAAC,IAA6B;IAC1D,MAAM,GAAG,GAAe;QACvB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,IAAI,GAAG,EAAE;QACjB,OAAO,EAAE,IAAI,GAAG,EAAE;QAClB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;KACX,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAA;QAEd,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,GAAG,CAAC,aAAa,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,EAAE,CAAA;QACf,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAA;YACzB,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAE,CAAA;YACzB,IAAI,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAEjC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAA;gBACpB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAA;YAC7B,CAAC;YAED,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAE7C,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC/B,MAAM,SAAS,GAAG,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;gBAC/C,MAAM,WAAW,GAAG,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;gBACjD,IAAI,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;gBAErC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACX,IAAI,GAAG,IAAI,GAAG,EAAE,CAAA;oBAChB,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;gBACjC,CAAC;gBAED,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACxD,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,aAAa,CAAC,YAA0D;IAMhF,MAAM,OAAO,GAAG,YAAY,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACxG,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,EAAE,CAAA;AACtG,CAAC;AA+BD,SAAS,yBAAyB,CAAC,KAAiB,EAAE,MAAmB;IACxE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,UAAU,IAAI,uBAAuB;YAC/C,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,eAAe;YACjC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,OAAO;gBACjB,KAAK;gBACL,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,UAAU,KAAK,wBAAwB,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;aAC9Q,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAiB,EAAE,MAAmB;IAC/D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAE5E,IAAI,WAAW,GAAG,uBAAuB;YAAE,SAAQ;QAEnD,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;YACjD,IAAI,UAAU,GAAG,sBAAsB;gBAAE,SAAQ;YAEjD,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxE,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,OAAO;oBACjB,KAAK;oBACL,UAAU,EAAE,KAAK;oBACjB,UAAU;oBACV,WAAW,EAAE,WAAW;oBACxB,MAAM,EAAE,UAAU,KAAK,sBAAsB,KAAK,IAAI,UAAU,2BAA2B,WAAW,6DAA6D;iBACnK,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAiB,EAAE,MAAmB;IACpE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,gBAAgB;YAClC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEhE,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,kBAAkB;gBACzB,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,WAAW,EAAE,iBAAiB;gBAC9B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,WAAW,YAAY,6BAA6B,gBAAgB,MAAM,QAAQ,CAAC,KAAK,+BAA+B,iBAAiB,MAAM,SAAS,CAAC,KAAK,sDAAsD;aAC3N,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB,EAAE,SAAwB;IAC9D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KACjE,CAAC,CAAC,CAAA;IAEH,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gCAAgC,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,mBAAmB;wBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,KAAK;wBACL,UAAU,EAAE,KAAK;wBACjB,UAAU,EAAE,KAAK;wBACjB,MAAM,EAAE,UAAU,KAAK,kBAAkB,IAAI,CAAC,EAAE,mBAAmB,KAAK,IAAI,KAAK,2BAA2B,IAAI,CAAC,OAAO,EAAE;qBAC1H,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB;IACrC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,gBAAgB;YACvB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,aAAa,uEAAuE;SACrG,CAAC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE9D,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yDAAyD;SAC1I,CAAC,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CACpB,IAAiE,EACjE,KAAiB,EACjB,KAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAA;IAE/D,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,EAAE;QAC3B,EAAE;QACF,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,yBAAyB,IAAI,CAAC,SAAS,IAAI;QAC3C,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,qBAAqB,KAAK,CAAC,QAAQ,EAAE;QACrC,wBAAwB,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;QAC3C,yBAAyB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAC7C,EAAE;QACF,eAAe,MAAM,CAAC,MAAM,iFAAiF;QAC7G,iBAAiB,KAAK,CAAC,MAAM,aAAa;QAC1C,EAAE;KACF,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAEpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAElD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACf,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAA;IACzD,MAAM,EAAE,CAAC,6BAA6B,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,eAAe,CAAc,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;IAEpF,MAAM,EAAE,CACP,KAAK,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU,CACnK,CAAA;IAED,MAAM,EAAE,CAAC,sBAAsB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAEtD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;IAElE,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,sBAAsB,SAAS,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,eAAe,CAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjF,MAAM,EAAE,CAAC,mBAAmB,CAAC,CAAA;IAE7B,MAAM,KAAK,GAAoB;QAC9B,GAAG,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3C,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC;QAClC,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC;QACvC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC;QAC/B,GAAG,WAAW,CAAC,KAAK,CAAC;KACrB,CAAA;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAEtH,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAErB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,aAAa,CACZ,OAAO,CAAC,OAAO,EACf,IAAI,CAAC,SAAS,CACb;YACC,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK;YACL,OAAO,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;aAC3D;SACD,EACD,IAAI,EACJ,CAAC,CACD,CACD,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IAEtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,gBAAgB,UAAU,YAAY,CAAC,CAAA;IACjD,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACzF,CAAC"}

package/out/src/tools/overlay-manifest.d.ts

@@ -1,39 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
- *   ADDS one shard parquet to a base corpus, keeping every base shard VERBATIM (pure overlay ADD),
- *   and re-roots base paths to /data (the Modal volume). Parameterized by --shard-parquet +
- *   --source so it works for any overlay shard (the fr-admin-split one is the original; #148's
- *   overture-multilocale is the second user).
- *
- *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new shard's source_id column is
- *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
- *
- *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
- *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-shard.ts
- *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
- *   --input <labeled> --output <NEW>/train/<shard-parquet> node
- *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
- *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
- *   --shard-parquet <shard-parquet> --source <source> --note "..."
- *
- *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
- */
-export interface OverlayManifestOptions {
-    base: string;
-    newDir: string;
-    modalRoot: string;
-    version: string;
-    shardParquet: string;
-    source: string;
-    note: string;
-}
-/**
- * Resolve a base manifest's shard path to the mounted corpus tree used by Modal.
- */
-export declare function rerootBaseShardPath(path: string, baseManifestPath: string): string;
-export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
-//# sourceMappingURL=overlay-manifest.d.ts.map

package/out/src/tools/overlay-manifest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../../src/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAiEH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCzF"}

package/out/src/tools/overlay-manifest.js

@@ -1,98 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
- *   ADDS one shard parquet to a base corpus, keeping every base shard VERBATIM (pure overlay ADD),
- *   and re-roots base paths to /data (the Modal volume). Parameterized by --shard-parquet +
- *   --source so it works for any overlay shard (the fr-admin-split one is the original; #148's
- *   overture-multilocale is the second user).
- *
- *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new shard's source_id column is
- *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
- *
- *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
- *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-shard.ts
- *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
- *   --input <labeled> --output <NEW>/train/<shard-parquet> node
- *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
- *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
- *   --shard-parquet <shard-parquet> --source <source> --note "..."
- *
- *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
- */
-import { readFileSync, statSync, writeFileSync } from "node:fs";
-import { basename, dirname, join } from "node:path";
-import { parseJSONStrict } from "@mailwoman/core/objects";
-import { sha256Hex } from "@mailwoman/core/utils";
-/**
- * Escape a path for single-quoted SQL string literals.
- */
-function sqlString(value) {
-    return value.replaceAll("'", "''");
-}
-async function descriptor(localPath, modalPath, split, source) {
-    // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const instance = await DuckDBInstance.create();
-    const db = await instance.connect();
-    const result = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${sqlString(localPath)}')`);
-    const sids = result.getRowObjects().map((r) => r.source_id);
-    return {
-        split,
-        path: modalPath,
-        format: "parquet",
-        compression: "SNAPPY",
-        rows: sids.length,
-        bytes: statSync(localPath).size,
-        sha256: sha256Hex(readFileSync(localPath)),
-        first_source_id: sids[0],
-        last_source_id: sids.at(-1),
-        source,
-    };
-}
-/**
- * Resolve a base manifest's shard path to the mounted corpus tree used by Modal.
- */
-export function rerootBaseShardPath(path, baseManifestPath) {
-    const versionedIndex = path.indexOf("/corpus/versioned/");
-    if (versionedIndex !== -1)
-        return "/data" + path.slice(versionedIndex);
-    if (/^\/data\/(?:train|val|test)\//u.test(path)) {
-        const localBaseDir = dirname(baseManifestPath);
-        const baseModalRoot = `/data/corpus/versioned/${basename(dirname(localBaseDir))}/${basename(localBaseDir)}`;
-        return baseModalRoot + path.slice("/data".length);
-    }
-    return path;
-}
-export async function assembleOverlayManifest(args) {
-    const base = parseJSONStrict(readFileSync(args.base, "utf8"));
-    if (base.shards.some((s) => s.source === args.source)) {
-        console.log(`WARN: base already contains source '${args.source}' — is this the right base?`);
-    }
-    const kept = base.shards.map((s) => ({ ...s, path: rerootBaseShardPath(s.path, args.base) }));
-    const newTrain = await descriptor(join(args.newDir, "train", args.shardParquet), `${args.modalRoot}/train/${args.shardParquet}`, "train", args.source);
-    const manifest = {
-        corpus_version: args.version,
-        overlay_base: base.corpus_version ?? null,
-        note: args.note || `${base.corpus_version} shards (all kept verbatim) + the ${args.source} shard. Pure overlay add.`,
-        schema: base.schema,
-        rows_per_shard: base.rows_per_shard,
-        row_group_size: base.row_group_size,
-        shards: [...kept, newTrain],
-        counts: {
-            train: base.counts.train + newTrain.rows,
-            val: base.counts.val,
-            test: base.counts.test,
-        },
-        total_rows: base.total_rows + newTrain.rows,
-    };
-    const out = join(args.newDir, "MANIFEST.json");
-    writeFileSync(out, JSON.stringify(manifest, null, 1) + "\n");
-    console.log(`wrote ${out}`);
-    console.log(`  shards: ${manifest.shards.length} (${kept.length} base kept, +1 ${args.source})`);
-    console.log(`  counts: ${JSON.stringify(manifest.counts)}  total: ${manifest.total_rows}`);
-    console.log(`  ${args.source} train: ${newTrain.rows} rows (${newTrain.bytes} bytes)`);
-}
-//# sourceMappingURL=overlay-manifest.js.map

package/out/src/tools/overlay-manifest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../../src/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAyBjD;;GAEG;AACH,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAED,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,gFAAgF;IAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAC9C,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IACtG,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,IAAI;QAC/B,MAAM,EAAE,SAAS,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAC1C,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,MAAM,IAAI,GAAG,eAAe,CAAe,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,MAAM,6BAA6B,CAAC,CAAA;IAC7F,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7F,MAAM,QAAQ,GAAG,MAAM,UAAU,CAChC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,EAC7C,GAAG,IAAI,CAAC,SAAS,UAAU,IAAI,CAAC,YAAY,EAAE,EAC9C,OAAO,EACP,IAAI,CAAC,MAAM,CACX,CAAA;IAED,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EACH,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,qCAAqC,IAAI,CAAC,MAAM,2BAA2B;QAC/G,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI;YACxC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI;KAC3C,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IAE5D,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,kBAAkB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAChG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,WAAW,QAAQ,CAAC,IAAI,UAAU,QAAQ,CAAC,KAAK,SAAS,CAAC,CAAA;AACvF,CAAC"}

package/out/src/tools/overture-subvenue.d.ts

@@ -1,112 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Overture Places source for the sub-venue lexicon (#35 wave 2) — read the sub-venue-bearing
- *   category slices of `poi.db` (spatial layer #1, `$MAILWOMAN_DATA_ROOT/poi/poi.db`) and yield
- *   {@link SubVenueHarvestRow}s the lexicon builder consumes exactly like an OSM extract.
- *
- *   ── WHY OVERTURE AT ALL, when the OSM extractor already exists ────────────────────────────────────
- *   Wave 1 measured `concourse` at 21 real Overture rows against 4 in the whole Great Britain OSM
- *   extract, 3 of which are a street called CONCOURSE WAY. Overture's `airport_terminal` category is
- *   curated venue-interior naming; OSM's `aeroway=terminal` is a building footprint that usually
- *   carries the AIRPORT's name. The two sources fail differently, so both are read.
- *
- *   ── THE CATEGORY SET IS MEASURED, NOT GUESSED ────────────────────────────────────────────────────
- *   A full scan of all 13,681,698 rows (2026-08-05, poi.db vintage 2026-05-20.0) counted, per
- *   category, how many named rows carry a designator token. The ranking is NOT what a category name
- *   predicts — `gas_station` leads the whole table with 12,996 hits, every one of them the token
- *   `station` inside "Holiday Station" / "Chevron Station Seward", and `shoe_store` contributes 708
- *   hits of `wing` because Red Wing sells boots. {@link OVERTURE_SUBVENUE_CATEGORIES} is the four
- *   categories whose hits survived reading the distribution; the rejects are listed below it so nobody
- *   re-proposes them.
- *
- *   ── poi.db IS FOUR COUNTRIES ─────────────────────────────────────────────────────────────────────
- *   Measured the same day: US 11,521,612 / CA 794,418 / FR 721,352 / MX 644,316, and nothing else. The
- *   shipped layer is not a world gazetteer, so Overture can attest en-US, en-CA, fr-FR and es-MX
- *   surfaces and NOTHING ELSE. Every non-Latin designator the corpus task asks for — `ターミナル`,
- *   `Halle`, `Flügel` — has to come from the OSM leg. Do not read a zero count here as evidence of
- *   absence in the world; it is evidence of absence in four countries.
- *
- *   ── The row shape fits; the PROVENANCE STAMP did not ─────────────────────────────────────────────
- *   Wave 1's `OSMSubVenueRow` was written to accept a non-OSM row, and it does: an Overture row is
- *   `{ designatorID, name }` with no `ref` and no `localizedNames`. What did NOT fit is
- *   `extractAttestedPhrases`, which hardcoded `osm:name` as the surface's `source`. Feeding Overture
- *   rows through it unchanged would have labelled every Overture surface as OSM-attested — a
- *   provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
- *   function and the source-neutral {@link SubVenueHarvestRow} name.
- */
-import type { SubVenueHarvestRow } from "./sub-venue-lexicon.ts";
-/**
- * Overture Places category → the designator its rows attest.
- *
- * Each entry carries the 2026-08-05 full-scan measurement in a comment: `rows` is the category's total, `hits` is how
- * many of its named rows contain any designator token, and the token list is the top of that distribution.
- *
- * REJECTED, with the number that rejected them — do not re-add without a fresh census:
- *
- * - `gas_station` (12,996 hits) — all `station` inside a brand name (Holiday Station, Chevron Station Seward).
- * - `fire_station` (10,377) — same, plus 330 `hall` from fire halls, which are venues rather than sub-venues.
- * - `town_hall` (5,528) — `City Hall` is a whole building, the `venue` tier, not interior structure.
- * - `building_supply_store` (4,030) — `building` inside "Allied Building Products".
- * - `jehovahs_witness_place_of_worship` (2,334) — every hit is "Kingdom Hall of Jehovah's Witnesses".
- * - `shoe_store` (909, of which 708 `wing`) — Red Wing. This one is a CONFOUND BOARD entry, not a source.
- * - `college_university` (3,697) — 2,082 `campus`, but the row names the whole institution; `campus_building` is the
- *   interior slice and is kept instead.
- * - `airport` (6,000 rows, 4,302 hits) — 4,071 of them are the token `airport` in the aerodrome's own name. Venue tier,
- *   already covered by OurAirports, and it drowns the interior signal.
- * - `transport_interchange` (1 row), `rail_facility_or_service` (81 rows, 2 hits) — too small to matter.
- * - `public_transit_facility_or_service` (2,274 rows, 185 hits) — 161 are `station` naming the station itself.
- */
-export declare const OVERTURE_SUBVENUE_CATEGORIES: Readonly<Record<string, string>>;
-/**
- * The layer's `source_vintage` — the Overture release the rows came from (`2026-05-20.0`), for the lexicon's
- * `sources[]`.
- *
- * Read off the database rather than passed in, because a vintage a caller types is a vintage that goes stale silently.
- * The layer-contract tables are part of every layer database by construction — see `docs/engineering/reference/
- * layer-contract.mdx` — so there is no version of poi.db where this is absent.
- */
-export declare function readOvertureLayerVintage(databasePath: string): Promise<string>;
-export interface ReadOvertureSubVenuesOptions {
-    /**
-     * Path to `poi.db`. Typically `dataRootPath("poi", "poi.db")`.
-     */
-    databasePath: string;
-    /**
-     * Category → designator map. Defaults to {@link OVERTURE_SUBVENUE_CATEGORIES}.
-     */
-    categories?: Readonly<Record<string, string>>;
-    /**
-     * Restrict to these ISO 3166-1 alpha-2 countries. Omit for all four the layer carries.
-     */
-    countries?: readonly string[];
-}
-/**
- * One Overture row plus the country it came from — {@link SubVenueHarvestRow} widened by the field the layer has and an
- * OSM extract does not (a Geofabrik extract's country is a property of the invocation, so OSM rows carry `country:
- * ""`).
- */
-export interface OvertureSubVenueRow extends SubVenueHarvestRow {
-    /**
-     * ISO 3166-1 alpha-2, straight off the Overture partition.
-     */
-    country: string;
-    /**
-     * The Overture category the row was read from, so a surface's provenance survives past the designator.
-     */
-    category: string;
-}
-/**
- * Read the sub-venue-bearing category slices of `poi.db`.
- *
- * Cold path, already async, no interface constraint — so Kysely, per the repo's inline-SQL rule.
- *
- * `category_id` is the SECOND component of poi.db's clustered `(h3_cell, category_id, …)` primary key, which reads like
- * a full 13.7M-row scan and is not one: SQLite skip-scans the leading `h3_cell` and the whole four-category read
- * returned 9,219 rows in **1.4 s** on the shipped layer (2026-08-05). A JS-side scan of every row to survey the same
- * question took 52 s, which is the number to remember if you are tempted to filter in JS instead.
- */
-export declare function readOvertureSubVenues(options: ReadOvertureSubVenuesOptions): Promise<OvertureSubVenueRow[]>;
-//# sourceMappingURL=overture-subvenue.d.ts.map

package/out/src/tools/overture-subvenue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"overture-subvenue.d.ts","sourceRoot":"","sources":["../../../src/tools/overture-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAMH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAEhE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,4BAA4B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAczE,CAAA;AA0BD;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAYpF;AAED,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC7C;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC9D;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,4BAA4B,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CA4CjH"}