@mailwoman/corpus 9.2.0 → 9.4.0

package/out/tools/lint-slice.js

@@ -0,0 +1,322 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
+ *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
+ *   venue templates.
+ *
+ *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
+ *
+ *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
+ *        majority label to the corpus's majority label. Flag when the corpus has a
+ *        confidently-established majority (>66%) AND the slice's majority differs AND both have
+ *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
+ *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
+ *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
+ *        we're introducing a novel association, not shifting a distribution.
+ *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
+ *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
+ *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
+ *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
+ *        mappings — flagging matches.
+ *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
+ *
+ *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
+ *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
+ *   flagged slice should require `lint_acknowledged: true` before training consumes it.
+ *
+ *   Usage: mailwoman dev lint corpus-slice\
+ *   --slice <new-slice.parquet>\
+ *   --stats <corpus-stats.json>\
+ *   [--rules <rules.json>]\
+ *   [--out-md /tmp/lint-report.md]\
+ *   [--out-json /tmp/lint-report.json]
+ */
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { writeLocalFile, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { resolvePackagePath } from "@mailwoman/core/module/resolvers";
+import { accumulateCooccurrences, createCooccurrenceStats, SLICE_STATS_SEP as SEP, streamTokenLabelRows, } from "#utils/slice-stats";
+/**
+ * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
+ */
+const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
+/**
+ * Examples printed per finding before the list is truncated.
+ */
+const MAX_LISTED_EXAMPLES = 20;
+/**
+ * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
+ * numbers miss.
+ */
+const CORPUS_CONFIDENCE_FLOOR = 0.66;
+const SLICE_MIN_COUNT = 50;
+const CORPUS_MIN_COUNT = 200;
+const VACUUM_SLICE_MIN_COUNT = 20;
+const VACUUM_CORPUS_MIN_COUNT = 100;
+const BIGRAM_MIN_COUNT = 10;
+const ALL_O_RATIO_CEILING = 0.9;
+/**
+ * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
+ * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
+ * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
+ * the common path.
+ */
+function defaultRulesPath() {
+    return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json");
+}
+async function statsFromSlice(rows) {
+    const co = createCooccurrenceStats();
+    const out = {
+        rowCount: 0,
+        tokens: co.tokens,
+        bigrams: co.bigrams,
+        truncatedRows: 0,
+        allORows: 0,
+    };
+    for await (const row of rows) {
+        out.rowCount++;
+        if (row.tokens.length !== row.labels.length) {
+            out.truncatedRows++;
+            continue;
+        }
+        if (row.labels.every((l) => l === "O")) {
+            out.allORows++;
+        }
+        accumulateCooccurrences(co, row.tokens, row.labels);
+    }
+    return out;
+}
+function majorityLabel(distribution) {
+    const entries = distribution instanceof Map ? [...distribution.entries()] : Object.entries(distribution);
+    let bestLabel = "";
+    let bestCount = 0;
+    let total = 0;
+    for (const [label, count] of entries) {
+        total += count;
+        if (count > bestCount) {
+            bestCount = count;
+            bestLabel = label;
+        }
+    }
+    return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total };
+}
+function checkDistributionOutliers(slice, corpus) {
+    const flags = [];
+    for (const [token, sliceLabelMap] of slice.tokens) {
+        const corpusLabelMap = corpus.tokens[token];
+        if (!corpusLabelMap)
+            continue;
+        const sliceMaj = majorityLabel(sliceLabelMap);
+        const corpusMaj = majorityLabel(corpusLabelMap);
+        if (corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
+            sliceMaj.label !== corpusMaj.label &&
+            sliceMaj.count >= SLICE_MIN_COUNT &&
+            corpusMaj.total >= CORPUS_MIN_COUNT) {
+            flags.push({
+                check: "distribution-outlier",
+                severity: "error",
+                token,
+                sliceLabel: sliceMaj.label,
+                corpusLabel: corpusMaj.label,
+                sliceCount: sliceMaj.count,
+                corpusCount: corpusMaj.count,
+                detail: `Token "${token}": slice majority is ${sliceMaj.label} (${sliceMaj.count}/${sliceMaj.total}, ${(sliceMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
+            });
+        }
+    }
+    return flags;
+}
+function checkLabelVacuum(slice, corpus) {
+    const flags = [];
+    for (const [token, sliceLabelMap] of slice.tokens) {
+        const corpusLabelMap = corpus.tokens[token];
+        if (!corpusLabelMap)
+            continue;
+        const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0);
+        if (corpusTotal < VACUUM_CORPUS_MIN_COUNT)
+            continue;
+        for (const [label, sliceCount] of sliceLabelMap) {
+            if (sliceCount < VACUUM_SLICE_MIN_COUNT)
+                continue;
+            if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
+                flags.push({
+                    check: "label-vacuum",
+                    severity: "error",
+                    token,
+                    sliceLabel: label,
+                    sliceCount,
+                    corpusCount: corpusTotal,
+                    detail: `Token "${token}": slice labels it ${label} ${sliceCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
+                });
+            }
+        }
+    }
+    return flags;
+}
+function checkBigramCollisions(slice, corpus) {
+    const flags = [];
+    for (const [bigram, sliceLabelMap] of slice.bigrams) {
+        const corpusLabelMap = corpus.bigrams[bigram];
+        if (!corpusLabelMap)
+            continue;
+        const sliceMaj = majorityLabel(sliceLabelMap);
+        const corpusMaj = majorityLabel(corpusLabelMap);
+        if (sliceMaj.label !== corpusMaj.label &&
+            sliceMaj.count >= BIGRAM_MIN_COUNT &&
+            corpusMaj.count >= BIGRAM_MIN_COUNT) {
+            const renderBigram = bigram.split(SEP).join(" ");
+            const renderSliceLabel = sliceMaj.label.split(SEP).join(" → ");
+            const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ");
+            flags.push({
+                check: "bigram-collision",
+                severity: "error",
+                bigram: renderBigram,
+                sliceLabel: renderSliceLabel,
+                corpusLabel: renderCorpusLabel,
+                sliceCount: sliceMaj.count,
+                corpusCount: corpusMaj.count,
+                detail: `Bigram "${renderBigram}": slice label-bigram is [${renderSliceLabel}] (${sliceMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
+            });
+        }
+    }
+    return flags;
+}
+function checkRules(slice, rulesFile) {
+    const flags = [];
+    const compiled = rulesFile.rules.map((r) => ({
+        rule: r,
+        regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
+    }));
+    for (const [token, labelMap] of slice.tokens) {
+        for (const { rule, regex } of compiled) {
+            if (!regex.test(token))
+                continue;
+            for (const [label, count] of labelMap) {
+                if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
+                    flags.push({
+                        check: "anti-pattern-rule",
+                        severity: rule.severity,
+                        ruleID: rule.id,
+                        token,
+                        sliceLabel: label,
+                        sliceCount: count,
+                        detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
+                    });
+                }
+            }
+        }
+    }
+    return flags;
+}
+function checkSanity(slice) {
+    const flags = [];
+    if (slice.truncatedRows > 0) {
+        flags.push({
+            check: "truncated-rows",
+            severity: "error",
+            detail: `${slice.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
+        });
+    }
+    const allORatio = slice.allORows / Math.max(1, slice.rowCount);
+    if (allORatio >= ALL_O_RATIO_CEILING) {
+        flags.push({
+            check: "all-O-slice",
+            severity: "warn",
+            detail: `${slice.allORows}/${slice.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Slice contributes no signal.`,
+        });
+    }
+    return flags;
+}
+function renderReport(opts, slice, flags) {
+    const errors = flags.filter((f) => f.severity === "error");
+    const warns = flags.filter((f) => f.severity === "warn");
+    const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠";
+    const lines = [
+        `# Corpus Lint: ${verdict}`,
+        "",
+        `- **Slice:** \`${opts.slicePath}\``,
+        `- **Corpus stats:** \`${opts.statsPath}\``,
+        `- **Rules:** \`${opts.rulesPath}\``,
+        `- **Slice rows:** ${slice.rowCount}`,
+        `- **Unique tokens:** ${slice.tokens.size}`,
+        `- **Unique bigrams:** ${slice.bigrams.size}`,
+        "",
+        `**Errors:** ${errors.length} (checks the slice's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
+        `**Warnings:** ${warns.length} (advisory)`,
+        "",
+    ];
+    if (!flags.length) {
+        lines.push("No anomalies detected.");
+        return lines.join("\n");
+    }
+    const byCheck = new Map();
+    for (const f of flags) {
+        const arr = byCheck.get(f.check) ?? [];
+        arr.push(f);
+        byCheck.set(f.check, arr);
+    }
+    for (const [check, list] of byCheck) {
+        lines.push(`## ${check} (${list.length})`);
+        lines.push("");
+        // Sort by sliceCount desc — highest-volume issues first
+        list.sort((a, b) => (b.sliceCount ?? 0) - (a.sliceCount ?? 0));
+        for (const f of list.slice(0, 20)) {
+            lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`);
+        }
+        if (list.length > MAX_LISTED_EXAMPLES) {
+            lines.push(`- ... and ${list.length - 20} more`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
+ */
+export async function lintCorpusSlice(options, report) {
+    const rulesPath = options.rulesPath ?? defaultRulesPath();
+    report?.(`Reading corpus stats from ${options.statsPath}...`);
+    const corpus = await readLocalJSONFile(options.statsPath);
+    report?.(`  ${corpus.row_count} rows from ${corpus.slice_paths.length} slice(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
+    report?.(`Reading slice from ${options.slicePath}...`);
+    const slice = await statsFromSlice(streamTokenLabelRows(options.slicePath));
+    report?.(`  ${slice.rowCount} rows`);
+    report?.(`Loading rules from ${rulesPath}...`);
+    const rulesFile = await readLocalJSONFile(rulesPath);
+    report?.(`Running checks...`);
+    const flags = [
+        ...checkDistributionOutliers(slice, corpus),
+        ...checkLabelVacuum(slice, corpus),
+        ...checkBigramCollisions(slice, corpus),
+        ...checkRules(slice, rulesFile),
+        ...checkSanity(slice),
+    ];
+    const rendered = renderReport({ slicePath: options.slicePath, statsPath: options.statsPath, rulesPath }, slice, flags);
+    console.log(rendered);
+    if (options.outMd) {
+        await writeLocalFile(rendered, options.outMd);
+    }
+    if (options.outJSON) {
+        await writeLocalJSONFile({
+            slice: options.slicePath,
+            stats: options.statsPath,
+            flags,
+            summary: {
+                errors: flags.filter((f) => f.severity === "error").length,
+                warnings: flags.filter((f) => f.severity === "warn").length,
+            },
+        }, options.outJSON);
+    }
+    const errorCount = flags.filter((f) => f.severity === "error").length;
+    const warningCount = flags.filter((f) => f.severity === "warn").length;
+    if (errorCount > 0) {
+        report?.(`LINT FAILED: ${errorCount} error(s).`);
+    }
+    else {
+        report?.("LINT PASSED.");
+    }
+    return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered };
+}
+//# sourceMappingURL=lint-slice.js.map

package/out/tools/lint-slice.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-slice.js","sourceRoot":"","sources":["../../lib/tools/lint-slice.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAErE,OAAO,EACN,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,IAAI,GAAG,EACtB,oBAAoB,GAEpB,MAAM,oBAAoB,CAAA;AAE3B;;GAEG;AACH,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B;;;GAGG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AACpC,MAAM,eAAe,GAAG,EAAE,CAAA;AAC1B,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAC5B,MAAM,sBAAsB,GAAG,EAAE,CAAA;AACjC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,mBAAmB,GAAG,GAAG,CAAA;AAE/B;;;;;GAKG;AACH,SAAS,gBAAgB;IACxB,OAAO,kBAAkB,CAAC,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAA;AAClF,CAAC;AAwDD,KAAK,UAAU,cAAc,CAAC,IAA6B;IAC1D,MAAM,EAAE,GAAG,uBAAuB,EAAE,CAAA;IAEpC,MAAM,GAAG,GAAe;QACvB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,CAAC,OAAO;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;KACX,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAA;QAEd,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,GAAG,CAAC,aAAa,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,EAAE,CAAA;QACf,CAAC;QAED,uBAAuB,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,aAAa,CAAC,YAA0D;IAMhF,MAAM,OAAO,GAAG,YAAY,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACxG,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,EAAE,CAAA;AACtG,CAAC;AA+BD,SAAS,yBAAyB,CAAC,KAAiB,EAAE,MAAmB;IACxE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,UAAU,IAAI,uBAAuB;YAC/C,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,eAAe;YACjC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,OAAO;gBACjB,KAAK;gBACL,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,UAAU,KAAK,wBAAwB,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;aAC9Q,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAiB,EAAE,MAAmB;IAC/D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAE5E,IAAI,WAAW,GAAG,uBAAuB;YAAE,SAAQ;QAEnD,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;YACjD,IAAI,UAAU,GAAG,sBAAsB;gBAAE,SAAQ;YAEjD,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxE,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,OAAO;oBACjB,KAAK;oBACL,UAAU,EAAE,KAAK;oBACjB,UAAU;oBACV,WAAW,EAAE,WAAW;oBACxB,MAAM,EAAE,UAAU,KAAK,sBAAsB,KAAK,IAAI,UAAU,2BAA2B,WAAW,6DAA6D;iBACnK,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAiB,EAAE,MAAmB;IACpE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,gBAAgB;YAClC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEhE,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,kBAAkB;gBACzB,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,WAAW,EAAE,iBAAiB;gBAC9B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,WAAW,YAAY,6BAA6B,gBAAgB,MAAM,QAAQ,CAAC,KAAK,+BAA+B,iBAAiB,MAAM,SAAS,CAAC,KAAK,sDAAsD;aAC3N,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB,EAAE,SAAwB;IAC9D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KACjE,CAAC,CAAC,CAAA;IAEH,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gCAAgC,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,mBAAmB;wBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,KAAK;wBACL,UAAU,EAAE,KAAK;wBACjB,UAAU,EAAE,KAAK;wBACjB,MAAM,EAAE,UAAU,KAAK,kBAAkB,IAAI,CAAC,EAAE,mBAAmB,KAAK,IAAI,KAAK,2BAA2B,IAAI,CAAC,OAAO,EAAE;qBAC1H,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB;IACrC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,gBAAgB;YACvB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,aAAa,uEAAuE;SACrG,CAAC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE9D,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yDAAyD;SAC1I,CAAC,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CACpB,IAAiE,EACjE,KAAiB,EACjB,KAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAA;IAE/D,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,EAAE;QAC3B,EAAE;QACF,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,yBAAyB,IAAI,CAAC,SAAS,IAAI;QAC3C,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,qBAAqB,KAAK,CAAC,QAAQ,EAAE;QACrC,wBAAwB,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;QAC3C,yBAAyB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAC7C,EAAE;QACF,eAAe,MAAM,CAAC,MAAM,kFAAkF;QAC9G,iBAAiB,KAAK,CAAC,MAAM,aAAa;QAC1C,EAAE;KACF,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAEpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAElD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACf,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAA;IACzD,MAAM,EAAE,CAAC,6BAA6B,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAc,OAAO,CAAC,SAAS,CAAC,CAAA;IAEtE,MAAM,EAAE,CACP,KAAK,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU,CACnK,CAAA;IAED,MAAM,EAAE,CAAC,sBAAsB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAEtD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;IAE3E,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,sBAAsB,SAAS,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAgB,SAAS,CAAC,CAAA;IAEnE,MAAM,EAAE,CAAC,mBAAmB,CAAC,CAAA;IAE7B,MAAM,KAAK,GAAoB;QAC9B,GAAG,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3C,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC;QAClC,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC;QACvC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC;QAC/B,GAAG,WAAW,CAAC,KAAK,CAAC;KACrB,CAAA;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAEtH,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAErB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,kBAAkB,CACvB;YACC,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK;YACL,OAAO,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;aAC3D;SACD,EACD,OAAO,CAAC,OAAO,CACf,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IAEtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,gBAAgB,UAAU,YAAY,CAAC,CAAA;IACjD,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACzF,CAAC"}

package/out/tools/overlay-manifest.d.ts

@@ -0,0 +1,39 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
+ *   ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
+ *   and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
+ *   --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
+ *   overture-multilocale is the second user).
+ *
+ *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
+ *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
+ *
+ *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
+ *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-slice.ts
+ *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
+ *   --input <labeled> --output <NEW>/train/<slice-parquet> node
+ *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
+ *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
+ *   --slice-parquet <slice-parquet> --source <source> --note "..."
+ *
+ *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
+ */
+export interface OverlayManifestOptions {
+    base: string;
+    newDir: string;
+    modalRoot: string;
+    version: string;
+    sliceParquet: string;
+    source: string;
+    note: string;
+}
+/**
+ * Resolve a base manifest's slice path to the mounted corpus tree used by Modal.
+ */
+export declare function rerootBaseSlicePath(path: string, baseManifestPath: string): string;
+export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
+//# sourceMappingURL=overlay-manifest.d.ts.map

package/out/tools/overlay-manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAwDH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCzF"}

package/out/tools/overlay-manifest.js

@@ -0,0 +1,90 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
+ *   ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
+ *   and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
+ *   --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
+ *   overture-multilocale is the second user).
+ *
+ *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
+ *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
+ *
+ *   Pipeline (the recipe rides the result): node scripts/build-overture-multilocale-canonical.mjs
+ *   --cap 150000 --out /tmp/ovl/overture-ml.canonical.jsonl node scripts/align-canonical-slice.ts
+ *   --input <canonical> --output <labeled> --corpus-version 0.5.0 mailwoman dev jsonl-to-parquet
+ *   --input <labeled> --output <NEW>/train/<slice-parquet> node
+ *   scripts/assemble-overlay-manifest.ts --base <BASE>/MANIFEST.json --new-dir <NEW>\
+ *   --modal-root /data/corpus/versioned/<ver>/<dir> --version <ver>\
+ *   --slice-parquet <slice-parquet> --source <source> --note "..."
+ *
+ *   # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
+ */
+import { readLocalBuffer, readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { sha256Hex } from "@mailwoman/core/hash";
+import { basename, dirname, join } from "path-ts";
+import { connectDuckDB, escapeSQLString } from "#utils/parquet";
+async function descriptor(localPath, modalPath, split, source) {
+    const db = await connectDuckDB();
+    const result = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${escapeSQLString(localPath)}')`);
+    const sids = result.getRowObjects().map((r) => r.source_id);
+    return {
+        split,
+        path: modalPath,
+        format: "parquet",
+        compression: "SNAPPY",
+        rows: sids.length,
+        bytes: (await statPath(localPath)).size,
+        sha256: sha256Hex(await readLocalBuffer(localPath)),
+        first_source_id: sids[0],
+        last_source_id: sids.at(-1),
+        source,
+    };
+}
+/**
+ * Resolve a base manifest's slice path to the mounted corpus tree used by Modal.
+ */
+export function rerootBaseSlicePath(path, baseManifestPath) {
+    const versionedIndex = path.indexOf("/corpus/versioned/");
+    if (versionedIndex !== -1)
+        return "/data" + path.slice(versionedIndex);
+    if (/^\/data\/(?:train|val|test)\//u.test(path)) {
+        const localBaseDir = dirname(baseManifestPath);
+        const baseModalRoot = `/data/corpus/versioned/${basename(dirname(localBaseDir))}/${basename(localBaseDir)}`;
+        return baseModalRoot + path.slice("/data".length);
+    }
+    return path;
+}
+export async function assembleOverlayManifest(args) {
+    const base = await readLocalJSONFile(args.base);
+    if (base.slices.some((s) => s.source === args.source)) {
+        console.log(`WARN: base already contains source '${args.source}' — is this the right base?`);
+    }
+    const kept = base.slices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }));
+    const newTrain = await descriptor(join(args.newDir, "train", args.sliceParquet), `${args.modalRoot}/train/${args.sliceParquet}`, "train", args.source);
+    const manifest = {
+        corpus_version: args.version,
+        overlay_base: base.corpus_version ?? null,
+        note: args.note || `${base.corpus_version} slices (all kept verbatim) + the ${args.source} slice. Pure overlay add.`,
+        schema: base.schema,
+        rows_per_slice: base.rows_per_slice,
+        row_group_size: base.row_group_size,
+        slices: [...kept, newTrain],
+        counts: {
+            train: base.counts.train + newTrain.rows,
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newTrain.rows,
+    };
+    const out = join(args.newDir, "MANIFEST.json");
+    await writeLocalJSONFile(manifest, out);
+    console.log(`wrote ${out}`);
+    console.log(`  slices: ${manifest.slices.length} (${kept.length} base kept, +1 ${args.source})`);
+    console.log(`  counts: ${JSON.stringify(manifest.counts)}  total: ${manifest.total_rows}`);
+    console.log(`  ${args.source} train: ${newTrain.rows} rows (${newTrain.bytes} bytes)`);
+}
+//# sourceMappingURL=overlay-manifest.js.map

package/out/tools/overlay-manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAyB/D,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5G,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,MAAM,6BAA6B,CAAC,CAAA;IAC7F,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7F,MAAM,QAAQ,GAAG,MAAM,UAAU,CAChC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,EAC7C,GAAG,IAAI,CAAC,SAAS,UAAU,IAAI,CAAC,YAAY,EAAE,EAC9C,OAAO,EACP,IAAI,CAAC,MAAM,CACX,CAAA;IAED,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EACH,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,qCAAqC,IAAI,CAAC,MAAM,2BAA2B;QAC/G,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,QAAQ,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI;YACxC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,IAAI;KAC3C,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,kBAAkB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;IAChG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,WAAW,QAAQ,CAAC,IAAI,UAAU,QAAQ,CAAC,KAAK,SAAS,CAAC,CAAA;AACvF,CAAC"}

package/out/tools/overture-subvenue.d.ts

@@ -0,0 +1,112 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Overture Places source for the sub-venue lexicon (#35 wave 2) — read the sub-venue-bearing
+ *   category slices of `poi.db` (spatial layer #1, `$MAILWOMAN_DATA_ROOT/poi/poi.db`) and yield
+ *   {@link SubVenueHarvestRow}s the lexicon builder consumes exactly like an OSM extract.
+ *
+ *   ── WHY OVERTURE AT ALL, when the OSM extractor already exists ────────────────────────────────────
+ *   Wave 1 measured `concourse` at 21 real Overture rows against 4 in the whole Great Britain OSM
+ *   extract, 3 of which are a street called CONCOURSE WAY. Overture's `airport_terminal` category is
+ *   curated venue-interior naming; OSM's `aeroway=terminal` is a building footprint that usually
+ *   carries the AIRPORT's name. The two sources fail differently, so both are read.
+ *
+ *   ── THE CATEGORY SET IS MEASURED, NOT GUESSED ────────────────────────────────────────────────────
+ *   A full scan of all 13,681,698 rows (2026-08-05, poi.db vintage 2026-05-20.0) counted, per
+ *   category, how many named rows carry a designator token. The ranking is NOT what a category name
+ *   predicts — `gas_station` leads the whole table with 12,996 hits, every one of them the token
+ *   `station` inside "Holiday Station" / "Chevron Station Seward", and `shoe_store` contributes 708
+ *   hits of `wing` because Red Wing sells boots. {@link OVERTURE_SUBVENUE_CATEGORIES} is the four
+ *   categories whose hits survived reading the distribution; the rejects are listed below it so nobody
+ *   re-proposes them.
+ *
+ *   ── poi.db IS FOUR COUNTRIES ─────────────────────────────────────────────────────────────────────
+ *   Measured the same day: US 11,521,612 / CA 794,418 / FR 721,352 / MX 644,316, and nothing else. The
+ *   shipped layer is not a world gazetteer, so Overture can attest en-US, en-CA, fr-FR and es-MX
+ *   surfaces and NOTHING ELSE. Every non-Latin designator the corpus task asks for — `ターミナル`,
+ *   `Halle`, `Flügel` — has to come from the OSM leg. Do not read a zero count here as evidence of
+ *   absence in the world; it is evidence of absence in four countries.
+ *
+ *   ── The row shape fits; the PROVENANCE STAMP did not ─────────────────────────────────────────────
+ *   Wave 1's `OSMSubVenueRow` was written to accept a non-OSM row, and it does: an Overture row is
+ *   `{ designatorID, name }` with no `ref` and no `localizedNames`. What did NOT fit is
+ *   `extractAttestedPhrases`, which hardcoded `osm:name` as the surface's `source`. Feeding Overture
+ *   rows through it unchanged would have labelled every Overture surface as OSM-attested — a
+ *   provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
+ *   function and the source-neutral {@link SubVenueHarvestRow} name.
+ */
+import type { SubVenueHarvestRow } from "#tools/sub-venue-lexicon";
+/**
+ * Overture Places category → the designator its rows attest.
+ *
+ * Each entry carries the 2026-08-05 full-scan measurement in a comment: `rows` is the category's total, `hits` is how
+ * many of its named rows contain any designator token, and the token list is the top of that distribution.
+ *
+ * REJECTED, with the number that rejected them — do not re-add without a fresh census:
+ *
+ * - `gas_station` (12,996 hits) — all `station` inside a brand name (Holiday Station, Chevron Station Seward).
+ * - `fire_station` (10,377) — same, plus 330 `hall` from fire halls, which are venues rather than sub-venues.
+ * - `town_hall` (5,528) — `City Hall` is a whole building, the `venue` tier, not interior structure.
+ * - `building_supply_store` (4,030) — `building` inside "Allied Building Products".
+ * - `jehovahs_witness_place_of_worship` (2,334) — every hit is "Kingdom Hall of Jehovah's Witnesses".
+ * - `shoe_store` (909, of which 708 `wing`) — Red Wing. This one is a CONFOUND BOARD entry, not a source.
+ * - `college_university` (3,697) — 2,082 `campus`, but the row names the whole institution; `campus_building` is the
+ *   interior slice and is kept instead.
+ * - `airport` (6,000 rows, 4,302 hits) — 4,071 of them are the token `airport` in the aerodrome's own name. Venue tier,
+ *   already covered by OurAirports, and it drowns the interior signal.
+ * - `transport_interchange` (1 row), `rail_facility_or_service` (81 rows, 2 hits) — too small to matter.
+ * - `public_transit_facility_or_service` (2,274 rows, 185 hits) — 161 are `station` naming the station itself.
+ */
+export declare const OVERTURE_SUBVENUE_CATEGORIES: Readonly<Record<string, string>>;
+/**
+ * The layer's `source_vintage` — the Overture release the rows came from (`2026-05-20.0`), for the lexicon's
+ * `sources[]`.
+ *
+ * Read off the database rather than passed in, because a vintage a caller types is a vintage that goes stale silently.
+ * The layer-contract tables are part of every layer database by construction — see `docs/engineering/reference/
+ * layer-contract.mdx` — so there is no version of poi.db where this is absent.
+ */
+export declare function readOvertureLayerVintage(databasePath: string): Promise<string>;
+export interface ReadOvertureSubVenuesOptions {
+    /**
+     * Path to `poi.db`. Typically `dataRootPath("poi", "poi.db")`.
+     */
+    databasePath: string;
+    /**
+     * Category → designator map. Defaults to {@link OVERTURE_SUBVENUE_CATEGORIES}.
+     */
+    categories?: Readonly<Record<string, string>>;
+    /**
+     * Restrict to these ISO 3166-1 alpha-2 countries. Omit for all four the layer carries.
+     */
+    countries?: readonly string[];
+}
+/**
+ * One Overture row plus the country it came from — {@link SubVenueHarvestRow} widened by the field the layer has and an
+ * OSM extract does not (a Geofabrik extract's country is a property of the invocation, so OSM rows carry `country:
+ * ""`).
+ */
+export interface OvertureSubVenueRow extends SubVenueHarvestRow {
+    /**
+     * ISO 3166-1 alpha-2, straight off the Overture partition.
+     */
+    country: string;
+    /**
+     * The Overture category the row was read from, so a surface's provenance survives past the designator.
+     */
+    category: string;
+}
+/**
+ * Read the sub-venue-bearing category slices of `poi.db`.
+ *
+ * Cold path, already async, no interface constraint — so Kysely, per the repo's inline-SQL rule.
+ *
+ * `category_id` is the SECOND component of poi.db's clustered `(h3_cell, category_id, …)` primary key, which reads like
+ * a full 13.7M-row scan and is not one: SQLite skip-scans the leading `h3_cell` and the whole four-category read
+ * returned 9,219 rows in **1.4 s** on the shipped layer (2026-08-05). A JS-side scan of every row to survey the same
+ * question took 52 s, which is the number to remember if you are tempted to filter in JS instead.
+ */
+export declare function readOvertureSubVenues(options: ReadOvertureSubVenuesOptions): Promise<OvertureSubVenueRow[]>;
+//# sourceMappingURL=overture-subvenue.d.ts.map

package/out/tools/overture-subvenue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"overture-subvenue.d.ts","sourceRoot":"","sources":["../../lib/tools/overture-subvenue.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAElE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,4BAA4B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAczE,CAAA;AA0BD;;;;;;;GAOG;AACH,wBAAsB,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAUpF;AAED,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC7C;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,mBAAoB,SAAQ,kBAAkB;IAC9D;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED;;;;;;;;;GASG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,4BAA4B,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC,CA0CjH"}