@mailwoman/corpus 10.0.0 → 10.1.0

package/out/tools/lint/{slice/index.js → recipe-output.js}

@@ -3,43 +3,20 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
+ * Corpus linter. Compares a new recipe output (one parquet file) against pre-computed corpus statistics and
+ * flags token-label distribution outliers, label-vacuum tokens, bigram-label collisions, anti-pattern rule
+ * matches and basic sanity problems.
  *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the slice's majority differs AND both have
- *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
- *   flagged slice should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-slice\
- *   --slice <new-slice.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
+ * The markdown report goes to stdout, with optional markdown and JSON sidecars. The command exits 0 when no
+ * errors are found and 1 when any are, because warnings do not refuse. A flagged recipe output should include
+ * `lint_acknowledged: true` in its manifest before training consumes it.
  */
 import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
 import { writeLocalFile, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
 import { resolvePackagePath } from "@mailwoman/core/module/resolvers";
-import { accumulateCooccurrences, createCooccurrenceStats, SLICE_STATS_SEP as SEP, streamTokenLabelRows, } from "#utils/slice-stats";
+import { accumulateCooccurrences, createCooccurrenceStats, COOCCURRENCE_KEY_SEP as SEP, streamTokenLabelRows, } from "#utils/cooccurrence-stats";
 /**
- * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
+ * Occurrences of a forbidden label before it is reported: one or two are noise, five is a pattern.
  */
 const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
 /**
@@ -47,26 +24,28 @@ const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
  */
 const MAX_LISTED_EXAMPLES = 20;
 /**
- * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
- * numbers miss.
+ * Calibrated thresholds, tunable if new failure modes surface that the current numbers miss.
  */
 const CORPUS_CONFIDENCE_FLOOR = 0.66;
-const SLICE_MIN_COUNT = 50;
+const OUTPUT_MIN_COUNT = 50;
 const CORPUS_MIN_COUNT = 200;
-const VACUUM_SLICE_MIN_COUNT = 20;
+const VACUUM_OUTPUT_MIN_COUNT = 20;
 const VACUUM_CORPUS_MIN_COUNT = 100;
 const BIGRAM_MIN_COUNT = 10;
 const ALL_O_RATIO_CEILING = 0.9;
 /**
- * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
- * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
- * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
- * the common path.
+ * Default `lint-rules.json` path — the rules ship beside this module in the source tree.
+ *
+ * Tsc does not emit readFileSync'd JSON into `out/`, so the compiled tree falls back
+ * to the source-tree copy (corpus/out/src/tools/ → corpus/src/tools/).
+ *
+ * In-repo the `node` exports condition loads this module from source anyway,
+ * so the sibling URL is the common path.
  */
 function defaultRulesPath() {
     return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json");
 }
-async function statsFromSlice(rows) {
+async function statsFromRecipeOutput(rows) {
     const co = createCooccurrenceStats();
     const out = {
         rowCount: 0,
@@ -102,94 +81,94 @@ function majorityLabel(distribution) {
     }
     return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total };
 }
-function checkDistributionOutliers(slice, corpus) {
+function checkDistributionOutliers(output, corpus) {
     const flags = [];
-    for (const [token, sliceLabelMap] of slice.tokens) {
+    for (const [token, outputLabelMap] of output.tokens) {
         const corpusLabelMap = corpus.tokens[token];
         if (!corpusLabelMap)
             continue;
-        const sliceMaj = majorityLabel(sliceLabelMap);
+        const outputMaj = majorityLabel(outputLabelMap);
         const corpusMaj = majorityLabel(corpusLabelMap);
         if (corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
-            sliceMaj.label !== corpusMaj.label &&
-            sliceMaj.count >= SLICE_MIN_COUNT &&
+            outputMaj.label !== corpusMaj.label &&
+            outputMaj.count >= OUTPUT_MIN_COUNT &&
             corpusMaj.total >= CORPUS_MIN_COUNT) {
             flags.push({
                 check: "distribution-outlier",
                 severity: "error",
                 token,
-                sliceLabel: sliceMaj.label,
+                outputLabel: outputMaj.label,
                 corpusLabel: corpusMaj.label,
-                sliceCount: sliceMaj.count,
+                outputCount: outputMaj.count,
                 corpusCount: corpusMaj.count,
-                detail: `Token "${token}": slice majority is ${sliceMaj.label} (${sliceMaj.count}/${sliceMaj.total}, ${(sliceMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
+                detail: `Token "${token}": recipe-output majority is ${outputMaj.label} (${outputMaj.count}/${outputMaj.total}, ${(outputMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
             });
         }
     }
     return flags;
 }
-function checkLabelVacuum(slice, corpus) {
+function checkLabelVacuum(output, corpus) {
     const flags = [];
-    for (const [token, sliceLabelMap] of slice.tokens) {
+    for (const [token, outputLabelMap] of output.tokens) {
         const corpusLabelMap = corpus.tokens[token];
         if (!corpusLabelMap)
             continue;
         const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0);
         if (corpusTotal < VACUUM_CORPUS_MIN_COUNT)
             continue;
-        for (const [label, sliceCount] of sliceLabelMap) {
-            if (sliceCount < VACUUM_SLICE_MIN_COUNT)
+        for (const [label, outputCount] of outputLabelMap) {
+            if (outputCount < VACUUM_OUTPUT_MIN_COUNT)
                 continue;
             if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
                 flags.push({
                     check: "label-vacuum",
                     severity: "error",
                     token,
-                    sliceLabel: label,
-                    sliceCount,
+                    outputLabel: label,
+                    outputCount,
                     corpusCount: corpusTotal,
-                    detail: `Token "${token}": slice labels it ${label} ${sliceCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
+                    detail: `Token "${token}": the recipe output labels it ${label} ${outputCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
                 });
             }
         }
     }
     return flags;
 }
-function checkBigramCollisions(slice, corpus) {
+function checkBigramCollisions(output, corpus) {
     const flags = [];
-    for (const [bigram, sliceLabelMap] of slice.bigrams) {
+    for (const [bigram, outputLabelMap] of output.bigrams) {
         const corpusLabelMap = corpus.bigrams[bigram];
         if (!corpusLabelMap)
             continue;
-        const sliceMaj = majorityLabel(sliceLabelMap);
+        const outputMaj = majorityLabel(outputLabelMap);
         const corpusMaj = majorityLabel(corpusLabelMap);
-        if (sliceMaj.label !== corpusMaj.label &&
-            sliceMaj.count >= BIGRAM_MIN_COUNT &&
+        if (outputMaj.label !== corpusMaj.label &&
+            outputMaj.count >= BIGRAM_MIN_COUNT &&
             corpusMaj.count >= BIGRAM_MIN_COUNT) {
             const renderBigram = bigram.split(SEP).join(" ");
-            const renderSliceLabel = sliceMaj.label.split(SEP).join(" → ");
+            const renderOutputLabel = outputMaj.label.split(SEP).join(" → ");
             const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ");
             flags.push({
                 check: "bigram-collision",
                 severity: "error",
                 bigram: renderBigram,
-                sliceLabel: renderSliceLabel,
+                outputLabel: renderOutputLabel,
                 corpusLabel: renderCorpusLabel,
-                sliceCount: sliceMaj.count,
+                outputCount: outputMaj.count,
                 corpusCount: corpusMaj.count,
-                detail: `Bigram "${renderBigram}": slice label-bigram is [${renderSliceLabel}] (${sliceMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
+                detail: `Bigram "${renderBigram}": recipe-output label-bigram is [${renderOutputLabel}] (${outputMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
             });
         }
     }
     return flags;
 }
-function checkRules(slice, rulesFile) {
+function checkRules(output, rulesFile) {
     const flags = [];
     const compiled = rulesFile.rules.map((r) => ({
         rule: r,
         regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
     }));
-    for (const [token, labelMap] of slice.tokens) {
+    for (const [token, labelMap] of output.tokens) {
         for (const { rule, regex } of compiled) {
             if (!regex.test(token))
                 continue;
@@ -200,8 +179,8 @@ function checkRules(slice, rulesFile) {
                         severity: rule.severity,
                         ruleID: rule.id,
                         token,
-                        sliceLabel: label,
-                        sliceCount: count,
+                        outputLabel: label,
+                        outputCount: count,
                         detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
                     });
                 }
@@ -210,40 +189,40 @@ function checkRules(slice, rulesFile) {
     }
     return flags;
 }
-function checkSanity(slice) {
+function checkSanity(output) {
     const flags = [];
-    if (slice.truncatedRows > 0) {
+    if (output.truncatedRows > 0) {
         flags.push({
             check: "truncated-rows",
             severity: "error",
-            detail: `${slice.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
+            detail: `${output.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
         });
     }
-    const allORatio = slice.allORows / Math.max(1, slice.rowCount);
+    const allORatio = output.allORows / Math.max(1, output.rowCount);
     if (allORatio >= ALL_O_RATIO_CEILING) {
         flags.push({
-            check: "all-O-slice",
+            check: "all-O-output",
             severity: "warn",
-            detail: `${slice.allORows}/${slice.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Slice contributes no signal.`,
+            detail: `${output.allORows}/${output.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. The recipe output contributes no signal.`,
         });
     }
     return flags;
 }
-function renderReport(opts, slice, flags) {
+function renderReport(opts, output, flags) {
     const errors = flags.filter((f) => f.severity === "error");
     const warns = flags.filter((f) => f.severity === "warn");
     const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠";
     const lines = [
         `# Corpus Lint: ${verdict}`,
         "",
-        `- **Slice:** \`${opts.slicePath}\``,
+        `- **Recipe output:** \`${opts.outputPath}\``,
         `- **Corpus stats:** \`${opts.statsPath}\``,
         `- **Rules:** \`${opts.rulesPath}\``,
-        `- **Slice rows:** ${slice.rowCount}`,
-        `- **Unique tokens:** ${slice.tokens.size}`,
-        `- **Unique bigrams:** ${slice.bigrams.size}`,
+        `- **Recipe output rows:** ${output.rowCount}`,
+        `- **Unique tokens:** ${output.tokens.size}`,
+        `- **Unique bigrams:** ${output.bigrams.size}`,
         "",
-        `**Errors:** ${errors.length} (checks the slice's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
+        `**Errors:** ${errors.length} (refuses the recipe output's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
         `**Warnings:** ${warns.length} (advisory)`,
         "",
     ];
@@ -260,8 +239,8 @@ function renderReport(opts, slice, flags) {
     for (const [check, list] of byCheck) {
         lines.push(`## ${check} (${list.length})`);
         lines.push("");
-        // Sort by sliceCount desc — highest-volume issues first
-        list.sort((a, b) => (b.sliceCount ?? 0) - (a.sliceCount ?? 0));
+        // Highest-volume issues first.
+        list.sort((a, b) => (b.outputCount ?? 0) - (a.outputCount ?? 0));
         for (const f of list.slice(0, 20)) {
             lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`);
         }
@@ -273,34 +252,35 @@ function renderReport(opts, slice, flags) {
     return lines.join("\n");
 }
 /**
- * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
+ * Lint a recipe output against corpus stats and the anti-pattern rules,
+ * printing the markdown report to stdout.
  */
-export async function lintCorpusSlice(options, report) {
+export async function lintRecipeOutput(options, report) {
     const rulesPath = options.rulesPath ?? defaultRulesPath();
     report?.(`Reading corpus stats from ${options.statsPath}...`);
     const corpus = await readLocalJSONFile(options.statsPath);
-    report?.(`  ${corpus.row_count} rows from ${corpus.slice_paths.length} slice(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
-    report?.(`Reading slice from ${options.slicePath}...`);
-    const slice = await statsFromSlice(streamTokenLabelRows(options.slicePath));
-    report?.(`  ${slice.rowCount} rows`);
+    report?.(`  ${corpus.row_count} rows from ${corpus.slice_paths.length} parquet file(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
+    report?.(`Reading recipe output from ${options.recipeOutputPath}...`);
+    const output = await statsFromRecipeOutput(streamTokenLabelRows(options.recipeOutputPath));
+    report?.(`  ${output.rowCount} rows`);
     report?.(`Loading rules from ${rulesPath}...`);
     const rulesFile = await readLocalJSONFile(rulesPath);
     report?.(`Running checks...`);
     const flags = [
-        ...checkDistributionOutliers(slice, corpus),
-        ...checkLabelVacuum(slice, corpus),
-        ...checkBigramCollisions(slice, corpus),
-        ...checkRules(slice, rulesFile),
-        ...checkSanity(slice),
+        ...checkDistributionOutliers(output, corpus),
+        ...checkLabelVacuum(output, corpus),
+        ...checkBigramCollisions(output, corpus),
+        ...checkRules(output, rulesFile),
+        ...checkSanity(output),
     ];
-    const rendered = renderReport({ slicePath: options.slicePath, statsPath: options.statsPath, rulesPath }, slice, flags);
+    const rendered = renderReport({ outputPath: options.recipeOutputPath, statsPath: options.statsPath, rulesPath }, output, flags);
     console.log(rendered);
     if (options.outMd) {
         await writeLocalFile(rendered, options.outMd);
     }
     if (options.outJSON) {
         await writeLocalJSONFile({
-            slice: options.slicePath,
+            recipe_output: options.recipeOutputPath,
             stats: options.statsPath,
             flags,
             summary: {
@@ -319,4 +299,4 @@ export async function lintCorpusSlice(options, report) {
     }
     return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered };
 }
-//# sourceMappingURL=index.js.map
+//# sourceMappingURL=recipe-output.js.map

package/out/tools/lint/recipe-output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-output.js","sourceRoot":"","sources":["../../../lib/tools/lint/recipe-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAErE,OAAO,EACN,uBAAuB,EACvB,uBAAuB,EACvB,oBAAoB,IAAI,GAAG,EAC3B,oBAAoB,GAEpB,MAAM,2BAA2B,CAAA;AAElC;;GAEG;AACH,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B;;GAEG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AACpC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAC5B,MAAM,uBAAuB,GAAG,EAAE,CAAA;AAClC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,mBAAmB,GAAG,GAAG,CAAA;AAE/B;;;;;;;;GAQG;AACH,SAAS,gBAAgB;IACxB,OAAO,kBAAkB,CAAC,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAA;AAClF,CAAC;AA0DD,KAAK,UAAU,qBAAqB,CAAC,IAAoC;IACxE,MAAM,EAAE,GAAG,uBAAuB,EAAE,CAAA;IAEpC,MAAM,GAAG,GAAsB;QAC9B,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,CAAC,OAAO;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;KACX,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAA;QAEd,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,GAAG,CAAC,aAAa,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,EAAE,CAAA;QACf,CAAC;QAED,uBAAuB,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,aAAa,CAAC,YAA0D;IAMhF,MAAM,OAAO,GAAG,YAAY,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACxG,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,EAAE,CAAA;AACtG,CAAC;AA+BD,SAAS,yBAAyB,CAAC,MAAyB,EAAE,MAAmB;IAChF,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAC/C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,UAAU,IAAI,uBAAuB;YAC/C,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YACnC,SAAS,CAAC,KAAK,IAAI,gBAAgB;YACnC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,OAAO;gBACjB,KAAK;gBACL,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,UAAU,KAAK,gCAAgC,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;aAC1R,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAyB,EAAE,MAAmB;IACvE,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAE5E,IAAI,WAAW,GAAG,uBAAuB;YAAE,SAAQ;QAEnD,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,cAAc,EAAE,CAAC;YACnD,IAAI,WAAW,GAAG,uBAAuB;gBAAE,SAAQ;YAEnD,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxE,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,OAAO;oBACjB,KAAK;oBACL,WAAW,EAAE,KAAK;oBAClB,WAAW;oBACX,WAAW,EAAE,WAAW;oBACxB,MAAM,EAAE,UAAU,KAAK,kCAAkC,KAAK,IAAI,WAAW,2BAA2B,WAAW,6DAA6D;iBAChL,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,qBAAqB,CAAC,MAAyB,EAAE,MAAmB;IAC5E,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACvD,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAC/C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YACnC,SAAS,CAAC,KAAK,IAAI,gBAAgB;YACnC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAChE,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEhE,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,kBAAkB;gBACzB,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,YAAY;gBACpB,WAAW,EAAE,iBAAiB;gBAC9B,WAAW,EAAE,iBAAiB;gBAC9B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,WAAW,YAAY,qCAAqC,iBAAiB,MAAM,SAAS,CAAC,KAAK,+BAA+B,iBAAiB,MAAM,SAAS,CAAC,KAAK,sDAAsD;aACrO,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,MAAyB,EAAE,SAAwB;IACtE,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KACjE,CAAC,CAAC,CAAA;IAEH,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gCAAgC,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,mBAAmB;wBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,KAAK;wBACL,WAAW,EAAE,KAAK;wBAClB,WAAW,EAAE,KAAK;wBAClB,MAAM,EAAE,UAAU,KAAK,kBAAkB,IAAI,CAAC,EAAE,mBAAmB,KAAK,IAAI,KAAK,2BAA2B,IAAI,CAAC,OAAO,EAAE;qBAC1H,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,MAAyB;IAC7C,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,IAAI,MAAM,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,gBAAgB;YACvB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,GAAG,MAAM,CAAC,aAAa,uEAAuE;SACtG,CAAC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;IAEhE,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,cAAc;YACrB,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qEAAqE;SACxJ,CAAC,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CACpB,IAAkE,EAClE,MAAyB,EACzB,KAAiB;IAEjB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAA;IAE/D,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,EAAE;QAC3B,EAAE;QACF,0BAA0B,IAAI,CAAC,UAAU,IAAI;QAC7C,yBAAyB,IAAI,CAAC,SAAS,IAAI;QAC3C,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,6BAA6B,MAAM,CAAC,QAAQ,EAAE;QAC9C,wBAAwB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QAC5C,yBAAyB,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE;QAC9C,EAAE;QACF,eAAe,MAAM,CAAC,MAAM,2FAA2F;QACvH,iBAAiB,KAAK,CAAC,MAAM,aAAa;QAC1C,EAAE;KACF,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAEpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsB,CAAA;IAE7C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,+BAA+B;QAC/B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAA;QAEhE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACf,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAA;IACzD,MAAM,EAAE,CAAC,6BAA6B,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAc,OAAO,CAAC,SAAS,CAAC,CAAA;IAEtE,MAAM,EAAE,CACP,KAAK,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,WAAW,CAAC,MAAM,qBAAqB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU,CAC1K,CAAA;IAED,MAAM,EAAE,CAAC,8BAA8B,OAAO,CAAC,gBAAgB,KAAK,CAAC,CAAA;IAErE,MAAM,MAAM,GAAG,MAAM,qBAAqB,CAAC,oBAAoB,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAE1F,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC,QAAQ,OAAO,CAAC,CAAA;IAErC,MAAM,EAAE,CAAC,sBAAsB,SAAS,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAgB,SAAS,CAAC,CAAA;IAEnE,MAAM,EAAE,CAAC,mBAAmB,CAAC,CAAA;IAE7B,MAAM,KAAK,GAAe;QACzB,GAAG,yBAAyB,CAAC,MAAM,EAAE,MAAM,CAAC;QAC5C,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC;QACnC,GAAG,qBAAqB,CAAC,MAAM,EAAE,MAAM,CAAC;QACxC,GAAG,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC;QAChC,GAAG,WAAW,CAAC,MAAM,CAAC;KACtB,CAAA;IAED,MAAM,QAAQ,GAAG,YAAY,CAC5B,EAAE,UAAU,EAAE,OAAO,CAAC,gBAAgB,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,EACjF,MAAM,EACN,KAAK,CACL,CAAA;IAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAErB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,kBAAkB,CACvB;YACC,aAAa,EAAE,OAAO,CAAC,gBAAgB;YACvC,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK;YACL,OAAO,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;aAC3D;SACD,EACD,OAAO,CAAC,OAAO,CACf,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IAEtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,gBAAgB,UAAU,YAAY,CAAC,CAAA;IACjD,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACzF,CAAC"}

package/out/tools/migrate/overlay-parquet.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert an overlay parquet to the current row schema, without its recipe output.
+ *   `mailwoman-derived-tuples` wherever the upstream was not recorded.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The register each overlay source's rows came from, or `mailwoman-derived-tuples`
+ * when the upstream was not recorded.
+ */
+export declare const OVERLAY_REGISTERS: Record<string, string | null>;
+/**
+ * What a parquet migration observed.
+ */
+export interface OverlayMigrationSummary {
+    rows: number;
+    bySource: Record<string, number>;
+}
+/**
+ * The register for one overlay source, refusing a source this map does not name.
+ */
+export declare function registerForSource(source: string): string | null;
+/**
+ * Rewrite one overlay parquet onto the current row schema, staged through
+ * {@linkcode jsonlToParquet} so it has the same column list as every other overlay.
+ */
+export declare function migrateOverlayParquet(input: PathBuilderLike, output: PathBuilderLike): Promise<OverlayMigrationSummary>;
+//# sourceMappingURL=overlay-parquet.d.ts.map

package/out/tools/migrate/overlay-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"overlay-parquet.d.ts","sourceRoot":"","sources":["../../../lib/tools/migrate/overlay-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAQ9C;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAgD3D,CAAA;AA+BD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAChC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAgB/D;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CAC1C,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,eAAe,GACrB,OAAO,CAAC,uBAAuB,CAAC,CAiClC"}

package/out/tools/migrate/overlay-parquet.js

@@ -0,0 +1,141 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert an overlay parquet to the current row schema, without its recipe output.
+ *   `mailwoman-derived-tuples` wherever the upstream was not recorded.
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { once } from "@mailwoman/core/utils/events";
+import { openParquetRowStream } from "#parquet/streams";
+import { retiredSourceName } from "#recipes/sources";
+import { SourceRegister } from "#registers";
+import { jsonlToParquet } from "#tools/jsonl-to-parquet";
+import { surfaceForSource } from "#tools/migrate/recipe-output";
+/**
+ * The register each overlay source's rows came from, or `mailwoman-derived-tuples`
+ * when the upstream was not recorded.
+ */
+export const OVERLAY_REGISTERS = {
+    // The recipe names its publication.
+    "synth-gb": SourceRegister.LandRegistryPricePaid,
+    "synth-german": SourceRegister.OpenAddresses,
+    "synth-nz": SourceRegister.OpenAddresses,
+    "synth-es": SourceRegister.OpenAddresses,
+    "synth-unit": SourceRegister.OpenAddresses,
+    "synth-unit-v30": SourceRegister.OpenAddresses,
+    "synth-country": SourceRegister.OpenAddresses,
+    "synth-fr-order": SourceRegister.BaseAdresseNationale,
+    "synth-fr-admin-split": SourceRegister.BaseAdresseNationale,
+    "synth-fr-bare-street": SourceRegister.BaseAdresseNationale,
+    "synth-fr-fragment": SourceRegister.BaseAdresseNationale,
+    "synth-fr-lieudit": SourceRegister.BaseAdresseNationale,
+    overture: SourceRegister.Overture,
+    "overture-latam": SourceRegister.Overture,
+    "synth-sg-register": SourceRegister.Overture,
+    gnaf: SourceRegister.GNAF,
+    osm: SourceRegister.OpenStreetMap,
+    "synth-pk-register": SourceRegister.OpenStreetMap,
+    "synth-bd-register": SourceRegister.OpenStreetMap,
+    "synth-bare-country": SourceRegister.Codex,
+    "synth-bare-postcode": SourceRegister.Codex,
+    "synth-reviewed-postcode-tail": SourceRegister.ReviewedByHand,
+    // The row names no published record at all.
+    "synth-po-box-cedex": null,
+    "synth-po-box-military": null,
+    "synth-intersection": null,
+    "synth-house-venue": null,
+    "deepseek-kryptonite": null,
+    "synth-anchor-absorption": null,
+    // The tuple extraction behind these was not recorded.
+    "synth-affix": SourceRegister.DerivedTuples,
+    "synth-suffix-boundary": SourceRegister.DerivedTuples,
+    "synth-cz-pcfirst-preposition": SourceRegister.DerivedTuples,
+    "synth-cz-pcfirst-preposition-v21": SourceRegister.DerivedTuples,
+    "synth-fr-bare-street-v22": SourceRegister.DerivedTuples,
+    "synth-si-bare-village": SourceRegister.DerivedTuples,
+    "synth-no-street-led": SourceRegister.DerivedTuples,
+    "synth-no-fragment": SourceRegister.DerivedTuples,
+    "synth-fragment": SourceRegister.DerivedTuples,
+    "synth-nl-postcode": SourceRegister.DerivedTuples,
+    "synth-sub-venue": SourceRegister.DerivedTuples,
+    "synth-trailing-region": SourceRegister.DerivedTuples,
+    "synth-trailing-region-us": SourceRegister.DerivedTuples,
+    "synth-trailing-region-ca-bare": SourceRegister.DerivedTuples,
+    "synth-trailing-region-structured": SourceRegister.DerivedTuples,
+    "synth-trailing-region-es-v28": SourceRegister.DerivedTuples,
+};
+/**
+ * Read an INT64 span offset back as a plain number.
+ *
+ * DuckDB returns INT64 values as `BigInt`.
+ * `stringifyJSON` refuses `BigInt` values.
+ *
+ * The current schema declares the span triple INT32 because `raw` is a short
+ * address string, so every in-range value is lossless.
+ *
+ * A value outside the safe-integer range is a corrupt offset, so this raises instead of truncating.
+ */
+function narrowBigInts(row) {
+    const narrow = (value) => {
+        if (typeof value === "bigint") {
+            if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < BigInt(Number.MIN_SAFE_INTEGER)) {
+                throw new Error(`span offset ${value} is outside the safe-integer range, so it is a corrupt offset`);
+            }
+            return Number(value);
+        }
+        if (Array.isArray(value))
+            return value.map(narrow);
+        return value;
+    };
+    return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, narrow(value)]));
+}
+/**
+ * The register for one overlay source, refusing a source this map does not name.
+ */
+export function registerForSource(source) {
+    // The map keys on the retired spelling.
+    // A corpus that uses the current spelling needs the retired key for this lookup.
+    // Otherwise each row fails as though it had no register.
+    // That failure misstates a renamed source as an unrecorded one.
+    const key = retiredSourceName(source);
+    if (!(key in OVERLAY_REGISTERS)) {
+        throw new Error(`No register recorded for overlay source ${stringifyJSON(source)}. Read the recipe that writes it and ` +
+            `add it to OVERLAY_REGISTERS: the publication it renders, or null when the row names no record, or ` +
+            `mailwoman-derived-tuples when its tuple extraction was not recorded.`);
+    }
+    return OVERLAY_REGISTERS[key] ?? null;
+}
+/**
+ * Rewrite one overlay parquet onto the current row schema, staged through
+ * {@linkcode jsonlToParquet} so it has the same column list as every other overlay.
+ */
+export async function migrateOverlayParquet(input, output) {
+    const summary = { rows: 0, bySource: {} };
+    await using staging = await temporaryDirectory("mw-migrate-overlay-");
+    const stagePath = staging.path("rows.jsonl");
+    const sink = staging.use(openWriteStream(stagePath, { encoding: "utf8" }));
+    for await (const row of openParquetRowStream(input)) {
+        const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = narrowBigInts(row);
+        const source = String(row["source"] ?? "");
+        summary.rows++;
+        summary.bySource[source] = (summary.bySource[source] ?? 0) + 1;
+        const migrated = {
+            ...rest,
+            recipe: recipe ?? null,
+            base_source_id: baseSourceID ?? null,
+            register: registerForSource(source),
+            surface: surfaceForSource(source),
+        };
+        if (!sink.write(`${stringifyJSON(migrated)}\n`)) {
+            await once(sink, "drain");
+        }
+    }
+    sink.end();
+    await once(sink, "finish");
+    await jsonlToParquet({ input: stagePath, output });
+    return summary;
+}
+//# sourceMappingURL=overlay-parquet.js.map

package/out/tools/migrate/overlay-parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"overlay-parquet.js","sourceRoot":"","sources":["../../../lib/tools/migrate/overlay-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAA;AAGnD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAA;AAE/D;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAkC;IAC/D,oCAAoC;IACpC,UAAU,EAAE,cAAc,CAAC,qBAAqB;IAChD,cAAc,EAAE,cAAc,CAAC,aAAa;IAC5C,UAAU,EAAE,cAAc,CAAC,aAAa;IACxC,UAAU,EAAE,cAAc,CAAC,aAAa;IACxC,YAAY,EAAE,cAAc,CAAC,aAAa;IAC1C,gBAAgB,EAAE,cAAc,CAAC,aAAa;IAC9C,eAAe,EAAE,cAAc,CAAC,aAAa;IAC7C,gBAAgB,EAAE,cAAc,CAAC,oBAAoB;IACrD,sBAAsB,EAAE,cAAc,CAAC,oBAAoB;IAC3D,sBAAsB,EAAE,cAAc,CAAC,oBAAoB;IAC3D,mBAAmB,EAAE,cAAc,CAAC,oBAAoB;IACxD,kBAAkB,EAAE,cAAc,CAAC,oBAAoB;IACvD,QAAQ,EAAE,cAAc,CAAC,QAAQ;IACjC,gBAAgB,EAAE,cAAc,CAAC,QAAQ;IACzC,mBAAmB,EAAE,cAAc,CAAC,QAAQ;IAC5C,IAAI,EAAE,cAAc,CAAC,IAAI;IACzB,GAAG,EAAE,cAAc,CAAC,aAAa;IACjC,mBAAmB,EAAE,cAAc,CAAC,aAAa;IACjD,mBAAmB,EAAE,cAAc,CAAC,aAAa;IACjD,oBAAoB,EAAE,cAAc,CAAC,KAAK;IAC1C,qBAAqB,EAAE,cAAc,CAAC,KAAK;IAC3C,8BAA8B,EAAE,cAAc,CAAC,cAAc;IAC7D,4CAA4C;IAC5C,oBAAoB,EAAE,IAAI;IAC1B,uBAAuB,EAAE,IAAI;IAC7B,oBAAoB,EAAE,IAAI;IAC1B,mBAAmB,EAAE,IAAI;IACzB,qBAAqB,EAAE,IAAI;IAC3B,yBAAyB,EAAE,IAAI;IAC/B,sDAAsD;IACtD,aAAa,EAAE,cAAc,CAAC,aAAa;IAC3C,uBAAuB,EAAE,cAAc,CAAC,aAAa;IACrD,8BAA8B,EAAE,cAAc,CAAC,aAAa;IAC5D,kCAAkC,EAAE,cAAc,CAAC,aAAa;IAChE,0BAA0B,EAAE,cAAc,CAAC,aAAa;IACxD,uBAAuB,EAAE,cAAc,CAAC,aAAa;IACrD,qBAAqB,EAAE,cAAc,CAAC,aAAa;IACnD,mBAAmB,EAAE,cAAc,CAAC,aAAa;IACjD,gBAAgB,EAAE,cAAc,CAAC,aAAa;IAC9C,mBAAmB,EAAE,cAAc,CAAC,aAAa;IACjD,iBAAiB,EAAE,cAAc,CAAC,aAAa;IAC/C,uBAAuB,EAAE,cAAc,CAAC,aAAa;IACrD,0BAA0B,EAAE,cAAc,CAAC,aAAa;IACxD,+BAA+B,EAAE,cAAc,CAAC,aAAa;IAC7D,kCAAkC,EAAE,cAAc,CAAC,aAAa;IAChE,8BAA8B,EAAE,cAAc,CAAC,aAAa;CAC5D,CAAA;AAED;;;;;;;;;;GAUG;AACH,SAAS,aAAa,CAAC,GAA4B;IAClD,MAAM,MAAM,GAAG,CAAC,KAAc,EAAW,EAAE;QAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/B,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACxF,MAAM,IAAI,KAAK,CAAC,eAAe,KAAK,+DAA+D,CAAC,CAAA;YACrG,CAAC;YAED,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;QACrB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAElD,OAAO,KAAK,CAAA;IACb,CAAC,CAAA;IAED,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3F,CAAC;AAUD;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,wCAAwC;IACxC,iFAAiF;IACjF,yDAAyD;IACzD,gEAAgE;IAChE,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAErC,IAAI,CAAC,CAAC,GAAG,IAAI,iBAAiB,CAAC,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CACd,2CAA2C,aAAa,CAAC,MAAM,CAAC,uCAAuC;YACtG,oGAAoG;YACpG,sEAAsE,CACvE,CAAA;IACF,CAAC;IAED,OAAO,iBAAiB,CAAC,GAAG,CAAC,IAAI,IAAI,CAAA;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,KAAsB,EACtB,MAAuB;IAEvB,MAAM,OAAO,GAA4B,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;IAElE,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,qBAAqB,CAAC,CAAA;IACrE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;IAC5C,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE1E,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAA0B,KAAK,CAAC,EAAE,CAAC;QAC9E,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QACzF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAE1C,OAAO,CAAC,IAAI,EAAE,CAAA;QACd,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAE9D,MAAM,QAAQ,GAAG;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,MAAM,IAAI,IAAI;YACtB,cAAc,EAAE,YAAY,IAAI,IAAI;YACpC,QAAQ,EAAE,iBAAiB,CAAC,MAAM,CAAC;YACnC,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC;SACjC,CAAA;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,IAAI,CAAC,GAAG,EAAE,CAAA;IACV,MAAM,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAE1B,MAAM,cAAc,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;IAElD,OAAO,OAAO,CAAA;AACf,CAAC"}

package/out/tools/migrate/recipe-output.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert a recipe output written before `recipe`/`register`/`surface` to the current row schema.
+ *
+ * `recipe` and `base_source_id` are renames of `synth_method` and `synth_base_id`, so they assert no new claim.
+ * `register` is `mailwoman-derived-tuples` for every row, because naming a publisher would record inference as fact.
+ */
+import type { PathBuilderLike } from "path-ts";
+import { SurfaceOrigin } from "#types";
+/**
+ * How each source id's recipe produced its surface.
+ *
+ * A source absent from this table refuses rather than taking a default.
+ */
+export declare const RECIPE_SURFACES: Record<string, SurfaceOrigin>;
+/**
+ * What a migration pass observed.
+ */
+export interface MigrationSummary {
+    rows: number;
+    bySource: Record<string, number>;
+    alreadyMigrated: number;
+}
+/**
+ * The surface for one source id, refusing a source the table does not name.
+ */
+export declare function surfaceForSource(source: string): SurfaceOrigin;
+/**
+ * Rewrite one recipe output onto the current row schema, passing an already-migrated
+ * row through untouched so a half-migrated file converges.
+ */
+export declare function migrateRecipeOutput(input: PathBuilderLike, output: PathBuilderLike): Promise<MigrationSummary>;
+//# sourceMappingURL=recipe-output.d.ts.map

package/out/tools/migrate/recipe-output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-output.d.ts","sourceRoot":"","sources":["../../../lib/tools/migrate/recipe-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAK9C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEtC;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAmDzD,CAAA;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,eAAe,EAAE,MAAM,CAAA;CACvB;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,aAAa,CAe9D;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAyCpH"}