@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/tools/lint/{slice/index.ts → recipe-output.ts}

@@ -3,36 +3,13 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
+ * Corpus linter. Compares a new recipe output (one parquet file) against pre-computed corpus statistics and
+ * flags token-label distribution outliers, label-vacuum tokens, bigram-label collisions, anti-pattern rule
+ * matches and basic sanity problems.
  *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the slice's majority differs AND both have
- *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
- *   flagged slice should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-slice\
- *   --slice <new-slice.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
+ * The markdown report goes to stdout, with optional markdown and JSON sidecars. The command exits 0 when no
+ * errors are found and 1 when any are, because warnings do not refuse. A flagged recipe output should include
+ * `lint_acknowledged: true` in its manifest before training consumes it.
  */
 
 import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
@@ -42,13 +19,13 @@ import { resolvePackagePath } from "@mailwoman/core/module/resolvers"
 import {
 	accumulateCooccurrences,
 	createCooccurrenceStats,
-	SLICE_STATS_SEP as SEP,
+	COOCCURRENCE_KEY_SEP as SEP,
 	streamTokenLabelRows,
-	type TokenLabelRow as SliceRow,
-} from "#utils/slice-stats"
+	type TokenLabelRow as RecipeOutputRow,
+} from "#utils/cooccurrence-stats"
 
 /**
- * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
+ * Occurrences of a forbidden label before it is reported: one or two are noise, five is a pattern.
  */
 const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5
 
@@ -58,41 +35,45 @@ const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5
 const MAX_LISTED_EXAMPLES = 20
 
 /**
- * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
- * numbers miss.
+ * Calibrated thresholds, tunable if new failure modes surface that the current numbers miss.
  */
 const CORPUS_CONFIDENCE_FLOOR = 0.66
-const SLICE_MIN_COUNT = 50
+const OUTPUT_MIN_COUNT = 50
 const CORPUS_MIN_COUNT = 200
-const VACUUM_SLICE_MIN_COUNT = 20
+const VACUUM_OUTPUT_MIN_COUNT = 20
 const VACUUM_CORPUS_MIN_COUNT = 100
 const BIGRAM_MIN_COUNT = 10
 const ALL_O_RATIO_CEILING = 0.9
 
 /**
- * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
- * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
- * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
- * the common path.
+ * Default `lint-rules.json` path — the rules ship beside this module in the source tree.
+ *
+ * Tsc does not emit readFileSync'd JSON into `out/`, so the compiled tree falls back
+ * to the source-tree copy (corpus/out/src/tools/ → corpus/src/tools/).
+ *
+ * In-repo the `node` exports condition loads this module from source anyway,
+ * so the sibling URL is the common path.
  */
 function defaultRulesPath(): string {
 	return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json")
 }
 
 /**
- * Options for {@linkcode lintCorpusSlice}.
+ * Options for {@linkcode lintRecipeOutput}.
  */
-export interface LintCorpusSliceOptions {
+export interface LintRecipeOutputOptions {
 	/**
-	 * The new slice parquet to lint.
+	 * The new recipe output parquet to lint.
 	 */
-	slicePath: string
+	recipeOutputPath: string
 	/**
 	 * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
 	 */
 	statsPath: string
 	/**
-	 * Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module.
+	 * Anti-pattern rules JSON.
+	 *
+	 * @defaultValue the `lint-rules.json` beside this module.
 	 */
 	rulesPath?: string
 	/**
@@ -125,7 +106,7 @@ interface LintRulesFile {
 	rules: LintRule[]
 }
 
-interface SliceStats {
+interface RecipeOutputStats {
 	rowCount: number
 	tokens: Map<string, Map<string, number>>
 	bigrams: Map<string, Map<string, number>>
@@ -133,10 +114,10 @@ interface SliceStats {
 	allORows: number
 }
 
-async function statsFromSlice(rows: AsyncIterable<SliceRow>): Promise<SliceStats> {
+async function statsFromRecipeOutput(rows: AsyncIterable<RecipeOutputRow>): Promise<RecipeOutputStats> {
 	const co = createCooccurrenceStats()
 
-	const out: SliceStats = {
+	const out: RecipeOutputStats = {
 		rowCount: 0,
 		tokens: co.tokens,
 		bigrams: co.bigrams,
@@ -189,57 +170,68 @@ function majorityLabel(distribution: Map<string, number> | Record<string, number
 /**
  * One lint flag emitted by a check.
  */
-export interface LintSliceFlag {
+export interface LintFlag {
 	check: string
 	severity: "error" | "warn"
-	token?: string
-	bigram?: string
-	sliceLabel?: string
-	corpusLabel?: string
-	sliceCount?: number
-	corpusCount?: number
+	token: string | null
+	bigram: string | null
+	outputLabel: string | null
+	corpusLabel: string | null
+	outputCount: number | null
+	corpusCount: number | null
 	detail: string
-	ruleID?: string
+	ruleID: string | null
 }
 
+const EMPTY_FLAG_FIELDS = {
+	token: null,
+	bigram: null,
+	outputLabel: null,
+	corpusLabel: null,
+	outputCount: null,
+	corpusCount: null,
+	ruleID: null,
+} as const satisfies Partial<LintFlag>
+
 /**
- * Findings summary returned by {@linkcode lintCorpusSlice}.
+ * Findings summary returned by {@linkcode lintRecipeOutput}.
  */
-export interface LintCorpusSliceSummary {
+export interface LintRecipeOutputSummary {
 	errors: number
 	warnings: number
-	findings: LintSliceFlag[]
+	findings: LintFlag[]
 	/**
 	 * The rendered markdown report (also printed to stdout).
 	 */
 	report: string
 }
 
-function checkDistributionOutliers(slice: SliceStats, corpus: CorpusStats): LintSliceFlag[] {
-	const flags: LintSliceFlag[] = []
+function checkDistributionOutliers(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
 
-	for (const [token, sliceLabelMap] of slice.tokens) {
+	for (const [token, outputLabelMap] of output.tokens) {
 		const corpusLabelMap = corpus.tokens[token]
 
 		if (!corpusLabelMap) continue
-		const sliceMaj = majorityLabel(sliceLabelMap)
+		const outputMaj = majorityLabel(outputLabelMap)
 		const corpusMaj = majorityLabel(corpusLabelMap)
 
 		if (
 			corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
-			sliceMaj.label !== corpusMaj.label &&
-			sliceMaj.count >= SLICE_MIN_COUNT &&
+			outputMaj.label !== corpusMaj.label &&
+			outputMaj.count >= OUTPUT_MIN_COUNT &&
 			corpusMaj.total >= CORPUS_MIN_COUNT
 		) {
 			flags.push({
+				...EMPTY_FLAG_FIELDS,
 				check: "distribution-outlier",
 				severity: "error",
 				token,
-				sliceLabel: sliceMaj.label,
+				outputLabel: outputMaj.label,
 				corpusLabel: corpusMaj.label,
-				sliceCount: sliceMaj.count,
+				outputCount: outputMaj.count,
 				corpusCount: corpusMaj.count,
-				detail: `Token "${token}": slice majority is ${sliceMaj.label} (${sliceMaj.count}/${sliceMaj.total}, ${(sliceMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
+				detail: `Token "${token}": recipe-output majority is ${outputMaj.label} (${outputMaj.count}/${outputMaj.total}, ${(outputMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
 			})
 		}
 	}
@@ -247,10 +239,10 @@ function checkDistributionOutliers(slice: SliceStats, corpus: CorpusStats): Lint
 	return flags
 }
 
-function checkLabelVacuum(slice: SliceStats, corpus: CorpusStats): LintSliceFlag[] {
-	const flags: LintSliceFlag[] = []
+function checkLabelVacuum(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
 
-	for (const [token, sliceLabelMap] of slice.tokens) {
+	for (const [token, outputLabelMap] of output.tokens) {
 		const corpusLabelMap = corpus.tokens[token]
 
 		if (!corpusLabelMap) continue
@@ -258,18 +250,19 @@ function checkLabelVacuum(slice: SliceStats, corpus: CorpusStats): LintSliceFlag
 
 		if (corpusTotal < VACUUM_CORPUS_MIN_COUNT) continue
 
-		for (const [label, sliceCount] of sliceLabelMap) {
-			if (sliceCount < VACUUM_SLICE_MIN_COUNT) continue
+		for (const [label, outputCount] of outputLabelMap) {
+			if (outputCount < VACUUM_OUTPUT_MIN_COUNT) continue
 
 			if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
 				flags.push({
+					...EMPTY_FLAG_FIELDS,
 					check: "label-vacuum",
 					severity: "error",
 					token,
-					sliceLabel: label,
-					sliceCount,
+					outputLabel: label,
+					outputCount,
 					corpusCount: corpusTotal,
-					detail: `Token "${token}": slice labels it ${label} ${sliceCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
+					detail: `Token "${token}": the recipe output labels it ${label} ${outputCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
 				})
 			}
 		}
@@ -278,34 +271,35 @@ function checkLabelVacuum(slice: SliceStats, corpus: CorpusStats): LintSliceFlag
 	return flags
 }
 
-function checkBigramCollisions(slice: SliceStats, corpus: CorpusStats): LintSliceFlag[] {
-	const flags: LintSliceFlag[] = []
+function checkBigramCollisions(output: RecipeOutputStats, corpus: CorpusStats): LintFlag[] {
+	const flags: LintFlag[] = []
 
-	for (const [bigram, sliceLabelMap] of slice.bigrams) {
+	for (const [bigram, outputLabelMap] of output.bigrams) {
 		const corpusLabelMap = corpus.bigrams[bigram]
 
 		if (!corpusLabelMap) continue
-		const sliceMaj = majorityLabel(sliceLabelMap)
+		const outputMaj = majorityLabel(outputLabelMap)
 		const corpusMaj = majorityLabel(corpusLabelMap)
 
 		if (
-			sliceMaj.label !== corpusMaj.label &&
-			sliceMaj.count >= BIGRAM_MIN_COUNT &&
+			outputMaj.label !== corpusMaj.label &&
+			outputMaj.count >= BIGRAM_MIN_COUNT &&
 			corpusMaj.count >= BIGRAM_MIN_COUNT
 		) {
 			const renderBigram = bigram.split(SEP).join(" ")
-			const renderSliceLabel = sliceMaj.label.split(SEP).join(" → ")
+			const renderOutputLabel = outputMaj.label.split(SEP).join(" → ")
 			const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ")
 
 			flags.push({
+				...EMPTY_FLAG_FIELDS,
 				check: "bigram-collision",
 				severity: "error",
 				bigram: renderBigram,
-				sliceLabel: renderSliceLabel,
+				outputLabel: renderOutputLabel,
 				corpusLabel: renderCorpusLabel,
-				sliceCount: sliceMaj.count,
+				outputCount: outputMaj.count,
 				corpusCount: corpusMaj.count,
-				detail: `Bigram "${renderBigram}": slice label-bigram is [${renderSliceLabel}] (${sliceMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
+				detail: `Bigram "${renderBigram}": recipe-output label-bigram is [${renderOutputLabel}] (${outputMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
 			})
 		}
 	}
@@ -313,27 +307,28 @@ function checkBigramCollisions(slice: SliceStats, corpus: CorpusStats): LintSlic
 	return flags
 }
 
-function checkRules(slice: SliceStats, rulesFile: LintRulesFile): LintSliceFlag[] {
-	const flags: LintSliceFlag[] = []
+function checkRules(output: RecipeOutputStats, rulesFile: LintRulesFile): LintFlag[] {
+	const flags: LintFlag[] = []
 
 	const compiled = rulesFile.rules.map((r) => ({
 		rule: r,
 		regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
 	}))
 
-	for (const [token, labelMap] of slice.tokens) {
+	for (const [token, labelMap] of output.tokens) {
 		for (const { rule, regex } of compiled) {
 			if (!regex.test(token)) continue
 
 			for (const [label, count] of labelMap) {
 				if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
 					flags.push({
+						...EMPTY_FLAG_FIELDS,
 						check: "anti-pattern-rule",
 						severity: rule.severity,
 						ruleID: rule.id,
 						token,
-						sliceLabel: label,
-						sliceCount: count,
+						outputLabel: label,
+						outputCount: count,
 						detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
 					})
 				}
@@ -344,34 +339,38 @@ function checkRules(slice: SliceStats, rulesFile: LintRulesFile): LintSliceFlag[
 	return flags
 }
 
-function checkSanity(slice: SliceStats): LintSliceFlag[] {
-	const flags: LintSliceFlag[] = []
+function checkSanity(output: RecipeOutputStats): LintFlag[] {
+	const flags: LintFlag[] = []
 
-	if (slice.truncatedRows > 0) {
+	if (output.truncatedRows > 0) {
 		flags.push({
+			...EMPTY_FLAG_FIELDS,
 			check: "truncated-rows",
 			severity: "error",
-			detail: `${slice.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
+			detail: `${output.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
 		})
 	}
 
-	const allORatio = slice.allORows / Math.max(1, slice.rowCount)
+	const allORatio = output.allORows / Math.max(1, output.rowCount)
 
 	if (allORatio >= ALL_O_RATIO_CEILING) {
 		flags.push({
-			check: "all-O-slice",
+			...EMPTY_FLAG_FIELDS,
+			check: "all-O-output",
 			severity: "warn",
-			detail: `${slice.allORows}/${slice.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Slice contributes no signal.`,
+			detail: `${output.allORows}/${output.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. The recipe output contributes no signal.`,
 		})
 	}
 
 	return flags
 }
 
+// repo-health-ignore private-name-shadows-export -- renders a recipe-lint report.
+// The dossier export renders a building dossier, an unrelated document.
 function renderReport(
-	opts: { slicePath: string; statsPath: string; rulesPath: string },
-	slice: SliceStats,
-	flags: LintSliceFlag[]
+	opts: { outputPath: string; statsPath: string; rulesPath: string },
+	output: RecipeOutputStats,
+	flags: LintFlag[]
 ): string {
 	const errors = flags.filter((f) => f.severity === "error")
 	const warns = flags.filter((f) => f.severity === "warn")
@@ -380,14 +379,14 @@ function renderReport(
 	const lines: string[] = [
 		`# Corpus Lint: ${verdict}`,
 		"",
-		`- **Slice:** \`${opts.slicePath}\``,
+		`- **Recipe output:** \`${opts.outputPath}\``,
 		`- **Corpus stats:** \`${opts.statsPath}\``,
 		`- **Rules:** \`${opts.rulesPath}\``,
-		`- **Slice rows:** ${slice.rowCount}`,
-		`- **Unique tokens:** ${slice.tokens.size}`,
-		`- **Unique bigrams:** ${slice.bigrams.size}`,
+		`- **Recipe output rows:** ${output.rowCount}`,
+		`- **Unique tokens:** ${output.tokens.size}`,
+		`- **Unique bigrams:** ${output.bigrams.size}`,
 		"",
-		`**Errors:** ${errors.length} (checks the slice's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
+		`**Errors:** ${errors.length} (refuses the recipe output's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
 		`**Warnings:** ${warns.length} (advisory)`,
 		"",
 	]
@@ -398,7 +397,7 @@ function renderReport(
 		return lines.join("\n")
 	}
 
-	const byCheck = new Map<string, LintSliceFlag[]>()
+	const byCheck = new Map<string, LintFlag[]>()
 
 	for (const f of flags) {
 		const arr = byCheck.get(f.check) ?? []
@@ -409,8 +408,8 @@ function renderReport(
 	for (const [check, list] of byCheck) {
 		lines.push(`## ${check} (${list.length})`)
 		lines.push("")
-		// Sort by sliceCount desc — highest-volume issues first
-		list.sort((a, b) => (b.sliceCount ?? 0) - (a.sliceCount ?? 0))
+		// Highest-volume issues first.
+		list.sort((a, b) => (b.outputCount ?? 0) - (a.outputCount ?? 0))
 
 		for (const f of list.slice(0, 20)) {
 			lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`)
@@ -427,40 +426,45 @@ function renderReport(
 }
 
 /**
- * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
+ * Lint a recipe output against corpus stats and the anti-pattern rules,
+ * printing the markdown report to stdout.
  */
-export async function lintCorpusSlice(
-	options: LintCorpusSliceOptions,
+export async function lintRecipeOutput(
+	options: LintRecipeOutputOptions,
 	report?: (line: string) => void
-): Promise<LintCorpusSliceSummary> {
+): Promise<LintRecipeOutputSummary> {
 	const rulesPath = options.rulesPath ?? defaultRulesPath()
 	report?.(`Reading corpus stats from ${options.statsPath}...`)
 	const corpus = await readLocalJSONFile<CorpusStats>(options.statsPath)
 
 	report?.(
-		`  ${corpus.row_count} rows from ${corpus.slice_paths.length} slice(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`
+		`  ${corpus.row_count} rows from ${corpus.slice_paths.length} parquet file(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`
 	)
 
-	report?.(`Reading slice from ${options.slicePath}...`)
+	report?.(`Reading recipe output from ${options.recipeOutputPath}...`)
 
-	const slice = await statsFromSlice(streamTokenLabelRows(options.slicePath))
+	const output = await statsFromRecipeOutput(streamTokenLabelRows(options.recipeOutputPath))
 
-	report?.(`  ${slice.rowCount} rows`)
+	report?.(`  ${output.rowCount} rows`)
 
 	report?.(`Loading rules from ${rulesPath}...`)
 	const rulesFile = await readLocalJSONFile<LintRulesFile>(rulesPath)
 
 	report?.(`Running checks...`)
 
-	const flags: LintSliceFlag[] = [
-		...checkDistributionOutliers(slice, corpus),
-		...checkLabelVacuum(slice, corpus),
-		...checkBigramCollisions(slice, corpus),
-		...checkRules(slice, rulesFile),
-		...checkSanity(slice),
+	const flags: LintFlag[] = [
+		...checkDistributionOutliers(output, corpus),
+		...checkLabelVacuum(output, corpus),
+		...checkBigramCollisions(output, corpus),
+		...checkRules(output, rulesFile),
+		...checkSanity(output),
 	]
 
-	const rendered = renderReport({ slicePath: options.slicePath, statsPath: options.statsPath, rulesPath }, slice, flags)
+	const rendered = renderReport(
+		{ outputPath: options.recipeOutputPath, statsPath: options.statsPath, rulesPath },
+		output,
+		flags
+	)
 
 	console.log(rendered)
 
@@ -471,7 +475,7 @@ export async function lintCorpusSlice(
 	if (options.outJSON) {
 		await writeLocalJSONFile(
 			{
-				slice: options.slicePath,
+				recipe_output: options.recipeOutputPath,
 				stats: options.statsPath,
 				flags,
 				summary: {

package/lib/tools/migrate/overlay-parquet.ts

@@ -0,0 +1,173 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert an overlay parquet to the current row schema, without its recipe output.
+ *   `mailwoman-derived-tuples` wherever the upstream was not recorded.
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { once } from "@mailwoman/core/utils/events"
+import type { PathBuilderLike } from "path-ts"
+
+import { openParquetRowStream } from "#parquet/streams"
+import { retiredSourceName } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { jsonlToParquet } from "#tools/jsonl-to-parquet"
+import { surfaceForSource } from "#tools/migrate/recipe-output"
+
+/**
+ * The register each overlay source's rows came from, or `mailwoman-derived-tuples`
+ * when the upstream was not recorded.
+ */
+export const OVERLAY_REGISTERS: Record<string, string | null> = {
+	// The recipe names its publication.
+	"synth-gb": SourceRegister.LandRegistryPricePaid,
+	"synth-german": SourceRegister.OpenAddresses,
+	"synth-nz": SourceRegister.OpenAddresses,
+	"synth-es": SourceRegister.OpenAddresses,
+	"synth-unit": SourceRegister.OpenAddresses,
+	"synth-unit-v30": SourceRegister.OpenAddresses,
+	"synth-country": SourceRegister.OpenAddresses,
+	"synth-fr-order": SourceRegister.BaseAdresseNationale,
+	"synth-fr-admin-split": SourceRegister.BaseAdresseNationale,
+	"synth-fr-bare-street": SourceRegister.BaseAdresseNationale,
+	"synth-fr-fragment": SourceRegister.BaseAdresseNationale,
+	"synth-fr-lieudit": SourceRegister.BaseAdresseNationale,
+	overture: SourceRegister.Overture,
+	"overture-latam": SourceRegister.Overture,
+	"synth-sg-register": SourceRegister.Overture,
+	gnaf: SourceRegister.GNAF,
+	osm: SourceRegister.OpenStreetMap,
+	"synth-pk-register": SourceRegister.OpenStreetMap,
+	"synth-bd-register": SourceRegister.OpenStreetMap,
+	"synth-bare-country": SourceRegister.Codex,
+	"synth-bare-postcode": SourceRegister.Codex,
+	"synth-reviewed-postcode-tail": SourceRegister.ReviewedByHand,
+	// The row names no published record at all.
+	"synth-po-box-cedex": null,
+	"synth-po-box-military": null,
+	"synth-intersection": null,
+	"synth-house-venue": null,
+	"deepseek-kryptonite": null,
+	"synth-anchor-absorption": null,
+	// The tuple extraction behind these was not recorded.
+	"synth-affix": SourceRegister.DerivedTuples,
+	"synth-suffix-boundary": SourceRegister.DerivedTuples,
+	"synth-cz-pcfirst-preposition": SourceRegister.DerivedTuples,
+	"synth-cz-pcfirst-preposition-v21": SourceRegister.DerivedTuples,
+	"synth-fr-bare-street-v22": SourceRegister.DerivedTuples,
+	"synth-si-bare-village": SourceRegister.DerivedTuples,
+	"synth-no-street-led": SourceRegister.DerivedTuples,
+	"synth-no-fragment": SourceRegister.DerivedTuples,
+	"synth-fragment": SourceRegister.DerivedTuples,
+	"synth-nl-postcode": SourceRegister.DerivedTuples,
+	"synth-sub-venue": SourceRegister.DerivedTuples,
+	"synth-trailing-region": SourceRegister.DerivedTuples,
+	"synth-trailing-region-us": SourceRegister.DerivedTuples,
+	"synth-trailing-region-ca-bare": SourceRegister.DerivedTuples,
+	"synth-trailing-region-structured": SourceRegister.DerivedTuples,
+	"synth-trailing-region-es-v28": SourceRegister.DerivedTuples,
+}
+
+/**
+ * Read an INT64 span offset back as a plain number.
+ *
+ * DuckDB returns INT64 values as `BigInt`.
+ * `stringifyJSON` refuses `BigInt` values.
+ *
+ * The current schema declares the span triple INT32 because `raw` is a short
+ * address string, so every in-range value is lossless.
+ *
+ * A value outside the safe-integer range is a corrupt offset, so this raises instead of truncating.
+ */
+function narrowBigInts(row: Record<string, unknown>): Record<string, unknown> {
+	const narrow = (value: unknown): unknown => {
+		if (typeof value === "bigint") {
+			if (value > BigInt(Number.MAX_SAFE_INTEGER) || value < BigInt(Number.MIN_SAFE_INTEGER)) {
+				throw new Error(`span offset ${value} is outside the safe-integer range, so it is a corrupt offset`)
+			}
+
+			return Number(value)
+		}
+
+		if (Array.isArray(value)) return value.map(narrow)
+
+		return value
+	}
+
+	return Object.fromEntries(Object.entries(row).map(([key, value]) => [key, narrow(value)]))
+}
+
+/**
+ * What a parquet migration observed.
+ */
+export interface OverlayMigrationSummary {
+	rows: number
+	bySource: Record<string, number>
+}
+
+/**
+ * The register for one overlay source, refusing a source this map does not name.
+ */
+export function registerForSource(source: string): string | null {
+	// The map keys on the retired spelling.
+	// A corpus that uses the current spelling needs the retired key for this lookup.
+	// Otherwise each row fails as though it had no register.
+	// That failure misstates a renamed source as an unrecorded one.
+	const key = retiredSourceName(source)
+
+	if (!(key in OVERLAY_REGISTERS)) {
+		throw new Error(
+			`No register recorded for overlay source ${stringifyJSON(source)}. Read the recipe that writes it and ` +
+				`add it to OVERLAY_REGISTERS: the publication it renders, or null when the row names no record, or ` +
+				`mailwoman-derived-tuples when its tuple extraction was not recorded.`
+		)
+	}
+
+	return OVERLAY_REGISTERS[key] ?? null
+}
+
+/**
+ * Rewrite one overlay parquet onto the current row schema, staged through
+ * {@linkcode jsonlToParquet} so it has the same column list as every other overlay.
+ */
+export async function migrateOverlayParquet(
+	input: PathBuilderLike,
+	output: PathBuilderLike
+): Promise<OverlayMigrationSummary> {
+	const summary: OverlayMigrationSummary = { rows: 0, bySource: {} }
+
+	await using staging = await temporaryDirectory("mw-migrate-overlay-")
+	const stagePath = staging.path("rows.jsonl")
+	const sink = staging.use(openWriteStream(stagePath, { encoding: "utf8" }))
+
+	for await (const row of openParquetRowStream<Record<string, unknown>>(input)) {
+		const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = narrowBigInts(row)
+		const source = String(row["source"] ?? "")
+
+		summary.rows++
+		summary.bySource[source] = (summary.bySource[source] ?? 0) + 1
+
+		const migrated = {
+			...rest,
+			recipe: recipe ?? null,
+			base_source_id: baseSourceID ?? null,
+			register: registerForSource(source),
+			surface: surfaceForSource(source),
+		}
+
+		if (!sink.write(`${stringifyJSON(migrated)}\n`)) {
+			await once(sink, "drain")
+		}
+	}
+
+	sink.end()
+	await once(sink, "finish")
+
+	await jsonlToParquet({ input: stagePath, output })
+
+	return summary
+}