@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/recipes/scaffold.ts

@@ -3,107 +3,94 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Shared scaffolding for the synthetic-corpus SLICE RECIPES — the common bits the 16
- *   `build-*-slice.mjs` scripts each re-implemented: the seeded LCG PRNG, the tuple reader, and the
- *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link CorpusRecipe}) supplies
- *   only its synthesis + filter; the `mailwoman corpus slice <recipe>` command supplies the I/O.
+ *   Shared scaffolding for corpus recipes: the seeded LCG PRNG, the tuple reader and
+ *   the canonical → `alignRow` → `LabeledRow` jsonl emit step. A recipe ({@link CorpusRecipe})
+ *   supplies only its row construction and filter. The `mailwoman corpus slice <recipe>` command
+ *   supplies the I/O.
  */
 
-import { pathExists } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { readZipEntry } from "@mailwoman/core/fs/zip"
-import { tryParsingJSON } from "@mailwoman/core/json"
+import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
 import type { PathBuilderLike } from "path-ts"
 import type { AsyncChunkIterator, AsyncDataResource } from "spliterator"
 import { AsyncSequence, CSVSpliterator, TextSpliterator } from "spliterator"
 
-import { stableSourceIDFromParts } from "#adapters/utils"
+import { stableSourceIDFromParts } from "#adapters/source-id"
+import type { SurfaceOrigin } from "#types"
 import { alignRow } from "#utils"
 
 /**
- * {@link stableSourceIDFromParts} under the name the recipes use: arbitrary disambiguator keys (e.g. a variant index
- * `v`) that aren't `ComponentTag`s, which is how the legacy builders kept per-variant ids unique.
+ * {@link stableSourceIDFromParts} under the name the recipes use, for arbitrary
+ * disambiguator keys (e.g. a variant index `v`) that are not `ComponentTag`s.
  */
-export function sliceSourceID(adapterID: string, parts: Record<string, string | undefined>): string {
+export function recipeSourceID(adapterID: string, parts: Record<string, string | undefined>): string {
 	return stableSourceIDFromParts(adapterID, parts)
 }
 
 /**
  * Where a country's convention writes the postcode inside the `«locality», «region»[, «country»]` admin tail.
  *
- * Position changes the tag assigned to the same digits. On the shipped model, `Heladería Frappé Manía, Avenida Country
- * Club, Barcelona 6001, Anzoátegui, Venezuela` tags `6001` as `house_number` and loses the locality into the street,
- * while the identical row written `… 6001 Barcelona, Anzoátegui, Venezuela` tags it `postcode` and recovers `locality:
- * Barcelona`. So a slice emitting one placement teaches one family of countries.
+ * The position changes which tag the same digits receive, so a recipe emitting
+ * one placement teaches one family of countries.
+ * Each value is attested by a gauntlet board row:
  *
- * Each is attested by a gauntlet board row, which is the bar for adding another:
- *
- * - `leading` — `«postcode» «locality», «region»`. `Rua da Praia, 15, 8600-315 Lagos, Algarve, Portugal`
- *   (`pt_structured`). The default, and what every tuple written before this field existed means.
- * - `after_locality` — `«locality» «postcode», «region»`. `…, Barcelona 6001, Anzoátegui, Venezuela`
- *   (`ve_city_postcode_trailing_state`).
- * - `after_region` — `«locality», «region» «postcode»`. `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India`
- *   (`in_structured`).
+ * - `leading` — `«postcode» «locality», «region»`.
+ *   This is the default placement.
+ *   It also describes every tuple written before this field existed.
+ * - `after_locality` — `«locality» «postcode», «region»`.
+ * - `after_region` — `«locality», «region» «postcode»`.
  */
 export type PostcodePlacement = "leading" | "after_locality" | "after_region"
 
 /**
  * A (locality, region, postcode, country) source tuple — the input to tuples-mode recipes.
  */
-export interface SliceTuple {
+export interface RecipeTuple {
 	locality?: string
 	/**
-	 * The segment before the locality, when the source has one. A slice whose every row BEGINS with the locality teaches
-	 * that the first named segment is the locality, which flips the model's default — measured on the v4.8.0 candidate,
-	 * where `Ye Three Lords, 27 Minories, London EC3N 1DE` came back `locality: "Ye Three Lords"`.
+	 * The source's segment before the locality.
+	 *
+	 * Its inclusion prevents every recipe row from beginning with the locality
+	 * and teaching that the first segment is the locality.
 	 */
 	dependentLocality?: string
 	region?: string
 	postcode?: string
 	country?: string
 	/**
-	 * Defaults to `leading` when absent, which is what every tuple file written before this field existed means — so an
-	 * old tuples file produces byte-identical rows.
+	 *
+	 * That matches every tuples file written before this field existed.
+	 *
+	 * @defaultValue `leading` when absent.
 	 */
 	postcodePlacement?: PostcodePlacement
+	/**
+	 * `source_id` of the record this tuple was extracted from, when the extraction included it.
+	 *
+	 * A recipe forwards it as `baseSourceID`, which is the only way a `composed`
+	 * row can name the record behind it.
+	 * Every tuples file written before this field existed omits it, so a recipe reading one passes `null`
+	 * and its rows are unlinkable to their source records by anything but the `raw` string (#2359).
+	 */
+	sourceID?: string
 	[k: string]: unknown
 }
 
 /**
- * The two seeded generators the legacy `build-*-slice` scripts used, re-exported from their home in
- * `@mailwoman/core/utils` so a recipe keeps importing everything it needs from this one scaffold module.
- *
- * - `makeLcg` (`s = s*1664525 + 1013904223 mod 2^32`) — what the street/po-box/anchor builders seeded.
- * - `makeMulberry32` — what the MAJORITY of them used (german, locale, boundary-stress, unit, fr-order, country-balanced,
- *   intersection, fr-admin-split, street-affix, street-bare, po-box-cedex).
+ * One CSV record, keyed by its lower-cased header name, every value trimmed.
  *
- * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
- * seed), or `--seed N` stops being byte-reproducible.
- */
-
-/**
- * One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
- * as `undefined`; a column the header declares but this record does not reach reads as `""`.
+ * An undeclared column reads as `undefined`, a declared but unreached column as `""`.
  */
 export type CSVRecord = Record<string, string | undefined>
 
 /**
- * Line breaks inside a value become single spaces, and every value is trimmed.
- *
- * A quote-aware parse is the first thing able to return a value CONTAINING a line break — `us/ia/statewide.csv` has 12,
- * all unit designators like `"#2\n#2"` — and every consumer synthesizes one-line address text from these cells with no
- * guard, because until that parse landed no value could carry one. Collapsing keeps the record (the address is fine;
- * the source's line break is not part of it) without emitting a training row with a newline inside it.
+ * Line breaks inside a value become single spaces.
+ * The function trims every value.
  *
- * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
- * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every slice
- * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
- * `scaffold.test.ts` pins both halves.
- *
- * This is what a CSV reader here still owns. Quote handling, object rows, and lower-case keys are `CSVSpliterator`'s
- * defaults, so a recipe reads a source with `CSVSpliterator.fromAsync(source).map(withoutLineBreaks)` and needs nothing
- * else.
- *
- * @category CSV
+ * The pattern matches exactly `\r` and `\n`: widening to `\s+` would silently rewrite values on
+ * rows with no line break at all (OA's IA extract writes `north`, three spaces, `main street`),
+ * and `scaffold.test.ts` pins both directions.
  */
 export function withoutLineBreaks(record: CSVRecord): CSVRecord {
 	const out: CSVRecord = {}
@@ -118,32 +105,23 @@ export function withoutLineBreaks(record: CSVRecord): CSVRecord {
 /**
  * Read a CSV as header-keyed records.
  *
- * Returns the spliterator's own {@linkcode AsyncSequence}, so a caller composes `take`/`drop`/`filter` onto it — those
- * ops fuse into one pull loop, and a `take` that is satisfied closes the source's file handle on the way out. Wrapping
- * this in an `async function*` costs an async frame per row and takes those ops away; don't.
- *
- * A source at or below the spliterator's 128 KiB bulk threshold is read whole and parsed by the synchronous engine, so
- * this is also the right reader for small sources — there is no buffered variant to reach for.
- *
- * @category CSV
+ * Returns the spliterator's own {@linkcode AsyncSequence} so a caller composes
+ * `take`/`drop`/`filter` onto it.
+ * An `async function*` wrapper would cost an async frame per row and remove those ops.
  */
 export function readCSVRecords(source: AsyncDataResource | AsyncChunkIterator): AsyncSequence<CSVRecord> {
 	return CSVSpliterator.fromAsync<CSVRecord>(source).map(withoutLineBreaks)
 }
 
 /**
- * {@link readCSVRecords} over one member of a zip archive — what every recipe reading a cached OA source wants.
+ * {@link readCSVRecords} over one member of a zip archive.
  *
- * A source a checkout has not cached yields nothing, after saying so. A lab holds the archives for the countries it has
- * built, so a recipe naming ten sources routinely finds three, and the `unzip -p` subprocesses these replaced behaved
- * the same way by accident — a non-zero exit warned and returned no rows. A recipe that ends up with NO tuples at all
- * still throws; that is the case where the cache, not the recipe, is the problem.
- *
- * @category CSV
+ * A source a checkout has not cached yields no rows after warning, while a recipe
+ * that ends up with no tuples at all still throws.
  */
 export function readZippedCSVRecords(archivePath: PathBuilderLike, entryName: string): AsyncSequence<CSVRecord> {
 	return AsyncSequence.from<CSVRecord>(async () => {
-		if (!(await pathExists(String(archivePath)))) {
+		if (!(await pathExists(archivePath))) {
 			console.error(`  WARN: ${archivePath} is not cached — skipping ${entryName}`)
 
 			return []
@@ -154,19 +132,19 @@ export function readZippedCSVRecords(archivePath: PathBuilderLike, entryName: st
 }
 
 /**
- * License stamped on the synthetic tuple-derived slices (`po-box`, `no-street`, `house-venue`): the output is
- * generated, but it inherits the terms of the real tuples it is derived from, so the attribution travels with it.
+ * License stamped on the synthetic tuple-derived recipe outputs.
+ *
+ * The output is generated but inherits the terms of the real tuples it derives from,
+ * so the attribution travels with it.
  */
 export const SYNTHETIC_TUPLE_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
 
 /**
- * The surface key shared by the Norwegian slices (`no-fragment`, `no-street-led`).
+ * The surface key shared by the Norwegian recipes (`no-fragment`, `no-street-led`).
  *
- * MUST match the NO digit board's `norm_surface`: NFC, lowercase, collapse whitespace — and KEEP diacritics.
- * fr-fragment's norm strips them (NFD + combining-mark removal), which is right for French but would collapse
- * `Tømmerlien` → `tommerlien` here, so a slice's exclusion check would never match the board's reserved `tømmerlien`
- * and the train/eval split would leak silently. Diacritic street heads (…vegen/…veien with ø/å/æ) are the whole point
- * of those slices' boundary; folding them away is not an option.
+ * Must match the Norwegian digit board's `norm_surface` (NFC, lowercase, collapse whitespace,
+ * keep diacritics); folding them away would collapse `Tømmerlien` to `tommerlien`,
+ * never match the board's reserved `tømmerlien`, and leak the train/eval split silently.
  */
 export const foldNOSurface = (value: string): string =>
 	value.normalize("NFC").toLowerCase().replaceAll(/\s+/g, " ").trim()
@@ -180,8 +158,8 @@ export interface OATupleSource {
 }
 
 /**
- * The four base fields every OA tuple reader extracts. `postcode` is `""` when the row carries none and
- * {@link ReadOATuplesOptions.requirePostcode} is unset.
+ * The four base fields every OA tuple reader extracts; `postcode` is `""` when the row
+ * contains no postcode and {@link ReadOATuplesOptions.requirePostcode} is unset.
  */
 export interface OATupleFields {
 	house_number: string
@@ -192,31 +170,34 @@ export interface OATupleFields {
 
 export interface ReadOATuplesOptions<T> {
 	/**
-	 * Stop after this many distinct tuples — the `break` closes the reader and releases the archive, which is what the
-	 * GB-scale countrywide extracts need.
+	 * Stop after this many distinct tuples.
+	 *
+	 * The `break` closes the reader and releases the archive.
+	 * GB-scale countrywide extracts need this behavior.
 	 */
 	limit?: number
 	/**
-	 * Drop rows without a postcode (reversed-order and balance readers, where the postcode drives the rendering).
+	 * Drop rows without a postcode, for the reversed-order and balance readers
+	 * where the postcode drives the rendering.
 	 */
 	requirePostcode?: boolean
 	/**
-	 * Fold the postcode into the dedup key (`fr-order`'s key). The default key is
-	 * `${house_number}|${street}|${locality}`, lower-cased.
+	 * Fold the postcode into the dedup key.
+	 *
+	 * The default key is `${house_number}|${street}|${locality}`, lower-cased.
 	 */
 	dedupIncludesPostcode?: boolean
 	/**
-	 * Shape the recipe's tuple from the base fields, the raw record, and the (lower-cased) dedup key.
+	 * Shape the recipe's tuple from the base fields, raw record and lower-cased dedup key.
 	 */
 	extra: (fields: OATupleFields, row: CSVRecord, key: string) => T
 }
 
 /**
- * Stream distinct tuples out of a cached OA zip — the reader the OA-skeleton recipes (`street-affix`, `unit`,
- * `country-balanced`, `fr-order`) share. Field reads, filters, dedup keys and row order are exactly what each recipe's
- * local copy did, so a recipe's slice stays byte-identical.
+ * Stream distinct tuples out of a cached OA zip, the reader the OA-skeleton recipes
+ * (`street-affix`, `unit`, `country-balanced`, `fr-order`) share.
  *
- * @category CSV
+ * Field reads, filters, dedup keys and row order keep each recipe's output byte-identical.
  */
 export async function readOATuples<T>(source: OATupleSource, options: ReadOATuplesOptions<T>): Promise<T[]> {
 	const tuples: T[] = []
@@ -250,16 +231,14 @@ export async function readOATuples<T>(source: OATupleSource, options: ReadOATupl
 }
 
 /**
- * Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped).
+ * Stream-parse a tuples jsonl file, yielding each parsed object (blank/invalid lines skipped).
  */
-export function readTuples(input: PathBuilderLike): AsyncSequence<SliceTuple> {
-	// TextSpliterator (not JSONSpliterator) keeps the reader's established tolerance: malformed lines are skipped rather
-	// than rejecting the sequence. These operators fuse into the source's pull loop instead of adding an async-generator
-	// frame per tuple.
+export function readTuples(input: PathBuilderLike): AsyncSequence<RecipeTuple> {
+	// TextSpliterator keeps the reader tolerant of malformed lines.
+	// JSONSpliterator would throw.
+	// These operators fuse into the source's pull loop.
 	return TextSpliterator.fromAsync(input)
-		.map((line) => line.trim())
-		.filter((line) => Boolean(line))
-		.map((line) => tryParsingJSON<SliceTuple>(line))
+		.map((line) => tryParsingJSON<RecipeTuple>(line))
 		.filter((tuple) => tuple !== null)
 		.map((tuple) => tuple!)
 }
@@ -267,7 +246,7 @@ export function readTuples(input: PathBuilderLike): AsyncSequence<SliceTuple> {
 /**
  * A canonical row as the recipes assemble it, before `alignRow` turns it into a `LabeledRow`.
  */
-export interface CanonicalSliceRow {
+export interface CanonicalRecipeRow {
 	raw: string
 	components: Record<string, string>
 	country: string
@@ -279,35 +258,136 @@ export interface CanonicalSliceRow {
 }
 
 /**
- * Run a canonical row through `alignRow` and, on success, write the `LabeledRow` (+ `synth_method` / `synth_base_id`)
- * as one JSONL line. Returns true if emitted, false if alignment quarantined it.
+ * Emit one line of a recipe's output.
+ *
+ * The delimiter is supplied separately, so pass only the content, without appending `"\n"`.
+ */
+export type WriteRecipeLine = (line: string) => void
+
+/**
+ * A sink the recipe writer sends each output line to.
+ * A `WriteStream` or a test's array push can satisfy it.
+ */
+export interface RecipeLineSink {
+	write(chunk: string): unknown
+}
+
+/**
+ * Bind {@linkcode WriteRecipeLine} to a sink, supplying the delimiter.
+ *
+ * A concatenation would stringify a non-string chunk through `toString()` and corrupt its bytes.
+ */
+export function createRecipeLineWriter(sink: RecipeLineSink): WriteRecipeLine {
+	return (line) => {
+		sink.write(line)
+		sink.write("\n")
+	}
+}
+
+/**
+ * What a recipe records about the rows it writes.
+ *
+ * `register` and `surface` answer separate questions.
+ * A collapsed result would report a real record as fabricated.
+ */
+export interface RecipeProvenance {
+	/**
+	 * The published register the underlying record came from, or `null` when the row names none.
+	 */
+	register: string | null
+
+	/**
+	 * How the written `raw` string was produced.
+	 */
+	surface: SurfaceOrigin
+
+	/**
+	 * `source_id` of the row this was derived from.
+	 *
+	 * This field is required.
+	 * `null` is a statement rather than an omission.
+	 *
+	 * A `composed` row reorders fields from a real record, so it has an underlying record by definition.
+	 * `null` means the recipe did not preserve that record's id.
+	 *
+	 * The cost of `null` is measurable.
+	 * Over `v0.6.0-register-surface` it is `null` for 4,176,539 of the 4,206,561 `composed` rows,
+	 * 99.3%, and every leak measurement over a composed source is therefore a lower bound: two rows
+	 * composed from one underlying address under different `source_id`s look like unrelated rows.
+	 *
+	 * `holdout-overlap.run.ts` then compares their `raw` strings.
+	 * That misses any pair the recipe rendered differently (#2359).
+	 */
+	baseSourceID: string | null
+}
+
+/**
+ * The register a tuple-reading recipe was invoked with.
+ *
+ * It throws when the flag is absent, naming the recipe, because a default would
+ * stamp one register's id on another register's records.
+ */
+export function requireRegister(opts: RecipeOptions, recipe: string): string {
+	if (!opts.register) {
+		throw new Error(
+			`${recipe} requires --register <id>: the register the --input tuples were extracted from, ` +
+				`e.g. whos-on-first, openstreetmap, fr-ban. It is written onto every row this recipe emits.`
+		)
+	}
+
+	return opts.register
+}
+
+/**
+ * Run a canonical row through `alignRow` and, on success, write the `LabeledRow`
+ * with its recipe id and provenance as one jsonl line.
+ *
+ * The provenance columns are flat because the parquet schema is flat.
  */
 export function alignAndWrite(
-	write: (line: string) => void,
-	canonical: CanonicalSliceRow,
-	synthMethod: string,
-	synthBaseID: string | null = null
+	write: WriteRecipeLine,
+	canonical: CanonicalRecipeRow,
+	recipe: string,
+	provenance: RecipeProvenance
 ): boolean {
 	const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
-	if (!aligned.row) return false
-	write(JSON.stringify({ ...aligned.row, synth_method: synthMethod, synth_base_id: synthBaseID }) + "\n")
+	// A quarantined result also has a `row`, the quarantine record, so the kind decides.
+	if (aligned.kind !== "labeled") return false
+
+	write(
+		stringifyJSON({
+			...aligned.row,
+			recipe,
+			register: provenance.register,
+			surface: provenance.surface,
+			base_source_id: provenance.baseSourceID ?? null,
+		})
+	)
 
 	return true
 }
 
 /**
- * Parsed options a recipe's `run` receives. Common fields + the union of recipe-specific flags.
+ * Parsed options a recipe's `run` receives.
+ *
+ * Common fields + the union of recipe-specific flags.
  */
-export interface SliceRecipeOpts {
+export interface RecipeOptions {
 	output: string
 	seed: number
 	variants: number
-	input?: string
+	input?: PathBuilderLike
 	count?: number
 	golden?: boolean
 	sourceName?: string
-	// recipe-specific (each recipe reads only what it needs):
+	/**
+	 * The published register the `--input` tuples were extracted from.
+	 *
+	 * {@link requireRegister} refuses a run that omits it rather than recording a guess.
+	 * A recipe that generates its rows from this repository's own tables declares its register in code.
+	 */
+	register?: string
 	houseNumberProb?: number
 	pmbRatio?: number
 	militaryRatio?: number
@@ -316,95 +396,124 @@ export interface SliceRecipeOpts {
 	country?: string
 	intlFraction?: number
 	/**
-	 * `german`: fraction of NATIVE-order rows rendered with no commas at all (`Neusser Str. 12 Nippes 50733 Köln`), the
-	 * single-line register #1946 found the model loses the district on. Default 0.3.
+	 * `german`: fraction of native-order rows rendered with no commas at all.
+	 *
+	 * @defaultValue `0.3`
 	 */
 	commaFreeFraction?: number
 	/**
-	 * `german`: fraction of rows that carry a WOF Ortsteil of the tuple's locality as `dependent_locality`. Default 0.3;
-	 * 0 when no admin database is readable.
+	 * `german`: fraction of rows whose `dependent_locality` is a WOF Ortsteil of the tuple's locality.
+	 *
+	 * It is 0 when no admin database is readable.
+	 *
+	 * @defaultValue `0.3`
 	 */
 	ortsteilFraction?: number
 	/**
-	 * `german`: the WOF admin database the Ortsteil pool is read from. Default
-	 * `$MAILWOMAN_DATA_ROOT/wof/admin-global-priority-importance.db`.
+	 * `german`: the WOF admin database the Ortsteil pool is read from.
+	 *
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/db/wof/admin-global-priority-importance.db`
 	 */
 	adminDB?: string
 	/**
-	 * `locale`: fraction of rows that append an explicit country surface form + a `country` component. Default 0.
+	 * `locale`: fraction of rows that append an explicit country surface form + a `country` component.
+	 *
+	 * @defaultValue `0`
 	 */
 	countryFraction?: number
 	/**
-	 * `locale`: tri-state override of the per-part `districtAsLocality` mapping for this invocation. `undefined` (flag
-	 * absent) leaves each `COUNTRY_SOURCES` part's own value untouched — every existing locale build stays
-	 * byte-identical. `true`/`false` forces that value on every part read this run. ES's pedanía slice
-	 * (`synth-es-pedania`) additionally uses `true` to select {@link LocaleCountrySource.pedaniaParts} instead of the
-	 * default `parts` — see `locale.ts`.
+	 * `locale`: tri-state override of the per-part `districtAsLocality` mapping.
+	 *
+	 * `undefined` leaves each `COUNTRY_SOURCES` part's own value untouched.
+	 * `true` or `false` forces that value on every part read this run.
 	 */
 	districtAsLocality?: boolean
 	bareProb?: number
 	hnProb?: number
 	communes?: string
 	/**
-	 * `fr-lieudit`: BAN `adresses-<dept>.csv` directory. Default `$MAILWOMAN_DATA_ROOT/corpus/sources/ban`.
+	 * `fr-lieudit`: BAN `adresses-<dept>.csv` directory.
+	 *
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/corpus/sources/ban`
 	 */
-	banDir?: string
+	banDir?: PathBuilderLike
 	multilocaleCount?: number
 	/**
-	 * `fr-fragment` / `no-fragment` / `no-street-led`: the eval board's reserved street-surface list. REQUIRED for those
-	 * recipes — a slice that trains on its own eval set measures memorization. See their docstrings.
+	 * `fr-fragment` / `no-fragment` / `no-street-led`: the eval board's reserved street-surface list.
+	 *
+	 * It is required for those recipes, because a recipe output that trains on
+	 * its own eval set measures memorization.
 	 */
-	excludeSurfaces?: string
+	excludeSurfaces?: PathBuilderLike
 	/**
-	 * `no-fragment`: share of rows that are counter-distribution (bare locality OR bare postcode).
+	 * `no-fragment`: share of rows that are counter-distribution (bare locality or bare postcode).
 	 */
 	counterProb?: number
 	/**
-	 * `no-fragment` knob 3: emit N copies of each street+number row whose number has >= longNumberMinDigits digits
-	 * (oversample the failing long-number class). Default 1 = no boost.
+	 * `no-fragment` knob 3: emit N copies of each street+number row whose number has >=
+	 * longNumberMinDigits digits (oversample the failing long-number class).
+	 *
+	 * @defaultValue `1` = no boost
 	 */
 	longNumberBoost?: number
 	/**
-	 * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3.
+	 * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted.
+	 *
+	 * @defaultValue `3`
 	 */
 	longNumberMinDigits?: number
 	/**
-	 * `sub-venue`: the sub-venue lexicon JSON. Default = the committed `corpus/data/sub-venue-lexicon.json`, resolved
-	 * through the package manifest so it works from the source tree and from `out/`.
+	 * `sub-venue`: the sub-venue lexicon JSON.
+	 *
+	 * @defaultValue = the committed `corpus/data/sub-venue-lexicon.json`, resolved through the package manifest so it works
+	 * from the source tree and from `out/`.
 	 */
 	lexicon?: string
 	/**
-	 * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region. Default
-	 * `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`.
+	 * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region.
+	 *
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`
 	 */
 	extractsDir?: string
 	/**
-	 * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
-	 * four countries this slice has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
-	 */
-	poiDB?: string
-	/**
-	 * `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
-	 * (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.
+	 * `sub-venue`: GB/US/FR address-context tuples jsonl.
+	 *
+	 * @defaultValue the house-venue v3 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`);
+	 * DE and ES read OpenAddresses directly.
 	 */
 	subVenueTuples?: string
 	/**
-	 * `sub-venue`: share of emitted rows that are confound NEGATIVES. Default 0.3.
+	 * `sub-venue`: share of emitted rows that are confound negatives.
+	 *
+	 * @defaultValue `0.3`
 	 */
 	negativeFraction?: number
+	/**
+	 * `venue` and `sub-venue`: the Overture place-names Parquet file with `name`,
+	 * `category` and `country` columns.
+	 *
+	 * @defaultValue the release the venue-head table is built from, under `$MAILWOMAN_DATA_ROOT/overture`
+	 */
+	venueNames?: PathBuilderLike
+	/**
+	 * `venue`: share of locality rows that also carry the tuple's region after the locality.
+	 *
+	 * @defaultValue `0.5`
+	 */
+	venueRegionFraction?: number
 }
 
 /**
  * Tally a recipe returns.
  */
-export interface SliceStats {
+export interface RecipeStats {
 	read?: number
 	emitted: number
 	skipped: number
 	/**
-	 * Rows dropped because their street SURFACE is reserved by an eval board (`--exclude-surfaces`). Separate from
-	 * `skipped` on purpose: a nonzero value is the audit trail that the train/eval split actually fired. Zero when a
-	 * recipe has no board split.
+	 * Rows dropped because their street surface is reserved by an eval board.
+	 *
+	 * It is separate from `skipped` so a nonzero value is the audit trail that the train/eval split fired.
 	 */
 	contaminated?: number
 }
@@ -412,13 +521,13 @@ export interface SliceStats {
 /**
  * A single declared recipe-specific option flag (for the command's --help).
  */
-export interface SliceRecipeOption {
+export interface RecipeOption {
 	flag: string
 	description: string
 }
 
 /**
- * A slice recipe: its identity, input mode, and its synthesis `run`.
+ * A corpus recipe: its identity, input mode and synthesis `run`.
  */
 export interface CorpusRecipe {
 	/**
@@ -430,16 +539,16 @@ export interface CorpusRecipe {
 	 */
 	description: string
 	/**
-	 * `tuples` reads `--input` JSONL; `generate` self-generates `--count` rows.
+	 * `tuples` reads `--input` jsonl; `generate` self-generates `--count` rows.
 	 */
 	mode: "tuples" | "generate"
 	/**
 	 * Recipe-specific flags this recipe honors (documentation only).
 	 */
-	options?: SliceRecipeOption[]
+	options?: RecipeOption[]
 	/**
-	 * Do the build: create the recipe's PRNG from `opts.seed` (its LEGACY generator — `makeLcg` or `makeMulberry32` — for
-	 * byte-reproducibility), synthesize, and emit each row via `write`.
+	 * Build by creating the recipe's PRNG from `opts.seed`.
+	 * Synthesize rows and emit each one through `write`.
 	 */
-	run(opts: SliceRecipeOpts, write: (line: string) => void): Promise<SliceStats>
+	run(opts: RecipeOptions, write: WriteRecipeLine): Promise<RecipeStats>
 }