@mailwoman/corpus 10.2.2 → 10.4.0

package/lib/tools/fetch/vlaanderen-ad.ts

@@ -146,7 +146,7 @@ export interface HarvestVlaanderenOptions {
 	 */
 	outputDir: PathBuilderLike
 	/**
-	 * Stop after this many address pages, for a probe rather than a full harvest.
+	 * Stop after this number of address pages, for a probe rather than a full harvest.
 	 */
 	maxPages?: number
 	/**

package/lib/tools/fetch/wfs-harvest.ts

@@ -3,7 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- * The paging, manifest and resumption machinery a paged WFS harvest shares.
+ * The paging, manifest and resumption code that every paged WFS harvest shares.
  *
  * `./inspire-addresses.ts` owns what an INSPIRE Addresses service exposes — its capabilities, its
  * feature-type names and its GeoJSON pages. This module owns what a harvest of any paged WFS does
@@ -75,7 +75,7 @@ export interface MarkupRoot {
 	 */
 	attributes: string
 	/**
-	 * The offset just past the opening tag.
+	 * The offset of the first character after the opening tag.
 	 */
 	contentStart: number
 }
@@ -158,7 +158,7 @@ export function rootContent(body: string, root: MarkupRoot, context: string): st
 /**
  * The attributes a page states about itself rather than about the feature type.
  *
- * A harvest assembles many pages under one root element, so the first page's counts
+ * A harvest assembles several pages under one root element, so the first page's counts
  * and timestamp would describe the assembled file wrongly.
  * They are dropped and the manifest records them per page.
  */
@@ -420,7 +420,7 @@ export interface PagedWFSHarvestOptions {
 	featureCount: FeatureCountStatement
 	pageSize: number
 	/**
-	 * Stop once the manifest holds this many pages, for a probe rather than a full harvest.
+	 * Stop once the manifest holds this number of pages, for a probe rather than a full harvest.
 	 *
 	 * A cap on the manifest rather than on the run, so a second run under the same cap makes no request.
 	 */
@@ -437,7 +437,7 @@ export interface PagedWFSHarvestOptions {
  * Does a recorded manifest describe the request being made now?
  *
  * A harvest taken at another page size, ordering, format or type cannot be continued:
- * its pages were cut at different boundaries, and appending to them would leave
+ * its pages were split at different boundaries, and appending to them would leave
  * a file whose manifest describes neither half.
  */
 function describesSameRequest(manifest: WFSHarvestManifest, options: PagedWFSHarvestOptions): boolean {
@@ -932,7 +932,7 @@ export async function runWFSHarvest(
 	if (options.client) {
 		manifest = await options.harvest(options.client)
 	} else {
-		// Bounded retry, because a harvest is many requests and one 5xx would otherwise end the run.
+		// Bounded retry, because a harvest is a long series of requests and one 5xx would otherwise end the run.
 		await using client = new APIClient({
 			displayName: options.displayName,
 			minRequestIntervalMs: options.minRequestIntervalMs,

package/lib/tools/fetch/zip-archive.ts

@@ -4,8 +4,8 @@
  * @author Teffen Ellis, et al.
  * @file Downloads one zip archive through an {@linkcode APIClient} and answers what arrived.
  *
- *   An INSPIRE ATOM service that publishes one archive per area is harvested through many small
- *   requests rather than through one large transfer, so the transfer runs on the client the harvest
+ *   An INSPIRE ATOM service that publishes one archive per area is harvested through one small
+ *   request per area rather than through one large transfer, so the transfer runs on the client the harvest
  *   already paces and a test can drive it over a stubbed transport. `downloadToFile` runs on global
  *   `fetch`, which a unit test cannot intercept.
  *

package/lib/tools/golden/expand.ts

@@ -231,7 +231,7 @@ async function loadSeeds(
 
 	// Stratified sampling: read all rows from all parquet files, group by source.
 	// Bounded by per-source reservoir: keep at most max(2*count, 5000) rows per source
-	// so we don't blow memory on train parquet files (1M rows × many files).
+	// so we don't blow memory on train parquet files (1M rows per file, across every file).
 	// Later samples are uniform within each pool.
 	const bySource = new Map<string, Seed[]>()
 	const PER_SOURCE_CAP = Math.max(2 * count, 5000)

package/lib/tools/golden/promote.ts

@@ -13,7 +13,7 @@
  *   3. Applies filters that drop candidates unlikely to be human-typed:
  *
  *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last. FR puts it
+ *        - Postcode-at-start with several other components (US/UK conventions put postcode last. FR puts it
  *                 before locality only)
  *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
  *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
@@ -47,7 +47,7 @@ import type { GoldenCandidateEntry as GoldenEntry } from "#utils/golden"
 // ── Types ──────────────────────────────────────────────────────────────────
 
 /**
- * Components a candidate needs before it is rich enough to be worth a golden-set slot.
+ * Components a candidate needs before it qualifies for a golden-set slot.
  */
 const MIN_GOLDEN_COMPONENTS = 5
 
@@ -153,7 +153,7 @@ function isSuspicious(entry: GoldenEntry): boolean {
 
 	if (raw.split('"').length > MAX_QUOTE_SEGMENTS) return true
 
-	// too many quote marks
+	// more quote marks than MAX_QUOTE_SEGMENTS admits
 	return false
 }
 

package/lib/tools/golden/relabel-street/directory.ts

@@ -36,7 +36,8 @@ import {
 export type GoldenRelabelCounts = Record<GoldenRelabelClass, number>
 
 /**
- * One line of the review deck: what a changed (or notably unchanged) row looked like before and after.
+ * One line of the review deck: what a changed row, or an unchanged row the deck
+ * lists for review, looked like before and after.
  */
 export interface GoldenRelabelDeckEntry {
 	file: string

package/lib/tools/lint/recipe-output/vocab.ts

@@ -47,7 +47,7 @@ function stripBIO(label: string): string {
  *
  * Pure-numeric tokens (house_number and postcode) are context-determined
  * rather than lexical vocab, so they are excluded.
- * `\p{Nd}` covers the decimal digits these address corpora actually contain.
+ * `\p{Nd}` covers the decimal digits these address corpora contain.
  */
 function isDigit(token: string): boolean {
 	return token.length > 0 && /^\p{Nd}+$/u.test(token)
@@ -148,7 +148,7 @@ async function readRows(con: DuckDBConnection, path: string): Promise<CorpusRow[
 }
 
 /**
- * Read just the first row's `source` value, used to group base parts for a proportional sample.
+ * Read only the first row's `source` value, used to group base parts for a proportional sample.
  */
 async function readSource(con: DuckDBConnection, path: string): Promise<string> {
 	const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`)

package/lib/tools/overlay/manifest.ts

@@ -3,7 +3,8 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Assembles a corpus manifest that keeps every file of a base corpus and adds new parquet files.
+ *   Assembles a corpus manifest from a base corpus's files and new parquet files.
+ *   A replaced source's base files are dropped.
  */
 
 import { dataRootPath } from "@mailwoman/core/data-root"
@@ -203,6 +204,10 @@ export interface OverlayPlan {
 	 * The filename was the only place that showed this (#2359).
 	 */
 	files: Array<{ parquet: string; source: string; split: SplitName; route: SplitRoute }>
+	/**
+	 * The base sources whose files this overlay dropped.
+	 */
+	replacedSources: string[]
 }
 
 /**
@@ -240,6 +245,12 @@ export interface OverlayManifestOptions {
 	 */
 	files: readonly OverlayFile[]
 	note: string
+	/**
+	 * Base sources whose files the overlay drops, so the added files take their place.
+	 *
+	 * Each source in this list must appear in the base, and its rows leave the split counts.
+	 */
+	replaceSources?: readonly string[]
 }
 
 /**
@@ -269,7 +280,9 @@ export function localManifestFilePath(path: string): string {
 }
 
 /**
- * Writes a corpus manifest that keeps every file of `args.base` and appends `args.files`.
+ * Writes a corpus manifest that keeps the files of `args.base` and appends `args.files`.
+ *
+ * A base file whose source `args.replaceSources` lists is dropped.
  *
  * A `val` or `test` file must use the `.<split>.parquet` suffix written by `splitOverlaySlice`.
  * The suffix shows that the holdout policy chose its rows.
@@ -291,8 +304,16 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		)
 	}
 
+	const replaced = new Set(args.replaceSources)
+
+	for (const source of replaced) {
+		if (!baseFiles.some((s) => s.source === source)) {
+			throw new Error(`${args.base} holds no file with source '${source}', so there is nothing for it to replace.`)
+		}
+	}
+
 	for (const file of args.files) {
-		if (baseFiles.some((s) => s.source === file.source)) {
+		if (!replaced.has(file.source) && baseFiles.some((s) => s.source === file.source)) {
 			console.log(`WARN: base already contains source '${file.source}' — is this the right base?`)
 		}
 	}
@@ -318,7 +339,24 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		)
 	}
 
-	const kept = baseFiles.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }))
+	const dropped = baseFiles.filter((s) => s.source !== undefined && replaced.has(s.source))
+	const droppedRows: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
+
+	for (const file of dropped) {
+		const split = file["split"] as SplitName
+		const rows = file["rows"]
+
+		if (typeof rows !== "number" || !(split in droppedRows)) {
+			throw new Error(`${file.path} records no row count or split, so dropping it would leave the counts unknown.`)
+		}
+
+		droppedRows[split] += rows
+	}
+
+	const kept = baseFiles
+		.filter((s) => !dropped.includes(s))
+		.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }))
+
 	const added: ParquetFileDescriptor[] = []
 
 	const newDir = PathBuilder.from(args.newDir)
@@ -345,20 +383,31 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 
 	const sources = args.files.map((file) => file.source).join(", ")
 
+	const defaultNote = replaced.size
+		? `${base.corpus_version} files without ${[...replaced].join(", ")} + ${sources}.`
+		: `${base.corpus_version} files (all kept verbatim) + ${sources}. Pure overlay add.`
+
 	const manifest = {
 		corpus_version: args.version,
 		overlay_base: base.corpus_version ?? null,
-		note: args.note || `${base.corpus_version} files (all kept verbatim) + ${sources}. Pure overlay add.`,
+		note: args.note || defaultNote,
 		schema: base.schema,
 		rows_per_slice: baseRowsPerFile(base),
 		row_group_size: base.row_group_size,
 		slices: [...kept, ...added],
 		counts: {
-			train: base.counts.train + addedRows.train,
-			val: base.counts.val + addedRows.val,
-			test: base.counts.test + addedRows.test,
+			train: base.counts.train - droppedRows.train + addedRows.train,
+			val: base.counts.val - droppedRows.val + addedRows.val,
+			test: base.counts.test - droppedRows.test + addedRows.test,
 		},
-		total_rows: base.total_rows + addedRows.train + addedRows.val + addedRows.test,
+		total_rows:
+			base.total_rows -
+			droppedRows.train -
+			droppedRows.val -
+			droppedRows.test +
+			addedRows.train +
+			addedRows.val +
+			addedRows.test,
 		// An overlay corpus's manifest reads no row's `license` column, so it states no license set.
 		// This value in the artifact prevents a reader from deriving an attribution table
 		// from `slices` and treating the base build's set as the whole.
@@ -389,6 +438,7 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 			split: file.split ?? "train",
 			route: splitRouteFor(file.parquet),
 		})),
+		replacedSources: [...replaced],
 	}
 
 	const planPath = newDir(OVERLAY_PLAN_FILE)
@@ -397,7 +447,9 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 
 	console.log(`wrote ${planPath}`)
 	console.log(`wrote ${out}`)
-	console.log(`  files: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`)
+	console.log(
+		`  files: ${manifest.slices.length} (${kept.length} base kept, ${dropped.length} base dropped, +${added.length} added)`
+	)
 	console.log(`  counts: ${stringifyJSON(manifest.counts)}  total: ${manifest.total_rows}`)
 
 	// The route prints beside the split, because a hand-placed split is otherwise visible only in a filename.

package/lib/tools/overlay/split-slice.ts

@@ -15,15 +15,7 @@ import type { ParquetRow } from "#parquet/schema"
 import { openParquetRowStream } from "#parquet/streams"
 import { writeParquetFile } from "#parquet/writers"
 import type { CanonicalRow } from "#types"
-import { type CountryHoldout, defaultHoldouts, type SplitName, splitForRow } from "#utils/split"
-
-/**
- * The component tags the holdout policy reads.
- *
- * `splitForRow` consults `region`, `postcode` and `locality` and no other field,
- * so reconstructing the rest would cost a string slice per span for no decision.
- */
-const HOLDOUT_TAGS = new Set<string>(["region", "postcode", "locality"])
+import { type CountryHoldout, defaultHoldouts, HOLDOUT_COMPONENT_TAGS, type SplitName, splitForRow } from "#utils/split"
 
 /**
  * The fields {@linkcode holdoutComponents} reads, with the span triple optional
@@ -63,7 +55,7 @@ export function holdoutComponents(row: SpannedRow, index: number, input: string)
 	const components: Record<string, string> = {}
 
 	for (const [span, tag] of tags.entries()) {
-		if (!HOLDOUT_TAGS.has(tag) || components[tag]) continue
+		if (!HOLDOUT_COMPONENT_TAGS.has(tag) || components[tag]) continue
 
 		components[tag] = raw.slice(starts[span]!, ends[span]!)
 	}

package/lib/tools/postcode-triples.ts

@@ -62,7 +62,7 @@ export const POSTCODE_CONVENTIONS: ReadonlyMap<
 	{ placement: PostcodePlacement; locale: string; localityColumn?: GeonamesLocalityColumn }
 > = new Map([
 	// `Rue de l'Église, 3, 29217 Plougonvelin, Bretagne, France` and its siblings in
-	// the structured slices attest this leading placement.
+	// the structured recipe outputs attest this leading placement.
 	["FR", { placement: "leading", locale: "fr-FR" }],
 	["DE", { placement: "leading", locale: "de-DE" }],
 	["ES", { placement: "leading", locale: "es-ES" }],
@@ -85,7 +85,7 @@ export const POSTCODE_CONVENTIONS: ReadonlyMap<
 ])
 
 /**
- * How many postcodes one locality may contribute.
+ * The number of postcodes one locality may contribute.
  *
  * A quota rather than a cutoff: it bounds repetition without deleting a locality.
  */

package/lib/tools/sub/venue/head-nouns.ts

@@ -28,7 +28,8 @@ function asciiFold(text: string): string {
 }
 
 /**
- * How many leading characters two ascii-folded forms must share for one to count as the other's inflection.
+ * The number of leading characters two ascii-folded forms must share for one
+ * to count as the other's inflection.
  *
  * Five, or the id's own length when shorter.
  * A limit of six loses Spanish `satélite`.
@@ -44,7 +45,7 @@ const HEAD_NOUN_PREFIX_FLOOR = 5
 const NON_LATIN_HEAD_MIN_LENGTH = 2
 
 /**
- * How many head-noun candidates one non-Latin record+language group may contribute.
+ * The number of head-noun candidates one non-Latin record+language group may contribute.
  *
  * Six, capped because the substring lattice of a nine-character label is large
  * and no entry past the sixth attests more than two surfaces.
@@ -67,7 +68,7 @@ const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Kat
 
 /**
  * Derive the head noun of every multi-part surface, so `terminal aeroportuaria`
- * contributes the form anyone actually writes on an envelope.
+ * contributes the form a sender writes on an envelope.
  *
  * Latin script uses a cognate test against the designator's own canonical id.
  * Han and Kana use a shared substring, because a token split finds no boundary there.

package/lib/types.ts

@@ -15,7 +15,7 @@ import type { PathBuilderLike } from "path-ts"
  * The values are wire values.
  *
  * Roles have different grammars.
- * For example, mailing addresses often use post-office boxes, so mixing roles in
+ * For example, some mailing addresses use post-office boxes, so mixing roles in
  * a corpus shifts what the premise parser learns.
  */
 export const AddressRole = {
@@ -277,7 +277,7 @@ export interface CanonicalRow extends SourceProvenance {
 /**
  * A row after alignment, with tokens, BIO labels and character spans.
  *
- * The span arrays are the source of truth.
+ * The span arrays are authoritative when they disagree with any other field.
  * Consumers can derive token labels from them.
  */
 export interface LabeledRow extends CanonicalRow {
@@ -399,7 +399,7 @@ export function countDropped(opts: Pick<AdapterOptions, "dropped">, reason: stri
 /**
  * The interface that every data source implements.
  *
- * Adapters stream rows, because many sources hold tens of millions of rows.
+ * Adapters stream rows, because some sources hold tens of millions of rows.
  */
 export interface CorpusAdapter {
 	/**

package/lib/us/adapters/state/tx-notaries/adapter.ts

@@ -6,7 +6,7 @@
  *   `state-tx-notaries`: Texas Notary Public Commissions CSV consumer.
  *
  *   The Texas Secretary of State publishes a registry of commissioned notaries public. Each row
- *   optionally includes a mailing address in free-form text (often multi-line with embedded
+ *   optionally includes a mailing address in free-form text (in some rows multi-line with embedded
  *   city/state/zip). Address fill rate is ~5-10%.
  *
  *   The adapter parses the embedded `Address` field for city/state/zip using a trailing `"city, ST

package/lib/us/adapters/usgov/irs-bmf/adapter.ts

@@ -10,7 +10,7 @@
  *   `https://www.irs.gov/charities-non-profits/exempt-organizations-business-master-file-extract-eo-bmf`
  *   (`eo1.csv`..`eo4.csv`, `eo_pr.csv`, `eo_xx.csv`). Each row includes an organization name plus
  *   its mailing address. It complements `usgov-nppes` with a different venue population
- *   (non-profits vs healthcare providers) and, notably, a high share of PO-box addresses — useful
+ *   (non-profits vs healthcare providers) and a high share of PO-box addresses — useful
  *   `po_box`-tag signal (a tag with historically low recall).
  *
  *   Output: one row per record with a usable city + postcode. name → `venue`; the street line becomes
@@ -40,7 +40,7 @@ export const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf"
 export const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain"
 
 /**
- * PO box in its many written forms: "PO BOX 12", "P.O.
+ * PO box in each of its written forms: "PO BOX 12", "P.O.
  *
  * BOX 12", "P O BOX 12", "POB 12", "BOX 12".
  */

package/lib/us/adapters/usgov/nppes/adapter.ts

@@ -38,7 +38,7 @@ interface NPPESRow {
 	"Provider Second Line Business Practice Location Address": string
 	"Provider Business Practice Location Address City Name": string
 	"Provider Business Practice Location Address State Name": string
-	// This is the publisher's exact column name, so it must keep the words "Postal Code".
+	// This is the publisher's exact column name, so the string keeps the publisher's spelling.
 	"Provider Business Practice Location Address Postal Code": string
 }
 

package/lib/us/synthesizers/intersection.ts

@@ -26,7 +26,7 @@ import { layoutAdminTail } from "#synthesizers/tail"
 import type { CanonicalRow } from "#types"
 
 /**
- * Street name cores (no suffix) — proper-noun streets that often appear bare.
+ * Street name cores (no suffix) — proper-noun streets that can appear bare.
  */
 
 /**
@@ -37,7 +37,7 @@ const MAX_DISTINCT_STREET_TRIES = 8
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches is the
    output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind identifiers. The genuine thresholds in these files are
+   would hide the distribution behind identifiers. The decision thresholds in these files are
     extracted as constants above. */
 
 const STREET_CORES = [
@@ -153,8 +153,8 @@ export function synthesizeIntersectionRow(
 ): SynthesizedIntersectionRow | null {
 	const random = opts.random ?? Math.random
 
-	// The junction connectors and the street vocabulary are US English,
-	// which is why this synthesizer sits under `us/`.
+	// The junction connectors and the street vocabulary are US English.
+	// This synthesizer therefore sits under `us/`.
 	// The admin tail renders through the country's layout.
 	if (base.country !== "US") return null
 

package/lib/us/synthesizers/street.ts

@@ -33,7 +33,7 @@ import { decomposeStreet } from "#us/adapters/tiger/street-decompose"
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches is the
    output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   would hide the distribution behind a wall of identifiers. Decision thresholds in these files are
     extracted as constants above. */
 
 const STREET_NAMES = [
@@ -221,8 +221,8 @@ export function synthesizeStreetRow(
 	const random = opts.random ?? Math.random
 	const includeHN = opts.includeHouseNumberProb ?? 0.85
 
-	// The street vocabulary (`STREET_NAMES`, `STREET_SUFFIXES`, the directionals) is
-	// US English, which is why this synthesizer sits under `us/`.
+	// The street vocabulary (`STREET_NAMES`, `STREET_SUFFIXES`, the directionals) is US English.
+	// This synthesizer therefore sits under `us/`.
 	// The admin tail renders through the country's layout.
 	if (base.country !== "US") return null
 

package/lib/us/tools/fetch/nad.ts

@@ -9,8 +9,8 @@
  *
  *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
  *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
+ *   - A `complete: true` flag that is set only when every page in the chunk fetched without error
+ *   - Built-in fetch with gzip/brotli decompression (no curl or Python subprocess per page)
  *   - Per-chunk manifest with sha256 + record count + error count
  *
  *   ## Source layout

package/lib/utils/split.ts

@@ -110,7 +110,7 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
 			regions: ["Cornwall"],
 		},
 		// The eight countries below enter training through OpenStreetMap rows. Most hold no
-		// region and many no postcode, so each holdout matches its place three ways: the region an
+		// region and some no postcode, so each holdout matches its place three ways: the region an
 		// admin row holds, the postcode prefix, and the city as the OSM rows spell it. The spellings
 		// are the ones the 2026-10-02 extracts write.
 		CN: {
@@ -204,6 +204,48 @@ export function defaultHoldouts(): Record<string, CountryHoldout> {
 	}
 }
 
+/**
+ * The component tags {@link splitForRow} reads.
+ *
+ * A reader that rebuilds components from spans needs only these.
+ */
+export const HOLDOUT_COMPONENT_TAGS: ReadonlySet<string> = new Set(["venue", "region", "postcode", "locality"])
+
+/**
+ * One place in this number is held out in a country without a {@link defaultHoldouts} entry.
+ */
+export const HASHED_PLACE_HOLDOUT_DIVISOR = 50
+
+/**
+ * The key a row is held out by under the hashed holdout: its venue, else its locality,
+ * else the first two characters of its postcode, else its region.
+ *
+ * A venue name keys before its place because the venue recipes write one name both with a locality and bare.
+ * A locality key would hold out the first form and train on the second.
+ *
+ * Returns null for a row with none of the four.
+ * That row stays in train.
+ */
+export function hashedHoldoutPlace(
+	components: Partial<Record<"venue" | "locality" | "postcode" | "region", string>>
+): string | null {
+	const venue = components.venue?.trim().toLowerCase()
+
+	if (venue) return `venue:${venue}`
+
+	const locality = components.locality?.trim().toLowerCase()
+
+	if (locality) return `locality:${locality}`
+
+	const postcode = components.postcode?.replaceAll(/\s+/g, "").toUpperCase()
+
+	if (postcode && postcode.length >= 2) return `postcode:${postcode.slice(0, 2)}`
+
+	const region = components.region?.trim().toLowerCase()
+
+	return region ? `region:${region}` : null
+}
+
 /**
  * Returns the matchers for one country, reading a bare array as a region list.
  */
@@ -225,13 +267,24 @@ export function splitForRow(
 	row: Pick<SplitInputRow, "source_id" | "country" | "components">,
 	holdouts: Record<string, CountryHoldout> = defaultHoldouts()
 ): SplitName {
-	const policy = holdoutPolicyFor(holdouts[row.country] ?? null)
+	const declared = holdouts[row.country]
 	const { region, postcode, locality } = row.components
+	let isHeldOut: boolean
+
+	if (declared) {
+		const policy = holdoutPolicyFor(declared)
 
-	const isHeldOut =
-		(region !== undefined && (policy.regions?.includes(region) ?? false)) ||
-		(postcode !== undefined && (policy.postcodePrefixes?.some((prefix) => postcode.startsWith(prefix)) ?? false)) ||
-		(locality !== undefined && (policy.localities?.includes(locality) ?? false))
+		isHeldOut =
+			(region !== undefined && (policy.regions?.includes(region) ?? false)) ||
+			(postcode !== undefined && (policy.postcodePrefixes?.some((prefix) => postcode.startsWith(prefix)) ?? false)) ||
+			(locality !== undefined && (policy.localities?.includes(locality) ?? false))
+	} else {
+		// A country without a declared holdout holds out whole places chosen by hash,
+		// so every country in the corpus reaches val and test.
+		const place = hashedHoldoutPlace(row.components)
+
+		isHeldOut = place !== null && hashBucket(`${row.country}|${place}`, HASHED_PLACE_HOLDOUT_DIVISOR) === 0
+	}
 
 	if (!isHeldOut) return "train"
 

package/lib/utils/wof-json.ts

@@ -75,7 +75,7 @@ export function isCurrentFeature(props: Record<string, unknown>): boolean {
 /**
  * Pull `name:*` localized variants off a WOF feature's properties, lifting the first
  * non-empty string from the array WOF stores them in (`["Saint Petersburg"]`);
- * Multiple-value variants are rare and usually historical aliases.
+ * Multiple-value variants are rare, and most of them are historical aliases.
  *
  * This function keeps them together.
  * An adapter that needs separate values can iterate the underlying array.

package/out/adapters/cordis/adapter.d.ts

@@ -71,7 +71,7 @@
  *
  * ## The street line
  *
- * `street` holds the whole street line as the participant typed it, usually but not always in the
+ * `street` holds the whole street line as the participant typed it, in most records but not all in the
  * country's own order: `THE WELLCOME TRUST LIMITED` writes `EUSTON ROAD 215 GIBBS BUILDING`, a
  * number-last line in a number-first country. The house number is therefore split off only on the
  * side the country's order puts it, and {@linkcode streetOrderForCountry} reads that side from codex's
@@ -110,11 +110,11 @@
  *
  * ## License
  *
- * The CORDIS legal notice at `https://cordis.europa.eu/about/legal`, read 2026-10-03, states:
- * "Unless otherwise noted (e.g. in individual copyright notices), the reuse of the editorial content on
- * this website owned by the EU is authorized under the Creative Commons Attribution 4.0 International
- * (CC BY 4.0) licence." The same notice records that "The Commission's reuse policy is implemented by
- * Commission Decision 2011/833/EU of 12 December 2011 on the reuse of Commission documents", and the
+ * The CORDIS legal notice at `https://cordis.europa.eu/about/legal`, read 2026-10-03, authorizes reuse
+ * of the EU-owned editorial content on the site under Creative Commons Attribution 4.0 International
+ * (CC BY 4.0), except where an individual copyright notice states otherwise. The same notice states
+ * that Commission Decision 2011/833/EU of 12 December 2011, on the reuse of Commission documents,
+ * implements the Commission's reuse policy, and the
  * data.europa.eu records for `cordisfp7projects`, `cordish2020projects` and
  * `cordis-eu-research-projects-under-horizon-europe-2021-2027` label each CSV distribution with that
  * decision (`COM_REUSE`). CC BY requires credit and a statement that changes were made, so every row
@@ -307,7 +307,7 @@ export interface CORDISRecordReading {
 /**
  * Every distinct published record under `inputPath`, read once each.
  *
- * A record is distinct on the five fields that reach a row, so an organization that took part in many
+ * A record is distinct on the five fields that reach a row, so an organization that took part in several
  * projects with one address is read once, and one whose address changed is read once per version.
  * `onDuplicate` is called once for every participation row a record already read covers.
  *