@mailwoman/corpus 10.3.0 → 10.5.0

package/lib/subvenue/harvest.ts

@@ -71,9 +71,10 @@ export interface SubVenueHarvestRow {
 /**
  * Classify an OSM `ref` into an {@link IdentifierShape} class.
  *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms
+ * The classes are the ones Berlin's airport gates produced, plus the two aviation forms
  * the corpus task names (`Terminal 2F` is digit-letter, `Concourse B` is letter).
- * `range` covers both separators OSM uses for a gate serving more than one stand: `16-18` and `0/1`.
+ * `range` covers both separators OSM uses for a boarding gate serving more
+ * than one stand: `16-18` and `0/1`.
  */
 export function classifyIdentifier(ref: string): string {
 	const value = ref.trim()
@@ -92,10 +93,10 @@ export function classifyIdentifier(ref: string): string {
 }
 
 /**
- * How many real `ref` values each {@link IdentifierShape} keeps.
+ * The number of real `ref` values each {@link IdentifierShape} keeps.
  *
  * Eight rather than "all" and not one.
- * The field exists so a recipe author can see what a class actually contains.
+ * The field exists so a recipe author can see what a class contains.
  *
  * GB's `other` class turned out to be semicolon multi-values (`1.2.3`, `13.14`),
  * which one example would have hidden and which the class name does not say.
@@ -128,7 +129,7 @@ export interface HarvestOptions {
 /**
  * Harvest attested phrases and identifier shapes out of a source's rows.
  *
- * `index` gates the name harvest.
+ * `index` decides whether the name harvest runs.
  * A name contributes only when it contains a phrase already in the table.
  *
  * That filter is the whole reason this function is safe to run over raw OSM:

package/lib/subvenue/promotions.ts

@@ -60,8 +60,8 @@ export interface SubVenuePromotion {
  * This ledger is the only input that sets `curated: true` on a machine-derived surface.
  * Rejections stay in the ledger so the system can reject duplicate proposals until new evidence arrives.
  *
- * The span proposer in `packages/neural/lib/venue/structure.ts` ships its English designators
- * without a locale gate, so a rejection here guides recipes only and does not stop the proposer.
+ * The span proposer in `packages/neural/lib/venue/structure.ts` ships its English designators without
+ * a locale restriction, so a rejection here guides recipes only and does not stop the proposer.
  */
 export const SUBVENUE_PROMOTIONS: readonly SubVenuePromotion[] = [
 	// wing
@@ -171,7 +171,7 @@ export const SUBVENUE_PROMOTIONS: readonly SubVenuePromotion[] = [
 		census: "france.jsonl 2026-08-05 (251,260 rows), 40 hits; all 19 shape hits enumerated",
 	},
 
-	// gate: the English surface ships without modifier eligibility.
+	// The airport gate designator: the English surface ships without modifier eligibility.
 	// These entries cover its localized forms.
 	{
 		designatorID: "gate",

package/lib/subvenue/surfaces.ts

@@ -42,7 +42,7 @@ export function normalizeSurface(text: string): string {
 }
 
 /**
- * A phrase-to-record index keyed by the normalized phrase.
+ * The normalized phrase keys this phrase-to-record index.
  * It attributes a matched phrase to its record.
  */
 export type SurfaceIndex = ReadonlyMap<string, { recordID: string; recordKind: "designator" | "modifier" }>

package/lib/subvenue/table.ts

@@ -104,13 +104,13 @@ export interface SubVenueSurface {
 	/**
 	 * Whether a human has approved this surface for parsing use in its region.
 	 *
-	 * Machine-derived entries start `false`; only a matching {@link SubVenuePromotion}
-	 * flips it, so a consumer that gates a parse must filter on this.
+	 * Machine-derived entries start `false`; only a matching {@link SubVenuePromotion} flips it,
+	 * so a consumer that accepts or refuses a parse by this table must filter on this.
 	 */
 	curated: boolean
 	/**
-	 * How many source features attested this exact phrase when the source counts (OSM, Overture);
-	 * Vocabulary sources attest phrase existence and use a count of `0`.
+	 * The number of source features that attested this exact phrase when the source counts
+	 * (OSM, Overture); Vocabulary sources attest phrase existence and use a count of `0`.
 	 * OSM and Overture sources count occurrences.
 	 */
 	observations: number
@@ -138,7 +138,7 @@ export interface IdentifierShape {
 	/**
 	 * ISO 3166-1 alpha-2 of the extract this distribution was measured in.
 	 *
-	 * It is per-region because the shapes differ (GB gates are 70% bare digits, Japanese platform
+	 * It is per-region because the shapes differ (GB airport gates are 70% bare digits, Japanese platform
 	 * refs a different range), so a recipe for a French address should sample France's distribution.
 	 */
 	region: string
@@ -190,7 +190,7 @@ export interface SubVenueLexiconTable {
  * below for the module docstring's dependency-direction reason.
  *
  * `tier` is added here: the seven WOF placetypes plus `terminal`/`gate` are all venue-interior,
- * while `campus` and `building` name a whole venue as often as a part of one
+ * while `campus` and `building` name a whole venue about as frequently as a part of one
  * but are marked `subvenue` because that is the role the span proposer uses them in.
  */
 export const SHIPPED_DESIGNATOR_SEED: ReadonlyArray<{

package/lib/surfaces/locale.ts

@@ -156,7 +156,7 @@ function tokenPresent(raw: string, value: string): boolean {
  * Render one tuple into a locale-ordered `{raw, components}` row via OpenCage templates,
  * with light variation (house number and postcode may be dropped).
  *
- * Returns `null` when the tuple is too thin or a component would not align cleanly.
+ * Returns `null` when the tuple is too thin or a component would not align to an exact span.
  *
  * Region handling depends on order:
  * - native omits region for alignment reliability

package/lib/synthesizers/anchor-absorption.ts

@@ -16,7 +16,7 @@ import { sample } from "@mailwoman/core/random"
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches defines the
    output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   would hide the distribution behind a wall of identifiers. Decision thresholds in these files are
     extracted as constants above. */
 
 /**

package/lib/synthesizers/boundary-stress.ts

@@ -20,7 +20,7 @@ import type { CanonicalRow } from "#types"
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches is the
    output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind identifiers. The genuine thresholds in these files are
+   would hide the distribution behind identifiers. The decision thresholds in these files are
     extracted as constants above. */
 
 /**
@@ -348,7 +348,8 @@ export function synthesizeBoundaryStressRow(
 
 		// The layout renders the tail in the country's order: `75005 Paris` for FR
 		// and `Boston, MA 02116` for the US.
-		// Dropping the layout's commas is a surface perturbation that applies to any country.
+		// The synthesizer drops the layout's commas.
+		// That is a surface perturbation that applies to any country.
 		const tail = layoutAdminTail(b.country, {
 			locality: b.locality,
 			region: b.region,

package/lib/synthesizers/no-street.ts

@@ -21,7 +21,7 @@ import type { CanonicalRow } from "#types"
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
     cutoffs rather than thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches defines the
    output distribution. The cascade shows each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   would hide the distribution behind a wall of identifiers. Decision thresholds in these files are
     extracted as constants above. */
 
 export interface NoStreetBaseTuple {

package/lib/synthesizers/po-box.ts

@@ -21,7 +21,7 @@ const MIN_DIGITS_FOR_HYPHEN_GROUPING = 3
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs rather than thresholds. `const r = random()` followed by `r < 0.4` branches defines the
    output distribution. The branches show each weight from top to bottom. A separate constant for each cutoff
-   would hide the distribution behind identifiers. Genuine thresholds in these files are
+   would hide the distribution behind identifiers. Decision thresholds in these files are
    constants defined above. */
 
 /**

package/lib/synthesizers/tail.ts

@@ -7,9 +7,9 @@
  *   synthesizer writes `Chicago, IL 60613`, `75005 Paris`, `London, EC3N 1DE` or
  *   `Maracaibo 4001, Zulia` from one call rather than from a template per country.
  *
- *   The layout decides which parts print: the FR layout prints no region, so a FR tail carries none
+ *   The layout decides which parts print: the FR layout prints no region, so a FR tail holds none
  *   and the returned components say so. The lines the layout writes are joined with the country's
- *   line join, which is the one-line form the corpus rows take.
+ *   line join. The result is the one-line form the corpus rows take.
  */
 
 import { formatAddressRow } from "@mailwoman/codex/address/format"

package/lib/test-kit/corpus-recipe.ts

@@ -70,7 +70,7 @@ export async function scratch(prefix: string, tuples: object[], surfaces: string
 export const TEST_REGISTER = "test-harness"
 
 /**
- * Bind a recipe and its seed to a runner the tests call with just the tuples and reserved surfaces.
+ * Bind a recipe and its seed to a runner the tests call with only the tuples and reserved surfaces.
  *
  * The seed is required because these suites assert on generated distributions.
  */

package/lib/tools/align-canonical.ts

@@ -7,8 +7,9 @@ import { createNewlineWriter, JSONSpliterator } from "spliterator"
  *
  * Most synthetic recipe outputs are generated on demand by a recipe (parametrized by --count),
  * so re-emitting them in a new label format is a re-run.
- * A few are fixed corpora with a hand/DeepSeek-authored canonical source that is never regenerated,
- * notably `deepseek-kryptonite` (the adversarial hard-case set) and the `deepseek-translit-*` variants.
+ * A few are fixed corpora with a hand/DeepSeek-authored canonical source that is never regenerated.
+ *
+ * They include `deepseek-kryptonite` (the adversarial hard-case set) and the `deepseek-translit-*` variants.
  *
  * Their committed parquets use the label format that was current when they were first built.
  *

package/lib/tools/audit.ts

@@ -130,7 +130,7 @@ async function scanParquetFiles(corpusDir: PathBuilderLike, sampleCount: number)
 }
 
 function inferSourceFromFilename(filename: string): string {
-	// Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet.
+	// Some corpus builds write part-<source>-<n>.parquet or part-<n>.parquet.
 	// The latter (current build at corpus-v0.3.0) gives no source signal in the filename.
 	// See manifestScan() for the authoritative path.
 	// Return "<unknown>" so the caller flags this case.

package/lib/tools/corpus-stats.ts

@@ -57,7 +57,7 @@ export interface CorpusStatsOptions {
 	parquetPath: string
 	outputPath: string
 	/**
-	 * Read at most this many rows per parquet file.
+	 * Read at most this number of rows per parquet file.
 	 */
 	limitPerFile?: number
 }

package/lib/tools/fetch/download/network.ts

@@ -82,7 +82,7 @@ export interface SourceManifest {
 }
 
 /**
- * A status worth retrying: rate limiting or a server-side failure.
+ * A status the download retries: rate limiting or a server-side failure.
  */
 /**
  * An HTTP failure that includes its status, so callers branch on `error.status`

package/lib/tools/fetch/geonames/dump.ts

@@ -116,7 +116,7 @@ const GAZETTEER_DUMP_COLUMNS = 19
  * The capitals builder already holds each country dump as a string.
  *
  * A byte-oriented spliterator would UTF-8 encode that input and allocate up to
- * another 350 MB just to inspect the first non-empty line.
+ * another 350 MB only to inspect the first non-empty line.
  */
 export function looksLikeGazetteerDump(text: string): boolean {
 	let start = 0

package/lib/tools/fetch/gleif.ts

@@ -14,7 +14,7 @@
  *
  * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
  *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
- *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a truncated body or a
  *    publish replaced mid-transfer is refused rather than recorded.
  * 2. **The host honors `Range`.** An interrupted transfer resumes through
  *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
@@ -79,7 +79,7 @@ export interface GLEIFPublishedFile {
 }
 
 /**
- * The receipt written beside the archive.
+ * The receipt that sits beside the archive.
  */
 export interface GLEIFGoldenCopyReceipt {
 	source: string

package/lib/tools/fetch/header.ts

@@ -6,7 +6,7 @@
  * This module reads a delimited file's header line and refuses one that drops a column an adapter
  * indexes.
  *
- * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * A fetch module checks the file it has written against the columns its adapter reads by name,
  * because a renamed column reaches the adapter as an empty string on every row rather than as an
  * error. The check belongs here rather than in each publisher's module: every caller needs the same
  * two steps, and each one that wrote them again would also be a second answer to how the file is

package/lib/tools/fetch/inspire-addresses.ts

@@ -131,7 +131,7 @@ export interface WFSCapabilities {
 /**
  * A value a GeoServer JSON response uses for a voidable property that holds no value.
  *
- * INSPIRE marks many attributes voidable.
+ * The INSPIRE schema marks a set of attributes voidable.
  * A voidable property must be present with either a value or a void with a reason.
  *
  * GeoServer encodes that void as an object rather than as `null`, so `String(value)`

package/lib/tools/fetch/vlaanderen-ad.ts

@@ -146,7 +146,7 @@ export interface HarvestVlaanderenOptions {
 	 */
 	outputDir: PathBuilderLike
 	/**
-	 * Stop after this many address pages, for a probe rather than a full harvest.
+	 * Stop after this number of address pages, for a probe rather than a full harvest.
 	 */
 	maxPages?: number
 	/**

package/lib/tools/fetch/wfs-harvest.ts

@@ -3,7 +3,7 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- * The paging, manifest and resumption machinery a paged WFS harvest shares.
+ * The paging, manifest and resumption code that every paged WFS harvest shares.
  *
  * `./inspire-addresses.ts` owns what an INSPIRE Addresses service exposes — its capabilities, its
  * feature-type names and its GeoJSON pages. This module owns what a harvest of any paged WFS does
@@ -75,7 +75,7 @@ export interface MarkupRoot {
 	 */
 	attributes: string
 	/**
-	 * The offset just past the opening tag.
+	 * The offset of the first character after the opening tag.
 	 */
 	contentStart: number
 }
@@ -158,7 +158,7 @@ export function rootContent(body: string, root: MarkupRoot, context: string): st
 /**
  * The attributes a page states about itself rather than about the feature type.
  *
- * A harvest assembles many pages under one root element, so the first page's counts
+ * A harvest assembles several pages under one root element, so the first page's counts
  * and timestamp would describe the assembled file wrongly.
  * They are dropped and the manifest records them per page.
  */
@@ -420,7 +420,7 @@ export interface PagedWFSHarvestOptions {
 	featureCount: FeatureCountStatement
 	pageSize: number
 	/**
-	 * Stop once the manifest holds this many pages, for a probe rather than a full harvest.
+	 * Stop once the manifest holds this number of pages, for a probe rather than a full harvest.
 	 *
 	 * A cap on the manifest rather than on the run, so a second run under the same cap makes no request.
 	 */
@@ -437,7 +437,7 @@ export interface PagedWFSHarvestOptions {
  * Does a recorded manifest describe the request being made now?
  *
  * A harvest taken at another page size, ordering, format or type cannot be continued:
- * its pages were cut at different boundaries, and appending to them would leave
+ * its pages were split at different boundaries, and appending to them would leave
  * a file whose manifest describes neither half.
  */
 function describesSameRequest(manifest: WFSHarvestManifest, options: PagedWFSHarvestOptions): boolean {
@@ -932,7 +932,7 @@ export async function runWFSHarvest(
 	if (options.client) {
 		manifest = await options.harvest(options.client)
 	} else {
-		// Bounded retry, because a harvest is many requests and one 5xx would otherwise end the run.
+		// Bounded retry, because a harvest is a long series of requests and one 5xx would otherwise end the run.
 		await using client = new APIClient({
 			displayName: options.displayName,
 			minRequestIntervalMs: options.minRequestIntervalMs,

package/lib/tools/fetch/zip-archive.ts

@@ -4,8 +4,8 @@
  * @author Teffen Ellis, et al.
  * @file Downloads one zip archive through an {@linkcode APIClient} and answers what arrived.
  *
- *   An INSPIRE ATOM service that publishes one archive per area is harvested through many small
- *   requests rather than through one large transfer, so the transfer runs on the client the harvest
+ *   An INSPIRE ATOM service that publishes one archive per area is harvested through one small
+ *   request per area rather than through one large transfer, so the transfer runs on the client the harvest
  *   already paces and a test can drive it over a stubbed transport. `downloadToFile` runs on global
  *   `fetch`, which a unit test cannot intercept.
  *

package/lib/tools/fetch.ts

@@ -19,6 +19,7 @@ import { fetchRyhti } from "#fi/tools/fetch/ryhti"
 import { fetchAnnuaireEducationOverseas } from "#fr/tools/fetch/annuaire-education"
 import { fetchBan } from "#fr/tools/fetch/ban"
 import { fetchFinessOverseas } from "#fr/tools/fetch/finess"
+import { fetchGBOSOpen } from "#gb/tools/fetch/os-open"
 import { fetchITANAC } from "#it/tools/fetch/anac"
 import { fetchHoujinJP } from "#jp/tools/fetch/houjin"
 import { fetchJusoKR } from "#kr/tools/fetch/juso"
@@ -67,6 +68,7 @@ export const FETCH_SOURCES = {
 	"es-navarra": fetchESNavarra,
 	"fr-annuaire-education-overseas": fetchAnnuaireEducationOverseas,
 	"fr-finess-overseas": fetchFinessOverseas,
+	"gb-os-open": fetchGBOSOpen,
 	"gcis-tw": fetchGCISTW,
 	gleif: fetchGLEIF,
 	"houjin-jp": fetchHoujinJP,

package/lib/tools/golden/expand.ts

@@ -231,7 +231,7 @@ async function loadSeeds(
 
 	// Stratified sampling: read all rows from all parquet files, group by source.
 	// Bounded by per-source reservoir: keep at most max(2*count, 5000) rows per source
-	// so we don't blow memory on train parquet files (1M rows × many files).
+	// so we don't blow memory on train parquet files (1M rows per file, across every file).
 	// Later samples are uniform within each pool.
 	const bySource = new Map<string, Seed[]>()
 	const PER_SOURCE_CAP = Math.max(2 * count, 5000)

package/lib/tools/golden/promote.ts

@@ -13,7 +13,7 @@
  *   3. Applies filters that drop candidates unlikely to be human-typed:
  *
  *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last. FR puts it
+ *        - Postcode-at-start with several other components (US/UK conventions put postcode last. FR puts it
  *                 before locality only)
  *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
  *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
@@ -47,7 +47,7 @@ import type { GoldenCandidateEntry as GoldenEntry } from "#utils/golden"
 // ── Types ──────────────────────────────────────────────────────────────────
 
 /**
- * Components a candidate needs before it is rich enough to be worth a golden-set slot.
+ * Components a candidate needs before it qualifies for a golden-set slot.
  */
 const MIN_GOLDEN_COMPONENTS = 5
 
@@ -153,7 +153,7 @@ function isSuspicious(entry: GoldenEntry): boolean {
 
 	if (raw.split('"').length > MAX_QUOTE_SEGMENTS) return true
 
-	// too many quote marks
+	// more quote marks than MAX_QUOTE_SEGMENTS admits
 	return false
 }
 

package/lib/tools/golden/relabel-street/directory.ts

@@ -36,7 +36,8 @@ import {
 export type GoldenRelabelCounts = Record<GoldenRelabelClass, number>
 
 /**
- * One line of the review deck: what a changed (or notably unchanged) row looked like before and after.
+ * One line of the review deck: what a changed row, or an unchanged row the deck
+ * lists for review, looked like before and after.
  */
 export interface GoldenRelabelDeckEntry {
 	file: string

package/lib/tools/lint/recipe-output/vocab.ts

@@ -47,7 +47,7 @@ function stripBIO(label: string): string {
  *
  * Pure-numeric tokens (house_number and postcode) are context-determined
  * rather than lexical vocab, so they are excluded.
- * `\p{Nd}` covers the decimal digits these address corpora actually contain.
+ * `\p{Nd}` covers the decimal digits these address corpora contain.
  */
 function isDigit(token: string): boolean {
 	return token.length > 0 && /^\p{Nd}+$/u.test(token)
@@ -148,7 +148,7 @@ async function readRows(con: DuckDBConnection, path: string): Promise<CorpusRow[
 }
 
 /**
- * Read just the first row's `source` value, used to group base parts for a proportional sample.
+ * Read only the first row's `source` value, used to group base parts for a proportional sample.
  */
 async function readSource(con: DuckDBConnection, path: string): Promise<string> {
 	const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`)

package/lib/tools/lint/recipe-output.ts

@@ -168,7 +168,7 @@ function majorityLabel(distribution: Map<string, number> | Record<string, number
 }
 
 /**
- * One lint flag emitted by a check.
+ * A check emits this lint flag.
  */
 export interface LintFlag {
 	check: string

package/lib/tools/migrate/source-names.ts

@@ -36,7 +36,7 @@ export interface SourceNameRewrite {
 	parquet: string
 	rows: number
 	/**
-	 * The mapping applied, keyed by the retired spelling.
+	 * The migration applied this mapping, keyed by the retired spelling.
 	 *
 	 * A file whose rows already use current spellings is reported with an empty map and left untouched.
 	 */

package/lib/tools/overlay/manifest.ts

@@ -3,7 +3,8 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Assembles a corpus manifest that keeps every file of a base corpus and adds new parquet files.
+ *   Assembles a corpus manifest from a base corpus's files and new parquet files.
+ *   A replaced source's base files are dropped.
  */
 
 import { dataRootPath } from "@mailwoman/core/data-root"
@@ -203,6 +204,10 @@ export interface OverlayPlan {
 	 * The filename was the only place that showed this (#2359).
 	 */
 	files: Array<{ parquet: string; source: string; split: SplitName; route: SplitRoute }>
+	/**
+	 * The base sources whose files this overlay dropped.
+	 */
+	replacedSources: string[]
 }
 
 /**
@@ -240,6 +245,12 @@ export interface OverlayManifestOptions {
 	 */
 	files: readonly OverlayFile[]
 	note: string
+	/**
+	 * Base sources whose files the overlay drops, so the added files take their place.
+	 *
+	 * Each source in this list must appear in the base, and its rows leave the split counts.
+	 */
+	replaceSources?: readonly string[]
 }
 
 /**
@@ -269,7 +280,9 @@ export function localManifestFilePath(path: string): string {
 }
 
 /**
- * Writes a corpus manifest that keeps every file of `args.base` and appends `args.files`.
+ * Writes a corpus manifest that keeps the files of `args.base` and appends `args.files`.
+ *
+ * A base file whose source `args.replaceSources` lists is dropped.
  *
  * A `val` or `test` file must use the `.<split>.parquet` suffix written by `splitOverlaySlice`.
  * The suffix shows that the holdout policy chose its rows.
@@ -291,8 +304,16 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		)
 	}
 
+	const replaced = new Set(args.replaceSources)
+
+	for (const source of replaced) {
+		if (!baseFiles.some((s) => s.source === source)) {
+			throw new Error(`${args.base} holds no file with source '${source}', so there is nothing for it to replace.`)
+		}
+	}
+
 	for (const file of args.files) {
-		if (baseFiles.some((s) => s.source === file.source)) {
+		if (!replaced.has(file.source) && baseFiles.some((s) => s.source === file.source)) {
 			console.log(`WARN: base already contains source '${file.source}' — is this the right base?`)
 		}
 	}
@@ -318,7 +339,24 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		)
 	}
 
-	const kept = baseFiles.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }))
+	const dropped = baseFiles.filter((s) => s.source !== undefined && replaced.has(s.source))
+	const droppedRows: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
+
+	for (const file of dropped) {
+		const split = file["split"] as SplitName
+		const rows = file["rows"]
+
+		if (typeof rows !== "number" || !(split in droppedRows)) {
+			throw new Error(`${file.path} records no row count or split, so dropping it would leave the counts unknown.`)
+		}
+
+		droppedRows[split] += rows
+	}
+
+	const kept = baseFiles
+		.filter((s) => !dropped.includes(s))
+		.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }))
+
 	const added: ParquetFileDescriptor[] = []
 
 	const newDir = PathBuilder.from(args.newDir)
@@ -345,20 +383,31 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 
 	const sources = args.files.map((file) => file.source).join(", ")
 
+	const defaultNote = replaced.size
+		? `${base.corpus_version} files without ${[...replaced].join(", ")} + ${sources}.`
+		: `${base.corpus_version} files (all kept verbatim) + ${sources}. Pure overlay add.`
+
 	const manifest = {
 		corpus_version: args.version,
 		overlay_base: base.corpus_version ?? null,
-		note: args.note || `${base.corpus_version} files (all kept verbatim) + ${sources}. Pure overlay add.`,
+		note: args.note || defaultNote,
 		schema: base.schema,
 		rows_per_slice: baseRowsPerFile(base),
 		row_group_size: base.row_group_size,
 		slices: [...kept, ...added],
 		counts: {
-			train: base.counts.train + addedRows.train,
-			val: base.counts.val + addedRows.val,
-			test: base.counts.test + addedRows.test,
+			train: base.counts.train - droppedRows.train + addedRows.train,
+			val: base.counts.val - droppedRows.val + addedRows.val,
+			test: base.counts.test - droppedRows.test + addedRows.test,
 		},
-		total_rows: base.total_rows + addedRows.train + addedRows.val + addedRows.test,
+		total_rows:
+			base.total_rows -
+			droppedRows.train -
+			droppedRows.val -
+			droppedRows.test +
+			addedRows.train +
+			addedRows.val +
+			addedRows.test,
 		// An overlay corpus's manifest reads no row's `license` column, so it states no license set.
 		// This value in the artifact prevents a reader from deriving an attribution table
 		// from `slices` and treating the base build's set as the whole.
@@ -389,6 +438,7 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 			split: file.split ?? "train",
 			route: splitRouteFor(file.parquet),
 		})),
+		replacedSources: [...replaced],
 	}
 
 	const planPath = newDir(OVERLAY_PLAN_FILE)
@@ -397,7 +447,9 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 
 	console.log(`wrote ${planPath}`)
 	console.log(`wrote ${out}`)
-	console.log(`  files: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`)
+	console.log(
+		`  files: ${manifest.slices.length} (${kept.length} base kept, ${dropped.length} base dropped, +${added.length} added)`
+	)
 	console.log(`  counts: ${stringifyJSON(manifest.counts)}  total: ${manifest.total_rows}`)
 
 	// The route prints beside the split, because a hand-placed split is otherwise visible only in a filename.

package/lib/tools/overlay/split-slice.ts

@@ -15,15 +15,7 @@ import type { ParquetRow } from "#parquet/schema"
 import { openParquetRowStream } from "#parquet/streams"
 import { writeParquetFile } from "#parquet/writers"
 import type { CanonicalRow } from "#types"
-import { type CountryHoldout, defaultHoldouts, type SplitName, splitForRow } from "#utils/split"
-
-/**
- * The component tags the holdout policy reads.
- *
- * `splitForRow` consults `region`, `postcode` and `locality` and no other field,
- * so reconstructing the rest would cost a string slice per span for no decision.
- */
-const HOLDOUT_TAGS = new Set<string>(["region", "postcode", "locality"])
+import { type CountryHoldout, defaultHoldouts, HOLDOUT_COMPONENT_TAGS, type SplitName, splitForRow } from "#utils/split"
 
 /**
  * The fields {@linkcode holdoutComponents} reads, with the span triple optional
@@ -63,7 +55,7 @@ export function holdoutComponents(row: SpannedRow, index: number, input: string)
 	const components: Record<string, string> = {}
 
 	for (const [span, tag] of tags.entries()) {
-		if (!HOLDOUT_TAGS.has(tag) || components[tag]) continue
+		if (!HOLDOUT_COMPONENT_TAGS.has(tag) || components[tag]) continue
 
 		components[tag] = raw.slice(starts[span]!, ends[span]!)
 	}