@mailwoman/corpus 7.2.0 → 7.3.0

package/src/shard-recipes/street.ts

@@ -0,0 +1,74 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `street` shard recipe — synthetic street-decomposition rows for Stage 3 (US-only): tuples →
+ *   {@link synthesizeStreetRow} → aligned LabeledRow. Ported from scripts/build-street-shard.mjs.
+ */
+
+import { stableSourceID } from "../adapter.ts"
+import { synthesizeStreetRow, type StreetBaseTuple } from "../synthesize-street.ts"
+import { alignAndWrite, makeLcg, readTuples, type ShardRecipe } from "./scaffold.ts"
+
+export const streetRecipe: ShardRecipe = {
+	name: "street",
+	description: "Street-decomposition rows (US): tuples → synthesizeStreetRow → aligned LabeledRow",
+	mode: "tuples",
+	options: [{ flag: "--house-number-prob <p>", description: "P(emit a house number). Default 0.85" }],
+	async run(opts, write) {
+		if (!opts.input) throw new Error("street recipe requires --input <tuples.jsonl>")
+		const random = makeLcg(opts.seed)
+		const includeHouseNumberProb = opts.houseNumberProb ?? 0.85
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			read++
+
+			if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
+				skipped++
+				continue
+			}
+
+			if (tuple.country !== "US") {
+				skipped++
+				continue
+			}
+
+			for (let v = 0; v < opts.variants; v++) {
+				const synth = synthesizeStreetRow(tuple as StreetBaseTuple, { random, includeHouseNumberProb })
+
+				if (!synth) continue
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: synth.raw,
+						components: synth.components,
+						country: tuple.country,
+						locale: synth.locale,
+						source: "synth-street",
+						source_id: stableSourceID("synth-street", {
+							locality: `${tuple.locality}#${v}`,
+							region: tuple.region,
+							postcode: tuple.postcode,
+							country: tuple.country,
+						}),
+						corpus_version: "0.4.0",
+						license: "Synthetic — public-domain street name + tuple combination",
+					},
+					"street-decomp"
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}

package/src/shard-recipes/unit.ts

@@ -0,0 +1,315 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `unit` shard recipe — US secondary-unit coverage (#451, the v0-parity `unit` gap). Onto REAL US
+ *   OpenAddresses skeletons (cached zips under `/tmp/oa-cache`) it INJECTS a USPS Pub-28 Appendix
+ *   C2 secondary-unit designator (the `@mailwoman/codex/us` table), varying the surface form
+ *   (canonical "Apartment" vs approved "Apt") AND the unit's POSITION (after-street / unit-first /
+ *   bare / venue-prefixed) per row, so the model learns to RECOGNIZE the designator wherever it
+ *   sits. The inline synthesis (the OA-CSV reader, the designator tables, `makeUnit`/`renderUnit`)
+ *   is ported faithfully from scripts/build-unit-shard.mjs.
+ *
+ *   `--golden`: a held-out eval over the VERMONT source only (the corpus `defaultHoldout`, never
+ *   trained) with a different seed, emitting `{raw, components, country}` for per-locale-f1. Train
+ *   uses every NON-Vermont US source. Designators are injected in both (OA carries none), so the
+ *   eval measures designator recognition on held-out addresses.
+ *
+ *   NOTE: this is a `generate`-mode recipe but it still reads REAL tuples off disk (`unzip` of the
+ *   cached OA zips) — `--count` bounds the OUTPUT, not the input. The passed `random` (the
+ *   framework LCG) is consumed in the exact call order the legacy script used.
+ */
+
+import { spawnSync } from "node:child_process"
+
+import { US_UNIT_DESIGNATOR_PREFERRED_ABBR, type USUnitDesignator } from "@mailwoman/codex/us"
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import { stableSourceID } from "../adapter.ts"
+import { alignRow } from "../align.ts"
+import type { CanonicalRow } from "../types.ts"
+import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+
+/** A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region. */
+interface UnitSource {
+	zip: string
+	csv: string
+	region: string
+}
+
+// OA REGION is empty for US per-state extracts — the region is implied by the file. Train sources are
+// every NON-Vermont state cached; eval is Vermont only (the corpus holdout).
+const TRAIN_SOURCES: readonly UnitSource[] = [
+	{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
+	{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
+	{ zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
+	{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
+	{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
+	{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
+	{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
+]
+const EVAL_SOURCE: UnitSource = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
+
+// USPS Pub-28 C2 designators that take a secondary identifier ("Apt 4B"). Weighted toward the common
+// ones the v0-parity arena failed on (Apt/Ste/Unit/Fl/Rm). Standalone designators (Basement, Lobby,
+// Penthouse) are emitted occasionally with no id.
+const ID_DESIGNATORS: readonly USUnitDesignator[] = [
+	"APARTMENT",
+	"SUITE",
+	"UNIT",
+	"FLOOR",
+	"ROOM",
+	"BUILDING",
+	"DEPARTMENT",
+	"SPACE",
+	"LOT",
+]
+const STANDALONE_DESIGNATORS: readonly USUnitDesignator[] = [
+	"BASEMENT",
+	"LOBBY",
+	"PENTHOUSE",
+	"FRONT",
+	"REAR",
+	"UPPER",
+	"LOWER",
+]
+const ID_WEIGHT = 0.85 // 85% id-bearing designators, 15% standalone
+const SYNTH_IDS: readonly string[] = ["4B", "200", "12", "3", "A", "101", "5", "2A", "310", "B", "7", "1500", "404"]
+
+/** A real US tuple read out of a cached OA zip (number/street/city/postcode + the bare OA unit id). */
+interface UnitTuple {
+	house_number: string
+	street: string
+	locality: string
+	region: string
+	postcode: string
+	oaUnit: string
+}
+
+/** Minimal RFC-4180-ish splitter (handles quoted fields). */
+function splitCSV(line: string): string[] {
+	const out: string[] = []
+	let cur = ""
+	let inQ = false
+
+	for (let i = 0; i < line.length; i++) {
+		const c = line[i]
+
+		if (inQ) {
+			if (c === '"') {
+				if (line[i + 1] === '"') {
+					cur += '"'
+					i++
+				} else {
+					inQ = false
+				}
+			} else {
+				cur += c
+			}
+		} else if (c === '"') {
+			inQ = true
+		} else if (c === ",") {
+			out.push(cur)
+			cur = ""
+		} else {
+			cur += c
+		}
+	}
+	out.push(cur)
+
+	return out
+}
+
+/** Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip. */
+function readTuples(source: UnitSource): UnitTuple[] {
+	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
+
+	if (r.status !== 0) {
+		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
+
+		return []
+	}
+	const lines = r.stdout.toString("utf8").split(/\r?\n/)
+
+	if (lines.length < 2) return []
+	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
+	const idx = (name: string): number => header.indexOf(name)
+	const iNum = idx("number"),
+		iStreet = idx("street"),
+		iUnit = idx("unit"),
+		iCity = idx("city"),
+		iPost = idx("postcode")
+	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
+	const tuples: UnitTuple[] = []
+	const seen = new Set<string>()
+
+	for (let li = 1; li < lines.length; li++) {
+		if (!lines[li]) continue
+		const cells = splitCSV(lines[li]!)
+		const street = get(cells, iStreet)
+		const locality = get(cells, iCity)
+		const house_number = get(cells, iNum)
+
+		if (!street || !locality || !house_number) continue
+		const key = `${house_number}|${street}|${locality}`.toLowerCase()
+
+		if (seen.has(key)) continue
+		seen.add(key)
+		tuples.push({
+			house_number,
+			street,
+			locality,
+			region: source.region,
+			postcode: get(cells, iPost),
+			oaUnit: get(cells, iUnit),
+		})
+	}
+
+	return tuples
+}
+
+/** Title-case a canonical/abbrev designator ("APARTMENT" → "Apartment", "APT" → "Apt"). */
+const title = (s: string): string => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase()
+
+/** Build an injected unit string ("Apt 4B"), varying canonical vs approved-abbrev form per row. */
+function makeUnit(random: () => number, oaUnit: string): string {
+	const standalone = random() >= ID_WEIGHT
+	const pool = standalone ? STANDALONE_DESIGNATORS : ID_DESIGNATORS
+	const canonical = pool[Math.floor(random() * pool.length)]!
+	// Vary the surface form 50/50 (this is the #454 expand/abbreviate variety, baked into the shard).
+	const designator = random() < 0.5 ? title(canonical) : title(US_UNIT_DESIGNATOR_PREFERRED_ABBR[canonical])
+
+	if (standalone) return designator
+	const id = oaUnit && oaUnit.length <= 6 ? oaUnit : SYNTH_IDS[Math.floor(random() * SYNTH_IDS.length)]!
+
+	return `${designator} ${id}`
+}
+
+/** Synthetic recipient/venue prefixes — the "JOHN DOE, ACME INC, ..." arena pattern. */
+const VENUES: readonly string[] = [
+	"John Doe",
+	"Jane Smith",
+	"Acme Inc",
+	"Wayne Enterprises",
+	"Stark Industries",
+	"Globex Corp",
+	"Maria Garcia",
+	"Robert Chen",
+	"Oak Street Dental",
+	"Riverside Clinic",
+]
+
+/** Address tail: "City, ST 12345" (or no postcode). */
+const tail = (loc: string, reg: string, pc: string): string => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`)
+
+/**
+ * Render a unit row in a RANDOM layout — units spread across positions, the city/state tail dropped on bare rows, a
+ * recipient/venue prefixed on the venue format — so the model learns to RECOGNIZE the designator wherever it sits.
+ * Returns {fmt, raw, components}.
+ */
+function renderUnit(
+	random: () => number,
+	base: UnitTuple,
+	unit: string
+): { fmt: string; raw: string; components: Partial<Record<ComponentTag, string>> } {
+	const hn = base.house_number,
+		street = base.street,
+		loc = base.locality,
+		reg = base.region,
+		pc = base.postcode
+	const road = `${hn} ${street}`
+	const full: Partial<Record<ComponentTag, string>> = {
+		house_number: hn,
+		street,
+		unit,
+		locality: loc,
+		region: reg,
+		...(pc ? { postcode: pc } : {}),
+	}
+	const r = random()
+
+	if (r < 0.34) return { fmt: "full-after", raw: `${road} ${unit}, ${tail(loc, reg, pc)}`, components: full }
+
+	if (r < 0.52) return { fmt: "full-first", raw: `${unit}, ${road}, ${tail(loc, reg, pc)}`, components: full }
+
+	if (r < 0.68) return { fmt: "bare-after", raw: `${road} ${unit}`, components: { house_number: hn, street, unit } }
+
+	if (r < 0.84) return { fmt: "bare-first", raw: `${unit} ${road}`, components: { house_number: hn, street, unit } }
+	const v = VENUES[Math.floor(random() * VENUES.length)]!
+
+	return { fmt: "venue", raw: `${v}, ${road} ${unit}, ${tail(loc, reg, pc)}`, components: { venue: v, ...full } }
+}
+
+export const unitRecipe: ShardRecipe = {
+	name: "unit",
+	description: "US secondary-unit rows (#451): real OA skeletons + injected USPS Pub-28 C2 unit designators",
+	mode: "generate",
+	options: [{ flag: "--golden", description: "Emit the held-out VT eval slice ({raw, components, country})" }],
+	async run(opts, write) {
+		if (opts.count == null) throw new Error("unit recipe requires --count <N>")
+		const count = opts.count
+		// Legacy build-unit-shard.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
+		const random = makeMulberry32(opts.seed)
+		const source = opts.sourceName ?? "synth-unit"
+		const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
+
+		const pool: UnitTuple[] = []
+
+		for (const s of sources) {
+			const t = readTuples(s)
+			console.error(`  ${s.csv}: ${t.length} unique tuples`)
+
+			for (const x of t) {
+				pool.push(x)
+			}
+		}
+
+		if (pool.length === 0) {
+			throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?")
+		}
+
+		let emitted = 0
+		let skipped = 0
+		let guard = 0
+		const N = pool.length
+
+		while (emitted < count && guard++ < count * 6) {
+			const base = pool[Math.floor(random() * N)]!
+			const unit = makeUnit(random, base.oaUnit)
+			const { raw, components } = renderUnit(random, base, unit)
+
+			// The unit must survive verbatim in raw, else alignment can't label it.
+			if (!raw.includes(unit)) {
+				skipped++
+				continue
+			}
+
+			if (opts.golden) {
+				write(JSON.stringify({ raw, components, country: "US" }) + "\n")
+				emitted++
+				continue
+			}
+			const canonical: CanonicalRow = {
+				raw,
+				components,
+				country: "US",
+				locale: "en-US",
+				source,
+				source_id: stableSourceID(source, components),
+				corpus_version: "0.4.0",
+				license: "OpenAddresses US (non-VT) skeletons + injected USPS Pub-28 C2 unit designators",
+			}
+			const aligned = alignRow(canonical)
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				skipped++
+				continue
+			}
+			write(JSON.stringify({ ...aligned.row, synth_method: "unit", synth_base_id: null }) + "\n")
+			emitted++
+		}
+
+		return { emitted, skipped }
+	},
+}

package/src/split.ts

@@ -0,0 +1,251 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+
+import { spawn } from "node:child_process"
+import { createHash } from "node:crypto"
+import { createWriteStream } from "node:fs"
+import { mkdir, unlink, writeFile } from "node:fs/promises"
+import { join } from "node:path"
+
+import { childEnv } from "@mailwoman/core/scripting/utils"
+import { JSONSpliterator } from "spliterator"
+
+import type { CanonicalRow, LabeledRow } from "./types.ts"
+
+export type SplitName = "train" | "val" | "test"
+
+export interface SplitOptions {
+	/**
+	 * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
+	 * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
+	 * `defaultHoldouts()`.
+	 */
+	holdouts?: Record<string, readonly string[]>
+}
+
+/** Output manifest: source_id lists per split. */
+export interface SplitManifest {
+	train: string[]
+	val: string[]
+	test: string[]
+	/** Echoes the holdouts used, so the manifest is self-describing. */
+	holdouts: Record<string, readonly string[]>
+	/** Corpus version stamped onto the manifest. Read from the first row. */
+	corpus_version: string
+	/** Counts for quick sanity checks. */
+	counts: { train: number; val: number; test: number; total: number }
+}
+
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
+ *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
+ *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
+ *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ */
+export function defaultHoldouts(): Record<string, readonly string[]> {
+	return {
+		US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
+		FR: ["Corse", "Lozère", "Lozere", "Creuse"],
+		DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+	}
+}
+
+type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">
+
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
+ * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
+ * array-based path so the decision is stable regardless of caller.
+ */
+export function splitForRow(
+	row: Pick<SplitInputRow, "source_id" | "country" | "components">,
+	holdouts: Record<string, readonly string[]> = defaultHoldouts()
+): SplitName {
+	const region = row.components.region
+	const countryHoldouts = holdouts[row.country] ?? []
+	const isHeldOut = region !== undefined && countryHoldouts.includes(region)
+
+	if (!isHeldOut) return "train"
+
+	// 50/50 deterministic by source_id hash. Same input always lands in the same split.
+	return hashBucket(row.source_id, 2) === 0 ? "val" : "test"
+}
+
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
+ * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
+ * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
+ * split membership in heap.
+ */
+export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}): SplitManifest {
+	const holdouts = opts.holdouts ?? defaultHoldouts()
+	const train: string[] = []
+	const val: string[] = []
+	const test: string[] = []
+	let corpus_version = ""
+
+	for (const row of rows) {
+		if (!corpus_version && row.corpus_version) {
+			corpus_version = row.corpus_version
+		}
+		const split = splitForRow(row, holdouts)
+
+		if (split === "train") {
+			train.push(row.source_id)
+		} else if (split === "val") {
+			val.push(row.source_id)
+		} else {
+			test.push(row.source_id)
+		}
+	}
+
+	const total = train.length + val.length + test.length
+
+	return {
+		train,
+		val,
+		test,
+		holdouts,
+		corpus_version,
+		counts: { train: train.length, val: val.length, test: test.length, total },
+	}
+}
+
+/** Lightweight deterministic 0..(n-1) bucket from a string id. */
+export function hashBucket(id: string, n: number): number {
+	const digest = createHash("sha256").update(id).digest()
+	// Read 4 bytes as uint32 to avoid bigint overhead.
+	const u = digest[0]! * 0x01_00_00_00 + digest[1]! * 0x01_00_00 + digest[2]! * 0x01_00 + digest[3]!
+
+	return u % n
+}
+
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
+ * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
+ * including holdouts + counts + corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export async function writeSplitManifests(manifest: SplitManifest, outputDir: string): Promise<void> {
+	await mkdir(outputDir, { recursive: true })
+
+	for (const name of ["train", "val", "test"] as const) {
+		const sorted = [...manifest[name]].sort()
+		await writeFile(join(outputDir, `${name}.txt`), sorted.join("\n") + (sorted.length ? "\n" : ""), "utf8")
+	}
+	const summary = {
+		corpus_version: manifest.corpus_version,
+		holdouts: manifest.holdouts,
+		counts: manifest.counts,
+	}
+	await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
+}
+
+/** Type re-export for callers that want to ingest LabeledRow specifically. */
+export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">
+
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
+ * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
+ * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
+ * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ */
+export async function writeSplitManifestsFromLabeledFiles(opts: {
+	labeledPaths: Record<SplitName, string>
+	outputDir: string
+	corpusVersion: string
+	counts: Record<SplitName, number>
+	holdouts?: Record<string, readonly string[]>
+}): Promise<SplitManifest["counts"]> {
+	await mkdir(opts.outputDir, { recursive: true })
+	const holdouts = opts.holdouts ?? defaultHoldouts()
+
+	for (const split of ["train", "val", "test"] as const) {
+		const labeledPath = opts.labeledPaths[split]
+		const outPath = join(opts.outputDir, `${split}.txt`)
+		await streamSortedSourceIds(labeledPath, outPath)
+	}
+
+	const total = opts.counts.train + opts.counts.val + opts.counts.test
+	const summary = {
+		corpus_version: opts.corpusVersion,
+		holdouts,
+		counts: { ...opts.counts, total },
+	}
+	await writeFile(join(opts.outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8")
+
+	return summary.counts
+}
+
+/**
+ * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
+ * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
+ */
+async function streamSortedSourceIds(labeledJsonlPath: string, outPath: string): Promise<void> {
+	const unsortedPath = `${outPath}.unsorted`
+	const out = createWriteStream(unsortedPath, { encoding: "utf8" })
+	const outClosed = new Promise<void>((resolve, reject) => {
+		out.on("close", () => resolve())
+		out.on("error", reject)
+	})
+
+	// JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
+	// SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
+	// always ends the write stream so `sort` reads a complete file even if the read throws.
+	try {
+		for await (const obj of JSONSpliterator.fromAsync<{ source_id?: string }>(labeledJsonlPath)) {
+			if (typeof obj.source_id === "string") {
+				out.write(`${obj.source_id}\n`)
+			}
+		}
+	} finally {
+		out.end()
+	}
+	await outClosed
+
+	await new Promise<void>((resolve, reject) => {
+		// LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+		const proc = spawn("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) })
+		proc.on("error", reject)
+		proc.on("exit", (code) => {
+			if (code === 0) {
+				resolve()
+			} else {
+				reject(new Error(`sort exited with code ${code}`))
+			}
+		})
+	})
+	await unlink(unsortedPath).catch(() => {})
+}