@mailwoman/corpus 8.5.0 → 9.0.0

package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts

@@ -28,11 +28,9 @@
  *   distribution terms.
  */
 
-import { createReadStream } from "node:fs"
+import { CSVSpliterator } from "spliterator"
 
-import { parse as csvParse } from "csv-parse"
-
-import { stableSourceID } from "../../adapter.ts"
+import { splitStreetLine, stableSourceID } from "../../adapter.ts"
 import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
 import { reconcileComponents } from "../../format.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
@@ -66,19 +64,6 @@ interface SamhsaSiteRow {
 	frid?: string
 }
 
-const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
-
-function splitAddress(address: string): { house_number?: string; street: string } | null {
-	const trimmed = address.trim()
-
-	if (!trimmed) return null
-	const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
-
-	if (m) return { house_number: m[1], street: m[2]!.trim() }
-
-	return { street: trimmed }
-}
-
 /**
  * Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
  * building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
@@ -148,76 +133,65 @@ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
 				throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
 			}
 
-			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
-
-			const parser = stream.pipe(
-				csvParse({
-					columns: true,
-					skip_empty_lines: true,
-					relax_quotes: true,
-					relax_column_count: true,
-				})
-			)
+			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
+				mode: "object",
+				normalizeKeys: false,
+				enableQuoteHandling: true,
+			})
 
 			let emitted = 0
 
-			try {
-				for await (const record of parser as AsyncIterable<SamhsaSiteRow>) {
-					if (opts.signal?.aborted) break
-
-					if (opts.limit !== undefined && emitted >= opts.limit) break
+			for await (const record of rows as AsyncIterable<SamhsaSiteRow>) {
+				if (opts.signal?.aborted) break
 
-					const venue = composeVenue(record.name1 ?? "", record.name2)
-					const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
-					const split = splitAddress(street)
-					const city = (record.city ?? "").trim()
-					const stateAbbr = (record.state ?? "").trim()
-					const postcode = (record.zip ?? "").trim()
+				if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					if (!venue || !split || !city || !postcode) continue
-					const state = lookupStateAbbreviation(stateAbbr)
+				const venue = composeVenue(record.name1 ?? "", record.name2)
+				const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
+				const split = splitStreetLine(street)
+				const city = (record.city ?? "").trim()
+				const stateAbbr = (record.state ?? "").trim()
+				const postcode = (record.zip ?? "").trim()
 
-					if (!state) continue
+				if (!venue || !split || !city || !postcode) continue
+				const state = lookupStateAbbreviation(stateAbbr)
 
-					// venue first — same kryptonite-defending insertion order as HRSA.
-					const components: CanonicalRow["components"] = {
-						venue,
-						...(split.house_number ? { house_number: split.house_number } : {}),
-						street: split.street,
-						locality: city,
-						region: state.abbreviation,
-						postcode,
-					}
+				if (!state) continue
 
-					const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
+				// venue first — same kryptonite-defending insertion order as HRSA.
+				const components: CanonicalRow["components"] = {
+					venue,
+					...(split.house_number ? { house_number: split.house_number } : {}),
+					street: split.street,
+					locality: city,
+					region: state.abbreviation,
+					postcode,
+				}
 
-					if (!raw) continue
+				const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
 
-					const aligned = reconcileComponents(components, raw)
+				if (!raw) continue
 
-					if (!Object.keys(aligned).length) continue
+				const aligned = reconcileComponents(components, raw)
 
-					const frID = (record.frid ?? "").trim()
+				if (!Object.keys(aligned).length) continue
 
-					const sourceID = frID
-						? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}`
-						: stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
+				const frID = (record.frid ?? "").trim()
 
-					yield {
-						raw,
-						components: aligned,
-						country: "US",
-						locale: "en-US",
-						source: USGOV_SAMHSA_ADAPTER_ID,
-						source_id: sourceID,
-						corpus_version: "",
-						license: USGOV_SAMHSA_DEFAULT_LICENSE,
-					}
+				const sourceID = frID ? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}` : stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
 
-					emitted++
+				yield {
+					raw,
+					components: aligned,
+					country: "US",
+					locale: "en-US",
+					source: USGOV_SAMHSA_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: USGOV_SAMHSA_DEFAULT_LICENSE,
 				}
-			} finally {
-				stream.destroy()
+
+				emitted++
 			}
 		},
 	}

package/src/adapters/wof-admin-jp/adapter.ts

@@ -59,15 +59,21 @@ interface NameRow {
 }
 
 /**
- * Walk parent chain up to 6 levels.
+ * Walk parent chain up to 6 levels, over a place table already resident in memory.
+ *
+ * `resolve` is the escape hatch for a parent outside the preloaded set (a chain that leaves the country); it is a point
+ * query, so keeping the common case out of it is the whole point.
  */
-function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, string>): PlaceRow[] {
-	const stmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
+function chainOf(
+	byID: Map<number, PlaceRow>,
+	resolve: (id: number) => PlaceRow | undefined,
+	startID: number
+): PlaceRow[] {
 	const out: PlaceRow[] = []
 	let id = startID
 
 	for (let i = 0; i < MAX_ANCESTRY_DEPTH && id > 0; i++) {
-		const row = stmt.get(id) as PlaceRow | undefined
+		const row = byID.get(id) ?? resolve(id)
 
 		if (!row) break
 		out.push(row)
@@ -151,9 +157,29 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 					}
 				}
 
-				const seeds = db.prepare(`SELECT id FROM spr WHERE country='JP' AND placetype='neighbourhood'`).all() as {
-					id: number
-				}[]
+				// One read of the JP place table instead of a fresh `prepare` + up to six point queries per
+				// seed; there are tens of thousands of neighbourhood seeds.
+				const byID = new Map<number, PlaceRow>()
+
+				for (const row of db
+					.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`)
+					.all() as unknown as PlaceRow[]) {
+					byID.set(row.id, row)
+				}
+
+				const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
+
+				const resolveOutside = (id: number): PlaceRow | undefined => {
+					const row = outsideStmt.get(id) as unknown as PlaceRow | undefined
+
+					if (row) {
+						byID.set(id, row)
+					}
+
+					return row
+				}
+
+				const seeds = [...byID.values()].filter((row) => row.placetype === "neighbourhood")
 
 				let emitted = 0
 
@@ -162,7 +188,7 @@ export function createWOFAdminJpAdapter(): CorpusAdapter {
 
 					if (opts.limit !== undefined && emitted >= opts.limit) break
 
-					const chain = chainOf(db, seed.id, jpnNames)
+					const chain = chainOf(byID, resolveOutside, seed.id)
 					const synth = synthesizeJpAddress(chain, jpnNames)
 
 					if (!synth) continue

package/src/build.ts

@@ -52,6 +52,7 @@ import { mkdir, writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
 import { $public } from "@mailwoman/core/env"
+import { parseJSONStrict } from "@mailwoman/core/objects"
 import { JSONSpliterator } from "spliterator"
 
 import { defaultAdapterRegistry } from "./adapter.ts"
@@ -188,7 +189,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 			existsSync(cachedManifest) &&
 			existsSync(join(adapterDir, "canonical.jsonl"))
 		) {
-			const cached = JSON.parse(readFileSync(cachedManifest, "utf8")) as AdapterRunManifest
+			const cached = parseJSONStrict<AdapterRunManifest>(readFileSync(cachedManifest, "utf8"))
 			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
 			adapterRuns.push(cached)
 

package/src/golden.ts

@@ -19,10 +19,11 @@
  *   creep in over time as new entries land.
  */
 
-import { readdir, readFile } from "node:fs/promises"
-import { extname, join } from "node:path"
+import { readdir } from "node:fs/promises"
 
 import { COMPONENT_TAGS, type ComponentTag } from "@mailwoman/core/types"
+import { join } from "path-ts"
+import { TextSpliterator } from "spliterator"
 
 import { reconcileComponents } from "./format.ts"
 
@@ -61,6 +62,9 @@ export interface GoldenReport {
  * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
  */
 export function parseGoldenLine(line: string): GoldenEntry {
+	// The throw IS the result: `validateGoldenFile` catches it and records the message against the line
+	// number, so a tolerant parse would report a corrupt row as valid.
+	// oxlint-disable-next-line no-restricted-properties
 	const obj = JSON.parse(line) as Partial<GoldenEntry> & Record<string, unknown>
 
 	if (typeof obj.raw !== "string" || !obj.raw.length) {
@@ -112,16 +116,23 @@ export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 
 /**
  * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ *
+ * Parses line by line over `TextSpliterator` rather than `JSONSpliterator`: every issue this returns carries the LINE
+ * NUMBER it was found on, and a malformed line has to be REPORTED rather than thrown. `JSONSpliterator` parses each row
+ * for you and throws on the first bad one — correct for consumers that want the rows, wrong for the validator whose
+ * whole job is locating the bad ones.
  */
 export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
-	const text = await readFile(path, "utf8")
-	const lines = text.split("\n")
 	const issues: GoldenIssue[] = []
+	// Counted over EVERY row including blanks, so the number matches what an editor shows.
+	let lineNumber = 0
 
-	for (let i = 0; i < lines.length; i++) {
-		const line = lines[i]!.trim()
+	for await (const raw of TextSpliterator.fromAsync(path, { skipEmpty: false })) {
+		lineNumber++
+		const line = raw.trim()
 
 		if (!line) continue
+		const i = lineNumber - 1
 
 		try {
 			const entry = parseGoldenLine(line)
@@ -146,7 +157,7 @@ export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
  * Validate every `.jsonl` in a golden directory.
  */
 export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
-	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").toSorted()
+	const files = (await readdir(dir)).filter((n) => n.endsWith(".jsonl")).toSorted()
 	const issues: GoldenIssue[] = []
 	let entries = 0
 
@@ -154,8 +165,12 @@ export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
 		const fullPath = join(dir, name)
 		const fileIssues = await validateGoldenFile(fullPath)
 		issues.push(...fileIssues)
-		const text = await readFile(fullPath, "utf8")
-		entries += text.split("\n").filter((l) => l.trim()).length
+
+		for await (const line of TextSpliterator.fromAsync(fullPath)) {
+			if (line.trim()) {
+				entries++
+			}
+		}
 	}
 
 	return { entries, files: files.length, issues }

package/src/parquet-wrapper/reader.ts

@@ -59,6 +59,25 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
 		return super[Symbol.asyncIterator]() as AsyncGenerator<T, void, unknown>
 	}
 
+	/**
+	 * Iterate a subset of the columns, narrowed to the keys asked for.
+	 *
+	 * Parquet is columnar, so a projection is read avoided rather than read-then-discarded: on a 14-column corpus shard,
+	 * two columns cost 93 ms against 253 ms for the whole row. The default iterator reads every column, which is what a
+	 * caller wanting the whole record should use.
+	 */
+	public async *project<K extends keyof T>(...columns: K[]): AsyncGenerator<Pick<T, K>, void, unknown> {
+		const cursor = this.getCursor(columns.map((column) => [column as string]))
+
+		for (;;) {
+			const row = (await cursor.next()) as Pick<T, K> | null
+
+			if (!row) return
+
+			yield row
+		}
+	}
+
 	public async [Symbol.asyncDispose]() {
 		return this.close()
 	}

package/src/parquet-wrapper/schema.ts

@@ -24,6 +24,10 @@ import { LRUCache } from "lru-cache"
 /**
  * A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
  */
+/**
+ * Shape a row type must satisfy. Declare yours as a `type` alias — TypeScript gives an alias an implicit index
+ * signature and an `interface` none, so an otherwise-identical interface does not satisfy this.
+ */
 export type ParquetRecordLike = Record<string, unknown | undefined>
 
 /**

package/src/parquet.ts

@@ -37,11 +37,12 @@
  *   close — cheap relative to writing it).
  */
 
-import { createHash } from "node:crypto"
 import { createReadStream } from "node:fs"
 import { mkdir, stat, writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
+import { sha256File } from "@mailwoman/core/utils"
+
 import { ParquetWriter, type ParquetSchemaDefinition } from "./parquet-wrapper/index.ts"
 import type { SplitName } from "./split.ts"
 import type { LabeledRow } from "./types.ts"
@@ -311,7 +312,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 
 			if (shardRows > 0) {
 				const fileStat = await stat(path)
-				const sha256 = await hashFile(path)
+				const sha256 = await sha256File(path)
 
 				shards.push({
 					split,
@@ -375,17 +376,3 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 
 	return manifest
 }
-
-/**
- * Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput.
- */
-async function hashFile(path: string): Promise<string> {
-	const hash = createHash("sha256")
-	const stream = createReadStream(path)
-
-	for await (const chunk of stream) {
-		hash.update(chunk as Buffer)
-	}
-
-	return hash.digest("hex")
-}

package/src/shard-recipes/anchor-absorption.ts

@@ -14,6 +14,7 @@
 
 import { readFileSync } from "node:fs"
 
+import { parseJSONStrict } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
 
 import { stableSourceID } from "../adapter.ts"
@@ -31,7 +32,7 @@ const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
  * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
  */
 function loadRealUsZips(path: string): string[] {
-	const d = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>
+	const d = parseJSONStrict<Record<string, unknown>>(readFileSync(path, "utf8"))
 	const zips: string[] = []
 
 	for (const [pc, v] of Object.entries(d)) {

package/src/shard-recipes/country-balanced.ts

@@ -28,11 +28,12 @@ import { spawnSync } from "node:child_process"
 
 import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
 import type { ComponentTag } from "@mailwoman/core/types"
+import { dataRootPath } from "@mailwoman/core/utils"
 
 import { stableSourceID } from "../adapter.ts"
 import { alignRow } from "../align.ts"
 import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 // v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
 // ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
@@ -72,24 +73,72 @@ interface CountrySource {
  * countrywide extracts (FR/IT/NL) read region from the CSV when present.
  */
 const SOURCES: readonly CountrySource[] = [
-	{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", iso2: "US", region: "IA", order: "us" },
-	{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
-	{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", iso2: "US", region: "MT", order: "us" },
-	{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", iso2: "US", region: "SD", order: "us" },
-	{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", iso2: "DE", region: "", order: "eu" },
-	{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv", iso2: "FR", region: "", order: "fr" },
+	{
+		zip: dataRootPath("oa-cache", "us__ia__statewide.zip"),
+		csv: "us/ia/statewide.csv",
+		iso2: "US",
+		region: "IA",
+		order: "us",
+	},
+	{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", iso2: "US", region: "IL", order: "us" },
+	{
+		zip: dataRootPath("oa-cache", "us__mt__statewide.zip"),
+		csv: "us/mt/statewide.csv",
+		iso2: "US",
+		region: "MT",
+		order: "us",
+	},
+	{
+		zip: dataRootPath("oa-cache", "us__sd__statewide.zip"),
+		csv: "us/sd/statewide.csv",
+		iso2: "US",
+		region: "SD",
+		order: "us",
+	},
+	{
+		zip: dataRootPath("oa-cache", "de__sn__statewide.zip"),
+		csv: "de/sn/statewide.csv",
+		iso2: "DE",
+		region: "",
+		order: "eu",
+	},
+	{
+		zip: dataRootPath("oa-cache", "fr__countrywide.zip"),
+		csv: "fr/countrywide.csv",
+		iso2: "FR",
+		region: "",
+		order: "fr",
+	},
 	// ES uses the Spanish IGN schema, not the OA standard columns — skipped here (codex still recognizes
 	// "España"/"Spain"). A dedicated IGN adapter is a follow-up.
-	{ zip: "/tmp/oa-cache/it__countrywide.zip", csv: "it/countrywide.csv", iso2: "IT", region: "", order: "eu" },
-	{ zip: "/tmp/oa-cache/nl__countrywide.zip", csv: "nl/countrywide.csv", iso2: "NL", region: "", order: "eu" },
+	{
+		zip: dataRootPath("oa-cache", "it__countrywide.zip"),
+		csv: "it/countrywide.csv",
+		iso2: "IT",
+		region: "",
+		order: "eu",
+	},
+	{
+		zip: dataRootPath("oa-cache", "nl__countrywide.zip"),
+		csv: "nl/countrywide.csv",
+		iso2: "NL",
+		region: "",
+		order: "eu",
+	},
 ]
 
 /**
  * Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
  */
 const EVAL_SOURCES: readonly CountrySource[] = [
-	{ zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", iso2: "US", region: "VT", order: "us" },
-	{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
+	{
+		zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
+		csv: "us/vt/statewide.csv",
+		iso2: "US",
+		region: "VT",
+		order: "us",
+	},
+	{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", iso2: "DE", region: "", order: "eu" },
 ]
 
 /**
@@ -105,45 +154,9 @@ interface CountryTuple {
 	order: string
 }
 
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-
-	let cur = "",
-		inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
-
 function readTuples(source: CountrySource, limit: number): CountryTuple[] {
 	// countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
-	// tuple to survive dedup/skips) so the toString stays under V8's string limit.
+	// tuple to survive dedup/skips) so the whole extract never has to be held in memory.
 	const maxLines = Math.max(limit * 8, 20_000) + 1
 
 	const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
@@ -157,29 +170,15 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
 		return []
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (n: string): number => header.indexOf(n)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iCity = idx("city"),
-		iRegion = idx("region"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: CountryTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length && tuples.length < limit; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
+	for (const row of readCSVRecords(r.stdout)) {
+		if (tuples.length >= limit) break
 
-		const street = get(cells, iStreet),
-			locality = get(cells, iCity),
-			house_number = get(cells, iNum)
+		const street = row.street ?? "",
+			locality = row.city ?? "",
+			house_number = row.number ?? ""
 
 		if (!street || !locality || !house_number) continue
 		const key = `${house_number}|${street}|${locality}`.toLowerCase()
@@ -191,8 +190,8 @@ function readTuples(source: CountrySource, limit: number): CountryTuple[] {
 			house_number,
 			street,
 			locality,
-			region: get(cells, iRegion) || source.region,
-			postcode: get(cells, iPost),
+			region: row.region || source.region,
+			postcode: row.postcode ?? "",
 			iso2: source.iso2,
 			order: source.order,
 		})
@@ -460,7 +459,7 @@ export const countryBalancedRecipe: ShardRecipe = {
 		}
 
 		if (!pool.length) {
-			throw new Error("No tuples — are the cached OA zips present in /tmp/oa-cache?")
+			throw new Error(`No tuples — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
 		}
 
 		let emitted = 0

package/src/shard-recipes/fr-fragment.ts

@@ -68,7 +68,7 @@
  *   blow-up). Disable that mask for any run including this shard.
  */
 
-import { readFileSync } from "node:fs"
+import { TextSpliterator } from "spliterator"
 
 import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
@@ -187,12 +187,15 @@ export const frFragmentRecipe: ShardRecipe = {
 			)
 		}
 
-		const excluded = new Set(
-			readFileSync(excludePath, "utf8")
-				.split("\n")
-				.map((line) => line.trim())
-				.filter((line) => line && !line.startsWith("#"))
-		)
+		const excluded = new Set<string>()
+
+		for await (const line of TextSpliterator.fromAsync(excludePath)) {
+			const trimmed = line.trim()
+
+			if (trimmed && !trimmed.startsWith("#")) {
+				excluded.add(trimmed)
+			}
+		}
 
 		if (!excluded.size) throw new Error(`fr-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)