@mailwoman/corpus 8.6.0 → 9.1.0

package/src/shard-recipes/unit.ts

@@ -4,7 +4,7 @@
  * @author Teffen Ellis, et al.
  *
  *   `unit` shard recipe — US secondary-unit coverage (#451, the v0-parity `unit` gap). Onto REAL US
- *   OpenAddresses skeletons (cached zips under `/tmp/oa-cache`) it INJECTS a USPS Pub-28 Appendix
+ *   OpenAddresses skeletons (cached zips under `$MAILWOMAN_DATA_ROOT/oa-cache`) it INJECTS a USPS Pub-28 Appendix
  *   C2 secondary-unit designator (the `@mailwoman/codex/us` table), varying the surface form
  *   (canonical "Apartment" vs approved "Apt") AND the unit's POSITION (after-street / unit-first /
  *   bare / venue-prefixed) per row, so the model learns to RECOGNIZE the designator wherever it
@@ -21,15 +21,14 @@
  *   framework LCG) is consumed in the exact call order the legacy script used.
  */
 
-import { spawnSync } from "node:child_process"
-
 import { US_UNIT_DESIGNATOR_PREFERRED_ABBR, type USUnitDesignator } from "@mailwoman/codex/us"
 import type { ComponentTag } from "@mailwoman/core/types"
+import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /**
  * A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region.
@@ -58,16 +57,20 @@ interface UnitSource {
  * state cached; eval is Vermont only (the corpus holdout).
  */
 const TRAIN_SOURCES: readonly UnitSource[] = [
-	{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
-	{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
-	{ zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
-	{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
-	{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
-	{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
-	{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
+	{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
+	{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
+	{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
+	{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
+	{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
 ]
 
-const EVAL_SOURCE: UnitSource = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
+const EVAL_SOURCE: UnitSource = {
+	zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
+	csv: "us/vt/statewide.csv",
+	region: "VT",
+}
 
 /**
  * USPS Pub-28 C2 designators that take a secondary identifier ("Apt 4B"). Weighted toward the common ones the v0-parity
@@ -114,78 +117,17 @@ interface UnitTuple {
 	oaUnit: string
 }
 
-/**
- * Minimal RFC-4180-ish splitter (handles quoted fields).
- */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
-
 /**
  * Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip.
  */
-function readTuples(source: UnitSource): UnitTuple[] {
-	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
-
-		return []
-	}
-
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (name: string): number => header.indexOf(name)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iUnit = idx("unit"),
-		iCity = idx("city"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
+async function readTuples(source: UnitSource): Promise<UnitTuple[]> {
 	const tuples: UnitTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-		const street = get(cells, iStreet)
-		const locality = get(cells, iCity)
-		const house_number = get(cells, iNum)
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
+		const street = row.street ?? ""
+		const locality = row.city ?? ""
+		const house_number = row.number ?? ""
 
 		if (!street || !locality || !house_number) continue
 		const key = `${house_number}|${street}|${locality}`.toLowerCase()
@@ -198,8 +140,8 @@ function readTuples(source: UnitSource): UnitTuple[] {
 			street,
 			locality,
 			region: source.region,
-			postcode: get(cells, iPost),
-			oaUnit: get(cells, iUnit),
+			postcode: row.postcode ?? "",
+			oaUnit: row.unit ?? "",
 		})
 	}
 
@@ -311,7 +253,7 @@ export const unitRecipe: ShardRecipe = {
 		const pool: UnitTuple[] = []
 
 		for (const s of sources) {
-			const t = readTuples(s)
+			const t = await readTuples(s)
 
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 
@@ -321,7 +263,7 @@ export const unitRecipe: ShardRecipe = {
 		}
 
 		if (!pool.length) {
-			throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?")
+			throw new Error(`No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
 		}
 
 		let emitted = 0

package/src/{synthesize-boundary-stress.ts → synthesizers/boundary-stress.ts}

@@ -46,10 +46,10 @@
  *   tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
  *   AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
  *   separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
- *   base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
+ *   base-locale shard. `synthesizers/boundary-stress.test.ts` proves the alignments.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the

package/src/{synthesize-german.ts → synthesizers/german.ts}

@@ -22,8 +22,8 @@
  *   `raw` so alignment lands.
  */
 
-import { formatAddress } from "./format.ts"
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { formatAddress } from "@mailwoman/formatter"
 
 /**
  * A real address tuple (e.g. one OpenAddresses row): street + locality required, rest optional.

package/src/{synthesize-house-venue.ts → synthesizers/house-venue.ts}

@@ -9,7 +9,7 @@
  *   house_number recall by ~4-5pp. DeepSeek's turn-8 root-cause:
  *
  *   1. Direct: `5th Avenue Theatre`-style adversarial venues teach the model that tokens like "5th"
- *        belong to venues, not house_numbers. (Fixed in `synthesize-no-street.ts` by removing
+ *        belong to venues, not house_numbers. (Fixed in `no-street.ts` by removing
  *        digit+ordinal venue patterns.)
  *   2. Distributional dilution: synth-no-street adds 122K rows where house_number is absent. The model's
  *        training distribution shifts toward "house_number is rare," and it under-emits the tag at
@@ -23,12 +23,12 @@
  *   Real-world shape: business cards, mailing labels, store directories — `"123 Main St, Sunrise
  *   Bakery, Springfield, IL 62701"` is a perfectly ordinary address form.
  *
- *   Venue pool: PLAIN_VENUES from `synthesize-no-street.ts` (re-exported here). Adversarial venues
+ *   Venue pool: PLAIN_VENUES from `no-street.ts` (re-exported here). Adversarial venues
  *   are deliberately NOT used here — the point is to teach co-occurrence, not to re-introduce
  *   decompose-mode pressure.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 export interface HouseVenueBaseTuple {
 	locality: string
@@ -61,7 +61,7 @@ export interface SynthesizedHouseVenueRow {
 
 /**
  * PLAIN venue names, carrying no street-typing tokens. This shard teaches house_number + venue coexistence, NOT
- * decompose-mode pressure — adversarial venue names live in `synthesize-no-street.ts`.
+ * decompose-mode pressure — adversarial venue names live in `no-street.ts`.
  */
 const PLAIN_VENUES: ReadonlyArray<string> = [
 	"Bob's Pizza",

package/src/{synthesize-intersection.ts → synthesizers/intersection.ts}

@@ -19,7 +19,7 @@
  *   US-idiomatic only (the harness intersection cases are US: "X & Y, City, ST ZIP").
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 /**
  * Street name cores (no suffix) — proper-noun streets that often appear bare.

package/src/{synthesize-no-street.ts → synthesizers/no-street.ts}

@@ -34,10 +34,10 @@
  *   counter-example signal the model is missing.
  *
  *   This complements (does not replace) the existing US-base-tuple source used by
- *   `synthesize-po-box.ts`; the same `NoStreetBaseTuple` shape is consumed.
+ *   `po-box.ts`; the same `NoStreetBaseTuple` shape is consumed.
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 //#region Types
 

package/src/{synthesize-po-box.ts → synthesizers/po-box.ts}

@@ -24,7 +24,7 @@
  *   - USPS DMM 508 §4.1.4 / §4.5.4 — PO Box and street-addressed PO Box
  */
 
-import type { CanonicalRow } from "./types.ts"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 /**
  * Digits a box number needs before a thousands comma is plausible (`1,234`).

package/src/{synthesize-street.ts → synthesizers/street.ts}

@@ -19,8 +19,8 @@
  *   adapter.
  */
 
-import { decomposeStreet } from "./adapters/tiger/street-decompose.ts"
-import type { CanonicalRow } from "./types.ts"
+import { decomposeStreet } from "@mailwoman/corpus/adapters/tiger/street-decompose"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
 
 // Hand-curated US street name pool. Real frequency-weighted street names — sampled
 // from US Census TIGER 2024 top-1000 by occurrence count. Keep ~50 entries so the

package/src/{synthesize.ts → synthesizers/utils.ts}

@@ -34,10 +34,11 @@ import {
 	matchTrailingSuffix,
 } from "@mailwoman/codex/us"
 import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
+import { mulberry32 } from "@mailwoman/core/utils"
+import type { CanonicalRow, LabeledRow, QuarantinedRow } from "@mailwoman/corpus/types"
+import { alignRow, assertSpanInvariants, type ComponentSpan } from "@mailwoman/corpus/utils"
 
-import { alignRow, assertSpanInvariants, type ComponentSpan } from "./align.ts"
-import { whitespaceTokenizer, type Tokenizer } from "./tokenize.ts"
-import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.ts"
+import { whitespaceTokenizer, type Tokenizer } from "../utils/tokenize.ts"
 
 /**
  * An augmentation transforms a single row. Return `null` if the augmentation doesn't apply (e.g. accent-strip on a row
@@ -213,21 +214,6 @@ function hashString(s: string): number {
 	return h >>> 0
 }
 
-/**
- * Mulberry32 — a tiny seeded PRNG. Same seed → same stream → reproducible typos.
- */
-function mulberry32(seed: number): () => number {
-	let a = seed >>> 0
-
-	return () => {
-		a = (a + 0x6d_2b_79_f5) | 0
-		let t = Math.imul(a ^ (a >>> 15), 1 | a)
-		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
-
-		return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296
-	}
-}
-
 /**
  * Inject ONE realistic typo — an adjacent-QWERTY-key substitution OR an adjacent-character transposition — into a
  * single alpha name component (street/locality/region…), teaching the model to recover from real-world misspellings

package/src/tools/align-shard.ts

@@ -26,8 +26,8 @@ import { createWriteStream } from "node:fs"
  * --output /tmp/kryptonite-labeled.jsonl\
  * --corpus-version 0.5.0
  */
-import { alignRow } from "@mailwoman/corpus"
-import { TextSpliterator } from "spliterator"
+import { alignRow } from "@mailwoman/corpus/utils"
+import { JSONSpliterator } from "spliterator"
 
 export interface AlignShardOptions {
 	input: string
@@ -36,16 +36,13 @@ export interface AlignShardOptions {
 }
 
 export async function alignCanonicalShard(args: AlignShardOptions): Promise<void> {
-	// Read phase only — the write path stays on createWriteStream. TextSpliterator + JSON.parse keeps the
-	// original tolerance: the `!line.trim()` guard skips blank lines and a trailing CR is valid JSON whitespace.
+	// Read phase only — the write path stays on createWriteStream.
 	const outStream = createWriteStream(args.output, { encoding: "utf8" })
 	let labeled = 0
 	let quarantined = 0
 	const quarantineReasons: Record<string, number> = {}
 
-	for await (const line of TextSpliterator.fromAsync(args.input)) {
-		if (!line.trim()) continue
-		const canonical = JSON.parse(line) as Parameters<typeof alignRow>[0]
+	for await (const canonical of JSONSpliterator.fromAsync<Parameters<typeof alignRow>[0]>(args.input)) {
 		// Stamp the target corpus version so the emitted row's provenance matches the run it joins.
 		canonical.corpus_version = args.corpusVersion
 		const result = alignRow(canonical)

package/src/tools/audit.ts

@@ -18,6 +18,9 @@
 import { existsSync, readFileSync, readdirSync } from "node:fs"
 import { basename, join } from "node:path"
 
+import { parseJSONStrict } from "@mailwoman/core/objects"
+import { TextSpliterator } from "spliterator"
+
 /**
  * Share of a shard one source may hold before the mix is flagged as dominated by it.
  */
@@ -66,13 +69,11 @@ interface ParsedConfig {
  */
 function parseConfig(configPath: string): ParsedConfig | null {
 	if (!existsSync(configPath)) return null
-	const text = readFileSync(configPath, "utf8")
-	const lines = text.split("\n")
 	const weights: Record<string, number> = {}
 	let inBlock = false
 	let blockIndent = -1
 
-	for (const raw of lines) {
+	for (const raw of TextSpliterator.from(readFileSync(configPath))) {
 		const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/)
 
 		if (sourceWeightsMatch) {
@@ -217,9 +218,9 @@ function manifestScan(corpusDir: string, knownPrefixes: readonly string[]): Shar
 
 	if (!existsSync(manifestPath)) return null
 
-	const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
+	const manifest = parseJSONStrict<{
 		shards?: Array<{ split: string; source?: string | null; first_source_id?: string | null }>
-	}
+	}>(readFileSync(manifestPath, "utf8"))
 
 	if (!Array.isArray(manifest.shards)) return null
 	const bySplit: Record<string, Record<string, number>> = {}

package/src/tools/corpus-stats.ts

@@ -32,14 +32,15 @@
  *
  *   For a quick local-corpus baseline (limited but useful for linter testing): node
  *   scripts/build-corpus-stats.ts\
- *   --shards /mnt/playpen/mailwoman-data/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --shards $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
  *   --output /tmp/corpus-stats-local.json
  */
 
-import { execSync } from "node:child_process"
 import { readdirSync, statSync, writeFileSync } from "node:fs"
 import { join } from "node:path"
 
+import { ParquetReader } from "../parquet-wrapper/index.ts"
+
 const SEP = ""
 const MIN_BIGRAM_COUNT = 2
 
@@ -65,36 +66,28 @@ function discoverShards(shardsArg: string): string[] {
 }
 
 /**
- * Use a Python subprocess to read parquet (pyarrow is heavier than parquet-wasm but already on the path here, and we
- * have nothing in the JS ecosystem that reads parquet cleanly at this scale). Emits one JSON object per line: `{tokens:
- * [...], labels: [...]}`.
+ * Stream a shard's `tokens`/`labels` columns.
+ *
+ * `limit` stops the iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
  */
-function streamShardRows(shardPath: string, limit?: number): Array<{ tokens: string[]; labels: string[] }> {
-	// Pipe the python script via stdin instead of `-c` to preserve newlines verbatim
-	// (JSON-encoding the script for -c collapses real newlines into literal `\n`).
-	const py = `
-import pyarrow.parquet as pq
-import json, sys
-t = pq.read_table(${JSON.stringify(shardPath)}, columns=['tokens', 'labels'])
-tokens_col = t['tokens'].to_pylist()
-labels_col = t['labels'].to_pylist()
-n = min(len(tokens_col), ${limit ?? "len(tokens_col)"})
-for i in range(n):
-    sys.stdout.write(json.dumps({"tokens": tokens_col[i], "labels": labels_col[i]}) + "\\n")
-`
-
-	const buf = execSync(`python3`, { input: py, maxBuffer: 1024 * 1024 * 1024 })
-	const rows: Array<{ tokens: string[]; labels: string[] }> = []
-
-	for (const line of buf.toString("utf8").split("\n")) {
-		if (!line) continue
-		rows.push(JSON.parse(line))
-	}
+async function* streamShardRows(
+	shardPath: string,
+	limit?: number
+): AsyncIterable<{ tokens: string[]; labels: string[] }> {
+	await using reader = await ParquetReader.openFile<{ tokens: string[]; labels: string[] }>(shardPath)
+
+	let emitted = 0
 
-	return rows
+	for await (const row of reader.project("tokens", "labels")) {
+		if (limit !== undefined && emitted >= limit) break
+
+		yield row
+
+		emitted++
+	}
 }
 
-export function buildCorpusStats(args: CorpusStatsOptions): void {
+export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void> {
 	const shardPaths = discoverShards(args.shardsArg)
 
 	console.error(`Discovered ${shardPaths.length} parquet shard(s)`)
@@ -106,11 +99,10 @@ export function buildCorpusStats(args: CorpusStatsOptions): void {
 	for (const path of shardPaths) {
 		console.error(`Reading ${path}...`)
 
-		const rows = streamShardRows(path, args.limitPerShard)
-		totalRows += rows.length
+		const before = totalRows
 
-		for (const row of rows) {
-			const { tokens, labels } = row
+		for await (const { tokens, labels } of streamShardRows(path, args.limitPerShard)) {
+			totalRows++
 
 			if (tokens.length !== labels.length) continue
 
@@ -143,7 +135,7 @@ export function buildCorpusStats(args: CorpusStatsOptions): void {
 		}
 
 		console.error(
-			`  ${rows.length} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`
+			`  ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`
 		)
 	}
 

package/src/tools/fetch/download.ts

@@ -12,6 +12,8 @@ import { existsSync } from "node:fs"
 import { readFile, writeFile } from "node:fs/promises"
 import { setTimeout as sleep } from "node:timers/promises"
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+
 /**
  * Rate limited — retryable, the server is asking us to back off.
  */
@@ -127,11 +129,11 @@ export async function downloadToFile(options: DownloadOptions): Promise<{ bytes:
 export async function readManifest<T>(path: string): Promise<T | null> {
 	if (!existsSync(path)) return null
 
-	try {
-		return JSON.parse(await readFile(path, "utf8")) as T
-	} catch {
-		return null
-	}
+	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
+	// JSON does; tryParsingJSON returns null for the non-string sentinel.
+	const text = await readFile(path, "utf8").catch(() => null)
+
+	return tryParsingJSON<T>(text)
 }
 
 /**

package/src/tools/fetch/imls-pls.ts

@@ -23,12 +23,11 @@
 /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
    procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
 
-import { execFile } from "node:child_process"
 import { existsSync, mkdirSync, statSync } from "node:fs"
 import { rm } from "node:fs/promises"
 import { basename, join } from "node:path"
-import { promisify } from "node:util"
 
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -40,8 +39,6 @@ import { downloadToFile, readManifest, writeManifest } from "./download.ts"
  */
 const BYTES_PER_KIB = 1024
 
-const execFileAsync = promisify(execFile)
-
 /**
  * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
  */
@@ -58,18 +55,6 @@ interface SourceManifest {
 	bytes: number
 }
 
-/**
- * Return the filenames listed inside a ZIP (the trailing column of each `unzip -l` row).
- */
-async function listZipEntries(zipPath: string): Promise<string[]> {
-	const listing = await execFileAsync("unzip", ["-l", zipPath])
-
-	return listing.stdout
-		.split("\n")
-		.map((line) => line.trim().split(/\s+/).pop() ?? "")
-		.filter((name) => name.length > 0)
-}
-
 export async function fetchIMLSPLS(
 	options: FetchIMLSPLSOptions,
 	report?: (line: string) => void
@@ -120,7 +105,7 @@ export async function fetchIMLSPLS(
 	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
 	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
 	report?.("  Inspecting ZIP contents ...")
-	const entries = await listZipEntries(zipDest)
+	const entries = (await listZipEntries(zipDest)).map((entry) => entry.name)
 
 	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
 
@@ -142,9 +127,10 @@ export async function fetchIMLSPLS(
 	}
 
 	report?.(`  Extracting outlet CSV: ${csvName}`)
-	await execFileAsync("unzip", ["-o", "-j", zipDest, csvName, "-d", destDir])
 
 	const csvDest = join(destDir, basename(csvName))
+
+	await extractZipEntry(zipDest, csvName, csvDest)
 	const csvSize = statSync(csvDest).size
 	const csvSha = await sha256File(csvDest)
 

package/src/tools/fetch/index.ts

@@ -40,6 +40,13 @@
  *       Tier A (US PD).
  *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
  *       — per-row filter.
+ *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
+ *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
+ *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
+ *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
+ *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
+ *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
+ *       family built on `APIClient` — see its docstring for why.
  *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
  *       Tier A (state PD-equivalent).
  *   - `state-hi-schools` — Hawaii DOE school directory (XLSX → CSV via openpyxl). Tier A (state
@@ -65,7 +72,7 @@
  *
  *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
  *   mailwoman corpus fetch openaddresses --country ca \
- *     --out-root /mnt/playpen/mailwoman-data/corpus/sources
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
  *
  *   # Or any other OA country code
  *   mailwoman corpus fetch openaddresses --country fr
@@ -91,9 +98,11 @@ import { fetchIMLSPLS } from "./imls-pls.ts"
 import { fetchNAD } from "./nad.ts"
 import { fetchNPPES } from "./nppes.ts"
 import { fetchOpenAddresses } from "./openaddresses.ts"
+import { fetchOurAirports } from "./ourairports.ts"
 import { fetchStateHISchools } from "./state-hi-schools.ts"
 import { fetchStateSources } from "./state-sources.ts"
 import { fetchTigerFull } from "./tiger-full.ts"
+import { fetchWikidataSubVenue } from "./wikidata-subvenue.ts"
 
 export * from "./ban.ts"
 export * from "./hrsa.ts"
@@ -101,9 +110,11 @@ export * from "./imls-pls.ts"
 export * from "./nad.ts"
 export * from "./nppes.ts"
 export * from "./openaddresses.ts"
+export * from "./ourairports.ts"
 export * from "./state-hi-schools.ts"
 export * from "./state-sources.ts"
 export * from "./tiger-full.ts"
+export * from "./wikidata-subvenue.ts"
 
 /**
  * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
@@ -115,9 +126,11 @@ export const FETCH_SOURCES = {
 	"imls-pls": fetchIMLSPLS,
 	nppes: fetchNPPES,
 	openaddresses: fetchOpenAddresses,
+	ourairports: fetchOurAirports,
 	"state-sources": fetchStateSources,
 	"state-hi-schools": fetchStateHISchools,
 	"tiger-full": fetchTigerFull,
+	"wikidata-subvenue": fetchWikidataSubVenue,
 } as const
 
 export type FetchSourceID = keyof typeof FETCH_SOURCES

package/src/tools/fetch/nad.ts

@@ -26,7 +26,7 @@
  *   ## Usage
  *
  *   ```sh
- *   mailwoman corpus fetch nad --out-root /mnt/playpen/mailwoman-data/corpus/sources
+ *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
  *
  *   # Resume from an OID
  *   mailwoman corpus fetch nad --start-oid 34400001