@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/fetch/openaddresses.ts

@@ -4,87 +4,36 @@ import { APIClient, isSuccessStatus } from "@mailwoman/core/api"
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
- *
- *   Source: https://batch.openaddresses.io
- *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
- *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
- *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
- *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
- *   adapter does the license filtering.
- *
- *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
- *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
- *   `gunzip` decompression, both nice/ionice-deprioritized).
- *
- *   ## Authentication note (2026-05-18)
- *
- *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
- *   still free at the "basic" tier (GeoJSON+LD output).
- *
- *   1. Register at https://batch.openaddresses.io/register
- *   2. Log in and go to Profile → "Create Token"
- *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
- *   4. Re-run the command.
- *
- *   The collection URL pattern (verified 2026-05-18):
- *
- *   - `POST /api/login {username, password}` → `{token}`
- *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
- *
- *   Collections are downloaded as a combined GeoJSON.gz via:
- *
- *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
- *
- *   Collection IDs discovered from `/api/collections`:
- *
- *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
- *
- *   ## Usage
- *
- *   ```sh
- *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
- *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Without token (will detect + print instructions, then report the failure):
- *   mailwoman corpus fetch openaddresses --country ca
- *   ```
+ * Fetch an OpenAddresses country collection from batch.openaddresses.io.
  */
 /* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
 import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { openReadStream, openWriteStream, pipeline } from "@mailwoman/core/fs/streams"
+import { openWriteStream, pipeline } from "@mailwoman/core/fs/streams"
 import { movePath, removePathIfPresent, makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
 import { runFile, spawnProcess } from "@mailwoman/core/process"
 import { isoSeconds } from "@mailwoman/core/utils"
-import { join } from "path-ts"
+import type { PathBuilderLike } from "path-ts"
+import { AsyncSpliterator } from "spliterator"
 
 import { $private } from "#env"
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
-import { streamDownload, writeManifest } from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { streamDownload, writeManifest } from "#tools/fetch/download"
 
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * A successful fetch; anything else is an error page or a redirect we did not follow.
- */
 const HTTP_OK = 200
 
 /**
- * Smallest plausible OpenAddresses slice. Below 10 KiB the file is a stub or an error body.
+ * Below this size the file is a stub or an error body rather than an archive.
  */
-const MIN_PLAUSIBLE_SLICE_BYTES = 10_240
+const MIN_PLAUSIBLE_ARCHIVE_BYTES = 10_240
 
 const OA_BASE = "https://batch.openaddresses.io"
 
 /**
- * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
- * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
+ * OA assigns stable integer collection IDs.
+ * Re-check `GET /api/collections` for a country absent here.
  */
 const OA_COLLECTION_IDS: Record<string, number> = {
 	ca: 6,
@@ -96,9 +45,6 @@ const OA_COLLECTION_IDS: Record<string, number> = {
 }
 
 export interface FetchOpenAddressesOptions extends BaseFetchOptions {
-	/**
-	 * OA country collection code. Default `ca`.
-	 */
 	country?: string
 }
 
@@ -109,27 +55,7 @@ interface OaCollection {
 	size?: number
 }
 
-/**
- * Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection).
- */
-async function countLines(path: string): Promise<number> {
-	let count = 0
-
-	for await (const chunk of openReadStream(path) as AsyncIterable<Buffer>) {
-		for (const byte of chunk) {
-			if (byte === 0x0a) {
-				count++
-			}
-		}
-	}
-
-	return count
-}
-
-/**
- * Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used.
- */
-async function gunzipToFile(src: string, dest: string): Promise<void> {
+async function gunzipToFile(src: PathBuilderLike, dest: PathBuilderLike): Promise<void> {
 	const child = spawnProcess("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
 		stdio: ["ignore", "pipe", "inherit"],
 	})
@@ -149,9 +75,9 @@ export async function fetchOpenAddresses(
 	const country = options.country ?? "ca"
 	const token = $private.OA_BATCH_TOKEN
 
-	const destDir = join(options.outRoot, "openaddresses", country)
-	const manifestPath = join(destDir, "MANIFEST.json")
-	const outputFile = join(destDir, "collection.geojsonl")
+	const destDir = options.outRoot("openaddresses", country)
+	const manifestPath = destDir("MANIFEST.json")
+	const outputFile = destDir("collection.geojsonl")
 
 	const fail = (code: string): FetchSummary => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] })
 
@@ -160,8 +86,6 @@ export async function fetchOpenAddresses(
 
 	await makeDirectories(destDir)
 
-	// MARK: Authentication check
-
 	if (!token) {
 		report?.(`
 ERROR: OA_BATCH_TOKEN is not set.
@@ -185,22 +109,18 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
 		return fail("OA_BATCH_TOKEN")
 	}
 
-	// MARK: Determine collection ID
-
 	let collectionID = OA_COLLECTION_IDS[country]
 
 	if (collectionID === undefined) {
 		report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`)
 
-		// The collections API only. The COLLECTION ARCHIVES stay on raw `fetch` — they stream multi-gigabyte
-		// bodies straight to disk, where response caching is nonsense and axios would buffer them in memory.
+		// The archive streams multi-gigabyte bodies to disk, where axios would buffer them in memory.
 		const res = await new APIClient({
 			displayName: "openaddresses-api",
 			retry: true,
 			axios: { headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" } },
 		})
-			// `validateStatus` keeps a non-2xx as a RESPONSE rather than a throw: the caller reports the status and
-			// returns `fail(country)`, a graceful path this must not turn into an exception.
+			// `validateStatus` keeps a non-2xx as a response rather than a throw: the caller reports the status and returns `fail(country)`, a graceful path this must not turn into an exception.
 			.fetch<OaCollection[]>({
 				url: `${OA_BASE}/api/collections`,
 				timeout: 30_000,
@@ -232,13 +152,11 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
 		report?.(`  Found collection id=${collectionID} for '${country}'`)
 	}
 
-	// MARK: Download via the collections download endpoint (302s to S3)
-
 	report?.(`  Resolving download URL for collection id=${collectionID}...`)
 	report?.(`  Attempting authenticated download...`)
 
-	const tmpGz = join(destDir, "collection.geojsonl.gz.tmp")
-	const tmpRaw = join(destDir, "collection.geojsonl.tmp")
+	const tmpGz = destDir("collection.geojsonl.gz.tmp")
+	const tmpRaw = destDir("collection.geojsonl.tmp")
 	const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`
 
 	let httpStatus = await streamDownload(sourceURL, tmpGz, {
@@ -249,7 +167,6 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
 	})
 
 	if (httpStatus !== HTTP_OK) {
-		// Try the geojsonl.gz directly with token as query param (alternate URL shape).
 		httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
 			timeoutMs: 7_200_000,
 			retries: 3,
@@ -280,8 +197,6 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
 		return fail(country)
 	}
 
-	// MARK: Decompress if the downloaded file is gzipped
-
 	const fileMagic = (await runFile("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout
 
 	if (/gzip|compressed/i.test(fileMagic)) {
@@ -290,17 +205,13 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
 		await removePathIfPresent(tmpGz)
 		await movePath(tmpRaw, outputFile)
 	} else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
-		// Already line-delimited GeoJSON.
 		await movePath(tmpGz, outputFile)
 		await removePathIfPresent(tmpRaw)
 	} else {
-		// Unknown type — keep as-is and let the operator inspect.
 		await movePath(tmpGz, outputFile)
 		report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`)
 	}
 
-	// MARK: Verify + write MANIFEST
-
 	if (!(await pathExists(outputFile))) {
 		report?.(`ERROR: Output file not found at ${outputFile} after download.`)
 
@@ -309,14 +220,14 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
 
 	const size = (await statPath(outputFile)).size
 
-	if (size < MIN_PLAUSIBLE_SLICE_BYTES) {
+	if (size < MIN_PLAUSIBLE_ARCHIVE_BYTES) {
 		report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`)
 
 		return fail(country)
 	}
 
 	const sha = await sha256File(outputFile)
-	const rowCount = await countLines(outputFile)
+	const rowCount = await AsyncSpliterator.count(outputFile)
 	const downloadedAt = isoSeconds()
 
 	const manifest = {
@@ -329,7 +240,7 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
 		bytes: size,
 		row_count: rowCount,
 		notes:
-			"batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
+			"batch.openaddresses.io requires a free registered account for downloads. The license is mixed per row, and the openaddresses adapter admits every row by default. Build with `mailwoman corpus build --license-policy share-alike-free` to refuse the share-alike rows, or pass `allowShareAlike: false` for an adapter-scoped drop.",
 	}
 
 	await writeManifest(manifestPath, manifest)

package/lib/tools/fetch/ourairports.ts

@@ -3,63 +3,30 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
+ * Fetch the OurAirports CSV dumps — the venue side of the sub-venue corpus arc.
  *
- *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
- *            mirror of the nightly export; `ourairports.com/data/` redirects here).
- *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
- *            courtesy credit — no attribution obligation rides on a derived slice, which makes this
- *            the one transport source in the arc with no licensing question at all. Tier A.
- *
- *   ## What it is good for, and what it is not
- *
- *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
- *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
- *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
- *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
- *   a spatial join.
- *
- *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
- *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
- *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
- *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
- *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
- *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
- *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
- *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
- *   accordingly; the split between the two is the one `AGENTS.md` draws.
- *
- *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
+ * Source: https://davidmegginson.github.io/ourairports-data
+ * License: public domain, with a courtesy credit requested.
  */
 
 import { APIClient } from "@mailwoman/core/api"
 import { makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
 
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
-import { downloadToFile, writeManifest } from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, writeManifest } from "#tools/fetch/download"
 
 const SLUG = "ourairports"
 
 /**
- * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
- * mirror directly saves a redirect and is the URL the project's own README gives.
+ * `ourairports.com/data/*.csv` redirects to the project's own GitHub Pages mirror.
  */
 const BASE_URL = "https://davidmegginson.github.io/ourairports-data"
 
 /**
- * The files worth having, and why each one.
+ * `airports.csv` is the payload.
  *
- * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
- * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
- * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
- * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
- * slice ever needs one.
+ * The other three join its codes to text or provide the negative class.
  */
 const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"] as const
 
@@ -71,9 +38,9 @@ interface OurAirportsFileEntry {
 	sha256: string
 	bytes: number
 	/**
-	 * The upstream `Last-Modified`, when the CDN gave one. This is the DATA's vintage; `downloaded_at` is only when we
-	 * asked. `corpus/AGENTS.md` has the standing warning that a file's mtime is not its data's vintage — recording the
-	 * upstream header is how a later refresh decision gets made on the right number.
+	 * The upstream `Last-Modified`, when the CDN gave one.
+	 *
+	 * It is the data's vintage; `downloaded_at` is only when we asked.
 	 */
 	last_modified: string | null
 }
@@ -87,8 +54,8 @@ interface OurAirportsManifest {
 }
 
 /**
- * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
- * and must never fail a download that otherwise succeeded.
+ * @returns `null` on any failure, because provenance metadata must never fail
+ * a download that otherwise succeeded.
  */
 async function readLastModified(url: string): Promise<string | null> {
 	try {
@@ -105,14 +72,14 @@ async function readLastModified(url: string): Promise<string | null> {
 }
 
 /**
- * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
- * origin URL, sha256, byte count and upstream `Last-Modified`.
+ * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `manifest.json`
+ * carrying each file's origin URL, sha256, byte count and upstream `Last-Modified`.
  */
 export async function fetchOurAirports(
 	options: FetchOurAirportsOptions,
 	report?: (line: string) => void
 ): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
+	const destDir = options.outRoot(SLUG)
 	await makeDirectories(destDir)
 
 	const entries: OurAirportsFileEntry[] = []
@@ -122,7 +89,7 @@ export async function fetchOurAirports(
 
 	for (const filename of FILES) {
 		const url = `${BASE_URL}/${filename}`
-		const dest = join(destDir, filename)
+		const dest = destDir(filename)
 
 		report?.(`=== ${SLUG} / ${filename}`)
 
@@ -164,7 +131,7 @@ export async function fetchOurAirports(
 		files: entries,
 	}
 
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+	await writeManifest(destDir("MANIFEST.json"), manifest)
 
 	return { fetched, skipped: 0, failed, failedCodes }
 }

package/lib/tools/fetch/ppd.ts

@@ -1,12 +1,12 @@
+import { titleCaseGB } from "@mailwoman/codex/gb"
 import { dataRootPath } from "@mailwoman/core/data-root"
 import { openReadStream, openWriteStream } from "@mailwoman/core/fs/streams"
+import { stringifyJSON } from "@mailwoman/core/json"
 import { runIfScript } from "@mailwoman/core/scripting"
 import { parseArguments } from "@mailwoman/core/scripting/arguments"
 import { resolvePath } from "path-ts"
 import { CSVSpliterator } from "spliterator"
 
-import { titleCaseGB } from "#tools/gb-title-case"
-
 const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/
 
 export interface PPDExtractStats {
@@ -20,9 +20,10 @@ export interface PPDExtractStats {
 const quote = (value: string): string => (value ? `"${value.replaceAll('"', '""')}"` : "")
 
 /**
- * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
- * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
- * of rows (tests) or a streamed async source (the real 31M-row extraction).
+ * Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
+ * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
+ *
+ * Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
  */
 export async function extractPPDTuples(
 	input: AsyncIterable<string[]> | Iterable<string[]>,
@@ -89,8 +90,9 @@ export async function extractPPDTuples(
  * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
  */
 async function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats> {
-	// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
-	// recipes/international/locale.ts). `header: false` yields every row as data — PPD ships no header row.
+	// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself
+	// (see readTuples in recipes/international/locale.ts).
+	// `header: false` yields every row as data — PPD ships no header row.
 	const rows = CSVSpliterator.fromAsync<string[]>(openReadStream(inputPath), {
 		header: false,
 	})
@@ -115,5 +117,5 @@ runIfScript(import.meta, async () => {
 
 	const stats = await runPPDExtract(values.input!, values.output!)
 
-	console.log(`[ppd] ${JSON.stringify(stats)}`)
+	console.log(`[ppd] ${stringifyJSON(stats)}`)
 })