@mailwoman/corpus 9.2.0 → 9.3.0

package/src/tools/fetch/openaddresses.ts

@@ -1,404 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
- *
- *   Source: https://batch.openaddresses.io
- *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
- *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
- *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
- *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
- *   adapter does the license gating.
- *
- *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
- *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
- *   `gunzip` decompression, both nice/ionice-deprioritized).
- *
- *   ## Authentication note (2026-05-18)
- *
- *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
- *   still free at the "basic" tier (GeoJSON+LD output).
- *
- *   1. Register at https://batch.openaddresses.io/register
- *   2. Log in and go to Profile → "Create Token"
- *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
- *   4. Re-run the command.
- *
- *   The collection URL pattern (verified 2026-05-18):
- *
- *   - `POST /api/login {username, password}` → `{token}`
- *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
- *
- *   Collections are downloaded as a combined GeoJSON.gz via:
- *
- *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
- *
- *   Collection IDs discovered from `/api/collections`:
- *
- *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
- *
- *   ## Usage
- *
- *   ```sh
- *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
- *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Without token (will detect + print instructions, then report the failure):
- *   mailwoman corpus fetch openaddresses --country ca
- *   ```
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { execFile, spawn } from "node:child_process"
-import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs"
-import { join } from "node:path"
-import { Readable } from "node:stream"
-import { pipeline } from "node:stream/promises"
-import { setTimeout as sleep } from "node:timers/promises"
-import { promisify } from "node:util"
-
-import { APIClient, isSuccessStatus } from "@mailwoman/core/api"
-import { $private } from "@mailwoman/core/env"
-import { ByteFormatter } from "@mailwoman/core/fs/utils"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { isTransientStatus, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * A successful fetch; anything else is an error page or a redirect we did not follow.
- */
-const HTTP_OK = 200
-
-/**
- * Smallest plausible OpenAddresses shard. Below 10 KiB the file is a stub or an error body.
- */
-const MIN_PLAUSIBLE_SHARD_BYTES = 10_240
-
-const execFileAsync = promisify(execFile)
-
-const OA_BASE = "https://batch.openaddresses.io"
-
-/**
- * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
- * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
- */
-const OA_COLLECTION_IDS: Record<string, number> = {
-	ca: 6,
-	"us-west": 4,
-	"us-south": 3,
-	"us-northeast": 2,
-	"us-midwest": 5,
-	global: 1,
-}
-
-export interface FetchOpenAddressesOptions extends BaseFetchOptions {
-	/**
-	 * OA country collection code. Default `ca`.
-	 */
-	country?: string
-}
-
-interface OaCollection {
-	name?: string
-	id?: number
-	human?: string
-	size?: number
-}
-
-/**
- * Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection).
- */
-async function countLines(path: string): Promise<number> {
-	let count = 0
-
-	for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {
-		for (const byte of chunk) {
-			if (byte === 0x0a) {
-				count++
-			}
-		}
-	}
-
-	return count
-}
-
-interface StreamDownloadOpts {
-	headers?: Record<string, string>
-	timeoutMs: number
-	retries: number
-	retryDelayMs: number
-}
-
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
- * redirects (the OA download endpoint 302s to a pre-signed S3 URL).
- *
- * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams a multi-GB body to disk (the
- * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the caller needs for
- * its two-URL fallback ladder.
- */
-async function streamDownload(url: string, dest: string, opts: StreamDownloadOpts): Promise<number> {
-	for (let attempt = 0; attempt <= opts.retries; attempt++) {
-		try {
-			const res = await fetch(url, {
-				headers: opts.headers ?? {},
-				redirect: "follow",
-				signal: AbortSignal.timeout(opts.timeoutMs),
-			})
-
-			if (res.ok && res.body) {
-				await pipeline(Readable.fromWeb(res.body), createWriteStream(dest))
-
-				return res.status
-			}
-
-			if (attempt < opts.retries && isTransientStatus(res.status)) {
-				await sleep(opts.retryDelayMs)
-
-				continue
-			}
-
-			return res.status
-		} catch {
-			if (attempt < opts.retries) {
-				await sleep(opts.retryDelayMs)
-
-				continue
-			}
-
-			return 0
-		}
-	}
-
-	return 0
-}
-
-/**
- * Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used.
- */
-async function gunzipToFile(src: string, dest: string): Promise<void> {
-	const child = spawn("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
-		stdio: ["ignore", "pipe", "inherit"],
-	})
-
-	await pipeline(child.stdout!, createWriteStream(dest))
-
-	await new Promise<void>((resolve, reject) => {
-		child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`gunzip exited with code ${code}`))))
-		child.on("error", reject)
-	})
-}
-
-export async function fetchOpenAddresses(
-	options: FetchOpenAddressesOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const country = options.country ?? "ca"
-	const token = $private.OA_BATCH_TOKEN
-
-	const destDir = join(options.outRoot, "openaddresses", country)
-	const manifestPath = join(destDir, "MANIFEST.json")
-	const outputFile = join(destDir, "collection.geojsonl")
-
-	const fail = (code: string): FetchSummary => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] })
-
-	report?.(`=== fetch openaddresses: country=${country}`)
-	report?.(`    dest: ${destDir}`)
-
-	mkdirSync(destDir, { recursive: true })
-
-	// MARK: Authentication check
-
-	if (!token) {
-		report?.(`
-ERROR: OA_BATCH_TOKEN is not set.
-
-As of 2026-05-18, batch.openaddresses.io requires a registered (free) account
-to download collection files.  Data remains openly licensed — the auth gate
-is there to prevent CDN abuse, not to restrict access.
-
-Steps to get a token:
-  1. Register at: https://batch.openaddresses.io/register
-  2. Verify your email and log in.
-  3. Go to Profile → "Create Token" → copy the token.
-  4. Export it in this shell:
-       export OA_BATCH_TOKEN=<your-token>
-  5. Re-run this command.
-
-The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
-(estimated), so budget ~20–45 minutes at typical cloud-to-host bandwidth.
-`)
-
-		return fail("OA_BATCH_TOKEN")
-	}
-
-	// MARK: Determine collection ID
-
-	let collectionID = OA_COLLECTION_IDS[country]
-
-	if (collectionID === undefined) {
-		report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`)
-
-		// The collections API only. The COLLECTION ARCHIVES stay on raw `fetch` — they stream multi-gigabyte
-		// bodies straight to disk, where response caching is nonsense and axios would buffer them in memory.
-		const res = await new APIClient({
-			displayName: "openaddresses-api",
-			retry: true,
-			axios: { headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" } },
-		})
-			// `validateStatus` keeps a non-2xx as a RESPONSE rather than a throw: the caller reports the status and
-			// returns `fail(country)`, a graceful path this must not turn into an exception.
-			.fetch<OaCollection[]>({
-				url: `${OA_BASE}/api/collections`,
-				timeout: 30_000,
-				validateStatus: () => true,
-			})
-
-		if (!isSuccessStatus(res.status)) {
-			report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`)
-
-			return fail(country)
-		}
-
-		const collections = res.data
-		const match = collections.find((item) => item.name === country)
-
-		if (match?.id === undefined) {
-			report?.(`ERROR: Could not find a collection named '${country}' in GET /api/collections.`)
-			report?.(`Available collections:`)
-
-			for (const item of collections) {
-				const size = (item.size ?? 0).toLocaleString()
-				report?.(`  ${(item.name ?? "").padEnd(20)}  id=${item.id}  ${item.human ?? ""}  size=${size} bytes`)
-			}
-
-			return fail(country)
-		}
-
-		collectionID = match.id
-		report?.(`  Found collection id=${collectionID} for '${country}'`)
-	}
-
-	// MARK: Download via the collections download endpoint (302s to S3)
-
-	report?.(`  Resolving download URL for collection id=${collectionID}...`)
-	report?.(`  Attempting authenticated download...`)
-
-	const tmpGz = join(destDir, "collection.geojsonl.gz.tmp")
-	const tmpRaw = join(destDir, "collection.geojsonl.tmp")
-	const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`
-
-	let httpStatus = await streamDownload(sourceURL, tmpGz, {
-		headers: { Authorization: `Bearer ${token}` },
-		timeoutMs: 7_200_000,
-		retries: 3,
-		retryDelayMs: 30_000,
-	})
-
-	if (httpStatus !== HTTP_OK) {
-		// Try the geojsonl.gz directly with token as query param (alternate URL shape).
-		httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
-			timeoutMs: 7_200_000,
-			retries: 3,
-			retryDelayMs: 30_000,
-		})
-	}
-
-	if (httpStatus !== HTTP_OK) {
-		rmSync(tmpGz, { force: true })
-
-		report?.(`
-ERROR: Download returned HTTP ${httpStatus}.
-
-Likely causes:
-  1. OA_BATCH_TOKEN is invalid or expired — re-create it at Profile → Tokens.
-  2. The collection download endpoint URL has changed (this module was written
-     against the 2026-05-18 batch.openaddresses.io API; it may need updating).
-  3. Network error or CDN outage.
-
-Manual download (after logging in to batch.openaddresses.io):
-  - Navigate to https://batch.openaddresses.io/collection/${collectionID}
-  - Click "GeoJSON+LD" to download the collection.
-  - Save as: ${outputFile}
-
-URL tried: ${OA_BASE}/api/collections/${collectionID}/download
-`)
-
-		return fail(country)
-	}
-
-	// MARK: Decompress if the downloaded file is gzipped
-
-	const fileMagic = (await execFileAsync("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout
-
-	if (/gzip|compressed/i.test(fileMagic)) {
-		report?.(`  Decompressing gzip archive...`)
-		await gunzipToFile(tmpGz, tmpRaw)
-		rmSync(tmpGz, { force: true })
-		renameSync(tmpRaw, outputFile)
-	} else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
-		// Already line-delimited GeoJSON.
-		renameSync(tmpGz, outputFile)
-		rmSync(tmpRaw, { force: true })
-	} else {
-		// Unknown type — keep as-is and let the operator inspect.
-		renameSync(tmpGz, outputFile)
-		report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`)
-	}
-
-	// MARK: Verify + write MANIFEST
-
-	if (!existsSync(outputFile)) {
-		report?.(`ERROR: Output file not found at ${outputFile} after download.`)
-
-		return fail(country)
-	}
-
-	const size = statSync(outputFile).size
-
-	if (size < MIN_PLAUSIBLE_SHARD_BYTES) {
-		report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`)
-
-		return fail(country)
-	}
-
-	const sha = await sha256File(outputFile)
-	const rowCount = await countLines(outputFile)
-	const downloadedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z")
-
-	const manifest = {
-		source_url: sourceURL,
-		collection_id: collectionID,
-		country,
-		filename: "collection.geojsonl",
-		downloaded_at: downloadedAt,
-		sha256: sha,
-		bytes: size,
-		row_count: rowCount,
-		notes:
-			"batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${ByteFormatter.formatIEC(size)}  rows=${rowCount}  sha256=${sha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-	report?.(`=== done`)
-	report?.(`Feed to the adapter:`)
-	report?.(`  mailwoman corpus run openaddresses \\`)
-	report?.(`    --input ${outputFile} \\`)
-	report?.(`    --country ${country.toUpperCase()} \\`)
-	report?.(`    --output ${options.outRoot}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/src/tools/fetch/ourairports.ts

@@ -1,171 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
- *
- *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
- *            mirror of the nightly export; `ourairports.com/data/` redirects here).
- *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
- *            courtesy credit — no attribution obligation rides on a derived shard, which makes this
- *            the one transport source in the arc with no licensing question at all. Tier A.
- *
- *   ## What it is good for, and what it is not
- *
- *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
- *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
- *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
- *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
- *   a spatial join.
- *
- *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
- *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
- *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
- *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
- *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
- *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
- *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
- *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
- *   accordingly; the split between the two is the one `AGENTS.md` draws.
- *
- *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
- */
-
-import { mkdirSync } from "node:fs"
-import { join } from "node:path"
-
-import { APIClient } from "@mailwoman/core/api"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, writeManifest } from "./download.ts"
-
-const SLUG = "ourairports"
-
-/**
- * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
- * mirror directly saves a redirect and is the URL the project's own README gives.
- */
-const BASE_URL = "https://davidmegginson.github.io/ourairports-data"
-
-/**
- * The files worth having, and why each one.
- *
- * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
- * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
- * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
- * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
- * shard ever needs one.
- */
-const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"] as const
-
-export type FetchOurAirportsOptions = BaseFetchOptions
-
-interface OurAirportsFileEntry {
-	filename: string
-	source_url: string
-	sha256: string
-	bytes: number
-	/**
-	 * The upstream `Last-Modified`, when the CDN gave one. This is the DATA's vintage; `downloaded_at` is only when we
-	 * asked. `corpus/AGENTS.md` has the standing warning that a file's mtime is not its data's vintage — recording the
-	 * upstream header is how a later refresh decision gets made on the right number.
-	 */
-	last_modified: string | null
-}
-
-interface OurAirportsManifest {
-	source: string
-	base_url: string
-	license: string
-	downloaded_at: string
-	files: OurAirportsFileEntry[]
-}
-
-/**
- * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
- * and must never fail a download that otherwise succeeded.
- */
-async function readLastModified(url: string): Promise<string | null> {
-	try {
-		const res = await new APIClient({ displayName: "ourairports", retry: true }).fetch({
-			url,
-			method: "head",
-			timeout: 30_000,
-		})
-
-		return (res.headers["last-modified"] as string | undefined) ?? null
-	} catch {
-		return null
-	}
-}
-
-/**
- * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
- * origin URL, sha256, byte count and upstream `Last-Modified`.
- */
-export async function fetchOurAirports(
-	options: FetchOurAirportsOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	mkdirSync(destDir, { recursive: true })
-
-	const entries: OurAirportsFileEntry[] = []
-	const failedCodes: string[] = []
-	let fetched = 0
-	let failed = 0
-
-	for (const filename of FILES) {
-		const url = `${BASE_URL}/${filename}`
-		const dest = join(destDir, filename)
-
-		report?.(`=== ${SLUG} / ${filename}`)
-
-		try {
-			const [{ bytes }, lastModified] = await Promise.all([
-				downloadToFile({
-					url,
-					dest,
-					timeoutMs: 600_000,
-					retries: 2,
-					headers: { "Accept-Encoding": "gzip, br" },
-					report,
-				}),
-				readLastModified(url),
-			])
-
-			entries.push({
-				filename,
-				source_url: url,
-				sha256: await sha256File(dest),
-				bytes,
-				last_modified: lastModified,
-			})
-
-			fetched++
-		} catch (error) {
-			report?.(`✗ ${filename}: ${error instanceof Error ? error.message : String(error)}`)
-			failedCodes.push(filename)
-
-			failed++
-		}
-	}
-
-	const manifest: OurAirportsManifest = {
-		source: "OurAirports",
-		base_url: BASE_URL,
-		license: "public domain (courtesy credit requested)",
-		downloaded_at: new Date().toISOString(),
-		files: entries,
-	}
-
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-	return { fetched, skipped: 0, failed, failedCodes }
-}