@mailwoman/corpus 9.1.0 → 9.3.0

package/out/tools/fetch/nppes.js

@@ -0,0 +1,126 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
+ *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
+ *   adapter. US Public Domain.
+ *
+ *   The file is published monthly by CMS. This module discovers the current filename by scraping the
+ *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
+ *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
+ *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
+ *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
+ *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
+ *   no resume, so a partial run re-downloads from the start.
+ *
+ *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
+ *   sha256 matches MANIFEST, skips download.
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { APIClient, pluckResponseData } from "@mailwoman/core/api";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { basename, join } from "path-ts";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
+const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html";
+const BASE_URL = "https://download.cms.gov/nppes";
+const SLUG = "usgov-nppes";
+/**
+ * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
+ * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
+ */
+async function discoverLatestZip() {
+    // `responseType: "text"` — the index is HTML, scraped by regex below.
+    const html = await new APIClient({
+        displayName: "nppes-index",
+        retry: true,
+        axios: { headers: { "Accept-Encoding": "gzip, br" } },
+    })
+        .fetch({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
+        .then(pluckResponseData);
+    for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
+        const name = match[0];
+        if (name && !/\d{6}_\d{6}/.test(name))
+            return name;
+    }
+    return undefined;
+}
+/**
+ * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
+ * change file.
+ */
+async function findNpidataCSV(zipPath) {
+    const entries = await listZipEntries(zipPath);
+    return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name;
+}
+export async function fetchNPPES(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    report?.(`=== ${SLUG}`);
+    report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`);
+    const zipFilename = await discoverLatestZip();
+    if (!zipFilename) {
+        report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const zipURL = `${BASE_URL}/${zipFilename}`;
+    const zipDest = join(destDir, zipFilename);
+    report?.(`  Latest full file: ${zipFilename}`);
+    // Idempotency check: if the main CSV already exists and sha matches,
+    // skip re-download.
+    const recorded = await readManifest(manifestPath);
+    if (recorded?.sha256 && recorded.filename) {
+        const recordedPath = join(destDir, recorded.filename);
+        if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
+            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+        }
+    }
+    // MARK: Download ZIP (large; 60-minute timeout)
+    report?.(`  Downloading ${zipURL} ...`);
+    const { bytes: zipSize } = await downloadToFile({
+        url: zipURL,
+        dest: zipDest,
+        timeoutMs: 3_600_000,
+        headers: { "Accept-Encoding": "gzip, br" },
+        report,
+    });
+    report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
+    // MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
+    report?.("  Extracting npidata_pfile CSV from ZIP ...");
+    const csvName = await findNpidataCSV(zipDest);
+    if (!csvName) {
+        report?.("  ✗ Could not find npidata_pfile CSV inside ZIP");
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Extracting: ${csvName}`);
+    const csvDest = join(destDir, basename(csvName));
+    await extractZipEntry(zipDest, csvName, csvDest);
+    const csvSize = (await statPath(csvDest)).size;
+    const csvSha = await sha256File(csvDest);
+    report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`);
+    // MARK: Remove the ZIP to reclaim ~1 GB
+    await removePathIfPresent(zipDest);
+    report?.("  Removed ZIP (CSV kept)");
+    // MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
+    const manifest = {
+        source_url: zipURL,
+        downloaded_at: new Date().toISOString(),
+        filename: csvName,
+        sha256: csvSha,
+        bytes: csvSize,
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=nppes.js.map

package/out/tools/fetch/nppes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../lib/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAI1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC;QAChC,WAAW,EAAE,aAAa;QAC1B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACxE,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAA;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IACvB,MAAM,EAAE,CAAC,kDAAkD,SAAS,MAAM,CAAC,CAAA;IAE3E,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAEjE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAA;IAE9C,qEAAqE;IACrE,oBAAoB;IACpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9F,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAEhD,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAA;IAEvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,iEAAiE;IAEjE,MAAM,EAAE,CAAC,6CAA6C,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAA;IAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEhE,wCAAwC;IAExC,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,gEAAgE;IAEhE,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/openaddresses.d.ts

@@ -0,0 +1,9 @@
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export interface FetchOpenAddressesOptions extends BaseFetchOptions {
+    /**
+     * OA country collection code. Default `ca`.
+     */
+    country?: string;
+}
+export declare function fetchOpenAddresses(options: FetchOpenAddressesOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=openaddresses.d.ts.map

package/out/tools/fetch/openaddresses.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaddresses.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAiEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAgC3E,MAAM,WAAW,yBAA0B,SAAQ,gBAAgB;IAClE;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AA0CD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,yBAAyB,EAClC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAuMvB"}

package/out/tools/fetch/openaddresses.js

@@ -0,0 +1,279 @@
+import { APIClient, isSuccessStatus } from "@mailwoman/core/api";
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
+ *
+ *   Source: https://batch.openaddresses.io
+ *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
+ *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
+ *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
+ *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
+ *   adapter does the license filtering.
+ *
+ *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
+ *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
+ *   `gunzip` decompression, both nice/ionice-deprioritized).
+ *
+ *   ## Authentication note (2026-05-18)
+ *
+ *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
+ *   still free at the "basic" tier (GeoJSON+LD output).
+ *
+ *   1. Register at https://batch.openaddresses.io/register
+ *   2. Log in and go to Profile → "Create Token"
+ *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
+ *   4. Re-run the command.
+ *
+ *   The collection URL pattern (verified 2026-05-18):
+ *
+ *   - `POST /api/login {username, password}` → `{token}`
+ *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
+ *
+ *   Collections are downloaded as a combined GeoJSON.gz via:
+ *
+ *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
+ *
+ *   Collection IDs discovered from `/api/collections`:
+ *
+ *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
+ *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Without token (will detect + print instructions, then report the failure):
+ *   mailwoman corpus fetch openaddresses --country ca
+ *   ```
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { openReadStream, openWriteStream, pipeline } from "@mailwoman/core/fs/streams";
+import { movePath, removePathIfPresent, makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { runFile, spawnProcess } from "@mailwoman/core/process";
+import { isoSeconds } from "@mailwoman/core/utils";
+import { join } from "path-ts";
+import { $private } from "#env";
+import { streamDownload, writeManifest } from "#tools/fetch/download";
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+/**
+ * A successful fetch; anything else is an error page or a redirect we did not follow.
+ */
+const HTTP_OK = 200;
+/**
+ * Smallest plausible OpenAddresses slice. Below 10 KiB the file is a stub or an error body.
+ */
+const MIN_PLAUSIBLE_SLICE_BYTES = 10_240;
+const OA_BASE = "https://batch.openaddresses.io";
+/**
+ * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
+ * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
+ */
+const OA_COLLECTION_IDS = {
+    ca: 6,
+    "us-west": 4,
+    "us-south": 3,
+    "us-northeast": 2,
+    "us-midwest": 5,
+    global: 1,
+};
+/**
+ * Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection).
+ */
+async function countLines(path) {
+    let count = 0;
+    for await (const chunk of openReadStream(path)) {
+        for (const byte of chunk) {
+            if (byte === 0x0a) {
+                count++;
+            }
+        }
+    }
+    return count;
+}
+/**
+ * Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used.
+ */
+async function gunzipToFile(src, dest) {
+    const child = spawnProcess("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
+        stdio: ["ignore", "pipe", "inherit"],
+    });
+    await pipeline(child.stdout, openWriteStream(dest));
+    await new Promise((resolve, reject) => {
+        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`gunzip exited with code ${code}`))));
+        child.on("error", reject);
+    });
+}
+export async function fetchOpenAddresses(options, report) {
+    const country = options.country ?? "ca";
+    const token = $private.OA_BATCH_TOKEN;
+    const destDir = join(options.outRoot, "openaddresses", country);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const outputFile = join(destDir, "collection.geojsonl");
+    const fail = (code) => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] });
+    report?.(`=== fetch openaddresses: country=${country}`);
+    report?.(`    dest: ${destDir}`);
+    await makeDirectories(destDir);
+    // MARK: Authentication check
+    if (!token) {
+        report?.(`
+ERROR: OA_BATCH_TOKEN is not set.
+
+As of 2026-05-18, batch.openaddresses.io requires a registered (free) account
+to download collection files.  Data remains openly licensed — the auth gate
+is there to prevent CDN abuse, not to restrict access.
+
+Steps to get a token:
+  1. Register at: https://batch.openaddresses.io/register
+  2. Verify your email and log in.
+  3. Go to Profile → "Create Token" → copy the token.
+  4. Export it in this shell:
+       export OA_BATCH_TOKEN=<your-token>
+  5. Re-run this command.
+
+The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
+(estimated), so budget ~20–45 minutes at typical cloud-to-host bandwidth.
+`);
+        return fail("OA_BATCH_TOKEN");
+    }
+    // MARK: Determine collection ID
+    let collectionID = OA_COLLECTION_IDS[country];
+    if (collectionID === undefined) {
+        report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`);
+        // The collections API only. The COLLECTION ARCHIVES stay on raw `fetch` — they stream multi-gigabyte
+        // bodies straight to disk, where response caching is nonsense and axios would buffer them in memory.
+        const res = await new APIClient({
+            displayName: "openaddresses-api",
+            retry: true,
+            axios: { headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" } },
+        })
+            // `validateStatus` keeps a non-2xx as a RESPONSE rather than a throw: the caller reports the status and
+            // returns `fail(country)`, a graceful path this must not turn into an exception.
+            .fetch({
+            url: `${OA_BASE}/api/collections`,
+            timeout: 30_000,
+            validateStatus: () => true,
+        });
+        if (!isSuccessStatus(res.status)) {
+            report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`);
+            return fail(country);
+        }
+        const collections = res.data;
+        const match = collections.find((item) => item.name === country);
+        if (match?.id === undefined) {
+            report?.(`ERROR: Could not find a collection named '${country}' in GET /api/collections.`);
+            report?.(`Available collections:`);
+            for (const item of collections) {
+                const size = (item.size ?? 0).toLocaleString();
+                report?.(`  ${(item.name ?? "").padEnd(20)}  id=${item.id}  ${item.human ?? ""}  size=${size} bytes`);
+            }
+            return fail(country);
+        }
+        collectionID = match.id;
+        report?.(`  Found collection id=${collectionID} for '${country}'`);
+    }
+    // MARK: Download via the collections download endpoint (302s to S3)
+    report?.(`  Resolving download URL for collection id=${collectionID}...`);
+    report?.(`  Attempting authenticated download...`);
+    const tmpGz = join(destDir, "collection.geojsonl.gz.tmp");
+    const tmpRaw = join(destDir, "collection.geojsonl.tmp");
+    const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`;
+    let httpStatus = await streamDownload(sourceURL, tmpGz, {
+        headers: { Authorization: `Bearer ${token}` },
+        timeoutMs: 7_200_000,
+        retries: 3,
+        retryDelayMs: 30_000,
+    });
+    if (httpStatus !== HTTP_OK) {
+        // Try the geojsonl.gz directly with token as query param (alternate URL shape).
+        httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
+            timeoutMs: 7_200_000,
+            retries: 3,
+            retryDelayMs: 30_000,
+        });
+    }
+    if (httpStatus !== HTTP_OK) {
+        await removePathIfPresent(tmpGz);
+        report?.(`
+ERROR: Download returned HTTP ${httpStatus}.
+
+Likely causes:
+  1. OA_BATCH_TOKEN is invalid or expired — re-create it at Profile → Tokens.
+  2. The collection download endpoint URL has changed (this module was written
+     against the 2026-05-18 batch.openaddresses.io API; it may need updating).
+  3. Network error or CDN outage.
+
+Manual download (after logging in to batch.openaddresses.io):
+  - Navigate to https://batch.openaddresses.io/collection/${collectionID}
+  - Click "GeoJSON+LD" to download the collection.
+  - Save as: ${outputFile}
+
+URL tried: ${OA_BASE}/api/collections/${collectionID}/download
+`);
+        return fail(country);
+    }
+    // MARK: Decompress if the downloaded file is gzipped
+    const fileMagic = (await runFile("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout;
+    if (/gzip|compressed/i.test(fileMagic)) {
+        report?.(`  Decompressing gzip archive...`);
+        await gunzipToFile(tmpGz, tmpRaw);
+        await removePathIfPresent(tmpGz);
+        await movePath(tmpRaw, outputFile);
+    }
+    else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
+        // Already line-delimited GeoJSON.
+        await movePath(tmpGz, outputFile);
+        await removePathIfPresent(tmpRaw);
+    }
+    else {
+        // Unknown type — keep as-is and let the operator inspect.
+        await movePath(tmpGz, outputFile);
+        report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`);
+    }
+    // MARK: Verify + write MANIFEST
+    if (!(await pathExists(outputFile))) {
+        report?.(`ERROR: Output file not found at ${outputFile} after download.`);
+        return fail(country);
+    }
+    const size = (await statPath(outputFile)).size;
+    if (size < MIN_PLAUSIBLE_SLICE_BYTES) {
+        report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`);
+        return fail(country);
+    }
+    const sha = await sha256File(outputFile);
+    const rowCount = await countLines(outputFile);
+    const downloadedAt = isoSeconds();
+    const manifest = {
+        source_url: sourceURL,
+        collection_id: collectionID,
+        country,
+        filename: "collection.geojsonl",
+        downloaded_at: downloadedAt,
+        sha256: sha,
+        bytes: size,
+        row_count: rowCount,
+        notes: "batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${ByteFormatter.formatIEC(size)}  rows=${rowCount}  sha256=${sha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    report?.(`=== done`);
+    report?.(`Feed to the adapter:`);
+    report?.(`  mailwoman corpus run openaddresses \\`);
+    report?.(`    --input ${outputFile} \\`);
+    report?.(`    --country ${country.toUpperCase()} \\`);
+    report?.(`    --output ${options.outRoot}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=openaddresses.js.map

package/out/tools/fetch/openaddresses.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openaddresses.js","sourceRoot":"","sources":["../../../lib/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAChE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH;kGACkG;AAClG,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACtF,OAAO,EAAE,QAAQ,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAE/B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,yBAAyB,GAAG,MAAM,CAAA;AAExC,MAAM,OAAO,GAAG,gCAAgC,CAAA;AAEhD;;;GAGG;AACH,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,CAAC;IACL,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,MAAM,EAAE,CAAC;CACT,CAAA;AAgBD;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,IAAI,CAA0B,EAAE,CAAC;QACzE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAA;YACR,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,IAAY;IACpD,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE;QAC1F,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC;KACpC,CAAC,CAAA;IAEF,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAO,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAA;IAEpD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5G,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC,EAClC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAA;IAErC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAA;IAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;IAEvD,MAAM,IAAI,GAAG,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEzG,MAAM,EAAE,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAEhC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,6BAA6B;IAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;;;;;;;;;;;;;;;;;CAiBV,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC9B,CAAC;IAED,gCAAgC;IAEhC,IAAI,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,EAAE,CAAC,yBAAyB,OAAO,2CAA2C,CAAC,CAAA;QAErF,qGAAqG;QACrG,qGAAqG;QACrG,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC;YAC/B,WAAW,EAAE,mBAAmB;YAChC,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;SACvF,CAAC;YACD,wGAAwG;YACxG,iFAAiF;aAChF,KAAK,CAAiB;YACtB,GAAG,EAAE,GAAG,OAAO,kBAAkB;YACjC,OAAO,EAAE,MAAM;YACf,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,CAAC,CAAA;QAEH,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,MAAM,EAAE,CAAC,6CAA6C,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;YAEpE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAA;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,EAAE,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC,6CAA6C,OAAO,4BAA4B,CAAC,CAAA;YAC1F,MAAM,EAAE,CAAC,wBAAwB,CAAC,CAAA;YAElC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAA;gBAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAA;YACtG,CAAC;YAED,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,YAAY,GAAG,KAAK,CAAC,EAAE,CAAA;QACvB,MAAM,EAAE,CAAC,yBAAyB,YAAY,SAAS,OAAO,GAAG,CAAC,CAAA;IACnE,CAAC;IAED,oEAAoE;IAEpE,MAAM,EAAE,CAAC,8CAA8C,YAAY,KAAK,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,wCAAwC,CAAC,CAAA;IAElD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,GAAG,OAAO,oBAAoB,YAAY,WAAW,CAAA;IAEvE,IAAI,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE;QACvD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE;QAC7C,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,MAAM;KACpB,CAAC,CAAA;IAEF,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,gFAAgF;QAChF,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,OAAO,oBAAoB,YAAY,sBAAsB,KAAK,EAAE,EAAE,KAAK,EAAE;YACjH,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAA;QAEhC,MAAM,EAAE,CAAC;gCACqB,UAAU;;;;;;;;;4DASkB,YAAY;;eAEzD,UAAU;;aAEZ,OAAO,oBAAoB,YAAY;CACnD,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,qDAAqD;IAErD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAElG,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,EAAE,CAAC,iCAAiC,CAAC,CAAA;QAC3C,MAAM,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,mBAAmB,CAAC,KAAK,CAAC,CAAA;QAChC,MAAM,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IACnC,CAAC;SAAM,IAAI,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAChD,kCAAkC;QAClC,MAAM,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QACjC,MAAM,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;SAAM,CAAC;QACP,0DAA0D;QAC1D,MAAM,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QACjC,MAAM,EAAE,CAAC,uCAAuC,SAAS,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAA;IACtG,CAAC;IAED,gCAAgC;IAEhC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC,mCAAmC,UAAU,kBAAkB,CAAC,CAAA;QAEzE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAA;IAE9C,IAAI,IAAI,GAAG,yBAAyB,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,sCAAsC,IAAI,qCAAqC,CAAC,CAAA;QAEzF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,UAAU,EAAE,CAAA;IAEjC,MAAM,QAAQ,GAAG;QAChB,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,YAAY;QAC3B,OAAO;QACP,QAAQ,EAAE,qBAAqB;QAC/B,aAAa,EAAE,YAAY;QAC3B,MAAM,EAAE,GAAG;QACX,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,QAAQ;QACnB,KAAK,EACJ,8LAA8L;KAC/L,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,QAAQ,YAAY,GAAG,EAAE,CAAC,CAAA;IACjF,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,UAAU,CAAC,CAAA;IACpB,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yCAAyC,CAAC,CAAA;IACnD,MAAM,EAAE,CAAC,eAAe,UAAU,KAAK,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,iBAAiB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/ourairports.d.ts

@@ -0,0 +1,45 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
+ *
+ *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
+ *            mirror of the nightly export; `ourairports.com/data/` redirects here).
+ *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
+ *            courtesy credit — no attribution obligation rides on a derived slice, which makes this
+ *            the one transport source in the arc with no licensing question at all. Tier A.
+ *
+ *   ## What it is good for, and what it is not
+ *
+ *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
+ *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
+ *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
+ *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
+ *   a spatial join.
+ *
+ *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
+ *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
+ *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
+ *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
+ *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
+ *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
+ *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
+ *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
+ *   accordingly; the split between the two is the one `AGENTS.md` draws.
+ *
+ *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchOurAirportsOptions = BaseFetchOptions;
+/**
+ * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
+ * origin URL, sha256, byte count and upstream `Last-Modified`.
+ */
+export declare function fetchOurAirports(options: FetchOurAirportsOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=ourairports.d.ts.map

package/out/tools/fetch/ourairports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ourairports.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAsB3E,MAAM,MAAM,uBAAuB,GAAG,gBAAgB,CAAA;AAyCtD;;;GAGG;AACH,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,uBAAuB,EAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAwDvB"}

package/out/tools/fetch/ourairports.js

@@ -0,0 +1,128 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
+ *
+ *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
+ *            mirror of the nightly export; `ourairports.com/data/` redirects here).
+ *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
+ *            courtesy credit — no attribution obligation rides on a derived slice, which makes this
+ *            the one transport source in the arc with no licensing question at all. Tier A.
+ *
+ *   ## What it is good for, and what it is not
+ *
+ *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
+ *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
+ *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
+ *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
+ *   a spatial join.
+ *
+ *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
+ *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
+ *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
+ *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
+ *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
+ *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
+ *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
+ *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
+ *   accordingly; the split between the two is the one `AGENTS.md` draws.
+ *
+ *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, writeManifest } from "#tools/fetch/download";
+const SLUG = "ourairports";
+/**
+ * The GitHub Pages mirror the project itself publishes. `ourairports.com/data/*.csv` 302s here, so pointing at the
+ * mirror directly saves a redirect and is the URL the project's own README gives.
+ */
+const BASE_URL = "https://davidmegginson.github.io/ourairports-data";
+/**
+ * The files worth having, and why each one.
+ *
+ * `airports.csv` is the payload. The other three are small joins that turn its codes into text: `countries.csv` and
+ * `regions.csv` expand `iso_country`/`iso_region` into names (a corpus line needs "Germany", not "DE"), and
+ * `runways.csv` is the only file carrying per-airport sub-structure of any kind — runway designators, which are NOT
+ * sub-venue designators (nobody addresses mail to a runway) but are worth having on disk as the negative class if the
+ * slice ever needs one.
+ */
+const FILES = ["airports.csv", "countries.csv", "regions.csv", "runways.csv"];
+/**
+ * Read the upstream `Last-Modified` with a HEAD. Returns `null` on any failure — provenance metadata is nice to have
+ * and must never fail a download that otherwise succeeded.
+ */
+async function readLastModified(url) {
+    try {
+        const res = await new APIClient({ displayName: "ourairports", retry: true }).fetch({
+            url,
+            method: "head",
+            timeout: 30_000,
+        });
+        return res.headers["last-modified"] ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
+ * origin URL, sha256, byte count and upstream `Last-Modified`.
+ */
+export async function fetchOurAirports(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const entries = [];
+    const failedCodes = [];
+    let fetched = 0;
+    let failed = 0;
+    for (const filename of FILES) {
+        const url = `${BASE_URL}/${filename}`;
+        const dest = join(destDir, filename);
+        report?.(`=== ${SLUG} / ${filename}`);
+        try {
+            const [{ bytes }, lastModified] = await Promise.all([
+                downloadToFile({
+                    url,
+                    dest,
+                    timeoutMs: 600_000,
+                    retries: 2,
+                    headers: { "Accept-Encoding": "gzip, br" },
+                    report,
+                }),
+                readLastModified(url),
+            ]);
+            entries.push({
+                filename,
+                source_url: url,
+                sha256: await sha256File(dest),
+                bytes,
+                last_modified: lastModified,
+            });
+            fetched++;
+        }
+        catch (error) {
+            report?.(`✗ ${filename}: ${error instanceof Error ? error.message : String(error)}`);
+            failedCodes.push(filename);
+            failed++;
+        }
+    }
+    const manifest = {
+        source: "OurAirports",
+        base_url: BASE_URL,
+        license: "public domain (courtesy credit requested)",
+        downloaded_at: new Date().toISOString(),
+        files: entries,
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched, skipped: 0, failed, failedCodes };
+}
+//# sourceMappingURL=ourairports.js.map

package/out/tools/fetch/ourairports.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ourairports.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE,MAAM,IAAI,GAAG,aAAa,CAAA;AAE1B;;;GAGG;AACH,MAAM,QAAQ,GAAG,mDAAmD,CAAA;AAEpE;;;;;;;;GAQG;AACH,MAAM,KAAK,GAAG,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,aAAa,CAAU,CAAA;AAyBtF;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC;YAClF,GAAG;YACH,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,MAAM;SACf,CAAC,CAAA;QAEF,OAAQ,GAAG,CAAC,OAAO,CAAC,eAAe,CAAwB,IAAI,IAAI,CAAA;IACpE,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAA;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,CAAC;YACJ,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACnD,cAAc,CAAC;oBACd,GAAG;oBACH,IAAI;oBACJ,SAAS,EAAE,OAAO;oBAClB,OAAO,EAAE,CAAC;oBACV,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;oBAC1C,MAAM;iBACN,CAAC;gBACF,gBAAgB,CAAC,GAAG,CAAC;aACrB,CAAC,CAAA;YAEF,OAAO,CAAC,IAAI,CAAC;gBACZ,QAAQ;gBACR,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;gBAC9B,KAAK;gBACL,aAAa,EAAE,YAAY;aAC3B,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,KAAK,QAAQ,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACpF,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAE1B,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAwB;QACrC,MAAM,EAAE,aAAa;QACrB,QAAQ,EAAE,QAAQ;QAClB,OAAO,EAAE,2CAA2C;QACpD,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/ppd.d.ts

@@ -0,0 +1,14 @@
+export interface PPDExtractStats {
+    kept: number;
+    skippedSAON: number;
+    skippedPAON: number;
+    skippedNoStreet: number;
+    skippedNoPostcode: number;
+}
+/**
+ * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
+ * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
+ * of rows (tests) or a streamed async source (the real 31M-row extraction).
+ */
+export declare function extractPPDTuples(input: AsyncIterable<string[]> | Iterable<string[]>, write: (line: string) => void): Promise<PPDExtractStats>;
+//# sourceMappingURL=ppd.d.ts.map

package/out/tools/fetch/ppd.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B"}