@mailwoman/corpus 9.2.0 → 9.4.0

package/out/src/tools/fetch/nppes.js

@@ -1,126 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
- *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
- *   adapter. US Public Domain.
- *
- *   The file is published monthly by CMS. This module discovers the current filename by scraping the
- *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
- *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
- *
- *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
- *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
- *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
- *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
- *   no resume, so a partial run re-downloads from the start.
- *
- *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
- *   sha256 matches MANIFEST, skips download.
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { existsSync, mkdirSync, statSync } from "node:fs";
-import { rm } from "node:fs/promises";
-import { basename, join } from "node:path";
-import { APIClient, pluckResponseData } from "@mailwoman/core/api";
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, readManifest, writeManifest } from "./download.js";
-const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html";
-const BASE_URL = "https://download.cms.gov/nppes";
-const SLUG = "usgov-nppes";
-/**
- * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
- * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
- */
-async function discoverLatestZip() {
-    // `responseType: "text"` — the index is HTML, scraped by regex below.
-    const html = await new APIClient({
-        displayName: "nppes-index",
-        retry: true,
-        axios: { headers: { "Accept-Encoding": "gzip, br" } },
-    })
-        .fetch({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
-        .then(pluckResponseData);
-    for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
-        const name = match[0];
-        if (name && !/\d{6}_\d{6}/.test(name))
-            return name;
-    }
-    return undefined;
-}
-/**
- * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
- * change file.
- */
-async function findNpidataCSV(zipPath) {
-    const entries = await listZipEntries(zipPath);
-    return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name;
-}
-export async function fetchNPPES(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    mkdirSync(destDir, { recursive: true });
-    const manifestPath = join(destDir, "MANIFEST.json");
-    report?.(`=== ${SLUG}`);
-    report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`);
-    const zipFilename = await discoverLatestZip();
-    if (!zipFilename) {
-        report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    const zipURL = `${BASE_URL}/${zipFilename}`;
-    const zipDest = join(destDir, zipFilename);
-    report?.(`  Latest full file: ${zipFilename}`);
-    // Idempotency check: if the main CSV already exists and sha matches,
-    // skip re-download.
-    const recorded = await readManifest(manifestPath);
-    if (recorded?.sha256 && recorded.filename) {
-        const recordedPath = join(destDir, recorded.filename);
-        if (existsSync(recordedPath) && (await sha256File(recordedPath)) === recorded.sha256) {
-            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
-            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
-        }
-    }
-    // MARK: Download ZIP (large; 60-minute timeout)
-    report?.(`  Downloading ${zipURL} ...`);
-    const { bytes: zipSize } = await downloadToFile({
-        url: zipURL,
-        dest: zipDest,
-        timeoutMs: 3_600_000,
-        headers: { "Accept-Encoding": "gzip, br" },
-        report,
-    });
-    report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
-    // MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
-    report?.("  Extracting npidata_pfile CSV from ZIP ...");
-    const csvName = await findNpidataCSV(zipDest);
-    if (!csvName) {
-        report?.("  ✗ Could not find npidata_pfile CSV inside ZIP");
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    report?.(`  Extracting: ${csvName}`);
-    const csvDest = join(destDir, basename(csvName));
-    await extractZipEntry(zipDest, csvName, csvDest);
-    const csvSize = statSync(csvDest).size;
-    const csvSha = await sha256File(csvDest);
-    report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`);
-    // MARK: Remove the ZIP to reclaim ~1 GB
-    await rm(zipDest, { force: true });
-    report?.("  Removed ZIP (CSV kept)");
-    // MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
-    const manifest = {
-        source_url: zipURL,
-        downloaded_at: new Date().toISOString(),
-        filename: csvName,
-        sha256: csvSha,
-        bytes: csvSize,
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=nppes.js.map

package/out/src/tools/fetch/nppes.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../src/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAY1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC;QAChC,WAAW,EAAE,aAAa;QAC1B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACxE,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAA;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IACvB,MAAM,EAAE,CAAC,kDAAkD,SAAS,MAAM,CAAC,CAAA;IAE3E,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAEjE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAA;IAE9C,qEAAqE;IACrE,oBAAoB;IACpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAEhD,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAA;IAEvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,iEAAiE;IAEjE,MAAM,EAAE,CAAC,6CAA6C,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAA;IAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEhE,wCAAwC;IAExC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,gEAAgE;IAEhE,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/openaddresses.d.ts

@@ -1,61 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
- *
- *   Source: https://batch.openaddresses.io
- *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
- *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
- *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
- *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
- *   adapter does the license gating.
- *
- *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
- *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
- *   `gunzip` decompression, both nice/ionice-deprioritized).
- *
- *   ## Authentication note (2026-05-18)
- *
- *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
- *   still free at the "basic" tier (GeoJSON+LD output).
- *
- *   1. Register at https://batch.openaddresses.io/register
- *   2. Log in and go to Profile → "Create Token"
- *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
- *   4. Re-run the command.
- *
- *   The collection URL pattern (verified 2026-05-18):
- *
- *   - `POST /api/login {username, password}` → `{token}`
- *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
- *
- *   Collections are downloaded as a combined GeoJSON.gz via:
- *
- *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
- *
- *   Collection IDs discovered from `/api/collections`:
- *
- *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
- *
- *   ## Usage
- *
- *   ```sh
- *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
- *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Without token (will detect + print instructions, then report the failure):
- *   mailwoman corpus fetch openaddresses --country ca
- *   ```
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export interface FetchOpenAddressesOptions extends BaseFetchOptions {
-    /**
-     * OA country collection code. Default `ca`.
-     */
-    country?: string;
-}
-export declare function fetchOpenAddresses(options: FetchOpenAddressesOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=openaddresses.d.ts.map

package/out/src/tools/fetch/openaddresses.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"openaddresses.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAkBH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAkCnE,MAAM,WAAW,yBAA0B,SAAQ,gBAAgB;IAClE;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AA6FD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,yBAAyB,EAClC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAuMvB"}

package/out/src/tools/fetch/openaddresses.js

@@ -1,317 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
- *
- *   Source: https://batch.openaddresses.io
- *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
- *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
- *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
- *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
- *   adapter does the license gating.
- *
- *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
- *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
- *   `gunzip` decompression, both nice/ionice-deprioritized).
- *
- *   ## Authentication note (2026-05-18)
- *
- *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
- *   still free at the "basic" tier (GeoJSON+LD output).
- *
- *   1. Register at https://batch.openaddresses.io/register
- *   2. Log in and go to Profile → "Create Token"
- *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
- *   4. Re-run the command.
- *
- *   The collection URL pattern (verified 2026-05-18):
- *
- *   - `POST /api/login {username, password}` → `{token}`
- *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
- *
- *   Collections are downloaded as a combined GeoJSON.gz via:
- *
- *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
- *
- *   Collection IDs discovered from `/api/collections`:
- *
- *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
- *
- *   ## Usage
- *
- *   ```sh
- *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
- *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Without token (will detect + print instructions, then report the failure):
- *   mailwoman corpus fetch openaddresses --country ca
- *   ```
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { execFile, spawn } from "node:child_process";
-import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs";
-import { join } from "node:path";
-import { Readable } from "node:stream";
-import { pipeline } from "node:stream/promises";
-import { setTimeout as sleep } from "node:timers/promises";
-import { promisify } from "node:util";
-import { APIClient, isSuccessStatus } from "@mailwoman/core/api";
-import { $private } from "@mailwoman/core/env";
-import { ByteFormatter } from "@mailwoman/core/fs/utils";
-import { sha256File } from "@mailwoman/core/utils";
-import { isTransientStatus, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * A successful fetch; anything else is an error page or a redirect we did not follow.
- */
-const HTTP_OK = 200;
-/**
- * Smallest plausible OpenAddresses shard. Below 10 KiB the file is a stub or an error body.
- */
-const MIN_PLAUSIBLE_SHARD_BYTES = 10_240;
-const execFileAsync = promisify(execFile);
-const OA_BASE = "https://batch.openaddresses.io";
-/**
- * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
- * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
- */
-const OA_COLLECTION_IDS = {
-    ca: 6,
-    "us-west": 4,
-    "us-south": 3,
-    "us-northeast": 2,
-    "us-midwest": 5,
-    global: 1,
-};
-/**
- * Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection).
- */
-async function countLines(path) {
-    let count = 0;
-    for await (const chunk of createReadStream(path)) {
-        for (const byte of chunk) {
-            if (byte === 0x0a) {
-                count++;
-            }
-        }
-    }
-    return count;
-}
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
- * redirects (the OA download endpoint 302s to a pre-signed S3 URL).
- *
- * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams a multi-GB body to disk (the
- * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the caller needs for
- * its two-URL fallback ladder.
- */
-async function streamDownload(url, dest, opts) {
-    for (let attempt = 0; attempt <= opts.retries; attempt++) {
-        try {
-            const res = await fetch(url, {
-                headers: opts.headers ?? {},
-                redirect: "follow",
-                signal: AbortSignal.timeout(opts.timeoutMs),
-            });
-            if (res.ok && res.body) {
-                await pipeline(Readable.fromWeb(res.body), createWriteStream(dest));
-                return res.status;
-            }
-            if (attempt < opts.retries && isTransientStatus(res.status)) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return res.status;
-        }
-        catch {
-            if (attempt < opts.retries) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-/**
- * Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used.
- */
-async function gunzipToFile(src, dest) {
-    const child = spawn("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
-        stdio: ["ignore", "pipe", "inherit"],
-    });
-    await pipeline(child.stdout, createWriteStream(dest));
-    await new Promise((resolve, reject) => {
-        child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`gunzip exited with code ${code}`))));
-        child.on("error", reject);
-    });
-}
-export async function fetchOpenAddresses(options, report) {
-    const country = options.country ?? "ca";
-    const token = $private.OA_BATCH_TOKEN;
-    const destDir = join(options.outRoot, "openaddresses", country);
-    const manifestPath = join(destDir, "MANIFEST.json");
-    const outputFile = join(destDir, "collection.geojsonl");
-    const fail = (code) => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] });
-    report?.(`=== fetch openaddresses: country=${country}`);
-    report?.(`    dest: ${destDir}`);
-    mkdirSync(destDir, { recursive: true });
-    // MARK: Authentication check
-    if (!token) {
-        report?.(`
-ERROR: OA_BATCH_TOKEN is not set.
-
-As of 2026-05-18, batch.openaddresses.io requires a registered (free) account
-to download collection files.  Data remains openly licensed — the auth gate
-is there to prevent CDN abuse, not to restrict access.
-
-Steps to get a token:
-  1. Register at: https://batch.openaddresses.io/register
-  2. Verify your email and log in.
-  3. Go to Profile → "Create Token" → copy the token.
-  4. Export it in this shell:
-       export OA_BATCH_TOKEN=<your-token>
-  5. Re-run this command.
-
-The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
-(estimated), so budget ~20–45 minutes at typical cloud-to-host bandwidth.
-`);
-        return fail("OA_BATCH_TOKEN");
-    }
-    // MARK: Determine collection ID
-    let collectionID = OA_COLLECTION_IDS[country];
-    if (collectionID === undefined) {
-        report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`);
-        // The collections API only. The COLLECTION ARCHIVES stay on raw `fetch` — they stream multi-gigabyte
-        // bodies straight to disk, where response caching is nonsense and axios would buffer them in memory.
-        const res = await new APIClient({
-            displayName: "openaddresses-api",
-            retry: true,
-            axios: { headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" } },
-        })
-            // `validateStatus` keeps a non-2xx as a RESPONSE rather than a throw: the caller reports the status and
-            // returns `fail(country)`, a graceful path this must not turn into an exception.
-            .fetch({
-            url: `${OA_BASE}/api/collections`,
-            timeout: 30_000,
-            validateStatus: () => true,
-        });
-        if (!isSuccessStatus(res.status)) {
-            report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`);
-            return fail(country);
-        }
-        const collections = res.data;
-        const match = collections.find((item) => item.name === country);
-        if (match?.id === undefined) {
-            report?.(`ERROR: Could not find a collection named '${country}' in GET /api/collections.`);
-            report?.(`Available collections:`);
-            for (const item of collections) {
-                const size = (item.size ?? 0).toLocaleString();
-                report?.(`  ${(item.name ?? "").padEnd(20)}  id=${item.id}  ${item.human ?? ""}  size=${size} bytes`);
-            }
-            return fail(country);
-        }
-        collectionID = match.id;
-        report?.(`  Found collection id=${collectionID} for '${country}'`);
-    }
-    // MARK: Download via the collections download endpoint (302s to S3)
-    report?.(`  Resolving download URL for collection id=${collectionID}...`);
-    report?.(`  Attempting authenticated download...`);
-    const tmpGz = join(destDir, "collection.geojsonl.gz.tmp");
-    const tmpRaw = join(destDir, "collection.geojsonl.tmp");
-    const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`;
-    let httpStatus = await streamDownload(sourceURL, tmpGz, {
-        headers: { Authorization: `Bearer ${token}` },
-        timeoutMs: 7_200_000,
-        retries: 3,
-        retryDelayMs: 30_000,
-    });
-    if (httpStatus !== HTTP_OK) {
-        // Try the geojsonl.gz directly with token as query param (alternate URL shape).
-        httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
-            timeoutMs: 7_200_000,
-            retries: 3,
-            retryDelayMs: 30_000,
-        });
-    }
-    if (httpStatus !== HTTP_OK) {
-        rmSync(tmpGz, { force: true });
-        report?.(`
-ERROR: Download returned HTTP ${httpStatus}.
-
-Likely causes:
-  1. OA_BATCH_TOKEN is invalid or expired — re-create it at Profile → Tokens.
-  2. The collection download endpoint URL has changed (this module was written
-     against the 2026-05-18 batch.openaddresses.io API; it may need updating).
-  3. Network error or CDN outage.
-
-Manual download (after logging in to batch.openaddresses.io):
-  - Navigate to https://batch.openaddresses.io/collection/${collectionID}
-  - Click "GeoJSON+LD" to download the collection.
-  - Save as: ${outputFile}
-
-URL tried: ${OA_BASE}/api/collections/${collectionID}/download
-`);
-        return fail(country);
-    }
-    // MARK: Decompress if the downloaded file is gzipped
-    const fileMagic = (await execFileAsync("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout;
-    if (/gzip|compressed/i.test(fileMagic)) {
-        report?.(`  Decompressing gzip archive...`);
-        await gunzipToFile(tmpGz, tmpRaw);
-        rmSync(tmpGz, { force: true });
-        renameSync(tmpRaw, outputFile);
-    }
-    else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
-        // Already line-delimited GeoJSON.
-        renameSync(tmpGz, outputFile);
-        rmSync(tmpRaw, { force: true });
-    }
-    else {
-        // Unknown type — keep as-is and let the operator inspect.
-        renameSync(tmpGz, outputFile);
-        report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`);
-    }
-    // MARK: Verify + write MANIFEST
-    if (!existsSync(outputFile)) {
-        report?.(`ERROR: Output file not found at ${outputFile} after download.`);
-        return fail(country);
-    }
-    const size = statSync(outputFile).size;
-    if (size < MIN_PLAUSIBLE_SHARD_BYTES) {
-        report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`);
-        return fail(country);
-    }
-    const sha = await sha256File(outputFile);
-    const rowCount = await countLines(outputFile);
-    const downloadedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
-    const manifest = {
-        source_url: sourceURL,
-        collection_id: collectionID,
-        country,
-        filename: "collection.geojsonl",
-        downloaded_at: downloadedAt,
-        sha256: sha,
-        bytes: size,
-        row_count: rowCount,
-        notes: "batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${ByteFormatter.formatIEC(size)}  rows=${rowCount}  sha256=${sha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    report?.(`=== done`);
-    report?.(`Feed to the adapter:`);
-    report?.(`  mailwoman corpus run openaddresses \\`);
-    report?.(`    --input ${outputFile} \\`);
-    report?.(`    --country ${country.toUpperCase()} \\`);
-    report?.(`    --output ${options.outRoot}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=openaddresses.js.map

package/out/src/tools/fetch/openaddresses.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"openaddresses.js","sourceRoot":"","sources":["../../../../src/tools/fetch/openaddresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH;kGACkG;AAElG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAEhE;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,yBAAyB,GAAG,MAAM,CAAA;AAExC,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAA;AAEzC,MAAM,OAAO,GAAG,gCAAgC,CAAA;AAEhD;;;GAGG;AACH,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,CAAC;IACL,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,CAAC;IACb,cAAc,EAAE,CAAC;IACjB,YAAY,EAAE,CAAC;IACf,MAAM,EAAE,CAAC;CACT,CAAA;AAgBD;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAA0B,EAAE,CAAC;QAC3E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBACnB,KAAK,EAAE,CAAA;YACR,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AASD;;;;;;;GAOG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAAwB;IAChF,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,IAAY;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,EAAE;QACnF,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC;KACpC,CAAC,CAAA;IAEF,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;IAEtD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5G,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAkC,EAClC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAA;IAErC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAA;IAC/D,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IACnD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAA;IAEvD,MAAM,IAAI,GAAG,CAAC,IAAY,EAAgB,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEzG,MAAM,EAAE,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAEhC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,6BAA6B;IAE7B,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;;;;;;;;;;;;;;;;;CAiBV,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAA;IAC9B,CAAC;IAED,gCAAgC;IAEhC,IAAI,YAAY,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,EAAE,CAAC,yBAAyB,OAAO,2CAA2C,CAAC,CAAA;QAErF,qGAAqG;QACrG,qGAAqG;QACrG,MAAM,GAAG,GAAG,MAAM,IAAI,SAAS,CAAC;YAC/B,WAAW,EAAE,mBAAmB;YAChC,KAAK,EAAE,IAAI;YACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;SACvF,CAAC;YACD,wGAAwG;YACxG,iFAAiF;aAChF,KAAK,CAAiB;YACtB,GAAG,EAAE,GAAG,OAAO,kBAAkB;YACjC,OAAO,EAAE,MAAM;YACf,cAAc,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,CAAC,CAAA;QAEH,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAClC,MAAM,EAAE,CAAC,6CAA6C,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;YAEpE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,CAAA;QAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,EAAE,KAAK,SAAS,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC,6CAA6C,OAAO,4BAA4B,CAAC,CAAA;YAC1F,MAAM,EAAE,CAAC,wBAAwB,CAAC,CAAA;YAElC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,EAAE,CAAA;gBAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,IAAI,EAAE,UAAU,IAAI,QAAQ,CAAC,CAAA;YACtG,CAAC;YAED,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;QACrB,CAAC;QAED,YAAY,GAAG,KAAK,CAAC,EAAE,CAAA;QACvB,MAAM,EAAE,CAAC,yBAAyB,YAAY,SAAS,OAAO,GAAG,CAAC,CAAA;IACnE,CAAC;IAED,oEAAoE;IAEpE,MAAM,EAAE,CAAC,8CAA8C,YAAY,KAAK,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,wCAAwC,CAAC,CAAA;IAElD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAA;IACzD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,CAAA;IACvD,MAAM,SAAS,GAAG,GAAG,OAAO,oBAAoB,YAAY,WAAW,CAAA;IAEvE,IAAI,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE;QACvD,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,EAAE,EAAE;QAC7C,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,CAAC;QACV,YAAY,EAAE,MAAM;KACpB,CAAC,CAAA;IAEF,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,gFAAgF;QAChF,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,OAAO,oBAAoB,YAAY,sBAAsB,KAAK,EAAE,EAAE,KAAK,EAAE;YACjH,SAAS,EAAE,SAAS;YACpB,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAE9B,MAAM,EAAE,CAAC;gCACqB,UAAU;;;;;;;;;4DASkB,YAAY;;eAEzD,UAAU;;aAEZ,OAAO,oBAAoB,YAAY;CACnD,CAAC,CAAA;QAEA,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,qDAAqD;IAErD,MAAM,SAAS,GAAG,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;IAExG,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACxC,MAAM,EAAE,CAAC,iCAAiC,CAAC,CAAA;QAC3C,MAAM,YAAY,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9B,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;IAC/B,CAAC;SAAM,IAAI,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QAChD,kCAAkC;QAClC,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAChC,CAAC;SAAM,CAAC;QACP,0DAA0D;QAC1D,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAA;QAC7B,MAAM,EAAE,CAAC,uCAAuC,SAAS,CAAC,IAAI,EAAE,oCAAoC,CAAC,CAAA;IACtG,CAAC;IAED,gCAAgC;IAEhC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,mCAAmC,UAAU,kBAAkB,CAAC,CAAA;QAEzE,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,CAAA;IAEtC,IAAI,IAAI,GAAG,yBAAyB,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,sCAAsC,IAAI,qCAAqC,CAAC,CAAA;QAEzF,OAAO,IAAI,CAAC,OAAO,CAAC,CAAA;IACrB,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAEvE,MAAM,QAAQ,GAAG;QAChB,UAAU,EAAE,SAAS;QACrB,aAAa,EAAE,YAAY;QAC3B,OAAO;QACP,QAAQ,EAAE,qBAAqB;QAC/B,aAAa,EAAE,YAAY;QAC3B,MAAM,EAAE,GAAG;QACX,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,QAAQ;QACnB,KAAK,EACJ,8LAA8L;KAC/L,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,QAAQ,YAAY,GAAG,EAAE,CAAC,CAAA;IACjF,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,UAAU,CAAC,CAAA;IACpB,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yCAAyC,CAAC,CAAA;IACnD,MAAM,EAAE,CAAC,eAAe,UAAU,KAAK,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,iBAAiB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/ourairports.d.ts

@@ -1,45 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the OurAirports CSV dumps — the VENUE side of the sub-venue corpus arc (#35).
- *
- *   Source : https://davidmegginson.github.io/ourairports-data/ (the project's own GitHub Pages
- *            mirror of the nightly export; `ourairports.com/data/` redirects here).
- *   License: PUBLIC DOMAIN. OurAirports places its data in the public domain and asks only for a
- *            courtesy credit — no attribution obligation rides on a derived shard, which makes this
- *            the one transport source in the arc with no licensing question at all. Tier A.
- *
- *   ## What it is good for, and what it is not
- *
- *   `airports.csv` is every airport on earth with ICAO/IATA codes, coordinates, `municipality`, and
- *   `iso_country` — 12.7 MB, ~83,000 rows as of 2026-08-04. That is the CONTAINING VENUE for a
- *   `<sub-venue>, <venue>, <street>, <locality>, <postcode>` corpus line, and it is better at that job
- *   than OSM: every row is named, the name is canonical, and `municipality` gives the locality without
- *   a spatial join.
- *
- *   It carries NO interior structure. There is no terminal, concourse, gate or pier table — the
- *   corpus task says as much ("Good for the venue side of each pair, weaker on interior structure")
- *   and a row-level read confirms it. Pair it with the OSM `aeroway` extractor
- *   (`@mailwoman/osm/sdk`'s `extractOSMSubVenues`), which is where the sub-venue half comes from.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   `AGENTS.md` routes HTTP through `APIClient`, and that rule is about API REQUESTS — small bodies,
- *   repeated calls, rate-limited hosts. This is four static file transfers against a GitHub Pages CDN
- *   with no rate limit and nothing to pace, run once per refresh. It uses the same `downloadToFile`
- *   every other module in this `fetch/` family uses, which is where the retry and timeout live.
- *   The Wikidata sibling (`wikidata-subvenue.ts`) IS an API client and is built on `APIClient`
- *   accordingly; the split between the two is the one `AGENTS.md` draws.
- *
- *   Invoke via `mailwoman corpus fetch ourairports --out-root <path>`.
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export type FetchOurAirportsOptions = BaseFetchOptions;
-/**
- * Download the OurAirports CSVs into `<outRoot>/ourairports/`, with a sibling `MANIFEST.json` carrying each file's
- * origin URL, sha256, byte count and upstream `Last-Modified`.
- */
-export declare function fetchOurAirports(options: FetchOurAirportsOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=ourairports.d.ts.map

package/out/src/tools/fetch/ourairports.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ourairports.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ourairports.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAsBnE,MAAM,MAAM,uBAAuB,GAAG,gBAAgB,CAAA;AAyCtD;;;GAGG;AACH,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,uBAAuB,EAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAwDvB"}