@mailwoman/corpus 9.1.0 → 9.3.0

package/out/src/tools/fetch/imls-pls.js

@@ -1,108 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { existsSync, mkdirSync, statSync } from "node:fs";
-import { rm } from "node:fs/promises";
-import { basename, join } from "node:path";
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
-import { sha256File } from "@mailwoman/core/utils";
-import { downloadToFile, readManifest, writeManifest } from "./download.js";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const BYTES_PER_KIB = 1024;
-/**
- * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
- */
-const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip";
-const SLUG = "usgov-imls-pls";
-export async function fetchIMLSPLS(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    mkdirSync(destDir, { recursive: true });
-    const zipDest = join(destDir, basename(ZIP_URL));
-    const manifestPath = join(destDir, "MANIFEST.json");
-    report?.(`=== ${SLUG}`);
-    // MARK: Idempotency check
-    const recorded = await readManifest(manifestPath);
-    if (recorded?.sha256 && recorded.filename) {
-        const recordedPath = join(destDir, recorded.filename);
-        if (existsSync(recordedPath) && (await sha256File(recordedPath)) === recorded.sha256) {
-            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
-            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
-        }
-    }
-    // MARK: Download ZIP
-    report?.(`  Downloading ${ZIP_URL} ...`);
-    const { bytes: zipSize } = await downloadToFile({
-        url: ZIP_URL,
-        dest: zipDest,
-        timeoutMs: 600_000,
-        headers: { "Accept-Encoding": "gzip, br" },
-        report,
-    });
-    report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
-    if (zipSize < BYTES_PER_KIB) {
-        report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Discover the outlet-level CSV inside the ZIP.
-    // Outlet files match: pls_fy*outlet*.csv (case-insensitive)
-    // Administrative-entity files match: pls_fy*ae*.csv — we skip those.
-    report?.("  Inspecting ZIP contents ...");
-    const entries = (await listZipEntries(zipDest)).map((entry) => entry.name);
-    let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name));
-    // Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
-    if (!csvName) {
-        csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name));
-    }
-    if (!csvName) {
-        report?.("  Available files in ZIP:");
-        for (const name of entries) {
-            report?.(`    ${name}`);
-        }
-        report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module");
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    report?.(`  Extracting outlet CSV: ${csvName}`);
-    const csvDest = join(destDir, basename(csvName));
-    await extractZipEntry(zipDest, csvName, csvDest);
-    const csvSize = statSync(csvDest).size;
-    const csvSha = await sha256File(csvDest);
-    // MARK: Remove ZIP (small, but keep destDir clean)
-    await rm(zipDest, { force: true });
-    report?.("  Removed ZIP (CSV kept)");
-    // MARK: Write MANIFEST
-    const manifest = {
-        source_url: ZIP_URL,
-        downloaded_at: new Date().toISOString(),
-        filename: basename(csvName),
-        sha256: csvSha,
-        bytes: csvSize,
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=imls-pls.js.map

package/out/src/tools/fetch/imls-pls.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../src/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAA;AACrC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAY7B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YACtF,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE1E,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/src/tools/fetch/index.d.ts

@@ -1,133 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
- *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
- *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
- *   and sha256 so downstream adapters can verify provenance.
- *
- *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
- *   operators to pre-download into an out-root and point adapters at the resulting files. These
- *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
- *   bootstrap).
- *
- *   ## Usage
- *
- *   ```sh
- *   # Default: writes under ./data/corpus/sources/ relative to the working directory
- *   mailwoman corpus fetch state-sources
- *   mailwoman corpus fetch hrsa
- *
- *   # Or point at the standard mailwoman data root
- *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
- *   ```
- *
- *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
- *   input was pulled from; these modules mirror those URLs in a single executable place.
- *
- *   ## Coverage
- *
- *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
- *       Ouverte 2.0).
- *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
- *       (US PD).
- *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
- *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
- *       Tier A (US PD).
- *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
- *       Tier A (US PD).
- *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
- *       — per-row filter.
- *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
- *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
- *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
- *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
- *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
- *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
- *       family built on `APIClient` — see its docstring for why.
- *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
- *       Tier A (state PD-equivalent).
- *   - `state-hi-schools` — Hawaii DOE school directory (XLSX → CSV via openpyxl). Tier A (state
- *       PD-equivalent).
- *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
- *
- *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
- *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
- *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
- *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
- *   dropped at ingest by default to protect proprietary-weights training.
- *
- *   ### OpenAddresses authentication (as of 2026-05-18)
- *
- *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth gate
- *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
- *   from the environment:
- *
- *   ```sh
- *   # One-time: register at https://batch.openaddresses.io/register
- *   # Log in → Profile → "Create Token" → copy token
- *   export OA_BATCH_TOKEN=<your-token>
- *
- *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
- *   mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Or any other OA country code
- *   mailwoman corpus fetch openaddresses --country fr
- *   ```
- *
- *   Without a token the command prints setup instructions and reports the failure.
- *
- *   ## Adding a new source
- *
- *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
- *      different family).
- *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
- *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
- *      open-data portals occasionally rotate Socrata view IDs.
- *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
- *      MANIFEST is well-formed.
- *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
- */
-import { fetchBan } from "./ban.ts";
-import { fetchHRSA } from "./hrsa.ts";
-import { fetchIMLSPLS } from "./imls-pls.ts";
-import { fetchNAD } from "./nad.ts";
-import { fetchNPPES } from "./nppes.ts";
-import { fetchOpenAddresses } from "./openaddresses.ts";
-import { fetchOurAirports } from "./ourairports.ts";
-import { fetchStateHISchools } from "./state-hi-schools.ts";
-import { fetchStateSources } from "./state-sources.ts";
-import { fetchTigerFull } from "./tiger-full.ts";
-import { fetchWikidataSubVenue } from "./wikidata-subvenue.ts";
-export * from "./ban.ts";
-export * from "./hrsa.ts";
-export * from "./imls-pls.ts";
-export * from "./nad.ts";
-export * from "./nppes.ts";
-export * from "./openaddresses.ts";
-export * from "./ourairports.ts";
-export * from "./state-hi-schools.ts";
-export * from "./state-sources.ts";
-export * from "./tiger-full.ts";
-export * from "./wikidata-subvenue.ts";
-/**
- * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
- */
-export declare const FETCH_SOURCES: {
-    readonly ban: typeof fetchBan;
-    readonly nad: typeof fetchNAD;
-    readonly hrsa: typeof fetchHRSA;
-    readonly "imls-pls": typeof fetchIMLSPLS;
-    readonly nppes: typeof fetchNPPES;
-    readonly openaddresses: typeof fetchOpenAddresses;
-    readonly ourairports: typeof fetchOurAirports;
-    readonly "state-sources": typeof fetchStateSources;
-    readonly "state-hi-schools": typeof fetchStateHISchools;
-    readonly "tiger-full": typeof fetchTigerFull;
-    readonly "wikidata-subvenue": typeof fetchWikidataSubVenue;
-};
-export type FetchSourceID = keyof typeof FETCH_SOURCES;
-//# sourceMappingURL=index.d.ts.map

package/out/src/tools/fetch/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE9D,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AAEtC;;GAEG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYhB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAA"}

package/out/src/tools/fetch/index.js

@@ -1,132 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
- *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
- *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
- *   and sha256 so downstream adapters can verify provenance.
- *
- *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
- *   operators to pre-download into an out-root and point adapters at the resulting files. These
- *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
- *   bootstrap).
- *
- *   ## Usage
- *
- *   ```sh
- *   # Default: writes under ./data/corpus/sources/ relative to the working directory
- *   mailwoman corpus fetch state-sources
- *   mailwoman corpus fetch hrsa
- *
- *   # Or point at the standard mailwoman data root
- *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
- *   ```
- *
- *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
- *   input was pulled from; these modules mirror those URLs in a single executable place.
- *
- *   ## Coverage
- *
- *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
- *       Ouverte 2.0).
- *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
- *       (US PD).
- *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
- *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
- *       Tier A (US PD).
- *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
- *       Tier A (US PD).
- *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
- *       — per-row filter.
- *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
- *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
- *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
- *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
- *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
- *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
- *       family built on `APIClient` — see its docstring for why.
- *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
- *       Tier A (state PD-equivalent).
- *   - `state-hi-schools` — Hawaii DOE school directory (XLSX → CSV via openpyxl). Tier A (state
- *       PD-equivalent).
- *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
- *
- *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
- *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
- *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
- *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
- *   dropped at ingest by default to protect proprietary-weights training.
- *
- *   ### OpenAddresses authentication (as of 2026-05-18)
- *
- *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth gate
- *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
- *   from the environment:
- *
- *   ```sh
- *   # One-time: register at https://batch.openaddresses.io/register
- *   # Log in → Profile → "Create Token" → copy token
- *   export OA_BATCH_TOKEN=<your-token>
- *
- *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
- *   mailwoman corpus fetch openaddresses --country ca \
- *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Or any other OA country code
- *   mailwoman corpus fetch openaddresses --country fr
- *   ```
- *
- *   Without a token the command prints setup instructions and reports the failure.
- *
- *   ## Adding a new source
- *
- *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
- *      different family).
- *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
- *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
- *      open-data portals occasionally rotate Socrata view IDs.
- *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
- *      MANIFEST is well-formed.
- *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
- */
-import { fetchBan } from "./ban.js";
-import { fetchHRSA } from "./hrsa.js";
-import { fetchIMLSPLS } from "./imls-pls.js";
-import { fetchNAD } from "./nad.js";
-import { fetchNPPES } from "./nppes.js";
-import { fetchOpenAddresses } from "./openaddresses.js";
-import { fetchOurAirports } from "./ourairports.js";
-import { fetchStateHISchools } from "./state-hi-schools.js";
-import { fetchStateSources } from "./state-sources.js";
-import { fetchTigerFull } from "./tiger-full.js";
-import { fetchWikidataSubVenue } from "./wikidata-subvenue.js";
-export * from "./ban.js";
-export * from "./hrsa.js";
-export * from "./imls-pls.js";
-export * from "./nad.js";
-export * from "./nppes.js";
-export * from "./openaddresses.js";
-export * from "./ourairports.js";
-export * from "./state-hi-schools.js";
-export * from "./state-sources.js";
-export * from "./tiger-full.js";
-export * from "./wikidata-subvenue.js";
-/**
- * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
- */
-export const FETCH_SOURCES = {
-    ban: fetchBan,
-    nad: fetchNAD,
-    hrsa: fetchHRSA,
-    "imls-pls": fetchIMLSPLS,
-    nppes: fetchNPPES,
-    openaddresses: fetchOpenAddresses,
-    ourairports: fetchOurAirports,
-    "state-sources": fetchStateSources,
-    "state-hi-schools": fetchStateHISchools,
-    "tiger-full": fetchTigerFull,
-    "wikidata-subvenue": fetchWikidataSubVenue,
-};
-//# sourceMappingURL=index.js.map

package/out/src/tools/fetch/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/tools/fetch/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4FG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAA;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAE9D,cAAc,UAAU,CAAA;AACxB,cAAc,WAAW,CAAA;AACzB,cAAc,eAAe,CAAA;AAC7B,cAAc,UAAU,CAAA;AACxB,cAAc,YAAY,CAAA;AAC1B,cAAc,oBAAoB,CAAA;AAClC,cAAc,kBAAkB,CAAA;AAChC,cAAc,uBAAuB,CAAA;AACrC,cAAc,oBAAoB,CAAA;AAClC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,wBAAwB,CAAA;AAEtC;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B,GAAG,EAAE,QAAQ;IACb,GAAG,EAAE,QAAQ;IACb,IAAI,EAAE,SAAS;IACf,UAAU,EAAE,YAAY;IACxB,KAAK,EAAE,UAAU;IACjB,aAAa,EAAE,kBAAkB;IACjC,WAAW,EAAE,gBAAgB;IAC7B,eAAe,EAAE,iBAAiB;IAClC,kBAAkB,EAAE,mBAAmB;IACvC,YAAY,EAAE,cAAc;IAC5B,mBAAmB,EAAE,qBAAqB;CACjC,CAAA"}

package/out/src/tools/fetch/nad.d.ts

@@ -1,70 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-import type { BaseFetchOptions, FetchSummary } from "./download.ts";
-export interface FetchNADOptions extends BaseFetchOptions {
-    /**
-     * Fetch strategy. Default `featureserver`.
-     */
-    mode?: "featureserver" | "bulk";
-    /**
-     * Pre-signed S3 URL for bulk mode.
-     */
-    nadURL?: string;
-    /**
-     * Records per output file. Default `100000`.
-     */
-    chunkSize?: number;
-    /**
-     * Records per HTTP request. Default `5000`.
-     */
-    pageSize?: number;
-    /**
-     * Parallel page fetches within a chunk. Default `4`.
-     */
-    concurrency?: number;
-    /**
-     * Start OBJECTID. Default `1`.
-     */
-    startOID?: number;
-    /**
-     * Stop before this OID. Default = total count.
-     */
-    endOID?: number;
-}
-export declare function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=nad.d.ts.map

package/out/src/tools/fetch/nad.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAQnE,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD;;OAEG;IACH,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAAA;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AA6OD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAY/G"}