@mailwoman/corpus 9.3.0 → 10.0.0

package/out/tools/fetch/geonames-postal.js

@@ -1,126 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
- *   `(postcode, locality, region)` triple with the NAMES inline.
- *
- *   Source : https://download.geonames.org/export/zip/<CC>.zip
- *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
- *
- *   ## Why this source and not a join
- *
- *   A postcode slice is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
- *   only one of them works everywhere:
- *
- *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
- *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
- *     `locality`/`localadmin`). It is also the ONLY slice that does: `postalcode-geonames-intl.db` and every
- *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
- *     have none.
- *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
- *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
- *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
- *     own. Under 50% for four of the five, which is not a join.
- *
- *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
- *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
- *
- *   ## Coverage is not universal, and the gap is the point
- *
- *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode slice cannot be built
- *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
- *   before assuming it is there; an absent country fails as one entry, never as the whole run.
- *
- *   ## Row counts from this source overstate, for some countries by exactly 2×
- *
- *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
- *   on the slice built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
- *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a slice from the row
- *   count doubles its estimate for those countries.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
- *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
- *
- *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
- */
-import { makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
-/**
- * The one status that means "the source does not publish this country" rather than "the transfer failed".
- */
-const HTTP_NOT_FOUND = 404;
-const SLUG = "geonames-postal";
-/**
- * GeoNames' own export directory. One zip per ISO alpha-2 code, each holding `<CC>.txt` plus the shared `readme.txt`.
- */
-const BASE_URL = "https://download.geonames.org/export/zip";
-/**
- * Countries fetched when the caller names none.
- *
- * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
- * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
- */
-export const GEONAMES_POSTAL_DEFAULT_COUNTRIES = ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
-/**
- * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
- * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
- *
- * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
- */
-export async function fetchGeonamesPostal(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const countries = (options.countries ?? GEONAMES_POSTAL_DEFAULT_COUNTRIES).map((code) => code.trim().toUpperCase());
-    const baseURL = options.baseURL ?? BASE_URL;
-    const retryDelayMs = options.retryDelayMs;
-    const entries = [];
-    const failedCodes = [];
-    const unavailable = [];
-    let fetched = 0;
-    let failed = 0;
-    for (const country of countries) {
-        const filename = `${country}.zip`;
-        const url = `${baseURL}/${filename}`;
-        const dest = join(destDir, filename);
-        report?.(`=== ${SLUG} / ${country}`);
-        try {
-            const { bytes } = await downloadToFile({ url, dest, timeoutMs: 300_000, retries: 2, retryDelayMs, report });
-            entries.push({ country, filename, source_url: url, sha256: await sha256File(dest), bytes });
-            fetched++;
-        }
-        catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            // A 404 here means GeoNames does not publish the country at all, which is a different finding from a failed
-            // transfer and the one a caller planning a slice needs to see. Branch on the TYPED status: matching message
-            // prose classified a 500 as "unpublished" whenever the URL happened to contain the substring 404 — an
-            // ephemeral test-server port did exactly that in CI.
-            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
-                report?.(`✗ ${country}: GeoNames does not publish a postal export for this country`);
-                unavailable.push(country);
-            }
-            else {
-                report?.(`✗ ${country}: ${message}`);
-            }
-            failedCodes.push(country);
-            failed++;
-        }
-    }
-    const manifest = {
-        source: "GeoNames postal codes",
-        base_url: baseURL,
-        license: "CC-BY-4.0",
-        attribution: "GeoNames",
-        downloaded_at: new Date().toISOString(),
-        files: entries,
-        unavailable,
-    };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-    return { fetched, skipped: 0, failed, failedCodes };
-}
-//# sourceMappingURL=geonames-postal.js.map

package/out/tools/fetch/geonames-postal.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"geonames-postal.js","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAE9B;;GAEG;AACH,MAAM,QAAQ,GAAG,0CAA0C,CAAA;AAE3D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU,CAAA;AAoCpG;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,iCAAiC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IACnH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;IACzC,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAA;YAE3G,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;YAE3F,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,4GAA4G;YAC5G,4GAA4G;YAC5G,sGAAsG;YACtG,qDAAqD;YACrD,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,8DAA8D,CAAC,CAAA;gBACpF,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEzB,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAA2B;QACxC,MAAM,EAAE,uBAAuB;QAC/B,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,WAAW;KACX,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/hrsa.d.ts

@@ -1,18 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
- *   adapter. US Public Domain.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
- *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
- *   downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export type FetchHRSAOptions = BaseFetchOptions;
-export declare function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=hrsa.d.ts.map

package/out/tools/fetch/hrsa.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"hrsa.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAA;AAO3F,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,CAAA;AAE/C,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CA8BjH"}

package/out/tools/fetch/hrsa.js

@@ -1,46 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
- *   adapter. US Public Domain.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
- *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
- *   downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
- */
-import { makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { downloadToFile, writeManifest } from "#tools/fetch/download";
-const SLUG = "usgov-hrsa-fqhc";
-const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv";
-const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`;
-export async function fetchHRSA(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const dest = join(destDir, FILENAME);
-    report?.(`=== ${SLUG} / ${FILENAME}`);
-    const { bytes } = await downloadToFile({
-        url: SOURCE_URL,
-        dest,
-        timeoutMs: 600_000,
-        headers: { "Accept-Encoding": "gzip, br" },
-        report,
-    });
-    const sha = await sha256File(dest);
-    const manifest = {
-        source_url: SOURCE_URL,
-        downloaded_at: new Date().toISOString(),
-        filename: FILENAME,
-        sha256: sha,
-        bytes,
-    };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-    report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=hrsa.js.map

package/out/tools/fetch/hrsa.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hrsa.js","sourceRoot":"","sources":["../../../lib/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAC9B,MAAM,QAAQ,GAAG,wDAAwD,CAAA;AACzE,MAAM,UAAU,GAAG,+CAA+C,QAAQ,EAAE,CAAA;AAI5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE,MAA+B;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IAErC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,GAAG,EAAE,UAAU;QACf,IAAI;QACJ,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAElC,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/imls-pls.d.ts

@@ -1,25 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export type FetchIMLSPLSOptions = BaseFetchOptions;
-export declare function fetchIMLSPLS(options: FetchIMLSPLSOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=imls-pls.d.ts.map

package/out/tools/fetch/imls-pls.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAA;AAc3F,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAA;AAElD,wBAAsB,YAAY,CACjC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAiGvB"}

package/out/tools/fetch/imls-pls.js

@@ -1,108 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
-import { statPath, pathExists } from "@mailwoman/core/fs/readers";
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
-import { sha256File } from "@mailwoman/core/hash";
-import { basename, join } from "path-ts";
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
- */
-const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip";
-const SLUG = "usgov-imls-pls";
-export async function fetchIMLSPLS(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const zipDest = join(destDir, basename(ZIP_URL));
-    const manifestPath = join(destDir, "MANIFEST.json");
-    report?.(`=== ${SLUG}`);
-    // MARK: Idempotency check
-    const recorded = await readManifest(manifestPath);
-    if (recorded?.sha256 && recorded.filename) {
-        const recordedPath = join(destDir, recorded.filename);
-        if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
-            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
-            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
-        }
-    }
-    // MARK: Download ZIP
-    report?.(`  Downloading ${ZIP_URL} ...`);
-    const { bytes: zipSize } = await downloadToFile({
-        url: ZIP_URL,
-        dest: zipDest,
-        timeoutMs: 600_000,
-        headers: { "Accept-Encoding": "gzip, br" },
-        report,
-    });
-    report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`);
-    if (zipSize < BYTES_PER_KIB) {
-        report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`);
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    // Discover the outlet-level CSV inside the ZIP.
-    // Outlet files match: pls_fy*outlet*.csv (case-insensitive)
-    // Administrative-entity files match: pls_fy*ae*.csv — we skip those.
-    report?.("  Inspecting ZIP contents ...");
-    const entries = (await listZipEntries(zipDest)).map((entry) => entry.name);
-    let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name));
-    // Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
-    if (!csvName) {
-        csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name));
-    }
-    if (!csvName) {
-        report?.("  Available files in ZIP:");
-        for (const name of entries) {
-            report?.(`    ${name}`);
-        }
-        report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module");
-        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
-    }
-    report?.(`  Extracting outlet CSV: ${csvName}`);
-    const csvDest = join(destDir, basename(csvName));
-    await extractZipEntry(zipDest, csvName, csvDest);
-    const csvSize = (await statPath(csvDest)).size;
-    const csvSha = await sha256File(csvDest);
-    // MARK: Remove ZIP (small, but keep destDir clean)
-    await removePathIfPresent(zipDest);
-    report?.("  Removed ZIP (CSV kept)");
-    // MARK: Write MANIFEST
-    const manifest = {
-        source_url: ZIP_URL,
-        downloaded_at: new Date().toISOString(),
-        filename: basename(csvName),
-        sha256: csvSha,
-        bytes: csvSize,
-    };
-    await writeManifest(manifestPath, manifest);
-    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
-    report?.(`  MANIFEST written to ${manifestPath}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-//# sourceMappingURL=imls-pls.js.map

package/out/tools/fetch/imls-pls.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../lib/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF;;;GAGG;AAEH;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAI7B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9F,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAE7D,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE1E,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/nad.d.ts

@@ -1,70 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export interface FetchNADOptions extends BaseFetchOptions {
-    /**
-     * Fetch strategy. Default `featureserver`.
-     */
-    mode?: "featureserver" | "bulk";
-    /**
-     * Pre-signed S3 URL for bulk mode.
-     */
-    nadURL?: string;
-    /**
-     * Records per output file. Default `100000`.
-     */
-    chunkSize?: number;
-    /**
-     * Records per HTTP request. Default `5000`.
-     */
-    pageSize?: number;
-    /**
-     * Parallel page fetches within a chunk. Default `4`.
-     */
-    concurrency?: number;
-    /**
-     * Start OBJECTID. Default `1`.
-     */
-    startOID?: number;
-    /**
-     * Stop before this OID. Default = total count.
-     */
-    endOID?: number;
-}
-export declare function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=nad.d.ts.map

package/out/tools/fetch/nad.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAQ3E,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD;;OAEG;IACH,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAAA;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAuPD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAY/G"}