@mailwoman/corpus 9.2.0 → 9.4.0

package/out/tools/fetch/geonames-dump.js

@@ -0,0 +1,171 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
+ *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
+ *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
+ *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
+ *
+ *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
+ *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
+ *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
+ *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
+ *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
+ *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
+ *   the basename) is reported as `wrong_format_present`, never counted as coverage.
+ */
+import { pathExists, readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { extractZipEntry } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404;
+const SLUG = "geonames-dump";
+/**
+ * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
+ * bare text file.
+ */
+const BASE_URL = "https://download.geonames.org/export/dump";
+/**
+ * Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
+ * `<CC>.txt` basename.
+ */
+const GAZETTEER_DUMP_COLUMNS = 19;
+/**
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
+ *
+ * Walk the string directly rather than constructing a spliterator: the capitals builder already holds whole country
+ * dumps as strings, and a byte-oriented spliterator would UTF-8 encode that input — allocating up to another 350 MB —
+ * merely to inspect its first non-empty line.
+ */
+export function looksLikeGazetteerDump(text) {
+    let start = 0;
+    while (start < text.length) {
+        const end = text.indexOf("\n", start);
+        const line = end === -1 ? text.slice(start) : text.slice(start, end);
+        if (line.trim()) {
+            let tabs = 0;
+            for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
+                tabs++;
+            }
+            return tabs === GAZETTEER_DUMP_COLUMNS - 1;
+        }
+        if (end === -1)
+            break;
+        start = end + 1;
+    }
+    return false;
+}
+/**
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
+ * file's own commentary.
+ */
+export function parseCountryInfo(text) {
+    const rows = [];
+    // oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
+    for (const line of text.split("\n")) {
+        if (line.startsWith("#") || !line.trim())
+            continue;
+        // oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
+        const cols = line.split("\t");
+        const country = cols[0]?.trim().toUpperCase();
+        if (country?.length === 2) {
+            rows.push({ country, capital: cols[5]?.trim() ?? "" });
+        }
+    }
+    return rows;
+}
+/**
+ * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
+ * reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
+ */
+const FORMAT_SNIFF_BYTES = 65_536;
+/**
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
+ */
+export async function fetchGeonamesDumps(options, report) {
+    await makeDirectories(options.outRoot);
+    const baseURL = options.baseURL ?? BASE_URL;
+    const countryInfoDest = join(options.outRoot, "countryInfo.txt");
+    await downloadToFile({
+        url: `${baseURL}/countryInfo.txt`,
+        dest: countryInfoDest,
+        timeoutMs: 120_000,
+        retries: 2,
+        report,
+    });
+    const catalog = parseCountryInfo(await readLocalTextFile(countryInfoDest));
+    const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country);
+    const entries = [];
+    const failedCodes = [];
+    const unavailable = [];
+    const skippedPresent = [];
+    const wrongFormatPresent = [];
+    let fetched = 0;
+    for (const country of countries) {
+        const txtDest = join(options.outRoot, `${country}.txt`);
+        if (!options.force && (await pathExists(txtDest))) {
+            if (looksLikeGazetteerDump(await readFileHead(txtDest, FORMAT_SNIFF_BYTES))) {
+                skippedPresent.push(country);
+            }
+            else {
+                report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`);
+                wrongFormatPresent.push(country);
+            }
+            continue;
+        }
+        const filename = `${country}.zip`;
+        const url = `${baseURL}/${filename}`;
+        const zipDest = join(options.outRoot, filename);
+        report?.(`=== ${SLUG} / ${country}`);
+        try {
+            await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report });
+            await extractZipEntry(zipDest, `${country}.txt`, txtDest);
+            await removePathIfPresent(zipDest);
+            entries.push({
+                country,
+                filename: `${country}.txt`,
+                source_url: url,
+                sha256: await sha256File(txtDest),
+                bytes: (await readLocalBuffer(txtDest)).byteLength,
+            });
+            fetched++;
+        }
+        catch (error) {
+            await removePathIfPresent(zipDest);
+            const message = error instanceof Error ? error.message : String(error);
+            // Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
+            // can contain any substring.
+            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+                report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`);
+                unavailable.push(country);
+            }
+            else {
+                report?.(`✗ ${country}: ${message}`);
+            }
+            failedCodes.push(country);
+        }
+    }
+    const manifest = {
+        source: SLUG,
+        base_url: baseURL,
+        license: "CC-BY-4.0",
+        attribution: "GeoNames",
+        downloaded_at: new Date().toISOString(),
+        files: entries,
+        skipped_present: skippedPresent.toSorted(),
+        unavailable,
+        wrong_format_present: wrongFormatPresent.toSorted(),
+    };
+    await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest);
+    return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent };
+}
+//# sourceMappingURL=geonames-dump.js.map

package/out/tools/fetch/geonames-dump.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geonames-dump.js","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACzG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,eAAe,CAAA;AAE5B;;;GAGG;AACH,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAmD5D;;;GAGG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAEjC;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEpE,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAA;YAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1E,IAAI,EAAE,CAAA;YACP,CAAC;YAED,OAAO,IAAI,KAAK,sBAAsB,GAAG,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QAErB,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAgD,EAAE,CAAA;IAE5D,6IAA6I;IAC7I,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAElD,6FAA6F;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7C,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAiC,EACjC,MAA+B;IAE/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;IAEhE,MAAM,cAAc,CAAC;QACpB,GAAG,EAAE,GAAG,OAAO,kBAAkB;QACjC,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,CAAC;QACV,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAA;IAC1E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAElH,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,kBAAkB,GAAa,EAAE,CAAA;IACvC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,CAAC,CAAA;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACnD,IAAI,sBAAsB,CAAC,MAAM,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBAC7E,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,iFAAiF,CAAC,CAAA;gBACvG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAE/C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YACpF,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,OAAO,CAAC,CAAA;YACzD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,OAAO,CAAC,IAAI,CAAC;gBACZ,OAAO;gBACP,QAAQ,EAAE,GAAG,OAAO,MAAM;gBAC1B,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;gBACjC,KAAK,EAAE,CAAC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;aAClD,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,qGAAqG;YACrG,6BAA6B;YAC7B,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,yDAAyD,CAAC,CAAA;gBAC/E,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAyB;QACtC,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,cAAc,CAAC,QAAQ,EAAE;QAC1C,WAAW;QACX,oBAAoB,EAAE,kBAAkB,CAAC,QAAQ,EAAE;KACnD,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;AAC5G,CAAC"}

package/out/tools/fetch/geonames-postal.d.ts

@@ -0,0 +1,76 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
+ *   `(postcode, locality, region)` triple with the NAMES inline.
+ *
+ *   Source : https://download.geonames.org/export/zip/<CC>.zip
+ *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
+ *
+ *   ## Why this source and not a join
+ *
+ *   A postcode slice is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
+ *   only one of them works everywhere:
+ *
+ *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
+ *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
+ *     `locality`/`localadmin`). It is also the ONLY slice that does: `postalcode-geonames-intl.db` and every
+ *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
+ *     have none.
+ *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
+ *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
+ *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
+ *     own. Under 50% for four of the five, which is not a join.
+ *
+ *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
+ *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
+ *
+ *   ## Coverage is not universal, and the gap is the point
+ *
+ *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode slice cannot be built
+ *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
+ *   before assuming it is there; an absent country fails as one entry, never as the whole run.
+ *
+ *   ## Row counts from this source overstate, for some countries by exactly 2×
+ *
+ *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
+ *   on the slice built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
+ *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a slice from the row
+ *   count doubles its estimate for those countries.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
+ *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
+ *
+ *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * Countries fetched when the caller names none.
+ *
+ * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
+ * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
+ */
+export declare const GEONAMES_POSTAL_DEFAULT_COUNTRIES: readonly ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
+export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
+    /**
+     * ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
+     */
+    countries?: readonly string[];
+    /**
+     * Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
+     * exercised against a local server rather than by asking GeoNames for a country it does not have.
+     */
+    baseURL?: string;
+}
+/**
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
+ * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
+ *
+ * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
+ */
+export declare function fetchGeonamesPostal(options: FetchGeonamesPostalOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=geonames-postal.d.ts.map

package/out/tools/fetch/geonames-postal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geonames-postal.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAe3E;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,qDAAsD,CAAA;AAEpG,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAwBD;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA2DvB"}

package/out/tools/fetch/geonames-postal.js

@@ -0,0 +1,126 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
+ *   `(postcode, locality, region)` triple with the NAMES inline.
+ *
+ *   Source : https://download.geonames.org/export/zip/<CC>.zip
+ *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
+ *
+ *   ## Why this source and not a join
+ *
+ *   A postcode slice is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
+ *   only one of them works everywhere:
+ *
+ *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
+ *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
+ *     `locality`/`localadmin`). It is also the ONLY slice that does: `postalcode-geonames-intl.db` and every
+ *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
+ *     have none.
+ *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
+ *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
+ *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
+ *     own. Under 50% for four of the five, which is not a join.
+ *
+ *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
+ *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
+ *
+ *   ## Coverage is not universal, and the gap is the point
+ *
+ *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode slice cannot be built
+ *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
+ *   before assuming it is there; an absent country fails as one entry, never as the whole run.
+ *
+ *   ## Row counts from this source overstate, for some countries by exactly 2×
+ *
+ *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
+ *   on the slice built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
+ *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a slice from the row
+ *   count doubles its estimate for those countries.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
+ *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
+ *
+ *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404;
+const SLUG = "geonames-postal";
+/**
+ * GeoNames' own export directory. One zip per ISO alpha-2 code, each holding `<CC>.txt` plus the shared `readme.txt`.
+ */
+const BASE_URL = "https://download.geonames.org/export/zip";
+/**
+ * Countries fetched when the caller names none.
+ *
+ * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
+ * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
+ */
+export const GEONAMES_POSTAL_DEFAULT_COUNTRIES = ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
+/**
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
+ * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
+ *
+ * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
+ */
+export async function fetchGeonamesPostal(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const countries = (options.countries ?? GEONAMES_POSTAL_DEFAULT_COUNTRIES).map((code) => code.trim().toUpperCase());
+    const baseURL = options.baseURL ?? BASE_URL;
+    const retryDelayMs = options.retryDelayMs;
+    const entries = [];
+    const failedCodes = [];
+    const unavailable = [];
+    let fetched = 0;
+    let failed = 0;
+    for (const country of countries) {
+        const filename = `${country}.zip`;
+        const url = `${baseURL}/${filename}`;
+        const dest = join(destDir, filename);
+        report?.(`=== ${SLUG} / ${country}`);
+        try {
+            const { bytes } = await downloadToFile({ url, dest, timeoutMs: 300_000, retries: 2, retryDelayMs, report });
+            entries.push({ country, filename, source_url: url, sha256: await sha256File(dest), bytes });
+            fetched++;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            // A 404 here means GeoNames does not publish the country at all, which is a different finding from a failed
+            // transfer and the one a caller planning a slice needs to see. Branch on the TYPED status: matching message
+            // prose classified a 500 as "unpublished" whenever the URL happened to contain the substring 404 — an
+            // ephemeral test-server port did exactly that in CI.
+            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+                report?.(`✗ ${country}: GeoNames does not publish a postal export for this country`);
+                unavailable.push(country);
+            }
+            else {
+                report?.(`✗ ${country}: ${message}`);
+            }
+            failedCodes.push(country);
+            failed++;
+        }
+    }
+    const manifest = {
+        source: "GeoNames postal codes",
+        base_url: baseURL,
+        license: "CC-BY-4.0",
+        attribution: "GeoNames",
+        downloaded_at: new Date().toISOString(),
+        files: entries,
+        unavailable,
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched, skipped: 0, failed, failedCodes };
+}
+//# sourceMappingURL=geonames-postal.js.map

package/out/tools/fetch/geonames-postal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geonames-postal.js","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAE9B;;GAEG;AACH,MAAM,QAAQ,GAAG,0CAA0C,CAAA;AAE3D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU,CAAA;AAoCpG;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,iCAAiC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IACnH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;IACzC,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAA;YAE3G,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;YAE3F,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,4GAA4G;YAC5G,4GAA4G;YAC5G,sGAAsG;YACtG,qDAAqD;YACrD,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,8DAA8D,CAAC,CAAA;gBACpF,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEzB,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAA2B;QACxC,MAAM,EAAE,uBAAuB;QAC/B,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,WAAW;KACX,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/houjin-jp.d.ts

@@ -0,0 +1,24 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchHoujinJPOptions = BaseFetchOptions;
+export declare function fetchHoujinJP(options: FetchHoujinJPOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=houjin-jp.d.ts.map

package/out/tools/fetch/houjin-jp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin-jp.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/houjin-jp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,MAAM,uBAAuB,CAAA;AAWrG,MAAM,MAAM,oBAAoB,GAAG,gBAAgB,CAAA;AAenD,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgFvB"}

package/out/tools/fetch/houjin-jp.js

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the National Tax Agency's corporate-number register (法人番号公表サイト 全件データ): every
+ *   corporation assigned a number, about 5,000,000 rows, with the head-office address as one string
+ *   (国内所在地) plus the prefecture and municipality split out and the postcode. The nationwide CSV
+ *   in Unicode is one zip of about 255 MB. This is the NOISY source for Japanese addresses; the
+ *   Overture rows the JP corpus is built from are the LABEL half.
+ *
+ *   The agency states the three published fields may be used freely by anyone ("どなたでも自由にご利用
+ *   いただくことができます"), with no attribution condition on the download page.
+ *
+ *   The file sits behind a form: the page hands out a per-session token, and a POST with that token,
+ *   `event=download` and the file number of the nationwide Unicode CSV answers the zip. The file
+ *   number is read off the page rather than pinned, because the agency re-issues the files monthly.
+ *
+ *   Invoke via `mailwoman corpus fetch houjin-jp --out-root <path>`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { attachmentFilename, cookieHeader, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
+const SLUG = "houjin-jp";
+const PAGE_URL = "https://www.houjin-bangou.nta.go.jp/download/zenken/";
+const FORM_URL = `${PAGE_URL}index.html`;
+const TOKEN_FIELD = "jp.go.nta.houjin_bangou.framework.web.common.CNSFWTokenProcessor.request.token";
+const LICENSE = "国税庁法人番号公表サイト — the three published fields are free for anyone to use (利用規約)";
+const ATTRIBUTION = "国税庁 法人番号公表サイト (National Tax Agency, Corporate Number Publication Site)";
+const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch";
+/**
+ * The nationwide row of the "CSV形式・Unicode" table carries the file number in its `doDownload(N)` handler.
+ */
+function nationwideUnicodeFileNumber(html) {
+    const start = html.indexOf('id="csv-unicode"');
+    if (start === -1)
+        return undefined;
+    const section = html.slice(start, html.indexOf('id="xml-unicode"', start));
+    const row = /全国[\s\S]{0,600}?doDownload\((\d+)\)/.exec(section);
+    return row?.[1];
+}
+export async function fetchHoujinJP(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const page = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+    if (!page.ok)
+        throw new Error(`houjin-jp: the download page answered HTTP ${page.status}`);
+    const html = await page.text();
+    const token = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(html)?.[1];
+    const fileNumber = nationwideUnicodeFileNumber(html);
+    if (!token || !fileNumber) {
+        report?.(`  ✗ the page carried ${token ? "a token" : "no token"} and ${fileNumber ? `file ${fileNumber}` : "no nationwide Unicode CSV row"}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    report?.(`=== ${SLUG}: nationwide CSV (Unicode), file ${fileNumber}`);
+    let filename = `zenkoku_all_${fileNumber}.zip`;
+    let bytes;
+    try {
+        bytes = await withRetries(async () => {
+            // A fresh page per attempt: the token is bound to the session cookie, and both may have lapsed.
+            const attemptPage = await fetch(PAGE_URL, { headers: { "user-agent": USER_AGENT, accept: "text/html" } });
+            const attemptToken = new RegExp(`${TOKEN_FIELD.replaceAll(".", "\\.")}" value="([^"]+)"`).exec(await attemptPage.text())?.[1];
+            if (!attemptToken)
+                throw new Error("the download page carried no token");
+            const res = await fetch(FORM_URL, {
+                method: "POST",
+                headers: {
+                    "user-agent": USER_AGENT,
+                    cookie: cookieHeader(attemptPage),
+                    referer: PAGE_URL,
+                    "content-type": "application/x-www-form-urlencoded",
+                },
+                body: new URLSearchParams({ [TOKEN_FIELD]: attemptToken, event: "download", selDlFileNo: fileNumber }),
+                signal: AbortSignal.timeout(1_800_000),
+            });
+            if (!res.ok || !(res.headers.get("content-type") ?? "").includes("octet-stream")) {
+                throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""}`);
+            }
+            filename = attachmentFilename(res, filename);
+            return streamBodyToFile(res, join(destDir, filename));
+        }, { report, label: filename });
+    }
+    catch (error) {
+        report?.(`  ✗ ${error.message}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: ["zenkoku-csv-unicode"] };
+    }
+    const dest = join(destDir, filename);
+    const sha = await sha256File(dest);
+    report?.(`  ✓ ${filename} ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+    const manifest = {
+        source: SLUG,
+        source_url: PAGE_URL,
+        license: LICENSE,
+        attribution: ATTRIBUTION,
+        downloaded_at: new Date().toISOString(),
+        files: [{ source_url: FORM_URL, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes }],
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=houjin-jp.js.map

package/out/tools/fetch/houjin-jp.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"houjin-jp.js","sourceRoot":"","sources":["../../../lib/tools/fetch/houjin-jp.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtH,MAAM,IAAI,GAAG,WAAW,CAAA;AACxB,MAAM,QAAQ,GAAG,sDAAsD,CAAA;AACvE,MAAM,QAAQ,GAAG,GAAG,QAAQ,YAAY,CAAA;AACxC,MAAM,WAAW,GAAG,gFAAgF,CAAA;AACpG,MAAM,OAAO,GAAG,6EAA6E,CAAA;AAC7F,MAAM,WAAW,GAAG,wEAAwE,CAAA;AAC5F,MAAM,UAAU,GAAG,wDAAwD,CAAA;AAI3E;;GAEG;AACH,SAAS,2BAA2B,CAAC,IAAY;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAA;IAE9C,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAA;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC,CAAA;IAC1E,MAAM,GAAG,GAAG,qCAAqC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAE/D,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;IAElG,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IAC1F,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAClG,MAAM,UAAU,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAA;IAEpD,IAAI,CAAC,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;QAC3B,MAAM,EAAE,CACP,wBAAwB,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,QAAQ,UAAU,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,CAAC,CAAC,CAAC,+BAA+B,EAAE,CACnI,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,EAAE,CAAC,OAAO,IAAI,oCAAoC,UAAU,EAAE,CAAC,CAAA;IAErE,IAAI,QAAQ,GAAG,eAAe,UAAU,MAAM,CAAA;IAC9C,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,WAAW,CACxB,KAAK,IAAI,EAAE;YACV,gGAAgG;YAChG,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAA;YAEzG,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAC7F,MAAM,WAAW,CAAC,IAAI,EAAE,CACxB,EAAE,CAAC,CAAC,CAAC,CAAA;YAEN,IAAI,CAAC,YAAY;gBAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;YAExE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;gBACjC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACR,YAAY,EAAE,UAAU;oBACxB,MAAM,EAAE,YAAY,CAAC,WAAW,CAAC;oBACjC,OAAO,EAAE,QAAQ;oBACjB,cAAc,EAAE,mCAAmC;iBACnD;gBACD,IAAI,EAAE,IAAI,eAAe,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC;gBACtG,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;aACtC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,QAAQ,GAAG,kBAAkB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;YAE5C,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAA;QACtD,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAC3B,CAAA;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAE3C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAA;IACnF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAEjF,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,QAAQ;QACpB,OAAO,EAAE,OAAO;QAChB,WAAW,EAAE,WAAW;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;KACxG,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/hrsa.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
+ *   adapter. US Public Domain.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
+ *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
+ *   downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchHRSAOptions = BaseFetchOptions;
+export declare function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=hrsa.d.ts.map

package/out/tools/fetch/hrsa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hrsa.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAA;AAO3F,MAAM,MAAM,gBAAgB,GAAG,gBAAgB,CAAA;AAE/C,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CA8BjH"}

package/out/tools/fetch/hrsa.js

@@ -0,0 +1,46 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
+ *   adapter. US Public Domain.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
+ *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
+ *   downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, writeManifest } from "#tools/fetch/download";
+const SLUG = "usgov-hrsa-fqhc";
+const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv";
+const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`;
+export async function fetchHRSA(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const dest = join(destDir, FILENAME);
+    report?.(`=== ${SLUG} / ${FILENAME}`);
+    const { bytes } = await downloadToFile({
+        url: SOURCE_URL,
+        dest,
+        timeoutMs: 600_000,
+        headers: { "Accept-Encoding": "gzip, br" },
+        report,
+    });
+    const sha = await sha256File(dest);
+    const manifest = {
+        source_url: SOURCE_URL,
+        downloaded_at: new Date().toISOString(),
+        filename: FILENAME,
+        sha256: sha,
+        bytes,
+    };
+    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
+    report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=hrsa.js.map

package/out/tools/fetch/hrsa.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hrsa.js","sourceRoot":"","sources":["../../../lib/tools/fetch/hrsa.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAErE,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAC9B,MAAM,QAAQ,GAAG,wDAAwD,CAAA;AACzE,MAAM,UAAU,GAAG,+CAA+C,QAAQ,EAAE,CAAA;AAI5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB,EAAE,MAA+B;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IAErC,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;QACtC,GAAG,EAAE,UAAU;QACf,IAAI;QACJ,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAElC,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/tools/fetch/imls-pls.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
+ *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
+ *   Public Domain (federal statistical survey).
+ *
+ *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
+ *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
+ *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
+ *   no per-branch address detail.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
+ *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
+ *   member listing + selective extraction).
+ *
+ *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
+ *   and sha matches MANIFEST, skips download.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export type FetchIMLSPLSOptions = BaseFetchOptions;
+export declare function fetchIMLSPLS(options: FetchIMLSPLSOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=imls-pls.d.ts.map

package/out/tools/fetch/imls-pls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"imls-pls.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,uBAAuB,CAAA;AAc3F,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,CAAA;AAElD,wBAAsB,YAAY,CACjC,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAiGvB"}