@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/geonames/dump.js

@@ -0,0 +1,181 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch GeoNames per-country gazetteer dumps. These are the 19-column `<CC>.txt` files under
+ *   `https://download.geonames.org/export/dump/`, the export directory rather than the postal export
+ *   at `export/zip/` that `geonames-postal.ts` handles. The dumps contain feature classes and codes
+ *   (column 8: `pplc` national capital, `ppla` first-order administrative seat), which is what the
+ *   capitals reference build consumes.
+ *
+ *   `countryInfo.txt` supplies the country catalog. It lists each country GeoNames publishes with one
+ *   row per ISO alpha-2 code. Each row also lists the capital. The capitals build checks it against
+ *   its `pplc` extraction. The tool fetches this file first and derives the country set from it. A dump
+ *   absent from disk then counts as a gap in the source's own catalog.
+ *   The dump directory may contain files this tool never fetched. The tool keeps present files unchanged.
+ *   It checks each present `<CC>.txt` for the 19-column gazetteer format. GeoNames postal exports share
+ *   the basename, so a postal export receives `wrong_format_present` and does not count as coverage.
+ */
+import { pathExists, readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { extractZipEntry } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { TSVSpliterator } from "spliterator";
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404;
+const SLUG = "geonames-dump";
+/**
+ * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`,
+ * plus `countryInfo.txt` as a bare text file.
+ */
+const BASE_URL = "https://download.geonames.org/export/dump";
+/**
+ * Column count of a gazetteer dump row.
+ *
+ * GeoNames postal exports have 12 columns and share the `<CC>.txt` basename.
+ */
+const GAZETTEER_DUMP_COLUMNS = 19;
+/**
+ * True when the first non-empty line contains the gazetteer dump's 19 tab-separated columns.
+ *
+ * Accepts a partial head read.
+ * The first line is the whole question, so callers need not hand it a resident 350 MB dump.
+ *
+ * Walk the string directly instead of constructing a spliterator.
+ * The capitals builder already holds each country dump as a string.
+ *
+ * A byte-oriented spliterator would UTF-8 encode that input and allocate up to
+ * another 350 MB just to inspect the first non-empty line.
+ */
+export function looksLikeGazetteerDump(text) {
+    let start = 0;
+    while (start < text.length) {
+        const end = text.indexOf("\n", start);
+        const line = end === -1 ? text.slice(start) : text.slice(start, end);
+        if (line.trim()) {
+            let tabs = 0;
+            for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
+                tabs++;
+            }
+            return tabs === GAZETTEER_DUMP_COLUMNS - 1;
+        }
+        if (end === -1)
+            break;
+        start = end + 1;
+    }
+    return false;
+}
+/**
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt`.
+ * `#`-prefixed lines are the file's own commentary.
+ */
+export function parseCountryInfo(text) {
+    const rows = [];
+    for (const cols of TSVSpliterator.from(text, { header: false, enableQuoteHandling: false })) {
+        const country = cols[0]?.toUpperCase();
+        if (!country || country.startsWith("#"))
+            continue;
+        if (country.length === 2) {
+            rows.push({ country, capital: cols[5] ?? "" });
+        }
+    }
+    return rows;
+}
+/**
+ * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names
+ * column runs long (they reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
+ */
+const FORMAT_SNIFF_BYTES = 65_536;
+/**
+ * Download `countryInfo.txt` and each missing `<CC>.zip`.
+ *
+ * Extract each dump to `<outRoot>/<CC>.txt` beside the hand-fetched files.
+ * The `manifest.json` lists fetched countries.
+ *
+ * It also lists countries skipped because their files were already present
+ * and countries unavailable from the source.
+ */
+export async function fetchGeonamesDumps(options, report) {
+    await makeDirectories(options.outRoot);
+    const baseURL = options.baseURL ?? BASE_URL;
+    const countryInfoDest = options.outRoot("countryInfo.txt");
+    await downloadToFile({
+        url: `${baseURL}/countryInfo.txt`,
+        dest: countryInfoDest,
+        timeoutMs: 120_000,
+        retries: 2,
+        report,
+    });
+    const catalog = parseCountryInfo(await readLocalTextFile(countryInfoDest));
+    const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country);
+    const entries = [];
+    const failedCodes = [];
+    const unavailable = [];
+    const skippedPresent = [];
+    const wrongFormatPresent = [];
+    let fetched = 0;
+    for (const country of countries) {
+        const txtDest = options.outRoot(`${country}.txt`);
+        if (!options.force && (await pathExists(txtDest))) {
+            if (looksLikeGazetteerDump(await readFileHead(txtDest, FORMAT_SNIFF_BYTES))) {
+                skippedPresent.push(country);
+            }
+            else {
+                report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`);
+                wrongFormatPresent.push(country);
+            }
+            continue;
+        }
+        const filename = `${country}.zip`;
+        const url = `${baseURL}/${filename}`;
+        const zipDest = options.outRoot(filename);
+        report?.(`=== ${SLUG} / ${country}`);
+        try {
+            await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report });
+            await extractZipEntry(zipDest, `${country}.txt`, txtDest);
+            await removePathIfPresent(zipDest);
+            entries.push({
+                country,
+                filename: `${country}.txt`,
+                source_url: url,
+                sha256: await sha256File(txtDest),
+                bytes: (await readLocalBuffer(txtDest)).byteLength,
+            });
+            fetched++;
+        }
+        catch (error) {
+            await removePathIfPresent(zipDest);
+            const message = error instanceof Error ? error.message : String(error);
+            // Branch on the typed status.
+            // The error message contains a URL.
+            // URLs can contain any substring.
+            // Branch on the typed status instead of matching message text.
+            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+                report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`);
+                unavailable.push(country);
+            }
+            else {
+                report?.(`✗ ${country}: ${message}`);
+            }
+            failedCodes.push(country);
+        }
+    }
+    const manifest = {
+        source: SLUG,
+        base_url: baseURL,
+        license: "CC-BY-4.0",
+        attribution: "GeoNames",
+        downloaded_at: new Date().toISOString(),
+        files: entries,
+        skipped_present: skippedPresent.toSorted(),
+        unavailable,
+        wrong_format_present: wrongFormatPresent.toSorted(),
+    };
+    await writeManifest(options.outRoot("MANIFEST.json"), manifest);
+    return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent };
+}
+//# sourceMappingURL=dump.js.map

package/out/tools/fetch/geonames/dump.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dump.js","sourceRoot":"","sources":["../../../../lib/tools/fetch/geonames/dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACzG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAG5C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,eAAe,CAAA;AAE5B;;;GAGG;AACH,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AA2D5D;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAEjC;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEpE,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAA;YAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1E,IAAI,EAAE,CAAA;YACP,CAAC;YAED,OAAO,IAAI,KAAK,sBAAsB,GAAG,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QAErB,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAgD,EAAE,CAAA;IAE5D,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAA;QAEtC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAQ;QAEjD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC/C,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAiC,EACjC,MAA+B;IAE/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAA;IAE1D,MAAM,cAAc,CAAC;QACpB,GAAG,EAAE,GAAG,OAAO,kBAAkB;QACjC,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,CAAC;QACV,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAA;IAC1E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAElH,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,kBAAkB,GAAa,EAAE,CAAA;IACvC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,MAAM,CAAC,CAAA;QAEjD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACnD,IAAI,sBAAsB,CAAC,MAAM,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBAC7E,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,iFAAiF,CAAC,CAAA;gBACvG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAEzC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YACpF,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,OAAO,CAAC,CAAA;YACzD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,OAAO,CAAC,IAAI,CAAC;gBACZ,OAAO;gBACP,QAAQ,EAAE,GAAG,OAAO,MAAM;gBAC1B,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;gBACjC,KAAK,EAAE,CAAC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;aAClD,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,8BAA8B;YAC9B,oCAAoC;YACpC,kCAAkC;YAClC,+DAA+D;YAC/D,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,yDAAyD,CAAC,CAAA;gBAC/E,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAyB;QACtC,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,cAAc,CAAC,QAAQ,EAAE;QAC1C,WAAW;QACX,oBAAoB,EAAE,kBAAkB,CAAC,QAAQ,EAAE;KACnD,CAAA;IAED,MAAM,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE/D,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;AAC5G,CAAC"}

package/out/tools/fetch/geonames/postal.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that includes a
+ *   `(postcode, locality, region)` triple with the names inline.
+ *
+ *   Source : https://download.geonames.org/export/zip/<CC>.zip
+ *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
+ *
+ *   GeoNames publishes roughly 80 countries. An absent country fails as one entry, never as the whole run.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * Countries fetched when the caller names none.
+ *
+ * Venezuela is deliberately absent because GeoNames does not publish it.
+ */
+export declare const GEONAMES_POSTAL_DEFAULT_COUNTRIES: readonly ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
+export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
+    /**
+     * ISO alpha-2 codes in any casing.
+     *
+     * Defaults to `{@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}`.
+     */
+    countries?: readonly string[];
+    /**
+     * Export directory to read from.
+     *
+     * Defaults to GeoNames' own, so the 404-is-coverage behavior can be exercised against a local server.
+     */
+    baseURL?: string;
+}
+/**
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/` with
+ * a sibling `manifest.json`; a country the source does not cover is counted as failed
+ * and recorded in `failedCodes` without stopping the rest.
+ */
+export declare function fetchGeonamesPostal(options: FetchGeonamesPostalOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=postal.d.ts.map

package/out/tools/fetch/geonames/postal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postal.d.ts","sourceRoot":"","sources":["../../../../lib/tools/fetch/geonames/postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAgB3E;;;;GAIG;AACH,eAAO,MAAM,iCAAiC,YAAI,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU,CAAA;AAEpG,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;;;OAIG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAyBD;;;;GAIG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA0DvB"}

package/out/tools/fetch/geonames/postal.js

@@ -0,0 +1,87 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that includes a
+ *   `(postcode, locality, region)` triple with the names inline.
+ *
+ *   Source : https://download.geonames.org/export/zip/<CC>.zip
+ *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
+ *
+ *   GeoNames publishes roughly 80 countries. An absent country fails as one entry, never as the whole run.
+ */
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
+/**
+ * The status that means the source does not publish this country rather than that the transfer failed.
+ */
+const HTTP_NOT_FOUND = 404;
+const SLUG = "geonames-postal";
+/**
+ * GeoNames' own export directory, one zip per ISO alpha-2 code holding `<CC>.txt`
+ * plus the shared `readme.txt`.
+ */
+const BASE_URL = "https://download.geonames.org/export/zip";
+/**
+ * Countries fetched when the caller names none.
+ *
+ * Venezuela is deliberately absent because GeoNames does not publish it.
+ */
+export const GEONAMES_POSTAL_DEFAULT_COUNTRIES = ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
+/**
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/` with
+ * a sibling `manifest.json`; a country the source does not cover is counted as failed
+ * and recorded in `failedCodes` without stopping the rest.
+ */
+export async function fetchGeonamesPostal(options, report) {
+    const destDir = options.outRoot(SLUG);
+    await makeDirectories(destDir);
+    const countries = (options.countries ?? GEONAMES_POSTAL_DEFAULT_COUNTRIES).map((code) => code.trim().toUpperCase());
+    const baseURL = options.baseURL ?? BASE_URL;
+    const retryDelayMs = options.retryDelayMs;
+    const entries = [];
+    const failedCodes = [];
+    const unavailable = [];
+    let fetched = 0;
+    let failed = 0;
+    for (const country of countries) {
+        const filename = `${country}.zip`;
+        const url = `${baseURL}/${filename}`;
+        const dest = destDir(filename);
+        report?.(`=== ${SLUG} / ${country}`);
+        try {
+            const { bytes } = await downloadToFile({ url, dest, timeoutMs: 300_000, retries: 2, retryDelayMs, report });
+            entries.push({ country, filename, source_url: url, sha256: await sha256File(dest), bytes });
+            fetched++;
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            // A 404 means GeoNames does not publish the country at all, a different
+            // finding from a failed transfer.
+            // Branch on the typed status rather than message prose.
+            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+                report?.(`✗ ${country}: GeoNames does not publish a postal export for this country`);
+                unavailable.push(country);
+            }
+            else {
+                report?.(`✗ ${country}: ${message}`);
+            }
+            failedCodes.push(country);
+            failed++;
+        }
+    }
+    const manifest = {
+        source: "GeoNames postal codes",
+        base_url: baseURL,
+        license: "CC-BY-4.0",
+        attribution: "GeoNames",
+        downloaded_at: new Date().toISOString(),
+        files: entries,
+        unavailable,
+    };
+    await writeManifest(destDir("MANIFEST.json"), manifest);
+    return { fetched, skipped: 0, failed, failedCodes };
+}
+//# sourceMappingURL=postal.js.map

package/out/tools/fetch/geonames/postal.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"postal.js","sourceRoot":"","sources":["../../../../lib/tools/fetch/geonames/postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAGjD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,iBAAiB,CAAA;AAE9B;;;GAGG;AACH,MAAM,QAAQ,GAAG,0CAA0C,CAAA;AAE3D;;;;GAIG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU,CAAA;AAwCpG;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,SAAS,IAAI,iCAAiC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IACnH,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;IACzC,MAAM,OAAO,GAA8B,EAAE,CAAA;IAC7C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;QAE9B,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC,CAAA;YAE3G,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;YAE3F,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,wEAAwE;YACxE,kCAAkC;YAClC,wDAAwD;YACxD,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,8DAA8D,CAAC,CAAA;gBACpF,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAEzB,MAAM,EAAE,CAAA;QACT,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAA2B;QACxC,MAAM,EAAE,uBAAuB;QAC/B,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,WAAW;KACX,CAAA;IAED,MAAM,aAAa,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAEvD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}

package/out/tools/fetch/gleif.d.ts

@@ -0,0 +1,137 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter.
+ *
+ * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at
+ * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the
+ * CDF version it follows, so this module reads that answer first and downloads the file it names.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the
+ * `2026-10-03 08:00:00` publish:
+ *
+ * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
+ *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    publish replaced mid-transfer is refused rather than recorded.
+ * 2. **The host honors `Range`.** An interrupted transfer resumes through
+ *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
+ *    compresses small responses on the fly and a range must count the archive's own bytes.
+ * 3. **Each publish has its own file name**, date-prefixed as
+ *    `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that
+ *    name, so a partial transfer of one publish never resumes into another, and the adapter reads the
+ *    newest name in the directory.
+ *
+ * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter
+ * streams the member out of the archive rather than reading an extracted copy.
+ *
+ * The receipt written beside the archive records the URL, the publish date, the byte count and the
+ * sha256 of the archive, which identify the bytes a corpus row was read from.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * The publish API's answer for the newest golden copy and its delta files.
+ */
+export declare const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest";
+/**
+ * The file the publish API names for the Level 1 golden copy in CSV.
+ */
+export interface GLEIFPublishedFile {
+    /**
+     * The publish timestamp the API states, as `2026-10-03 08:00:00`.
+     */
+    publishDate: string;
+    url: string;
+    /**
+     * The archive's length in bytes, as the API states it.
+     */
+    bytes: number;
+    recordCount: number;
+    cdfVersion: string;
+}
+/**
+ * The receipt written beside the archive.
+ */
+export interface GLEIFGoldenCopyReceipt {
+    source: string;
+    source_url: string;
+    api_url: string;
+    publish_date: string;
+    cdf_version: string;
+    /**
+     * The record count the API states for this publish.
+     */
+    record_count: number;
+    downloaded_at: string;
+    filename: string;
+    bytes: number;
+    sha256: string;
+    /**
+     * The CSV member inside the archive, and its decompressed length as the central directory states it.
+     */
+    csv_member: string;
+    csv_member_bytes: number;
+    license: string;
+    license_url: string;
+    attribution: string;
+}
+/**
+ * Read which file the publish API names as the current Level 1 golden copy in CSV.
+ *
+ * @throws When the answer omits any of the five fields, naming the one it lacks,
+ * rather than downloading a file whose length or date cannot be checked.
+ */
+export declare function readGLEIFLatestPublish(client: Pick<APIClient, "fetch">): Promise<GLEIFPublishedFile>;
+/**
+ * The archive's file name, which is the last path segment of the URL the API names.
+ */
+export declare function gleifArchiveFilename(url: string): string;
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export declare function gleifInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+/**
+ * Whether the archive on disk is the publish the API names and the length the receipt records.
+ *
+ * Exported so a test can exercise the decision without a transfer.
+ */
+export declare function isGLEIFArchiveCurrent(recorded: GLEIFGoldenCopyReceipt | null, published: GLEIFPublishedFile, path: PathBuilderLike, verifyDigest: boolean): Promise<boolean>;
+export interface DownloadGLEIFOptions {
+    /**
+     * Where the archive and the receipt are written.
+     * The directory itself is the adapter's `inputPath`.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Re-read the archive's sha256 on a re-run instead of comparing its byte count.
+     */
+    verifyDigest?: boolean;
+    /**
+     * Download even where the receipt already records the current publish.
+     */
+    force?: boolean;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+}
+export type FetchGLEIFOptions = BaseFetchOptions & Omit<DownloadGLEIFOptions, "outputDir" | "report">;
+/**
+ * Download the current Level 1 golden copy into `options.outputDir` and write its receipt.
+ *
+ * Re-runnable: a receipt naming the current publish, with the archive still on disk
+ * at the recorded length, makes no request for the body.
+ * Archives of earlier publishes are removed once the new one is verified,
+ * so the directory holds one publish.
+ */
+export declare function downloadGLEIF(client: Pick<APIClient, "fetch">, options: DownloadGLEIFOptions): Promise<FetchSummary>;
+/**
+ * Download the current golden copy into `<outRoot>/gleif/`.
+ *
+ * The registry entry point.
+ * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself.
+ */
+export declare function fetchGLEIF(options: FetchGLEIFOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=gleif.d.ts.map

package/out/tools/fetch/gleif.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gleif.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/gleif.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAM/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAU3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAG3E;;GAEG;AACH,eAAO,MAAM,wBAAwB,uEAAuE,CAAA;AAO5G;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB,EAAE,MAAM,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;CACnB;AAgBD;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA4B1G;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAQxD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAEpF;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,CAC1C,QAAQ,EAAE,sBAAsB,GAAG,IAAI,EACvC,SAAS,EAAE,kBAAkB,EAC7B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,OAAO,GACnB,OAAO,CAAC,OAAO,CAAC,CAYlB;AAED,MAAM,WAAW,oBAAoB;IACpC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,QAAQ,CAAC,CAAA;AAErG;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAClC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,oBAAoB,GAC3B,OAAO,CAAC,YAAY,CAAC,CA6GvB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAWnH"}