@mailwoman/corpus 9.2.0 → 9.4.0

package/out/tools/fetch/ban.d.ts

@@ -0,0 +1,31 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+export interface BanManifestEntry {
+    dept_code: string;
+    filename: string;
+    source_url: string;
+    downloaded_at: string;
+    sha256: string;
+    bytes: number;
+}
+export type FetchBanOptions = BaseFetchOptions;
+export declare function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=ban.d.ts.map

package/out/tools/fetch/ban.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ban.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAsH3E,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,MAAM,eAAe,GAAG,gBAAgB,CAAA;AAE9C,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAmI/G"}

package/out/tools/fetch/ban.js

@@ -0,0 +1,239 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+import { gunzip } from "@mailwoman/core/fs/compression";
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { isFile, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePath, writeLocalFile } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { sleep } from "@mailwoman/core/utils/sleep";
+import { join } from "path-ts";
+import { downloadToFile, loadManifestEntries, writeManifest } from "#tools/fetch/download";
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv";
+/**
+ * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
+ * change.
+ */
+const DEPT_CODES = [
+    "01",
+    "02",
+    "03",
+    "04",
+    "05",
+    "06",
+    "07",
+    "08",
+    "09",
+    "10",
+    "11",
+    "12",
+    "13",
+    "14",
+    "15",
+    "16",
+    "17",
+    "18",
+    "19",
+    "21",
+    "22",
+    "23",
+    "24",
+    "25",
+    "26",
+    "27",
+    "28",
+    "29",
+    "2A",
+    "2B",
+    "30",
+    "31",
+    "32",
+    "33",
+    "34",
+    "35",
+    "36",
+    "37",
+    "38",
+    "39",
+    "40",
+    "41",
+    "42",
+    "43",
+    "44",
+    "45",
+    "46",
+    "47",
+    "48",
+    "49",
+    "50",
+    "51",
+    "52",
+    "53",
+    "54",
+    "55",
+    "56",
+    "57",
+    "58",
+    "59",
+    "60",
+    "61",
+    "62",
+    "63",
+    "64",
+    "65",
+    "66",
+    "67",
+    "68",
+    "69",
+    "70",
+    "71",
+    "72",
+    "73",
+    "74",
+    "75",
+    "76",
+    "77",
+    "78",
+    "79",
+    "80",
+    "81",
+    "82",
+    "83",
+    "84",
+    "85",
+    "86",
+    "87",
+    "88",
+    "89",
+    "90",
+    "91",
+    "92",
+    "93",
+    "94",
+    "95",
+    "971",
+    "972",
+    "973",
+    "974",
+    "976",
+];
+export async function fetchBan(options, report) {
+    const banDir = join(options.outRoot, "ban");
+    const manifestPath = join(banDir, "MANIFEST.json");
+    await makeDirectories(banDir);
+    // Load existing entries (code -> entry): skip detection + preservation of untouched codes.
+    const entries = await loadManifestEntries(manifestPath, (entry) => entry.dept_code);
+    let fetched = 0;
+    let skipped = 0;
+    let failed = 0;
+    const failedCodes = [];
+    for (const code of DEPT_CODES) {
+        const filename = `adresses-${code}.csv`;
+        const gzFile = join(banDir, `${filename}.gz`);
+        const csvFile = join(banDir, filename);
+        const url = `${BASE_URL}/adresses-${code}.csv.gz`;
+        report?.(`=== dept ${code}`);
+        // If the CSV already exists, compare its sha256 against the manifest.
+        if (await isFile(csvFile)) {
+            const existingSha = await sha256File(csvFile);
+            const recordedSha = entries.get(code)?.sha256;
+            if (recordedSha && existingSha === recordedSha) {
+                report?.(`  → already present + sha matches — skipping`);
+                skipped++;
+                continue;
+            }
+            report?.(`  → present but sha mismatch or no manifest entry — re-fetching`);
+            await removePath(csvFile);
+        }
+        // Download the gzipped CSV.
+        try {
+            await downloadToFile({
+                url,
+                dest: gzFile,
+                timeoutMs: 600_000,
+                headers: { "Accept-Encoding": "gzip, br" },
+                report,
+            });
+        }
+        catch (error) {
+            report?.(`  ✗ download failed: ${url} (${error.message})`);
+            failed++;
+            failedCodes.push(code);
+            continue;
+        }
+        // Guard against truncated 404/error pages.
+        const gzSize = (await tryStat(gzFile))?.size ?? 0;
+        if (gzSize < BYTES_PER_KIB) {
+            report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`);
+            await removePath(gzFile);
+            failed++;
+            failedCodes.push(code);
+            continue;
+        }
+        // Decompress in-place; delete the .gz.
+        try {
+            await writeLocalFile(await gunzip(await readLocalBuffer(gzFile)), csvFile);
+        }
+        catch (error) {
+            report?.(`  ✗ decompress failed: ${error.message}`);
+            await removePath(gzFile);
+            failed++;
+            failedCodes.push(code);
+            continue;
+        }
+        await removePath(gzFile);
+        if (!(await isFile(csvFile))) {
+            report?.(`  ✗ decompressed file not found at ${csvFile}`);
+            failed++;
+            failedCodes.push(code);
+            continue;
+        }
+        const bytes = (await tryStat(csvFile))?.size ?? 0;
+        const sha = await sha256File(csvFile);
+        entries.set(code, {
+            dept_code: code,
+            filename,
+            source_url: url,
+            downloaded_at: new Date().toISOString(),
+            sha256: sha,
+            bytes,
+        });
+        report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`);
+        fetched++;
+        // Be a polite citizen — short pause between requests.
+        await sleep(200);
+    }
+    // Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
+    const sorted = [...entries.values()].toSorted((a, b) => a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0);
+    await writeManifest(manifestPath, sorted);
+    report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`);
+    report?.(`=== summary ===`);
+    report?.(`fetched:  ${fetched}`);
+    report?.(`skipped:  ${skipped} (already present + sha matched)`);
+    report?.(`failed:   ${failed}`);
+    if (failedCodes.length) {
+        report?.(`failed codes: ${failedCodes.join(" ")}`);
+    }
+    return { fetched, skipped, failed, failedCodes };
+}
+//# sourceMappingURL=ban.js.map

package/out/tools/fetch/ban.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ban.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ban.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,gCAAgC,CAAA;AACvD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AAC7E,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AACxF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAE1F;;;GAGG;AAEH,MAAM,QAAQ,GAAG,2DAA2D,CAAA;AAE5E;;;GAGG;AACH,MAAM,UAAU,GAAG;IAClB,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACL,CAAA;AAaD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAClD,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAE7B,2FAA2F;IAC3F,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAmB,YAAY,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAErG,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,YAAY,IAAI,MAAM,CAAA;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAA;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACtC,MAAM,GAAG,GAAG,GAAG,QAAQ,aAAa,IAAI,SAAS,CAAA;QAEjD,MAAM,EAAE,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;QAE5B,sEAAsE;QACtE,IAAI,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;YAE7C,IAAI,WAAW,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChD,MAAM,EAAE,CAAC,8CAA8C,CAAC,CAAA;gBAExD,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,CAAC,iEAAiE,CAAC,CAAA;YAC3E,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC;gBACpB,GAAG;gBACH,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,wBAAwB,GAAG,KAAM,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;YAErE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,2CAA2C;QAC3C,MAAM,MAAM,GAAG,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAA;QAEjD,IAAI,MAAM,GAAG,aAAa,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,2BAA2B,MAAM,qCAAqC,CAAC,CAAA;YAChF,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;YAExB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,uCAAuC;QACvC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,MAAM,MAAM,CAAC,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;QAC3E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;YAExB,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;QAExB,IAAI,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAC9B,MAAM,EAAE,CAAC,sCAAsC,OAAO,EAAE,CAAC,CAAA;YAEzD,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAA;QACjD,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;QAErC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE;YACjB,SAAS,EAAE,IAAI;YACf,QAAQ;YACR,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,OAAO,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,GAAG,EAAE,CAAC,CAAA;QAEhE,OAAO,EAAE,CAAA;QAET,sDAAsD;QACtD,MAAM,KAAK,CAAC,GAAG,CAAC,CAAA;IACjB,CAAC;IAED,uFAAuF;IACvF,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtD,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAClE,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,SAAS,YAAY,SAAS,MAAM,CAAC,MAAM,WAAW,CAAC,CAAA;IAEhE,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,aAAa,OAAO,EAAE,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,aAAa,OAAO,kCAAkC,CAAC,CAAA;IAChE,MAAM,EAAE,CAAC,aAAa,MAAM,EAAE,CAAC,CAAA;IAE/B,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,CAAC,iBAAiB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACnD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACjD,CAAC"}

package/out/tools/fetch/download.d.ts

@@ -0,0 +1,182 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The option base every `mailwoman corpus fetch <source>` module extends.
+ */
+/**
+ * How long a failed transfer waits before the next attempt.
+ */
+export declare const DEFAULT_RETRY_DELAY_MS = 5000;
+export interface BaseFetchOptions {
+    /**
+     * Destination root for downloaded source data. Each source writes its own subdirectory.
+     */
+    outRoot: PathBuilderLike;
+    /**
+     * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
+     *
+     * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two
+     * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the
+     * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten.
+     */
+    retryDelayMs?: number;
+}
+/**
+ * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
+ */
+export interface FetchSummary {
+    fetched: number;
+    skipped: number;
+    failed: number;
+    failedCodes: string[];
+}
+/**
+ * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count +
+ * sha256, so downstream adapters can verify provenance.
+ */
+export interface SourceManifest {
+    source_url: string;
+    downloaded_at: string;
+    filename: string;
+    sha256: string;
+    bytes: number;
+}
+/**
+ * A status worth retrying: rate limiting or a server-side failure.
+ */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
+ * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
+ * run and vanishes locally.
+ */
+export declare class HTTPStatusError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+export declare function isTransientStatus(status: number): boolean;
+export interface DownloadOptions {
+    url: string;
+    dest: string;
+    /**
+     * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
+     */
+    timeoutMs?: number;
+    /**
+     * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
+     */
+    retries?: number;
+    /**
+     * Delay between attempts. Default 5s.
+     */
+    retryDelayMs?: number;
+    headers?: Record<string, string>;
+    report?: (line: string) => void;
+}
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export declare function downloadToFile(options: DownloadOptions): Promise<{
+    bytes: number;
+}>;
+export interface StreamDownloadOptions {
+    headers?: Record<string, string>;
+    timeoutMs: number;
+    retries: number;
+    retryDelayMs: number;
+}
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
+ *
+ * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
+ * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
+ * two-URL fallback ladders consume.
+ */
+export declare function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise<number>;
+/**
+ * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
+ */
+export declare function readManifest<T>(path: string): Promise<T | null>;
+/**
+ * Load manifest entries into a map so untouched keys survive a partial re-fetch.
+ */
+export declare function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>>;
+/**
+ * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
+ */
+export declare function writeManifest(path: string, manifest: unknown): Promise<void>;
+/**
+ * The sibling `MANIFEST.json` shape for a source that is a COLLECTION of files behind one portal (a monthly register
+ * published per region, per industry, or per first letter). It carries what a trained artifact has to be able to cite
+ * later: the license the portal labels the data with, the attribution wording it requires, and one
+ * {@link SourceManifest} per file.
+ */
+export interface SourceCollectionManifest {
+    source: string;
+    source_url: string;
+    license: string;
+    attribution: string;
+    downloaded_at: string;
+    files: SourceManifest[];
+}
+/**
+ * Pipe a response body to `dest` and answer the byte count. The one primitive the portal fetchers share when the
+ * request is not a bare GET — a session cookie, a CSRF header, or a form POST stands between the listing and the file,
+ * so {@link streamDownload}'s URL-only contract does not fit and each module builds its own `Response` first. Writes a
+ * `.tmp` sibling and renames, so an interrupted transfer never lands at the final path looking complete.
+ */
+export declare function streamBodyToFile(res: Response, dest: string): Promise<number>;
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name, or an empty map when the manifest is
+ * missing or not in the collection shape, so a re-run after an interruption fetches only what is missing. The
+ * single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export declare function loadCollectionFiles(path: string): Promise<Map<string, SourceManifest>>;
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export declare function withRetries<T>(transfer: () => Promise<T>, options?: {
+    retries?: number;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+    label?: string;
+}): Promise<T>;
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export declare function resumableDownload(options: {
+    url: string;
+    dest: string;
+    headers?: Record<string, string>;
+    maxConnections?: number;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+}): Promise<number>;
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export declare function cookieHeader(res: Response): string;
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export declare function attachmentFilename(res: Response, fallback: string): string;
+//# sourceMappingURL=download.d.ts.map

package/out/tools/fetch/download.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AA4B9C;;GAEG;AACH;;GAEG;AACH,eAAO,MAAM,sBAAsB,OAAO,CAAA;AAE1C,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,eAAe,CAAA;IACxB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;gBAEX,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK3C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsDzF;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAkC5G;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF;AAED;;;;;GAKG;AACH,MAAM,WAAW,wBAAwB;IACxC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,cAAc,EAAE,CAAA;CACvB;AAED;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAgBnF;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAS5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,QAAQ,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC1B,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACxG,OAAO,CAAC,CAAC,CAAC,CAkBZ;AAWD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,EAAE;IAChD,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B,GAAG,OAAO,CAAC,MAAM,CAAC,CAmElB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,CAKlD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU1E"}