@mailwoman/corpus 9.4.0 → 10.0.0

package/out/tools/fetch/download/network.d.ts

@@ -0,0 +1,105 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The option base every `mailwoman corpus fetch <source>` module extends.
+ */
+/**
+ * How long a failed transfer waits before the next attempt.
+ */
+export declare const DEFAULT_RETRY_DELAY_MS = 5000;
+export interface BaseFetchOptions {
+    /**
+     * Destination root for downloaded source data. Each source writes its own subdirectory.
+     */
+    outRoot: PathBuilderLike;
+    /**
+     * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
+     *
+     * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two
+     * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the
+     * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten.
+     */
+    retryDelayMs?: number;
+}
+/**
+ * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
+ */
+export interface FetchSummary {
+    fetched: number;
+    skipped: number;
+    failed: number;
+    failedCodes: string[];
+}
+/**
+ * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count +
+ * sha256, so downstream adapters can verify provenance.
+ */
+export interface SourceManifest {
+    source_url: string;
+    downloaded_at: string;
+    filename: string;
+    sha256: string;
+    bytes: number;
+}
+/**
+ * A status worth retrying: rate limiting or a server-side failure.
+ */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
+ * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
+ * run and vanishes locally.
+ */
+export declare class HTTPStatusError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
+export declare function isTransientStatus(status: number): boolean;
+export interface DownloadOptions {
+    url: string;
+    dest: string;
+    /**
+     * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
+     */
+    timeoutMs?: number;
+    /**
+     * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
+     */
+    retries?: number;
+    /**
+     * Delay between attempts. Default 5s.
+     */
+    retryDelayMs?: number;
+    headers?: Record<string, string>;
+    report?: (line: string) => void;
+}
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export declare function downloadToFile(options: DownloadOptions): Promise<{
+    bytes: number;
+}>;
+export interface StreamDownloadOptions {
+    headers?: Record<string, string>;
+    timeoutMs: number;
+    retries: number;
+    retryDelayMs: number;
+}
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
+ *
+ * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
+ * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
+ * two-URL fallback ladders consume.
+ */
+export declare function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise<number>;
+//# sourceMappingURL=network.d.ts.map

package/out/tools/fetch/download/network.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"network.d.ts","sourceRoot":"","sources":["../../../../lib/tools/fetch/download/network.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAiB9C;;GAEG;AACH;;GAEG;AACH,eAAO,MAAM,sBAAsB,OAAO,CAAA;AAE1C,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,eAAe,CAAA;IACxB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IAEvB,YAAY,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAI1C;CACD;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsDzF;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAkC5G"}

package/out/tools/fetch/download/network.js

@@ -0,0 +1,130 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
+ */
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
+import { writeLocalFile } from "@mailwoman/core/fs/writers";
+import { sleep } from "@mailwoman/core/utils/sleep";
+/**
+ * Rate limited — retryable, the server is asking us to back off.
+ */
+const HTTP_TOO_MANY_REQUESTS = 429;
+/**
+ * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
+ */
+const HTTP_SERVER_ERROR_MIN = 500;
+/**
+ * Highest 5xx status.
+ */
+const HTTP_SERVER_ERROR_MAX = 599;
+/**
+ * The option base every `mailwoman corpus fetch <source>` module extends.
+ */
+/**
+ * How long a failed transfer waits before the next attempt.
+ */
+export const DEFAULT_RETRY_DELAY_MS = 5000;
+/**
+ * A status worth retrying: rate limiting or a server-side failure.
+ */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
+ * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
+ * run and vanishes locally.
+ */
+export class HTTPStatusError extends Error {
+    status;
+    constructor(status, message) {
+        super(message);
+        this.name = "HTTPStatusError";
+        this.status = status;
+    }
+}
+export function isTransientStatus(status) {
+    return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX);
+}
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export async function downloadToFile(options) {
+    const { url, dest, timeoutMs = 600_000, retries = 0, retryDelayMs = DEFAULT_RETRY_DELAY_MS, headers, report, } = options;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        if (attempt > 0) {
+            report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`);
+            await sleep(retryDelayMs);
+        }
+        let res;
+        try {
+            // Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
+            // `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
+            // non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
+            res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
+        }
+        catch (error) {
+            // AbortSignal timeouts and network-level failures are retryable.
+            lastError = error;
+            continue;
+        }
+        if (!res.ok) {
+            const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`);
+            if (!isTransientStatus(res.status))
+                throw error;
+            lastError = error;
+            continue;
+        }
+        try {
+            const buffer = Buffer.from(await res.arrayBuffer());
+            await writeLocalFile(buffer, dest);
+            return { bytes: buffer.byteLength };
+        }
+        catch (error) {
+            // A mid-stream abort while reading the body is retryable too.
+            lastError = error;
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
+ *
+ * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
+ * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
+ * two-URL fallback ladders consume.
+ */
+export async function streamDownload(url, dest, opts) {
+    for (let attempt = 0; attempt <= opts.retries; attempt++) {
+        try {
+            const res = await fetch(url, {
+                headers: opts.headers ?? {},
+                redirect: "follow",
+                signal: AbortSignal.timeout(opts.timeoutMs),
+            });
+            if (res.ok && res.body) {
+                await pipeline(Readable.fromWeb(res.body), openWriteStream(dest));
+                return res.status;
+            }
+            if (attempt < opts.retries && isTransientStatus(res.status)) {
+                await sleep(opts.retryDelayMs);
+                continue;
+            }
+            return res.status;
+        }
+        catch {
+            if (attempt < opts.retries) {
+                await sleep(opts.retryDelayMs);
+                continue;
+            }
+            return 0;
+        }
+    }
+    return 0;
+}
+//# sourceMappingURL=network.js.map

package/out/tools/fetch/download/network.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"network.js","sourceRoot":"","sources":["../../../../lib/tools/fetch/download/network.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAA;AAC3D,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAGnD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAA;AAuC1C;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EACL,GAAG,EACH,IAAI,EACJ,SAAS,GAAG,OAAO,EACnB,OAAO,GAAG,CAAC,EACX,YAAY,GAAG,sBAAsB,EACrC,OAAO,EACP,MAAM,GACN,GAAG,OAAO,CAAA;IAEX,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAElC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAA2B;IAC1F,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEjE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC"}

package/out/tools/fetch/download/retry.d.ts

@@ -0,0 +1,45 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Retries and resumes interrupted corpus downloads.
+ */
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export declare function withRetries<T>(transfer: () => Promise<T>, options?: {
+    retries?: number;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+    label?: string;
+}): Promise<T>;
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export declare function resumableDownload(options: {
+    url: string;
+    dest: string;
+    headers?: Record<string, string>;
+    maxConnections?: number;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+}): Promise<number>;
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export declare function cookieHeader(res: Response): string;
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export declare function attachmentFilename(res: Response, fallback: string): string;
+//# sourceMappingURL=retry.d.ts.map

package/out/tools/fetch/download/retry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../../../lib/tools/fetch/download/retry.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAoBH;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,CAAC,EAClC,QAAQ,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAC1B,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACxG,OAAO,CAAC,CAAC,CAAC,CAkBZ;AAWD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,EAAE;IAChD,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B,GAAG,OAAO,CAAC,MAAM,CAAC,CAmElB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,QAAQ,GAAG,MAAM,CAKlD;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAU1E"}

package/out/tools/fetch/download/retry.js

@@ -0,0 +1,136 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Retries and resumes interrupted corpus downloads.
+ */
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
+import { movePath } from "@mailwoman/core/fs/writers";
+import { sleep } from "@mailwoman/core/utils/sleep";
+import { DEFAULT_RETRY_DELAY_MS, HTTPStatusError } from "#tools/fetch/download/network";
+/**
+ * The answer to a range that starts at or past the end of the body: the resume point is already the whole file.
+ */
+const HTTP_RANGE_NOT_SATISFIABLE = 416;
+/**
+ * The answer to a satisfiable `Range` request; anything else means the host ignored the range and would send the whole
+ * body again.
+ */
+const HTTP_PARTIAL_CONTENT = 206;
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export async function withRetries(transfer, options = {}) {
+    const { retries = 3, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report, label = "transfer" } = options;
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        if (attempt > 0) {
+            report?.(`  retry ${attempt}/${retries} after ${retryDelayMs}ms — ${label}`);
+            await sleep(retryDelayMs);
+        }
+        try {
+            return await transfer();
+        }
+        catch (error) {
+            lastError = error;
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+}
+/**
+ * The byte count already at `path`, or 0 when nothing is there — the resume point of an interrupted range download.
+ */
+async function bytesOnDisk(path) {
+    const stat = await tryStat(path);
+    return stat?.size ?? 0;
+}
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export async function resumableDownload(options) {
+    const { url, dest, headers = {}, maxConnections = 400, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report } = options;
+    const tmp = dest + ".tmp";
+    let have = await bytesOnDisk(tmp);
+    let total;
+    for (let connection = 0; connection < maxConnections; connection++) {
+        if (total !== undefined && have >= total)
+            break;
+        let res;
+        try {
+            res = await fetch(url, { headers: { ...headers, range: `bytes=${have}-` }, signal: AbortSignal.timeout(600_000) });
+        }
+        catch {
+            await sleep(retryDelayMs);
+            continue;
+        }
+        if (res.status === HTTP_RANGE_NOT_SATISFIABLE && have > 0) {
+            // Nothing past `have`: the file on disk is already the whole body (a parallel filler or an earlier run
+            // landed it), and `Content-Range: bytes */<total>` says how long it is.
+            const whole = /\*\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1];
+            total = whole ? Number(whole) : have;
+            await res.body?.cancel();
+            break;
+        }
+        if (res.status !== HTTP_PARTIAL_CONTENT) {
+            throw new HTTPStatusError(res.status, `HTTP ${res.status} for a range request — ${url}`);
+        }
+        const range = /bytes \d+-\d+\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1];
+        if (range) {
+            total = Number(range);
+        }
+        const before = have;
+        try {
+            // Append: the stream opens the file for appending so a partial body extends what earlier connections left.
+            await pipeline(Readable.fromWeb(res.body), openWriteStream(tmp, { flags: "a" }));
+        }
+        catch {
+            // The connection dropped mid-body; the bytes that landed are on disk, and the next range starts after them.
+        }
+        have = await bytesOnDisk(tmp);
+        if (have === before) {
+            await sleep(retryDelayMs);
+        }
+        else {
+            report?.(`  ${(have / 1024 / 1024).toFixed(1)} MB${total ? ` of ${(total / 1024 / 1024).toFixed(1)} MB` : ""}`);
+        }
+    }
+    if (total === undefined || have < total) {
+        throw new Error(`resumable download stalled at ${have} of ${total ?? "?"} bytes — ${url}`);
+    }
+    await movePath(tmp, dest);
+    return have;
+}
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export function cookieHeader(res) {
+    return res.headers
+        .getSetCookie()
+        .map((cookie) => cookie.split(";")[0])
+        .join("; ");
+}
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export function attachmentFilename(res, fallback) {
+    const disposition = res.headers.get("content-disposition") ?? "";
+    const extended = /filename\*=(?:utf-8|UTF-8)'[^']*'([^;]+)/.exec(disposition)?.[1];
+    if (extended)
+        return decodeURIComponent(extended.trim());
+    const plain = /filename="?([^";]+)"?/.exec(disposition)?.[1];
+    if (plain)
+        return decodeURIComponent(plain.trim());
+    return fallback;
+}
+//# sourceMappingURL=retry.js.map

package/out/tools/fetch/download/retry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../../lib/tools/fetch/download/retry.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAEnD,OAAO,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAA;AAEvF;;GAEG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;;GAGG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEhC;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,QAA0B,EAC1B,OAAO,GAAiG,EAAE;IAE1G,MAAM,EAAE,OAAO,GAAG,CAAC,EAAE,YAAY,GAAG,sBAAsB,EAAE,MAAM,EAAE,KAAK,GAAG,UAAU,EAAE,GAAG,OAAO,CAAA;IAClG,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,WAAW,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,KAAK,EAAE,CAAC,CAAA;YAC5E,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,CAAC;YACJ,OAAO,MAAM,QAAQ,EAAE,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,OAAO,IAAI,EAAE,IAAI,IAAI,CAAC,CAAA;AACvB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAOvC;IACA,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,GAAG,EAAE,EAAE,cAAc,GAAG,GAAG,EAAE,YAAY,GAAG,sBAAsB,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAChH,MAAM,GAAG,GAAG,IAAI,GAAG,MAAM,CAAA;IACzB,IAAI,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;IACjC,IAAI,KAAyB,CAAA;IAE7B,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,UAAU,EAAE,EAAE,CAAC;QACpE,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,IAAI,KAAK;YAAE,MAAK;QAE/C,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,SAAS,IAAI,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACnH,CAAC;QAAC,MAAM,CAAC;YACR,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,0BAA0B,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YAC3D,uGAAuG;YACvG,wEAAwE;YACxE,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC3E,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;YACpC,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAA;YAExB,MAAK;QACN,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,0BAA0B,GAAG,EAAE,CAAC,CAAA;QACzF,CAAC;QAED,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAEtF,IAAI,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QACtB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAA;QAEnB,IAAI,CAAC;YACJ,2GAA2G;YAC3G,MAAM,QAAQ,CACb,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAA8C,CAAC,EACpE,eAAe,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CACpC,CAAA;QACF,CAAC;QAAC,MAAM,CAAC;YACR,4GAA4G;QAC7G,CAAC;QAED,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,CAAA;QAE7B,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YACrB,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;aAAM,CAAC;YACP,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAChH,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,OAAO,KAAK,IAAI,GAAG,YAAY,GAAG,EAAE,CAAC,CAAA;IAC3F,CAAC;IAED,MAAM,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAEzB,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAa;IACzC,OAAO,GAAG,CAAC,OAAO;SAChB,YAAY,EAAE;SACd,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SACrC,IAAI,CAAC,IAAI,CAAC,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAa,EAAE,QAAgB;IACjE,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,EAAE,CAAA;IAChE,MAAM,QAAQ,GAAG,0CAA0C,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAElF,IAAI,QAAQ;QAAE,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAE5D,IAAI,KAAK;QAAE,OAAO,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IAElD,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/tools/fetch/geonames/dump.d.ts

@@ -0,0 +1,91 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
+ *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
+ *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
+ *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
+ *
+ *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
+ *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
+ *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
+ *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
+ *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
+ *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
+ *   the basename) is reported as `wrong_format_present`, never counted as coverage.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index";
+export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
+    /**
+     * ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
+     */
+    countries?: readonly string[];
+    /**
+     * Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
+     * against a local server.
+     */
+    baseURL?: string;
+    /**
+     * Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
+     */
+    force?: boolean;
+}
+interface GeonamesDumpFileEntry {
+    country: string;
+    filename: string;
+    source_url: string;
+    sha256: string;
+    bytes: number;
+}
+export interface GeonamesDumpManifest {
+    source: string;
+    base_url: string;
+    license: string;
+    attribution: string;
+    downloaded_at: string;
+    files: GeonamesDumpFileEntry[];
+    /**
+     * `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
+     */
+    skipped_present: string[];
+    /**
+     * Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
+     * recorded so a later reader does not spend the fetch to rediscover it.
+     */
+    unavailable: string[];
+    /**
+     * Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
+     * and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
+     * capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
+     * own home and rerun.
+     */
+    wrong_format_present: string[];
+}
+/**
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
+ *
+ * Walk the string directly rather than constructing a spliterator: the capitals builder already holds whole country
+ * dumps as strings, and a byte-oriented spliterator would UTF-8 encode that input — allocating up to another 350 MB —
+ * merely to inspect its first non-empty line.
+ */
+export declare function looksLikeGazetteerDump(text: string): boolean;
+/**
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
+ * file's own commentary.
+ */
+export declare function parseCountryInfo(text: string): Array<{
+    country: string;
+    capital: string;
+}>;
+/**
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
+ */
+export declare function fetchGeonamesDumps(options: FetchGeonamesDumpOptions, report?: (line: string) => void): Promise<FetchSummary & {
+    skippedPresent: string[];
+}>;
+export {};
+//# sourceMappingURL=dump.d.ts.map

package/out/tools/fetch/geonames/dump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dump.d.ts","sourceRoot":"","sources":["../../../../lib/tools/fetch/geonames/dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAA;AAgBjF,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IACjE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,qBAAqB,EAAE,CAAA;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB;;;OAGG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB;;;;;OAKG;IACH,oBAAoB,EAAE,MAAM,EAAE,CAAA;CAC9B;AAQD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAuB5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB1F;AAQD;;;GAGG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,GAAG;IAAE,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA2FtD"}