@mailwoman/corpus 9.3.0 → 10.0.0

package/out/tools/fetch/download.d.ts

@@ -1,117 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
- */
-import type { PathBuilderLike } from "path-ts";
-/**
- * The option base every `mailwoman corpus fetch <source>` module extends.
- */
-/**
- * How long a failed transfer waits before the next attempt.
- */
-export declare const DEFAULT_RETRY_DELAY_MS = 5000;
-export interface BaseFetchOptions {
-    /**
-     * Destination root for downloaded source data. Each source writes its own subdirectory.
-     */
-    outRoot: PathBuilderLike;
-    /**
-     * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
-     *
-     * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two
-     * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the
-     * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten.
-     */
-    retryDelayMs?: number;
-}
-/**
- * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
- */
-export interface FetchSummary {
-    fetched: number;
-    skipped: number;
-    failed: number;
-    failedCodes: string[];
-}
-/**
- * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count +
- * sha256, so downstream adapters can verify provenance.
- */
-export interface SourceManifest {
-    source_url: string;
-    downloaded_at: string;
-    filename: string;
-    sha256: string;
-    bytes: number;
-}
-/**
- * A status worth retrying: rate limiting or a server-side failure.
- */
-/**
- * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
- * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
- * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
- * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
- * run and vanishes locally.
- */
-export declare class HTTPStatusError extends Error {
-    readonly status: number;
-    constructor(status: number, message: string);
-}
-export declare function isTransientStatus(status: number): boolean;
-export interface DownloadOptions {
-    url: string;
-    dest: string;
-    /**
-     * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
-     */
-    timeoutMs?: number;
-    /**
-     * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
-     */
-    retries?: number;
-    /**
-     * Delay between attempts. Default 5s.
-     */
-    retryDelayMs?: number;
-    headers?: Record<string, string>;
-    report?: (line: string) => void;
-}
-/**
- * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
- * or once retries are exhausted. Returns the byte count written.
- */
-export declare function downloadToFile(options: DownloadOptions): Promise<{
-    bytes: number;
-}>;
-export interface StreamDownloadOptions {
-    headers?: Record<string, string>;
-    timeoutMs: number;
-    retries: number;
-    retryDelayMs: number;
-}
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
- * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
- *
- * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
- * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
- * two-URL fallback ladders consume.
- */
-export declare function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise<number>;
-/**
- * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
- */
-export declare function readManifest<T>(path: string): Promise<T | null>;
-/**
- * Load manifest entries into a map so untouched keys survive a partial re-fetch.
- */
-export declare function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>>;
-/**
- * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
- */
-export declare function writeManifest(path: string, manifest: unknown): Promise<void>;
-//# sourceMappingURL=download.d.ts.map

package/out/tools/fetch/download.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAiB9C;;GAEG;AACH;;GAEG;AACH,eAAO,MAAM,sBAAsB,OAAO,CAAA;AAE1C,MAAM,WAAW,gBAAgB;IAChC;;OAEG;IACH,OAAO,EAAE,eAAe,CAAA;IACxB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,KAAK;IACzC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;gBAEX,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK3C;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAsDzF;AAED,MAAM,WAAW,qBAAqB;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;CACpB;AAED;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,MAAM,CAAC,CAkC5G;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAQrE;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAS7G;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF"}

package/out/tools/fetch/download.js

@@ -1,160 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
- */
-import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers";
-import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams";
-import { writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers";
-import { tryParsingJSON } from "@mailwoman/core/json";
-import { sleep } from "@mailwoman/core/utils/sleep";
-/**
- * Rate limited — retryable, the server is asking us to back off.
- */
-const HTTP_TOO_MANY_REQUESTS = 429;
-/**
- * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
- */
-const HTTP_SERVER_ERROR_MIN = 500;
-/**
- * Highest 5xx status.
- */
-const HTTP_SERVER_ERROR_MAX = 599;
-/**
- * The option base every `mailwoman corpus fetch <source>` module extends.
- */
-/**
- * How long a failed transfer waits before the next attempt.
- */
-export const DEFAULT_RETRY_DELAY_MS = 5000;
-/**
- * A status worth retrying: rate limiting or a server-side failure.
- */
-/**
- * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
- * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
- * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
- * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
- * run and vanishes locally.
- */
-export class HTTPStatusError extends Error {
-    status;
-    constructor(status, message) {
-        super(message);
-        this.name = "HTTPStatusError";
-        this.status = status;
-    }
-}
-export function isTransientStatus(status) {
-    return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX);
-}
-/**
- * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
- * or once retries are exhausted. Returns the byte count written.
- */
-export async function downloadToFile(options) {
-    const { url, dest, timeoutMs = 600_000, retries = 0, retryDelayMs = DEFAULT_RETRY_DELAY_MS, headers, report, } = options;
-    let lastError;
-    for (let attempt = 0; attempt <= retries; attempt++) {
-        if (attempt > 0) {
-            report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`);
-            await sleep(retryDelayMs);
-        }
-        let res;
-        try {
-            // Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
-            // `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
-            // non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
-            res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) });
-        }
-        catch (error) {
-            // AbortSignal timeouts and network-level failures are retryable.
-            lastError = error;
-            continue;
-        }
-        if (!res.ok) {
-            const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`);
-            if (!isTransientStatus(res.status))
-                throw error;
-            lastError = error;
-            continue;
-        }
-        try {
-            const buffer = Buffer.from(await res.arrayBuffer());
-            await writeLocalFile(buffer, dest);
-            return { bytes: buffer.byteLength };
-        }
-        catch (error) {
-            // A mid-stream abort while reading the body is retryable too.
-            lastError = error;
-        }
-    }
-    throw lastError instanceof Error ? lastError : new Error(String(lastError));
-}
-/**
- * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
- * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
- *
- * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
- * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
- * two-URL fallback ladders consume.
- */
-export async function streamDownload(url, dest, opts) {
-    for (let attempt = 0; attempt <= opts.retries; attempt++) {
-        try {
-            const res = await fetch(url, {
-                headers: opts.headers ?? {},
-                redirect: "follow",
-                signal: AbortSignal.timeout(opts.timeoutMs),
-            });
-            if (res.ok && res.body) {
-                await pipeline(Readable.fromWeb(res.body), openWriteStream(dest));
-                return res.status;
-            }
-            if (attempt < opts.retries && isTransientStatus(res.status)) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return res.status;
-        }
-        catch {
-            if (attempt < opts.retries) {
-                await sleep(opts.retryDelayMs);
-                continue;
-            }
-            return 0;
-        }
-    }
-    return 0;
-}
-/**
- * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
- */
-export async function readManifest(path) {
-    if (!(await pathExists(path)))
-        return null;
-    // A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
-    // JSON does; tryParsingJSON returns null for the non-string sentinel.
-    const text = await readLocalTextFile(path).catch(() => null);
-    return tryParsingJSON(text);
-}
-/**
- * Load manifest entries into a map so untouched keys survive a partial re-fetch.
- */
-export async function loadManifestEntries(path, key) {
-    const entries = new Map();
-    const parsed = await readManifest(path);
-    for (const entry of parsed ?? []) {
-        entries.set(key(entry), entry);
-    }
-    return entries;
-}
-/**
- * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
- */
-export async function writeManifest(path, manifest) {
-    await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path);
-}
-//# sourceMappingURL=download.js.map

package/out/tools/fetch/download.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"download.js","sourceRoot":"","sources":["../../../lib/tools/fetch/download.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AAGnD;;GAEG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAA;AAuC1C;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAChC,MAAM,CAAQ;IAEvB,YAAY,MAAc,EAAE,OAAe;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;CACD;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC/C,OAAO,MAAM,KAAK,sBAAsB,IAAI,CAAC,MAAM,IAAI,qBAAqB,IAAI,MAAM,IAAI,qBAAqB,CAAC,CAAA;AACjH,CAAC;AAqBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAAwB;IAC5D,MAAM,EACL,GAAG,EACH,IAAI,EACJ,SAAS,GAAG,OAAO,EACnB,OAAO,GAAG,CAAC,EACX,YAAY,GAAG,sBAAsB,EACrC,OAAO,EACP,MAAM,GACN,GAAG,OAAO,CAAA;IAEX,IAAI,SAAkB,CAAA;IAEtB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,EAAE,CAAC,SAAS,OAAO,IAAI,OAAO,UAAU,YAAY,QAAQ,GAAG,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,CAAC,YAAY,CAAC,CAAA;QAC1B,CAAC;QAED,IAAI,GAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,qGAAqG;YACrG,kGAAkG;YAClG,+FAA+F;YAC/F,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,iEAAiE;YACjE,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,MAAM,GAAG,EAAE,CAAC,CAAA;YAE9F,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/C,SAAS,GAAG,KAAK,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAA;YACnD,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAElC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,CAAA;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,8DAA8D;YAC9D,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;AAC5E,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY,EAAE,IAA2B;IAC1F,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;aAC3C,CAAC,CAAA;YAEF,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEjE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,IAAY;IACjD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,+FAA+F;IAC/F,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAE5D,OAAO,cAAc,CAAI,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAI,IAAY,EAAE,GAAyB;IACnF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAa,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAM,IAAI,CAAC,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,EAAE,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,QAAiB;IAClE,MAAM,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,CAAA;AACzE,CAAC"}

package/out/tools/fetch/geonames-dump.d.ts

@@ -1,91 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
- *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
- *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
- *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
- *
- *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
- *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
- *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
- *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
- *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
- *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
- *   the basename) is reported as `wrong_format_present`, never counted as coverage.
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
-    /**
-     * ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
-     */
-    countries?: readonly string[];
-    /**
-     * Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
-     * against a local server.
-     */
-    baseURL?: string;
-    /**
-     * Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
-     */
-    force?: boolean;
-}
-interface GeonamesDumpFileEntry {
-    country: string;
-    filename: string;
-    source_url: string;
-    sha256: string;
-    bytes: number;
-}
-export interface GeonamesDumpManifest {
-    source: string;
-    base_url: string;
-    license: string;
-    attribution: string;
-    downloaded_at: string;
-    files: GeonamesDumpFileEntry[];
-    /**
-     * `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
-     */
-    skipped_present: string[];
-    /**
-     * Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
-     * recorded so a later reader does not spend the fetch to rediscover it.
-     */
-    unavailable: string[];
-    /**
-     * Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
-     * and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
-     * capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
-     * own home and rerun.
-     */
-    wrong_format_present: string[];
-}
-/**
- * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
- * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
- *
- * Walk the string directly rather than constructing a spliterator: the capitals builder already holds whole country
- * dumps as strings, and a byte-oriented spliterator would UTF-8 encode that input — allocating up to another 350 MB —
- * merely to inspect its first non-empty line.
- */
-export declare function looksLikeGazetteerDump(text: string): boolean;
-/**
- * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
- * file's own commentary.
- */
-export declare function parseCountryInfo(text: string): Array<{
-    country: string;
-    capital: string;
-}>;
-/**
- * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
- * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
- */
-export declare function fetchGeonamesDumps(options: FetchGeonamesDumpOptions, report?: (line: string) => void): Promise<FetchSummary & {
-    skippedPresent: string[];
-}>;
-export {};
-//# sourceMappingURL=geonames-dump.d.ts.map

package/out/tools/fetch/geonames-dump.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"geonames-dump.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAgB3E,MAAM,WAAW,wBAAyB,SAAQ,gBAAgB;IACjE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;CACf;AAED,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,oBAAoB;IACpC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,KAAK,EAAE,qBAAqB,EAAE,CAAA;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB;;;OAGG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB;;;;;OAKG;IACH,oBAAoB,EAAE,MAAM,EAAE,CAAA;CAC9B;AAQD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAuB5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAiB1F;AAQD;;;GAGG;AACH,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,GAAG;IAAE,cAAc,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA2FtD"}

package/out/tools/fetch/geonames-dump.js

@@ -1,171 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
- *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
- *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
- *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
- *
- *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
- *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
- *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
- *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
- *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
- *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
- *   the basename) is reported as `wrong_format_present`, never counted as coverage.
- */
-import { pathExists, readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers";
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
-import { extractZipEntry } from "@mailwoman/core/fs/zip";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download";
-/**
- * The one status that means "the source does not publish this country" rather than "the transfer failed".
- */
-const HTTP_NOT_FOUND = 404;
-const SLUG = "geonames-dump";
-/**
- * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
- * bare text file.
- */
-const BASE_URL = "https://download.geonames.org/export/dump";
-/**
- * Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
- * `<CC>.txt` basename.
- */
-const GAZETTEER_DUMP_COLUMNS = 19;
-/**
- * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
- * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
- *
- * Walk the string directly rather than constructing a spliterator: the capitals builder already holds whole country
- * dumps as strings, and a byte-oriented spliterator would UTF-8 encode that input — allocating up to another 350 MB —
- * merely to inspect its first non-empty line.
- */
-export function looksLikeGazetteerDump(text) {
-    let start = 0;
-    while (start < text.length) {
-        const end = text.indexOf("\n", start);
-        const line = end === -1 ? text.slice(start) : text.slice(start, end);
-        if (line.trim()) {
-            let tabs = 0;
-            for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
-                tabs++;
-            }
-            return tabs === GAZETTEER_DUMP_COLUMNS - 1;
-        }
-        if (end === -1)
-            break;
-        start = end + 1;
-    }
-    return false;
-}
-/**
- * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
- * file's own commentary.
- */
-export function parseCountryInfo(text) {
-    const rows = [];
-    // oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
-    for (const line of text.split("\n")) {
-        if (line.startsWith("#") || !line.trim())
-            continue;
-        // oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
-        const cols = line.split("\t");
-        const country = cols[0]?.trim().toUpperCase();
-        if (country?.length === 2) {
-            rows.push({ country, capital: cols[5]?.trim() ?? "" });
-        }
-    }
-    return rows;
-}
-/**
- * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
- * reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
- */
-const FORMAT_SNIFF_BYTES = 65_536;
-/**
- * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
- * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
- */
-export async function fetchGeonamesDumps(options, report) {
-    await makeDirectories(options.outRoot);
-    const baseURL = options.baseURL ?? BASE_URL;
-    const countryInfoDest = join(options.outRoot, "countryInfo.txt");
-    await downloadToFile({
-        url: `${baseURL}/countryInfo.txt`,
-        dest: countryInfoDest,
-        timeoutMs: 120_000,
-        retries: 2,
-        report,
-    });
-    const catalog = parseCountryInfo(await readLocalTextFile(countryInfoDest));
-    const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country);
-    const entries = [];
-    const failedCodes = [];
-    const unavailable = [];
-    const skippedPresent = [];
-    const wrongFormatPresent = [];
-    let fetched = 0;
-    for (const country of countries) {
-        const txtDest = join(options.outRoot, `${country}.txt`);
-        if (!options.force && (await pathExists(txtDest))) {
-            if (looksLikeGazetteerDump(await readFileHead(txtDest, FORMAT_SNIFF_BYTES))) {
-                skippedPresent.push(country);
-            }
-            else {
-                report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`);
-                wrongFormatPresent.push(country);
-            }
-            continue;
-        }
-        const filename = `${country}.zip`;
-        const url = `${baseURL}/${filename}`;
-        const zipDest = join(options.outRoot, filename);
-        report?.(`=== ${SLUG} / ${country}`);
-        try {
-            await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report });
-            await extractZipEntry(zipDest, `${country}.txt`, txtDest);
-            await removePathIfPresent(zipDest);
-            entries.push({
-                country,
-                filename: `${country}.txt`,
-                source_url: url,
-                sha256: await sha256File(txtDest),
-                bytes: (await readLocalBuffer(txtDest)).byteLength,
-            });
-            fetched++;
-        }
-        catch (error) {
-            await removePathIfPresent(zipDest);
-            const message = error instanceof Error ? error.message : String(error);
-            // Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
-            // can contain any substring.
-            if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
-                report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`);
-                unavailable.push(country);
-            }
-            else {
-                report?.(`✗ ${country}: ${message}`);
-            }
-            failedCodes.push(country);
-        }
-    }
-    const manifest = {
-        source: SLUG,
-        base_url: baseURL,
-        license: "CC-BY-4.0",
-        attribution: "GeoNames",
-        downloaded_at: new Date().toISOString(),
-        files: entries,
-        skipped_present: skippedPresent.toSorted(),
-        unavailable,
-        wrong_format_present: wrongFormatPresent.toSorted(),
-    };
-    await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest);
-    return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent };
-}
-//# sourceMappingURL=geonames-dump.js.map

package/out/tools/fetch/geonames-dump.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"geonames-dump.js","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACzG,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,cAAc,GAAG,GAAG,CAAA;AAE1B,MAAM,IAAI,GAAG,eAAe,CAAA;AAE5B;;;GAGG;AACH,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAmD5D;;;GAGG;AACH,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAEjC;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;QAEpE,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAA;YAEZ,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1E,IAAI,EAAE,CAAA;YACP,CAAC;YAED,OAAO,IAAI,KAAK,sBAAsB,GAAG,CAAC,CAAA;QAC3C,CAAC;QAED,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QAErB,KAAK,GAAG,GAAG,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,IAAI,GAAgD,EAAE,CAAA;IAE5D,6IAA6I;IAC7I,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAElD,6FAA6F;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7C,IAAI,OAAO,EAAE,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,kBAAkB,GAAG,MAAM,CAAA;AAEjC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAAiC,EACjC,MAA+B;IAE/B,MAAM,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAA;IAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAA;IAEhE,MAAM,cAAc,CAAC;QACpB,GAAG,EAAE,GAAG,OAAO,kBAAkB;QACjC,IAAI,EAAE,eAAe;QACrB,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,CAAC;QACV,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,iBAAiB,CAAC,eAAe,CAAC,CAAC,CAAA;IAC1E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAElH,MAAM,OAAO,GAA4B,EAAE,CAAA;IAC3C,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,cAAc,GAAa,EAAE,CAAA;IACnC,MAAM,kBAAkB,GAAa,EAAE,CAAA;IACvC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,CAAC,CAAA;QAEvD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YACnD,IAAI,sBAAsB,CAAC,MAAM,YAAY,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,EAAE,CAAC;gBAC7E,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC7B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,iFAAiF,CAAC,CAAA;gBACvG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACjC,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,OAAO,MAAM,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAE/C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,EAAE,CAAC,CAAA;QAEpC,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YACpF,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,OAAO,CAAC,CAAA;YACzD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,OAAO,CAAC,IAAI,CAAC;gBACZ,OAAO;gBACP,QAAQ,EAAE,GAAG,OAAO,MAAM;gBAC1B,UAAU,EAAE,GAAG;gBACf,MAAM,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC;gBACjC,KAAK,EAAE,CAAC,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU;aAClD,CAAC,CAAA;YAEF,OAAO,EAAE,CAAA;QACV,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;YAElC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAEtE,qGAAqG;YACrG,6BAA6B;YAC7B,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;gBACzE,MAAM,EAAE,CAAC,KAAK,OAAO,yDAAyD,CAAC,CAAA;gBAC/E,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAC1B,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,KAAK,OAAO,KAAK,OAAO,EAAE,CAAC,CAAA;YACrC,CAAC;YAED,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAyB;QACtC,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,WAAW;QACpB,WAAW,EAAE,UAAU;QACvB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK,EAAE,OAAO;QACd,eAAe,EAAE,cAAc,CAAC,QAAQ,EAAE;QAC1C,WAAW;QACX,oBAAoB,EAAE,kBAAkB,CAAC,QAAQ,EAAE;KACnD,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAErE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,EAAE,CAAA;AAC5G,CAAC"}

package/out/tools/fetch/geonames-postal.d.ts

@@ -1,76 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
- *   `(postcode, locality, region)` triple with the NAMES inline.
- *
- *   Source : https://download.geonames.org/export/zip/<CC>.zip
- *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
- *
- *   ## Why this source and not a join
- *
- *   A postcode slice is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
- *   only one of them works everywhere:
- *
- *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
- *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
- *     `locality`/`localadmin`). It is also the ONLY slice that does: `postalcode-geonames-intl.db` and every
- *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
- *     have none.
- *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
- *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
- *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
- *     own. Under 50% for four of the five, which is not a join.
- *
- *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
- *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
- *
- *   ## Coverage is not universal, and the gap is the point
- *
- *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode slice cannot be built
- *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
- *   before assuming it is there; an absent country fails as one entry, never as the whole run.
- *
- *   ## Row counts from this source overstate, for some countries by exactly 2×
- *
- *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
- *   on the slice built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
- *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a slice from the row
- *   count doubles its estimate for those countries.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
- *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
- *
- *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-/**
- * Countries fetched when the caller names none.
- *
- * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
- * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
- */
-export declare const GEONAMES_POSTAL_DEFAULT_COUNTRIES: readonly ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"];
-export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
-    /**
-     * ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
-     */
-    countries?: readonly string[];
-    /**
-     * Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
-     * exercised against a local server rather than by asking GeoNames for a country it does not have.
-     */
-    baseURL?: string;
-}
-/**
- * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
- * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
- *
- * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
- */
-export declare function fetchGeonamesPostal(options: FetchGeonamesPostalOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=geonames-postal.d.ts.map

package/out/tools/fetch/geonames-postal.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"geonames-postal.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/geonames-postal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAMH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAe3E;;;;;GAKG;AACH,eAAO,MAAM,iCAAiC,qDAAsD,CAAA;AAEpG,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAwBD;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA2DvB"}