@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/juso-kr.js

@@ -1,120 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the Korean road-name address register (도로명주소 주소DB) from the ministry's address-industry
- *   portal, business.juso.go.kr. This is the LABEL source for Korean addresses: about 6,000,000
- *   road-name addresses and 8,000,000 lot-number (지번) records, split by the publisher into four
- *   pipe-delimited CP949 text files per region (도로명코드 / 주소 / 지번 / 부가정보), with the postcode in
- *   the supplementary file. No coordinates. The public-data portal labels it "이용허락범위 제한 없음"
- *   (no restriction on the scope of use), and it is a direct download with no application step — the
- *   distinction that matters against the ministry's entrance-coordinate products, which are provided
- *   only after a purpose-of-use review (`docs/superpowers/plans/counsel-dossier.md` §4).
- *
- *   The portal is a single-page app; the file behind it is reached in two calls. `selectAttrbDBDwldList`
- *   lists the monthly full files for one product (`rtlDtaDtlSn` 8 is the 주소DB, 2 the English
- *   road-name DB), and `/api/jst/download` streams one by the parameters the listing carried. The
- *   English DB rides along because it is the Latin-script half of the same register.
- *
- *   Invoke via `mailwoman corpus fetch juso-kr --out-root <path>`.
- */
-import { makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { resumableDownload, writeManifest } from "#tools/fetch/download";
-const SLUG = "juso-kr";
-const PORTAL = "https://business.juso.go.kr";
-const LIST_URL = `${PORTAL}/api/jst/selectAttrbDBDwldList`;
-const DOWNLOAD_URL = `${PORTAL}/api/jst/download`;
-const REFERER = `${PORTAL}/jst/jstAddressDownload?menu=30`;
-/**
- * The portal's product ids for the two registers this module fetches.
- */
-const PRODUCTS = [
-    { sn: "8", name: "주소DB" },
-    { sn: "2", name: "도로명주소 영문" },
-];
-const LICENSE = "이용허락범위 제한 없음 (data.go.kr label for 행정안전부_도로명주소 주소DB); 공공누리 attribution to 행정안전부";
-const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 도로명주소 (juso.go.kr)";
-const BROWSER_HEADERS = {
-    "user-agent": "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch",
-    referer: REFERER,
-};
-/**
- * The monthly full files the portal lists for one product over the twelve months ending at `year`/`month`.
- */
-async function listMonthlyFiles(sn, year, month) {
-    const res = await fetch(LIST_URL, {
-        method: "POST",
-        headers: { ...BROWSER_HEADERS, accept: "application/json", "content-type": "application/json;charset=UTF-8" },
-        body: JSON.stringify({ rtlDtaDtlSn: sn, year, month, expand: "Y" }),
-    });
-    if (!res.ok)
-        throw new Error(`juso-kr: listing product ${sn} answered HTTP ${res.status}`);
-    const json = (await res.json());
-    return (json.results?.allMonthFileList ?? []).filter((file) => file.isExist === "Y");
-}
-function downloadURL(file, regYmd) {
-    const query = new URLSearchParams({
-        reqType: file.fileTypeNm,
-        ctprvnCd: file.ctpvClsfCd,
-        stdde: file.crtrYm,
-        fileName: file.fileNm,
-        realFileName: file.tmprFileNm,
-        intFileNo: file.fileSn ?? "0",
-        intNum: file.atflNo ?? "0",
-        regYmd,
-    });
-    return `${DOWNLOAD_URL}?${query.toString()}`;
-}
-export async function fetchJusoKR(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const now = new Date();
-    const files = [];
-    const failedCodes = [];
-    for (const product of PRODUCTS) {
-        const listed = await listMonthlyFiles(product.sn, now.getFullYear(), now.getMonth() + 1);
-        const file = options.month ? listed.find((entry) => entry.crtrYm === options.month) : listed.at(-1);
-        if (!file) {
-            report?.(`  ✗ ${product.name}: no full file listed${options.month ? ` for ${options.month}` : ""}`);
-            failedCodes.push(product.name);
-            continue;
-        }
-        const url = downloadURL(file, String(now.getFullYear()));
-        const dest = join(destDir, file.tmprFileNm);
-        report?.(`=== ${SLUG} / ${product.name} ${file.crtrYm} → ${file.tmprFileNm}`);
-        let bytes;
-        try {
-            // The portal drops a connection every few megabytes and answers Range with 206, so the transfer resumes
-            // from what landed rather than starting the 181 MB over.
-            bytes = await resumableDownload({ url, dest, headers: BROWSER_HEADERS, report });
-        }
-        catch (error) {
-            report?.(`  ✗ ${error.message}`);
-            failedCodes.push(product.name);
-            continue;
-        }
-        const sha = await sha256File(dest);
-        files.push({
-            source_url: url,
-            downloaded_at: new Date().toISOString(),
-            filename: file.tmprFileNm,
-            sha256: sha,
-            bytes,
-        });
-        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-    }
-    const manifest = {
-        source: SLUG,
-        source_url: REFERER,
-        license: LICENSE,
-        attribution: ATTRIBUTION,
-        downloaded_at: new Date().toISOString(),
-        files,
-    };
-    await writeManifest(join(destDir, "MANIFEST.json"), manifest);
-    return { fetched: files.length, skipped: 0, failed: failedCodes.length, failedCodes };
-}
-//# sourceMappingURL=juso-kr.js.map

package/out/tools/fetch/juso-kr.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"juso-kr.js","sourceRoot":"","sources":["../../../lib/tools/fetch/juso-kr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAExE,MAAM,IAAI,GAAG,SAAS,CAAA;AACtB,MAAM,MAAM,GAAG,6BAA6B,CAAA;AAC5C,MAAM,QAAQ,GAAG,GAAG,MAAM,gCAAgC,CAAA;AAC1D,MAAM,YAAY,GAAG,GAAG,MAAM,mBAAmB,CAAA;AACjD,MAAM,OAAO,GAAG,GAAG,MAAM,iCAAiC,CAAA;AAE1D;;GAEG;AACH,MAAM,QAAQ,GAAG;IAChB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE;IACzB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE;CACpB,CAAA;AAEV,MAAM,OAAO,GACZ,iFAAiF,CAAA;AAClF,MAAM,WAAW,GAAG,iEAAiE,CAAA;AAwBrF,MAAM,eAAe,GAAG;IACvB,YAAY,EAAE,wDAAwD;IACtE,OAAO,EAAE,OAAO;CAChB,CAAA;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,EAAU,EAAE,IAAY,EAAE,KAAa;IACtE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;QACjC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,GAAG,eAAe,EAAE,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,gCAAgC,EAAE;QAC7G,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;KACnE,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAC1F,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiB,CAAA;IAE/C,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,GAAG,CAAC,CAAA;AACrF,CAAC;AAED,SAAS,WAAW,CAAC,IAAgB,EAAE,MAAc;IACpD,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC;QACjC,OAAO,EAAE,IAAI,CAAC,UAAU;QACxB,QAAQ,EAAE,IAAI,CAAC,UAAU;QACzB,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,YAAY,EAAE,IAAI,CAAC,UAAU;QAC7B,SAAS,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG;QAC7B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,GAAG;QAC1B,MAAM;KACN,CAAC,CAAA;IAEF,OAAO,GAAG,YAAY,IAAI,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B,EAAE,MAA+B;IAC7F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAA;IACtB,MAAM,KAAK,GAAqB,EAAE,CAAA;IAClC,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAA;QACxF,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAEnG,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,MAAM,EAAE,CAAC,OAAO,OAAO,CAAC,IAAI,wBAAwB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QACxD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;QAC3C,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,CAAA;QAC7E,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,wGAAwG;YACxG,yDAAyD;YACzD,KAAK,GAAG,MAAM,iBAAiB,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,CAAC,CAAA;QACjF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3C,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,KAAK,CAAC,IAAI,CAAC;YACV,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,IAAI,CAAC,UAAU;YACzB,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAC,CAAA;QACF,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IACtE,CAAC;IAED,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,IAAI;QACZ,UAAU,EAAE,OAAO;QACnB,OAAO,EAAE,OAAO;QAChB,WAAW,EAAE,WAAW;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,KAAK;KACL,CAAA;IAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;IAE7D,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACtF,CAAC"}

package/out/tools/fetch/localdata-kr.d.ts

@@ -1,30 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch Korea's local-government permit registry (지방행정인허가데이터): one CSV per permit category,
- *   every business a local authority has licensed, about 195 categories. Each row carries BOTH address
- *   forms of the same premises — `소재지전체주소` (the lot-number form, 지번) and `도로명전체주소` (the
- *   road-name form) — plus both postcodes and a planar coordinate (`좌표정보(x/y)`, EPSG:5174). That
- *   pairing is the cheapest dual-format signal Korean addresses offer, and the coordinate is the
- *   board's coordinate half, which is why this is the NOISY source for Korean and not a side dish.
- *
- *   The public-data portal labels the category files "이용허락범위 제한 없음" and links them to
- *   `file.localdata.go.kr`, which serves them behind a session: the category page sets the XSRF cookie,
- *   `/file/validate/download-count` is the portal's own rate check (429 when it wants a pause), and
- *   `/file/download/<slug>/info` streams the CSV. The files are CP949 as delivered; the adapter
- *   decodes. The restaurant category alone is about 700 MB.
- *
- *   Invoke via `mailwoman corpus fetch localdata-kr --out-root <path>`, optionally with
- *   `--categories general_restaurants,rest_cafes`.
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export interface FetchLocaldataKROptions extends BaseFetchOptions {
-    /**
-     * Category slugs to fetch (the path segment of `/file/<slug>/info`). Defaults to every category the portal lists.
-     */
-    categories?: string[];
-}
-export declare function fetchLocaldataKR(options: FetchLocaldataKROptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=localdata-kr.d.ts.map

package/out/tools/fetch/localdata-kr.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"localdata-kr.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/localdata-kr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4C,MAAM,uBAAuB,CAAA;AAWrH,MAAM,WAAW,uBAAwB,SAAQ,gBAAgB;IAChE;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;CACrB;AAqDD,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,uBAAuB,EAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA4FvB"}

package/out/tools/fetch/localdata-kr.js

@@ -1,139 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch Korea's local-government permit registry (지방행정인허가데이터): one CSV per permit category,
- *   every business a local authority has licensed, about 195 categories. Each row carries BOTH address
- *   forms of the same premises — `소재지전체주소` (the lot-number form, 지번) and `도로명전체주소` (the
- *   road-name form) — plus both postcodes and a planar coordinate (`좌표정보(x/y)`, EPSG:5174). That
- *   pairing is the cheapest dual-format signal Korean addresses offer, and the coordinate is the
- *   board's coordinate half, which is why this is the NOISY source for Korean and not a side dish.
- *
- *   The public-data portal labels the category files "이용허락범위 제한 없음" and links them to
- *   `file.localdata.go.kr`, which serves them behind a session: the category page sets the XSRF cookie,
- *   `/file/validate/download-count` is the portal's own rate check (429 when it wants a pause), and
- *   `/file/download/<slug>/info` streams the CSV. The files are CP949 as delivered; the adapter
- *   decodes. The restaurant category alone is about 700 MB.
- *
- *   Invoke via `mailwoman corpus fetch localdata-kr --out-root <path>`, optionally with
- *   `--categories general_restaurants,rest_cafes`.
- */
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters";
-import { pathExists } from "@mailwoman/core/fs/readers";
-import { makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { sleep } from "@mailwoman/core/utils/sleep";
-import { join } from "path-ts";
-import { cookieHeader, loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download";
-const SLUG = "localdata-kr";
-const PORTAL = "https://file.localdata.go.kr";
-const INDEX_URL = `${PORTAL}/file/general_restaurants/info`;
-const LICENSE = "이용허락범위 제한 없음 (data.go.kr label on each category file, e.g. 행정안전부_식품_일반음식점)";
-const ATTRIBUTION = "행정안전부 (Ministry of the Interior and Safety), 지방행정인허가데이터";
-const HTTP_TOO_MANY_REQUESTS = 429;
-const RATE_PAUSE_MS = 30_000;
-const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) mailwoman-corpus-fetch";
-/**
- * Open a session on the portal: the category page answers the XSRF cookie and the navigation that names every category.
- */
-async function openSession() {
-    const res = await fetch(INDEX_URL, {
-        headers: { "user-agent": USER_AGENT, accept: "text/html", referer: "https://www.data.go.kr/" },
-    });
-    if (!res.ok)
-        throw new Error(`localdata-kr: the category page answered HTTP ${res.status}`);
-    const html = await res.text();
-    const cookie = cookieHeader(res);
-    const xsrf = /XSRF-TOKEN=([^;]+)/.exec(cookie)?.[1];
-    const categories = [];
-    const seen = new Set();
-    for (const [, slug, name] of html.matchAll(/href="\/file\/([a-z0-9_]+)\/info"[^>]*>\s*<span>([^<]+)<\/span>/g)) {
-        if (!slug || !name || seen.has(slug))
-            continue;
-        seen.add(slug);
-        categories.push({ slug, name: name.trim() });
-    }
-    return { cookie, xsrf, categories };
-}
-/**
- * The portal's own rate check; a 429 carries the pause it asks for in prose, so the caller sleeps and retries.
- */
-async function validateDownloadCount(session) {
-    const headers = { "user-agent": USER_AGENT, cookie: session.cookie, referer: INDEX_URL };
-    if (session.xsrf) {
-        headers["X-XSRF-TOKEN"] = session.xsrf;
-    }
-    const res = await fetch(`${PORTAL}/file/validate/download-count`, { headers });
-    return res.status !== HTTP_TOO_MANY_REQUESTS;
-}
-export async function fetchLocaldataKR(options, report) {
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const manifestPath = join(destDir, "MANIFEST.json");
-    const session = await openSession();
-    const wanted = options.categories?.length
-        ? session.categories.filter((category) => options.categories.includes(category.slug))
-        : session.categories;
-    report?.(`=== ${SLUG}: ${wanted.length} of ${session.categories.length} categories`);
-    // A previous run's entries survive, keyed by file name, so a re-run after an interruption fetches only what is missing.
-    const previous = await loadCollectionFiles(manifestPath);
-    const files = new Map();
-    let fetched = 0;
-    let skipped = 0;
-    const failedCodes = [];
-    for (const category of wanted) {
-        const filename = `${category.slug}.csv`;
-        const dest = join(destDir, filename);
-        const url = `${PORTAL}/file/download/${category.slug}/info`;
-        const before = previous.get(filename);
-        if (before && (await pathExists(dest))) {
-            files.set(filename, before);
-            skipped++;
-            continue;
-        }
-        report?.(`--- ${category.name} (${category.slug})`);
-        while (!(await validateDownloadCount(session))) {
-            report?.(`  portal asks for a pause — sleeping ${RATE_PAUSE_MS / 1000}s`);
-            await sleep(RATE_PAUSE_MS);
-        }
-        let bytes;
-        try {
-            bytes = await withRetries(async () => {
-                const res = await fetch(url, {
-                    headers: { "user-agent": USER_AGENT, cookie: session.cookie, referer: INDEX_URL, accept: "*/*" },
-                    signal: AbortSignal.timeout(1_800_000),
-                });
-                if (!res.ok || !(res.headers.get("content-type") ?? "").includes("text/csv")) {
-                    throw new Error(`HTTP ${res.status} ${res.headers.get("content-type") ?? ""} for ${category.slug}`);
-                }
-                return streamBodyToFile(res, dest);
-            }, { report, label: category.slug });
-        }
-        catch (error) {
-            report?.(`  ✗ ${error.message}`);
-            failedCodes.push(category.slug);
-            continue;
-        }
-        if (bytes < BYTES_PER_KIB) {
-            report?.(`  ✗ ${bytes} bytes — an error page, not the register`);
-            failedCodes.push(category.slug);
-            continue;
-        }
-        const sha = await sha256File(dest);
-        files.set(filename, { source_url: url, downloaded_at: new Date().toISOString(), filename, sha256: sha, bytes });
-        fetched++;
-        report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`);
-        const manifest = {
-            source: SLUG,
-            source_url: INDEX_URL,
-            license: LICENSE,
-            attribution: ATTRIBUTION,
-            downloaded_at: new Date().toISOString(),
-            files: [...files.values()],
-        };
-        await writeManifest(manifestPath, manifest);
-    }
-    return { fetched, skipped, failed: failedCodes.length, failedCodes };
-}
-//# sourceMappingURL=localdata-kr.js.map

package/out/tools/fetch/localdata-kr.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"localdata-kr.js","sourceRoot":"","sources":["../../../lib/tools/fetch/localdata-kr.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,KAAK,EAAE,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEvH,MAAM,IAAI,GAAG,cAAc,CAAA;AAC3B,MAAM,MAAM,GAAG,8BAA8B,CAAA;AAC7C,MAAM,SAAS,GAAG,GAAG,MAAM,gCAAgC,CAAA;AAC3D,MAAM,OAAO,GAAG,4EAA4E,CAAA;AAC5F,MAAM,WAAW,GAAG,yDAAyD,CAAA;AAC7E,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAClC,MAAM,aAAa,GAAG,MAAM,CAAA;AAoB5B,MAAM,UAAU,GAAG,wDAAwD,CAAA;AAE3E;;GAEG;AACH,KAAK,UAAU,WAAW;IACzB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE;QAClC,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,yBAAyB,EAAE;KAC9F,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3F,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAA;IAC7B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;IAChC,MAAM,IAAI,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IACnD,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kEAAkE,CAAC,EAAE,CAAC;QAChH,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC9C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAAC,OAAgB;IACpD,MAAM,OAAO,GAA2B,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;IAEhH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,OAAO,CAAC,cAAc,CAAC,GAAG,OAAO,CAAC,IAAI,CAAA;IACvC,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,+BAA+B,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAE9E,OAAO,GAAG,CAAC,MAAM,KAAK,sBAAsB,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAgC,EAChC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAA;IAEnC,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,MAAM;QACxC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,UAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtF,CAAC,CAAC,OAAO,CAAC,UAAU,CAAA;IAErB,MAAM,EAAE,CAAC,OAAO,IAAI,KAAK,MAAM,CAAC,MAAM,OAAO,OAAO,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,CAAA;IAEpF,wHAAwH;IACxH,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAA;IACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAA;IAC/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,QAAQ,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,GAAG,QAAQ,CAAC,IAAI,MAAM,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACpC,MAAM,GAAG,GAAG,GAAG,MAAM,kBAAkB,QAAQ,CAAC,IAAI,OAAO,CAAA;QAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAErC,IAAI,MAAM,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAE3B,OAAO,EAAE,CAAA;YAET,SAAQ;QACT,CAAC;QAED,MAAM,EAAE,CAAC,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAA;QAEnD,OAAO,CAAC,CAAC,MAAM,qBAAqB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;YAChD,MAAM,EAAE,CAAC,wCAAwC,aAAa,GAAG,IAAI,GAAG,CAAC,CAAA;YACzE,MAAM,KAAK,CAAC,aAAa,CAAC,CAAA;QAC3B,CAAC;QAED,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,WAAW,CACxB,KAAK,IAAI,EAAE;gBACV,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBAC5B,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE;oBAChG,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;iBACtC,CAAC,CAAA;gBAEF,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC9E,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;gBACpG,CAAC;gBAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACnC,CAAC,EACD,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,CAChC,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,OAAQ,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC3C,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YAE/B,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,OAAO,KAAK,0CAA0C,CAAC,CAAA;YAChE,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;YAE/B,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAClC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAA;QAE/G,OAAO,EAAE,CAAA;QACT,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,MAAM,QAAQ,GAA6B;YAC1C,MAAM,EAAE,IAAI;YACZ,UAAU,EAAE,SAAS;YACrB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,WAAW;YACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;SAC1B,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,CAAA;AACrE,CAAC"}

package/out/tools/fetch/nad.d.ts

@@ -1,70 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
-export interface FetchNADOptions extends BaseFetchOptions {
-    /**
-     * Fetch strategy. Default `featureserver`.
-     */
-    mode?: "featureserver" | "bulk";
-    /**
-     * Pre-signed S3 URL for bulk mode.
-     */
-    nadURL?: string;
-    /**
-     * Records per output file. Default `100000`.
-     */
-    chunkSize?: number;
-    /**
-     * Records per HTTP request. Default `5000`.
-     */
-    pageSize?: number;
-    /**
-     * Parallel page fetches within a chunk. Default `4`.
-     */
-    concurrency?: number;
-    /**
-     * Start OBJECTID. Default `1`.
-     */
-    startOID?: number;
-    /**
-     * Stop before this OID. Default = total count.
-     */
-    endOID?: number;
-}
-export declare function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary>;
-//# sourceMappingURL=nad.d.ts.map

package/out/tools/fetch/nad.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAQ3E,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD;;OAEG;IACH,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAAA;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAuPD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAY/G"}

package/out/tools/fetch/nad.js

@@ -1,234 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-import { APIClient, pluckResponseData } from "@mailwoman/core/api";
-import { statPath, pathExists } from "@mailwoman/core/fs/readers";
-import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download";
-const SLUG = "usgov-nad";
-const FEATURE_SERVICE_URL = "https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0";
-/**
- * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
- * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
- * records server-side.
- */
-const nadClient = new APIClient({
-    displayName: "nad",
-    retry: true,
-    axios: { headers: { "Accept-Encoding": "gzip, br" } },
-});
-async function fetchPage(startOID, endOID, pageSize) {
-    const url = new URL(`${FEATURE_SERVICE_URL}/query`);
-    url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`);
-    url.searchParams.set("outFields", "*");
-    url.searchParams.set("f", "json");
-    url.searchParams.set("resultRecordCount", String(pageSize));
-    const data = await nadClient
-        .fetch({
-        url: url.toString(),
-        timeout: 120_000,
-    })
-        .then(pluckResponseData);
-    if (data.error)
-        throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`);
-    return (data.features ?? []).map((f) => f.attributes);
-}
-async function discoverTotalCount() {
-    const url = new URL(`${FEATURE_SERVICE_URL}/query`);
-    url.searchParams.set("where", "1=1");
-    url.searchParams.set("returnCountOnly", "true");
-    url.searchParams.set("f", "json");
-    const data = await nadClient
-        .fetch({ url: url.toString(), timeout: 30_000 })
-        .then(pluckResponseData);
-    if (typeof data.count !== "number")
-        throw new Error("NAD count query returned no count field");
-    return data.count;
-}
-/**
- * Fetch a single chunk by paging through its OID range with bounded concurrency. Returns the count of records written
- * and the count of pages that errored. The caller decides whether to mark the chunk complete based on errors === 0.
- *
- * NOTE(phase1): this is a JSON API pager, not a file download — the shared `downloadToFile` doesn't apply here.
- */
-async function fetchChunk(chunkPath, chunkStart, chunkEnd, pageSize, concurrency, report) {
-    const pageRanges = [];
-    for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
-        pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)]);
-    }
-    // Run bounded-concurrency page fetches. Results indexed by page slot for in-order write.
-    const pageResults = pageRanges.map(() => ({
-        rows: [],
-        error: null,
-    }));
-    let nextSlot = 0;
-    const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
-        while (true) {
-            const slot = nextSlot++;
-            if (slot >= pageRanges.length)
-                return;
-            const [s, e] = pageRanges[slot];
-            try {
-                pageResults[slot].rows = await fetchPage(s, e, pageSize);
-            }
-            catch (error) {
-                pageResults[slot].error = error;
-                report?.(`    ✗ page ${s}-${e}: ${error.message}`);
-            }
-        }
-    });
-    await Promise.all(workers);
-    // Single-writer phase — write all pages in OID order to keep NDJSON deterministic.
-    const lines = [];
-    let errors = 0;
-    for (const { rows, error } of pageResults) {
-        if (error) {
-            errors++;
-            continue;
-        }
-        for (const row of rows) {
-            lines.push(JSON.stringify(row));
-        }
-    }
-    await writeLocalTextFile(!lines.length ? "" : lines.join("\n") + "\n", chunkPath);
-    return { recordCount: lines.length, errors };
-}
-async function featureserverMode(options, report) {
-    const chunkSize = options.chunkSize ?? 100_000;
-    const pageSize = options.pageSize ?? 5000;
-    const concurrency = options.concurrency ?? 4;
-    const startOID = options.startOID ?? 1;
-    const chunkDir = join(options.outRoot, SLUG, "featureserver");
-    await makeDirectories(chunkDir);
-    report?.(`=== ${SLUG} / featureserver`);
-    report?.(`  Discovering record count ...`);
-    const totalCount = await discoverTotalCount();
-    const endOID = options.endOID ?? totalCount;
-    report?.(`  Total records: ${totalCount.toLocaleString()}`);
-    report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`);
-    report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`);
-    let fetched = 0;
-    let skipped = 0;
-    let totalRecords = 0;
-    let totalErrors = 0;
-    const failedCodes = [];
-    for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
-        const chunkEnd = Math.min(cursor + chunkSize - 1, endOID);
-        const chunkName = `oids_${cursor}-${chunkEnd}`;
-        const chunkPath = join(chunkDir, `${chunkName}.ndjson`);
-        const manifestPath = join(chunkDir, `${chunkName}.manifest.json`);
-        // Idempotency: skip a chunk only if it's marked complete (the bash version's bug was
-        // marking complete on partial-failure runs; we now only set complete after a clean fetch).
-        if (await pathExists(chunkPath)) {
-            const recorded = await readManifest(manifestPath);
-            if (recorded?.complete) {
-                skipped++;
-                continue;
-            }
-        }
-        report?.(`  Fetching ${chunkName} ...`);
-        const t0 = Date.now();
-        const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report);
-        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
-        const bytes = (await statPath(chunkPath)).size;
-        const sha = await sha256File(chunkPath);
-        const manifest = {
-            source_url: FEATURE_SERVICE_URL,
-            oid_range: [cursor, chunkEnd],
-            downloaded_at: new Date().toISOString(),
-            filename: `${chunkName}.ndjson`,
-            sha256: sha,
-            bytes,
-            record_count: recordCount,
-            page_errors: errors,
-            complete: errors === 0,
-        };
-        await writeManifest(manifestPath, manifest);
-        const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`;
-        report?.(`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`);
-        fetched++;
-        totalRecords += recordCount;
-        totalErrors += errors;
-        if (errors > 0) {
-            failedCodes.push(chunkName);
-        }
-    }
-    report?.(`=== featureserver summary ===`);
-    report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`);
-    report?.(`total records: ${totalRecords.toLocaleString()}`);
-    report?.(`page errors:   ${totalErrors}`);
-    report?.(`output:        ${chunkDir}`);
-    // `failed` counts page errors (the old exit-1 condition), `failedCodes` names the dirty chunks.
-    return { fetched, skipped, failed: totalErrors, failedCodes };
-}
-async function bulkMode(options, report) {
-    if (!options.nadURL) {
-        throw new Error(`bulk mode requires --nad-url. The DOT page is Akamai-gated; ` +
-            `visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
-            `accept the disclaimer, and re-run with the pre-signed S3 URL.`);
-    }
-    const destDir = join(options.outRoot, SLUG);
-    await makeDirectories(destDir);
-    const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip";
-    const destPath = join(destDir, filename);
-    report?.(`=== ${SLUG} / ${filename}`);
-    report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`);
-    const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report });
-    const sha = await sha256File(destPath);
-    await writeManifest(join(destDir, "MANIFEST.json"), {
-        source_url: options.nadURL,
-        downloaded_at: new Date().toISOString(),
-        filename,
-        sha256: sha,
-        bytes,
-    });
-    report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`);
-    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
-}
-export async function fetchNAD(options, report) {
-    const mode = options.mode ?? "featureserver";
-    if (mode === "featureserver") {
-        return featureserverMode(options, report);
-    }
-    if (mode === "bulk") {
-        return bulkMode(options, report);
-    }
-    throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`);
-}
-//# sourceMappingURL=nad.js.map

package/out/tools/fetch/nad.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"nad.js","sourceRoot":"","sources":["../../../lib/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEnF,MAAM,IAAI,GAAG,WAAW,CAAA;AAExB,MAAM,mBAAmB,GACxB,sIAAsI,CAAA;AA6CvI;;;;GAIG;AACH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;IAC/B,WAAW,EAAE,KAAK;IAClB,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;CACrD,CAAC,CAAA;AAEF,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,MAAc,EAAE,QAAgB;IAC1E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,QAAQ,QAAQ,MAAM,EAAE,CAAC,CAAA;IAC3E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IACtC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE3D,MAAM,IAAI,GAAG,MAAM,SAAS;SAC1B,KAAK,CAA6E;QAClF,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,OAAO,EAAE,OAAO;KAChB,CAAC;SACD,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;IAEnG,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAA;AACtD,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACpC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAA;IAC/C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAEjC,MAAM,IAAI,GAAG,MAAM,SAAS;SAC1B,KAAK,CAAqB,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnE,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAE9F,OAAO,IAAI,CAAC,KAAK,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,QAAgB,EAChB,WAAmB,EACnB,MAA+B;IAE/B,MAAM,UAAU,GAA4B,EAAE,CAAA;IAE9C,KAAK,IAAI,MAAM,GAAG,UAAU,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,CAAC;QACtE,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAA;IACrE,CAAC;IAED,yFAAyF;IACzF,MAAM,WAAW,GAAoD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1F,IAAI,EAAE,EAAE;QACR,KAAK,EAAE,IAAI;KACX,CAAC,CAAC,CAAA;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAA;YAEvB,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM;gBAAE,OAAM;YACrC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,CAAE,CAAA;YAEhC,IAAI,CAAC;gBACJ,WAAW,CAAC,IAAI,CAAE,CAAC,IAAI,GAAG,MAAM,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;YAC1D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,WAAW,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,KAAc,CAAA;gBACzC,MAAM,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,mFAAmF;IACnF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,CAAA;YAER,SAAQ;QACT,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QAChC,CAAC;IACF,CAAC;IAED,MAAM,kBAAkB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,SAAS,CAAC,CAAA;IAEjF,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC7C,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAwB,EAAE,MAA+B;IACzF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAA;IAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAA;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,CAAA;IAC7D,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAE/B,MAAM,EAAE,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAA;IACvC,MAAM,EAAE,CAAC,gCAAgC,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAA;IAC3C,MAAM,EAAE,CAAC,oBAAoB,UAAU,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,cAAc,EAAE,OAAO,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IACnF,MAAM,EAAE,CAAC,iBAAiB,SAAS,gBAAgB,QAAQ,kBAAkB,WAAW,EAAE,CAAC,CAAA;IAE3F,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,IAAI,MAAM,GAAG,QAAQ,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,EAAE,MAAM,CAAC,CAAA;QACzD,MAAM,SAAS,GAAG,QAAQ,MAAM,IAAI,QAAQ,EAAE,CAAA;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,SAAS,CAAC,CAAA;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,gBAAgB,CAAC,CAAA;QAEjE,qFAAqF;QACrF,2FAA2F;QAC3F,IAAI,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAgB,YAAY,CAAC,CAAA;YAEhE,IAAI,QAAQ,EAAE,QAAQ,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;QACF,CAAC;QAED,MAAM,EAAE,CAAC,cAAc,SAAS,MAAM,CAAC,CAAA;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACrB,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;QAC5G,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;QAC9C,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAA;QAEvC,MAAM,QAAQ,GAAkB;YAC/B,UAAU,EAAE,mBAAmB;YAC/B,SAAS,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;YAC7B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,GAAG,SAAS,SAAS;YAC/B,MAAM,EAAE,GAAG;YACX,KAAK;YACL,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,QAAQ,EAAE,MAAM,KAAK,CAAC;SACtB,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;QAE3C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,cAAc,CAAA;QAE7D,MAAM,EAAE,CACP,OAAO,MAAM,KAAK,WAAW,CAAC,cAAc,EAAE,eAAe,OAAO,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CACjH,CAAA;QAED,OAAO,EAAE,CAAA;QACT,YAAY,IAAI,WAAW,CAAA;QAC3B,WAAW,IAAI,MAAM,CAAA;QAErB,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,mBAAmB,OAAO,cAAc,OAAO,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,WAAW,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAA;IAEtC,gGAAgG;IAChG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9D,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IAChF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACd,8DAA8D;YAC7D,mFAAmF;YACnF,+DAA+D,CAChE,CAAA;IACF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,SAAS,CAAA;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAExC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,UAAU,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE3F,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;IAEnH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAA;IAEtC,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE;QACnD,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ;QACR,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,eAAe,CAAA;IAE5C,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACrB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACjC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;AAChF,CAAC"}