@mailwoman/corpus 9.4.0 → 10.0.0

package/out/us/tools/fetch/imls-pls.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"imls-pls.js","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/imls-pls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;kGACkG;AAElG,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC5E,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAEzF;;;GAGG;AAEH;;GAEG;AACH,MAAM,OAAO,GAAG,qEAAqE,CAAA;AACrF,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAI7B,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAA4B,EAC5B,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAChD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,0BAA0B;IAE1B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9F,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,qBAAqB;IAErB,MAAM,EAAE,CAAC,iBAAiB,OAAO,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,OAAO;QACZ,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,OAAO;QAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,aAAa,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAE7D,IAAI,OAAO,GAAG,aAAa,EAAE,CAAC;QAC7B,MAAM,EAAE,CAAC,2BAA2B,OAAO,+BAA+B,CAAC,CAAA;QAE3E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,gDAAgD;IAChD,4DAA4D;IAC5D,qEAAqE;IACrE,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE1E,IAAI,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAEzE,4EAA4E;IAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;QAErC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;QAED,MAAM,EAAE,CAAC,kFAAkF,CAAC,CAAA;QAE5F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,4BAA4B,OAAO,EAAE,CAAC,CAAA;IAE/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,mDAAmD;IAEnD,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,uBAAuB;IAEvB,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,OAAO;QACnB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC;QAC3B,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/us/tools/fetch/nad.d.ts

@@ -0,0 +1,70 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
+ *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
+ *   US Public Domain (17 U.S.C. § 105).
+ *
+ *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
+ *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
+ *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
+ *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
+ *   - Per-chunk manifest with sha256 + record count + error count
+ *
+ *   ## Source layout
+ *
+ *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
+ *
+ *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
+ *       `--nad-url <presigned>` from a browser visit to
+ *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
+ *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
+ *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Resume from an OID
+ *   mailwoman corpus fetch nad --start-oid 34400001
+ *
+ *   # Increase concurrency on a fast link
+ *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
+ *   ```
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index";
+export interface FetchNADOptions extends BaseFetchOptions {
+    /**
+     * Fetch strategy. Default `featureserver`.
+     */
+    mode?: "featureserver" | "bulk";
+    /**
+     * Pre-signed S3 URL for bulk mode.
+     */
+    nadURL?: string;
+    /**
+     * Records per output file. Default `100000`.
+     */
+    chunkSize?: number;
+    /**
+     * Records per HTTP request. Default `5000`.
+     */
+    pageSize?: number;
+    /**
+     * Parallel page fetches within a chunk. Default `4`.
+     */
+    concurrency?: number;
+    /**
+     * Start OBJECTID. Default `1`.
+     */
+    startOID?: number;
+    /**
+     * Stop before this OID. Default = total count.
+     */
+    endOID?: number;
+}
+export declare function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=nad.d.ts.map

package/out/us/tools/fetch/nad.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nad.d.ts","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAA;AAQjF,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACxD;;OAEG;IACH,IAAI,CAAC,EAAE,eAAe,GAAG,MAAM,CAAA;IAC/B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAuPD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAY/G"}

package/out/us/tools/fetch/nad.js

@@ -0,0 +1,234 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
+ *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
+ *   US Public Domain (17 U.S.C. § 105).
+ *
+ *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
+ *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
+ *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
+ *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
+ *   - Per-chunk manifest with sha256 + record count + error count
+ *
+ *   ## Source layout
+ *
+ *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
+ *
+ *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
+ *       `--nad-url <presigned>` from a browser visit to
+ *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
+ *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
+ *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Resume from an OID
+ *   mailwoman corpus fetch nad --start-oid 34400001
+ *
+ *   # Increase concurrency on a fast link
+ *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
+ *   ```
+ */
+import { APIClient, pluckResponseData } from "@mailwoman/core/api";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download/index";
+const SLUG = "usgov-nad";
+const FEATURE_SERVICE_URL = "https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0";
+/**
+ * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
+ * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
+ * records server-side.
+ */
+const nadClient = new APIClient({
+    displayName: "nad",
+    retry: true,
+    axios: { headers: { "Accept-Encoding": "gzip, br" } },
+});
+async function fetchPage(startOID, endOID, pageSize) {
+    const url = new URL(`${FEATURE_SERVICE_URL}/query`);
+    url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`);
+    url.searchParams.set("outFields", "*");
+    url.searchParams.set("f", "json");
+    url.searchParams.set("resultRecordCount", String(pageSize));
+    const data = await nadClient
+        .fetch({
+        url: url.toString(),
+        timeout: 120_000,
+    })
+        .then(pluckResponseData);
+    if (data.error)
+        throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`);
+    return (data.features ?? []).map((f) => f.attributes);
+}
+async function discoverTotalCount() {
+    const url = new URL(`${FEATURE_SERVICE_URL}/query`);
+    url.searchParams.set("where", "1=1");
+    url.searchParams.set("returnCountOnly", "true");
+    url.searchParams.set("f", "json");
+    const data = await nadClient
+        .fetch({ url: url.toString(), timeout: 30_000 })
+        .then(pluckResponseData);
+    if (typeof data.count !== "number")
+        throw new Error("NAD count query returned no count field");
+    return data.count;
+}
+/**
+ * Fetch a single chunk by paging through its OID range with bounded concurrency. Returns the count of records written
+ * and the count of pages that errored. The caller decides whether to mark the chunk complete based on errors === 0.
+ *
+ * NOTE(phase1): this is a JSON API pager, not a file download — the shared `downloadToFile` doesn't apply here.
+ */
+async function fetchChunk(chunkPath, chunkStart, chunkEnd, pageSize, concurrency, report) {
+    const pageRanges = [];
+    for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
+        pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)]);
+    }
+    // Run bounded-concurrency page fetches. Results indexed by page slot for in-order write.
+    const pageResults = pageRanges.map(() => ({
+        rows: [],
+        error: null,
+    }));
+    let nextSlot = 0;
+    const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
+        while (true) {
+            const slot = nextSlot++;
+            if (slot >= pageRanges.length)
+                return;
+            const [s, e] = pageRanges[slot];
+            try {
+                pageResults[slot].rows = await fetchPage(s, e, pageSize);
+            }
+            catch (error) {
+                pageResults[slot].error = error;
+                report?.(`    ✗ page ${s}-${e}: ${error.message}`);
+            }
+        }
+    });
+    await Promise.all(workers);
+    // Single-writer phase — write all pages in OID order to keep NDJSON deterministic.
+    const lines = [];
+    let errors = 0;
+    for (const { rows, error } of pageResults) {
+        if (error) {
+            errors++;
+            continue;
+        }
+        for (const row of rows) {
+            lines.push(JSON.stringify(row));
+        }
+    }
+    await writeLocalTextFile(!lines.length ? "" : lines.join("\n") + "\n", chunkPath);
+    return { recordCount: lines.length, errors };
+}
+async function featureserverMode(options, report) {
+    const chunkSize = options.chunkSize ?? 100_000;
+    const pageSize = options.pageSize ?? 5000;
+    const concurrency = options.concurrency ?? 4;
+    const startOID = options.startOID ?? 1;
+    const chunkDir = join(options.outRoot, SLUG, "featureserver");
+    await makeDirectories(chunkDir);
+    report?.(`=== ${SLUG} / featureserver`);
+    report?.(`  Discovering record count ...`);
+    const totalCount = await discoverTotalCount();
+    const endOID = options.endOID ?? totalCount;
+    report?.(`  Total records: ${totalCount.toLocaleString()}`);
+    report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`);
+    report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`);
+    let fetched = 0;
+    let skipped = 0;
+    let totalRecords = 0;
+    let totalErrors = 0;
+    const failedCodes = [];
+    for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
+        const chunkEnd = Math.min(cursor + chunkSize - 1, endOID);
+        const chunkName = `oids_${cursor}-${chunkEnd}`;
+        const chunkPath = join(chunkDir, `${chunkName}.ndjson`);
+        const manifestPath = join(chunkDir, `${chunkName}.manifest.json`);
+        // Idempotency: skip a chunk only if it's marked complete (the bash version's bug was
+        // marking complete on partial-failure runs; we now only set complete after a clean fetch).
+        if (await pathExists(chunkPath)) {
+            const recorded = await readManifest(manifestPath);
+            if (recorded?.complete) {
+                skipped++;
+                continue;
+            }
+        }
+        report?.(`  Fetching ${chunkName} ...`);
+        const t0 = Date.now();
+        const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report);
+        const elapsed = ((Date.now() - t0) / 1000).toFixed(1);
+        const bytes = (await statPath(chunkPath)).size;
+        const sha = await sha256File(chunkPath);
+        const manifest = {
+            source_url: FEATURE_SERVICE_URL,
+            oid_range: [cursor, chunkEnd],
+            downloaded_at: new Date().toISOString(),
+            filename: `${chunkName}.ndjson`,
+            sha256: sha,
+            bytes,
+            record_count: recordCount,
+            page_errors: errors,
+            complete: errors === 0,
+        };
+        await writeManifest(manifestPath, manifest);
+        const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`;
+        report?.(`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`);
+        fetched++;
+        totalRecords += recordCount;
+        totalErrors += errors;
+        if (errors > 0) {
+            failedCodes.push(chunkName);
+        }
+    }
+    report?.(`=== featureserver summary ===`);
+    report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`);
+    report?.(`total records: ${totalRecords.toLocaleString()}`);
+    report?.(`page errors:   ${totalErrors}`);
+    report?.(`output:        ${chunkDir}`);
+    // `failed` counts page errors (the old exit-1 condition), `failedCodes` names the dirty chunks.
+    return { fetched, skipped, failed: totalErrors, failedCodes };
+}
+async function bulkMode(options, report) {
+    if (!options.nadURL) {
+        throw new Error(`bulk mode requires --nad-url. Akamai protects the DOT page; ` +
+            `visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
+            `accept the disclaimer, and re-run with the pre-signed S3 URL.`);
+    }
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip";
+    const destPath = join(destDir, filename);
+    report?.(`=== ${SLUG} / ${filename}`);
+    report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`);
+    const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report });
+    const sha = await sha256File(destPath);
+    await writeManifest(join(destDir, "MANIFEST.json"), {
+        source_url: options.nadURL,
+        downloaded_at: new Date().toISOString(),
+        filename,
+        sha256: sha,
+        bytes,
+    });
+    report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+export async function fetchNAD(options, report) {
+    const mode = options.mode ?? "featureserver";
+    if (mode === "featureserver") {
+        return featureserverMode(options, report);
+    }
+    if (mode === "bulk") {
+        return bulkMode(options, report);
+    }
+    throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`);
+}
+//# sourceMappingURL=nad.js.map

package/out/us/tools/fetch/nad.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nad.js","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/nad.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAG9B,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAEzF,MAAM,IAAI,GAAG,WAAW,CAAA;AAExB,MAAM,mBAAmB,GACxB,sIAAsI,CAAA;AA6CvI;;;;GAIG;AACH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC;IAC/B,WAAW,EAAE,KAAK;IAClB,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;CACrD,CAAC,CAAA;AAEF,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,MAAc,EAAE,QAAgB;IAC1E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,oBAAoB,QAAQ,QAAQ,MAAM,EAAE,CAAC,CAAA;IAC3E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IACtC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,mBAAmB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE3D,MAAM,IAAI,GAAG,MAAM,SAAS;SAC1B,KAAK,CAA6E;QAClF,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,OAAO,EAAE,OAAO;KAChB,CAAC;SACD,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,IAAI,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,IAAI,MAAM,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;IAEnG,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAA;AACtD,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,mBAAmB,QAAQ,CAAC,CAAA;IACnD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACpC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAA;IAC/C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAEjC,MAAM,IAAI,GAAG,MAAM,SAAS;SAC1B,KAAK,CAAqB,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACnE,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAE9F,OAAO,IAAI,CAAC,KAAK,CAAA;AAClB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,UAAkB,EAClB,QAAgB,EAChB,QAAgB,EAChB,WAAmB,EACnB,MAA+B;IAE/B,MAAM,UAAU,GAA4B,EAAE,CAAA;IAE9C,KAAK,IAAI,MAAM,GAAG,UAAU,EAAE,MAAM,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,EAAE,CAAC;QACtE,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAA;IACrE,CAAC;IAED,yFAAyF;IACzF,MAAM,WAAW,GAAoD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1F,IAAI,EAAE,EAAE;QACR,KAAK,EAAE,IAAI;KACX,CAAC,CAAC,CAAA;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAA;IAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAA;YAEvB,IAAI,IAAI,IAAI,UAAU,CAAC,MAAM;gBAAE,OAAM;YACrC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,IAAI,CAAE,CAAA;YAEhC,IAAI,CAAC;gBACJ,WAAW,CAAC,IAAI,CAAE,CAAC,IAAI,GAAG,MAAM,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAA;YAC1D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,WAAW,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,KAAc,CAAA;gBACzC,MAAM,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,mFAAmF;IACnF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,WAAW,EAAE,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,CAAA;YAER,SAAQ;QACT,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QAChC,CAAC;IACF,CAAC;IAED,MAAM,kBAAkB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,SAAS,CAAC,CAAA;IAEjF,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAC7C,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,OAAwB,EAAE,MAA+B;IACzF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAA;IAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAA;IACzC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,CAAA;IAC7D,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAE/B,MAAM,EAAE,CAAC,OAAO,IAAI,kBAAkB,CAAC,CAAA;IACvC,MAAM,EAAE,CAAC,gCAAgC,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAA;IAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAA;IAC3C,MAAM,EAAE,CAAC,oBAAoB,UAAU,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,cAAc,EAAE,OAAO,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IACnF,MAAM,EAAE,CAAC,iBAAiB,SAAS,gBAAgB,QAAQ,kBAAkB,WAAW,EAAE,CAAC,CAAA;IAE3F,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,IAAI,MAAM,GAAG,QAAQ,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;QACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,EAAE,MAAM,CAAC,CAAA;QACzD,MAAM,SAAS,GAAG,QAAQ,MAAM,IAAI,QAAQ,EAAE,CAAA;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,SAAS,CAAC,CAAA;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,SAAS,gBAAgB,CAAC,CAAA;QAEjE,qFAAqF;QACrF,2FAA2F;QAC3F,IAAI,MAAM,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAgB,YAAY,CAAC,CAAA;YAEhE,IAAI,QAAQ,EAAE,QAAQ,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;QACF,CAAC;QAED,MAAM,EAAE,CAAC,cAAc,SAAS,MAAM,CAAC,CAAA;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACrB,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;QAC5G,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAA;QAC9C,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAA;QAEvC,MAAM,QAAQ,GAAkB;YAC/B,UAAU,EAAE,mBAAmB;YAC/B,SAAS,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;YAC7B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ,EAAE,GAAG,SAAS,SAAS;YAC/B,MAAM,EAAE,GAAG;YACX,KAAK;YACL,YAAY,EAAE,WAAW;YACzB,WAAW,EAAE,MAAM;YACnB,QAAQ,EAAE,MAAM,KAAK,CAAC;SACtB,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;QAE3C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,cAAc,CAAA;QAE7D,MAAM,EAAE,CACP,OAAO,MAAM,KAAK,WAAW,CAAC,cAAc,EAAE,eAAe,OAAO,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CACjH,CAAA;QAED,OAAO,EAAE,CAAA;QACT,YAAY,IAAI,WAAW,CAAA;QAC3B,WAAW,IAAI,MAAM,CAAA;QAErB,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YAChB,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,+BAA+B,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,mBAAmB,OAAO,cAAc,OAAO,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,YAAY,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,CAAC,kBAAkB,WAAW,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAA;IAEtC,gGAAgG;IAChG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9D,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IAChF,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACd,8DAA8D;YAC7D,mFAAmF;YACnF,+DAA+D,CAChE,CAAA;IACF,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,SAAS,CAAA;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;IAExC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,UAAU,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE3F,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CAAA;IAEnH,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAA;IAEtC,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE;QACnD,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ;QACR,MAAM,EAAE,GAAG;QACX,KAAK;KACL,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;IAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAwB,EAAE,MAA+B;IACvF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,eAAe,CAAA;IAE5C,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;QAC9B,OAAO,iBAAiB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1C,CAAC;IAED,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACrB,OAAO,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACjC,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,iBAAiB,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;AAChF,CAAC"}

package/out/us/tools/fetch/nppes.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
+ *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
+ *   adapter. US Public Domain.
+ *
+ *   The file is published monthly by CMS. This module discovers the current filename by scraping the
+ *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
+ *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
+ *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
+ *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
+ *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
+ *   no resume, so a partial run re-downloads from the start.
+ *
+ *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
+ *   sha256 matches MANIFEST, skips download.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index";
+export type FetchNPPESOptions = BaseFetchOptions;
+export declare function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=nppes.d.ts.map

package/out/us/tools/fetch/nppes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"nppes.d.ts","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,6BAA6B,CAAA;AAOjG,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,CAAA;AAmChD,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAyFnH"}

package/out/us/tools/fetch/nppes.js

@@ -0,0 +1,126 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
+ *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
+ *   adapter. US Public Domain.
+ *
+ *   The file is published monthly by CMS. This module discovers the current filename by scraping the
+ *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
+ *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
+ *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
+ *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
+ *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
+ *   no resume, so a partial run re-downloads from the start.
+ *
+ *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
+ *   sha256 matches MANIFEST, skips download.
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { APIClient, pluckResponseData } from "@mailwoman/core/api";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { basename, join } from "path-ts";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download/index";
+const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html";
+const BASE_URL = "https://download.cms.gov/nppes";
+const SLUG = "usgov-nppes";
+/**
+ * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
+ * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
+ */
+async function discoverLatestZip() {
+    // `responseType: "text"` — the index is HTML, scraped by regex below.
+    const html = await new APIClient({
+        displayName: "nppes-index",
+        retry: true,
+        axios: { headers: { "Accept-Encoding": "gzip, br" } },
+    })
+        .fetch({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
+        .then(pluckResponseData);
+    for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
+        const name = match[0];
+        if (name && !/\d{6}_\d{6}/.test(name))
+            return name;
+    }
+    return undefined;
+}
+/**
+ * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
+ * change file.
+ */
+async function findNpidataCSV(zipPath) {
+    const entries = await listZipEntries(zipPath);
+    return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name;
+}
+export async function fetchNPPES(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    report?.(`=== ${SLUG}`);
+    report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`);
+    const zipFilename = await discoverLatestZip();
+    if (!zipFilename) {
+        report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const zipURL = `${BASE_URL}/${zipFilename}`;
+    const zipDest = join(destDir, zipFilename);
+    report?.(`  Latest full file: ${zipFilename}`);
+    // Idempotency check: if the main CSV already exists and sha matches,
+    // skip re-download.
+    const recorded = await readManifest(manifestPath);
+    if (recorded?.sha256 && recorded.filename) {
+        const recordedPath = join(destDir, recorded.filename);
+        if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+            report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.");
+            return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+        }
+    }
+    // MARK: Download ZIP (large; 60-minute timeout)
+    report?.(`  Downloading ${zipURL} ...`);
+    const { bytes: zipSize } = await downloadToFile({
+        url: zipURL,
+        dest: zipDest,
+        timeoutMs: 3_600_000,
+        headers: { "Accept-Encoding": "gzip, br" },
+        report,
+    });
+    report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`);
+    // MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
+    report?.("  Extracting npidata_pfile CSV from ZIP ...");
+    const csvName = await findNpidataCSV(zipDest);
+    if (!csvName) {
+        report?.("  ✗ Could not find npidata_pfile CSV inside ZIP");
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    report?.(`  Extracting: ${csvName}`);
+    const csvDest = join(destDir, basename(csvName));
+    await extractZipEntry(zipDest, csvName, csvDest);
+    const csvSize = (await statPath(csvDest)).size;
+    const csvSha = await sha256File(csvDest);
+    report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`);
+    // MARK: Remove the ZIP to reclaim ~1 GB
+    await removePathIfPresent(zipDest);
+    report?.("  Removed ZIP (CSV kept)");
+    // MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
+    const manifest = {
+        source_url: zipURL,
+        downloaded_at: new Date().toISOString(),
+        filename: csvName,
+        sha256: csvSha,
+        bytes: csvSize,
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=nppes.js.map

package/out/us/tools/fetch/nppes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"nppes.js","sourceRoot":"","sources":["../../../../lib/us/tools/fetch/nppes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH;kGACkG;AAElG,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACxE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAEzF,MAAM,SAAS,GAAG,+CAA+C,CAAA;AACjE,MAAM,QAAQ,GAAG,gCAAgC,CAAA;AACjD,MAAM,IAAI,GAAG,aAAa,CAAA;AAI1B;;;GAGG;AACH,KAAK,UAAU,iBAAiB;IAC/B,sEAAsE;IACtE,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC;QAChC,WAAW,EAAE,aAAa;QAC1B,KAAK,EAAE,IAAI;QACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,EAAE;KACrD,CAAC;SACA,KAAK,CAAS,EAAE,GAAG,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SACxE,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAEzB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QAErB,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IACnD,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAAC,OAAe;IAC5C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAA;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IACvB,MAAM,EAAE,CAAC,kDAAkD,SAAS,MAAM,CAAC,CAAA;IAE3E,MAAM,WAAW,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE7C,IAAI,CAAC,WAAW,EAAE,CAAC;QAClB,MAAM,EAAE,CAAC,4CAA4C,SAAS,EAAE,CAAC,CAAA;QAEjE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;IAC1C,MAAM,EAAE,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAA;IAE9C,qEAAqE;IACrE,oBAAoB;IACpB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAA0B,YAAY,CAAC,CAAA;IAE1E,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAErD,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9F,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;YAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAEhD,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,CAAC,CAAA;IAEvC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC;QAC/C,GAAG,EAAE,MAAM;QACX,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,SAAS;QACpB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM;KACN,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,iBAAiB,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAElE,iEAAiE;IAEjE,MAAM,EAAE,CAAC,6CAA6C,CAAC,CAAA;IACvD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAA;IAE7C,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,EAAE,CAAC,iDAAiD,CAAC,CAAA;QAE3D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAA;IAEpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAEhD,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;IAChD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,EAAE,CAAC,eAAe,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAEhE,wCAAwC;IAExC,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,gEAAgE;IAEhE,MAAM,QAAQ,GAAmB;QAChC,UAAU,EAAE,MAAM;QAClB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,OAAO;QACjB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;KACd,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,MAAM,EAAE,CAAC,CAAA;IAC1E,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}

package/out/us/tools/fetch/state/hi-schools.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches MANIFEST, skips download.
+ */
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index";
+export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
+    /**
+     * Workbook URL. Defaults to the Hawaii DOE source; overridable for an isolated fetch test.
+     */
+    sourceURL?: string;
+}
+export declare function fetchStateHISchools(options: FetchStateHISchoolsOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=hi-schools.d.ts.map

package/out/us/tools/fetch/state/hi-schools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hi-schools.d.ts","sourceRoot":"","sources":["../../../../../lib/us/tools/fetch/state/hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAA;AAQjF,MAAM,WAAW,0BAA2B,SAAQ,gBAAgB;IACnE;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAqCD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAkFvB"}

package/out/us/tools/fetch/state/hi-schools.js

@@ -0,0 +1,112 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches MANIFEST, skips download.
+ */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { statPath, pathExists } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { XLSXSpliterator } from "spliterator";
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download/index";
+import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#us/adapters/state/hi-schools/workbook";
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
+const SLUG = "state-hi-schools";
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx";
+async function validateWorkbook(path) {
+    for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+        const rows = await XLSXSpliterator.fromAsync(path, {
+            sheet,
+            header: false,
+            take: 2,
+        }).toArray();
+        const header = rows[0];
+        if (!header || rows.length < 2) {
+            throw new Error(`sheet ${sheet} must contain a header and at least one school row`);
+        }
+        // Exact casing is part of the adapter contract: object mode preserves the workbook's header names, so accepting
+        // `Code` here while the reader asks for `record.code` would validate a workbook whose every row is later skipped.
+        const columns = new Set(header.map((cell) => String(cell ?? "").trim()));
+        const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column));
+        if (missing.length) {
+            throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`);
+        }
+    }
+}
+export async function fetchStateHISchools(options, report) {
+    const destDir = join(options.outRoot, SLUG);
+    await makeDirectories(destDir);
+    const xlsxDest = join(destDir, XLSX_FILENAME);
+    const manifestPath = join(destDir, "MANIFEST.json");
+    const sourceURL = options.sourceURL ?? SOURCE_URL;
+    report?.(`=== ${SLUG}`);
+    // Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
+    if (await pathExists(xlsxDest)) {
+        const recorded = await readManifest(manifestPath);
+        if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
+            const actualSha = await sha256File(xlsxDest);
+            if (actualSha === recorded.sha256) {
+                report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`);
+                return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] };
+            }
+        }
+    }
+    // Download XLSX.
+    report?.(`  Downloading ${sourceURL} ...`);
+    try {
+        await downloadToFile({
+            url: sourceURL,
+            dest: xlsxDest,
+            timeoutMs: 600_000,
+            headers: { "Accept-Encoding": "gzip, br" },
+            report,
+        });
+    }
+    catch (error) {
+        report?.(`  ✗ Download failed (${error.message})`);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const xlsxSize = (await statPath(xlsxDest)).size;
+    report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`);
+    if (xlsxSize < BYTES_PER_KIB) {
+        report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`);
+        await removePathIfPresent(xlsxDest);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    try {
+        await validateWorkbook(xlsxDest);
+    }
+    catch (error) {
+        report?.(`  ✗ Workbook validation failed (${error.message})`);
+        await removePathIfPresent(xlsxDest);
+        return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
+    }
+    const xlsxSha = await sha256File(xlsxDest);
+    // Write MANIFEST.
+    const manifest = {
+        source_url: sourceURL,
+        downloaded_at: new Date().toISOString(),
+        filename: XLSX_FILENAME,
+        sha256: xlsxSha,
+        bytes: xlsxSize,
+        notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`);
+    report?.(`  MANIFEST written to ${manifestPath}`);
+    return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] };
+}
+//# sourceMappingURL=hi-schools.js.map