@mailwoman/corpus 9.2.0 → 9.3.0

package/lib/tools/fetch/index.ts

@@ -0,0 +1,147 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The `mailwoman corpus fetch <source>` family — reproducible bulk-download recovery modules for
+ *   the open-data sources the corpus build pipeline consumes. Each module writes the raw download
+ *   files **plus** a sibling `MANIFEST.json` capturing the origin URL, fetch timestamp, byte count,
+ *   and sha256 so downstream adapters can verify provenance.
+ *
+ *   The corpus build pipeline itself does NOT call these modules — the existing convention is for
+ *   operators to pre-download into an out-root and point adapters at the resulting files. These
+ *   modules exist for **reproducibility** (disk-loss recovery, weekly refresh, fresh-environment
+ *   bootstrap).
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # Default: writes under ./data/corpus/sources/ relative to the working directory
+ *   mailwoman corpus fetch state-sources
+ *   mailwoman corpus fetch hrsa
+ *
+ *   # Or point at the standard mailwoman data root
+ *   mailwoman corpus fetch state-sources --out-root /data/corpus/sources
+ *   ```
+ *
+ *   Each adapter under `corpus/src/adapters/<adapter>/README.md` documents the specific URL its
+ *   input was pulled from; these modules mirror those URLs in a single executable place.
+ *
+ *   ## Coverage
+ *
+ *   - `ban` — French BAN (Base Adresse Nationale), all départements incl. DOM/TOM. Tier B (Licence
+ *       Ouverte 2.0).
+ *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
+ *       (US PD).
+ *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
+ *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
+ *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
+ *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
+ *       GeoNames does not publish every country — Venezuela 404s.
+ *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
+ *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
+ *       Tier A (US PD).
+ *   - `nppes` — NPPES NPI registry, full monthly dissemination (~7M provider venue+address rows).
+ *       Tier A (US PD).
+ *   - `openaddresses` — OpenAddresses country collections (default: Canada / `ca`). Tier B/C mixed
+ *       — per-row filter.
+ *   - `ourairports` — OurAirports global airport CSVs (~83K airports + the country/region/runway
+ *       joins). Tier A (public domain). The VENUE half of the sub-venue arc (#35); it carries no
+ *       interior structure, which comes from `@mailwoman/osm/sdk`'s `extractOSMSubVenues`.
+ *   - `wikidata-subvenue` — the multilingual sub-venue DESIGNATOR vocabulary, pulled as the labels
+ *       and aliases of eight Wikidata concepts (terminal, gate, concourse, campus, …) plus the
+ *       airport-terminal instance labels as attested usage. Tier A (CC0). The only module in this
+ *       family built on `APIClient` — see its docstring for why.
+ *   - `state-sources` — NY/TX/DE/OR notaries, IA contractors, WA health providers, HI lobbyists.
+ *       Tier A (state PD-equivalent).
+ *   - `state-hi-schools` — Hawaii DOE school directory (original XLSX; HIDOE + PCS sheets). Tier A
+ *       (state PD-equivalent).
+ *   - `tiger-full` — US Census TIGER 2024 ADDRFEAT, all US counties. Tier A (US PD).
+ *
+ *   License tiers per `docs/licensing-strategy.md` (or the playpen knowledge base mirror at
+ *   `docs/docs/projects/mailwoman/licensing-strategy.md`). `openaddresses` is a **Tier-mixed**
+ *   source: the downloaded collection includes CC0, CC-BY, OGL, and ODbL/CC-BY-SA rows. The per-row
+ *   `LICENSE` filter in the `openaddresses` adapter is essential — Tier-C (ODbL, CC-BY-SA) rows are
+ *   dropped at ingest by default to protect proprietary-weights training.
+ *
+ *   ### OpenAddresses authentication (as of 2026-05-18)
+ *
+ *   `batch.openaddresses.io` now requires a free registered account for bulk downloads (auth check
+ *   prevents CDN abuse; data remains openly licensed). `fetchOpenAddresses` reads `OA_BATCH_TOKEN`
+ *   from the environment:
+ *
+ *   ```sh
+ *   # One-time: register at https://batch.openaddresses.io/register
+ *   # Log in → Profile → "Create Token" → copy token
+ *   export OA_BATCH_TOKEN=<your-token>
+ *
+ *   # Download Canada (~2 GiB compressed, ~7 GiB uncompressed)
+ *   mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Or any other OA country code
+ *   mailwoman corpus fetch openaddresses --country fr
+ *   ```
+ *
+ *   Without a token the command prints setup instructions and reports the failure.
+ *
+ *   ## Adding a new source
+ *
+ *   1. Pick the right module (or create a sibling one if the source is from a meaningfully
+ *      different family).
+ *   2. Append to the `SOURCES` array: `{ slug, filename, url }`.
+ *   3. Confirm the destination URL via `curl -sI -L <url> | head` before committing — state
+ *      open-data portals occasionally rotate Socrata view IDs.
+ *   4. Run the command against a scratch `--out-root` to verify the download succeeds + the
+ *      MANIFEST is well-formed.
+ *   5. Add the source's adapter (or extend an existing one) under `corpus/src/adapters/`.
+ */
+
+import { fetchBan } from "#tools/fetch/ban"
+import { fetchGeonamesDumps } from "#tools/fetch/geonames-dump"
+import { fetchGeonamesPostal } from "#tools/fetch/geonames-postal"
+import { fetchHRSA } from "#tools/fetch/hrsa"
+import { fetchIMLSPLS } from "#tools/fetch/imls-pls"
+import { fetchNAD } from "#tools/fetch/nad"
+import { fetchNPPES } from "#tools/fetch/nppes"
+import { fetchOpenAddresses } from "#tools/fetch/openaddresses"
+import { fetchOurAirports } from "#tools/fetch/ourairports"
+import { fetchStateHISchools } from "#tools/fetch/state-hi-schools"
+import { fetchStateSources } from "#tools/fetch/state-sources"
+import { fetchTigerFull } from "#tools/fetch/tiger-full"
+import { fetchWikidataSubVenue } from "#tools/fetch/wikidata-subvenue"
+
+export * from "#tools/fetch/ban"
+export * from "#tools/fetch/geonames-dump"
+export * from "#tools/fetch/geonames-postal"
+export * from "#tools/fetch/hrsa"
+export * from "#tools/fetch/imls-pls"
+export * from "#tools/fetch/nad"
+export * from "#tools/fetch/nppes"
+export * from "#tools/fetch/openaddresses"
+export * from "#tools/fetch/ourairports"
+export * from "#tools/fetch/state-hi-schools"
+export * from "#tools/fetch/state-sources"
+export * from "#tools/fetch/tiger-full"
+export * from "#tools/fetch/wikidata-subvenue"
+
+/**
+ * The fetch-source registry: id → module entry point. Each entry point takes its own options interface.
+ */
+export const FETCH_SOURCES = {
+	ban: fetchBan,
+	nad: fetchNAD,
+	"geonames-dump": fetchGeonamesDumps,
+	"geonames-postal": fetchGeonamesPostal,
+	hrsa: fetchHRSA,
+	"imls-pls": fetchIMLSPLS,
+	nppes: fetchNPPES,
+	openaddresses: fetchOpenAddresses,
+	ourairports: fetchOurAirports,
+	"state-sources": fetchStateSources,
+	"state-hi-schools": fetchStateHISchools,
+	"tiger-full": fetchTigerFull,
+	"wikidata-subvenue": fetchWikidataSubVenue,
+} as const
+
+export type FetchSourceID = keyof typeof FETCH_SOURCES

package/lib/tools/fetch/nad.ts

@@ -0,0 +1,341 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
+ *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
+ *   US Public Domain (17 U.S.C. § 105).
+ *
+ *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
+ *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
+ *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
+ *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
+ *   - Per-chunk manifest with sha256 + record count + error count
+ *
+ *   ## Source layout
+ *
+ *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
+ *
+ *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
+ *       `--nad-url <presigned>` from a browser visit to
+ *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
+ *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
+ *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Resume from an OID
+ *   mailwoman corpus fetch nad --start-oid 34400001
+ *
+ *   # Increase concurrency on a fast link
+ *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
+ *   ```
+ */
+
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+const SLUG = "usgov-nad"
+
+const FEATURE_SERVICE_URL =
+	"https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0"
+
+export interface FetchNADOptions extends BaseFetchOptions {
+	/**
+	 * Fetch strategy. Default `featureserver`.
+	 */
+	mode?: "featureserver" | "bulk"
+	/**
+	 * Pre-signed S3 URL for bulk mode.
+	 */
+	nadURL?: string
+	/**
+	 * Records per output file. Default `100000`.
+	 */
+	chunkSize?: number
+	/**
+	 * Records per HTTP request. Default `5000`.
+	 */
+	pageSize?: number
+	/**
+	 * Parallel page fetches within a chunk. Default `4`.
+	 */
+	concurrency?: number
+	/**
+	 * Start OBJECTID. Default `1`.
+	 */
+	startOID?: number
+	/**
+	 * Stop before this OID. Default = total count.
+	 */
+	endOID?: number
+}
+
+interface ChunkManifest {
+	source_url: string
+	oid_range: [number, number]
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+	record_count: number
+	page_errors: number
+	complete: boolean
+}
+
+/**
+ * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
+ * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
+ * records server-side.
+ */
+const nadClient = new APIClient({
+	displayName: "nad",
+	retry: true,
+	axios: { headers: { "Accept-Encoding": "gzip, br" } },
+})
+
+async function fetchPage(startOID: number, endOID: number, pageSize: number): Promise<unknown[]> {
+	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
+	url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`)
+	url.searchParams.set("outFields", "*")
+	url.searchParams.set("f", "json")
+	url.searchParams.set("resultRecordCount", String(pageSize))
+
+	const data = await nadClient
+		.fetch<{ features?: Array<{ attributes: unknown }>; error?: { message: string } }>({
+			url: url.toString(),
+			timeout: 120_000,
+		})
+		.then(pluckResponseData)
+
+	if (data.error) throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`)
+
+	return (data.features ?? []).map((f) => f.attributes)
+}
+
+async function discoverTotalCount(): Promise<number> {
+	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
+	url.searchParams.set("where", "1=1")
+	url.searchParams.set("returnCountOnly", "true")
+	url.searchParams.set("f", "json")
+
+	const data = await nadClient
+		.fetch<{ count?: number }>({ url: url.toString(), timeout: 30_000 })
+		.then(pluckResponseData)
+
+	if (typeof data.count !== "number") throw new Error("NAD count query returned no count field")
+
+	return data.count
+}
+
+/**
+ * Fetch a single chunk by paging through its OID range with bounded concurrency. Returns the count of records written
+ * and the count of pages that errored. The caller decides whether to mark the chunk complete based on errors === 0.
+ *
+ * NOTE(phase1): this is a JSON API pager, not a file download — the shared `downloadToFile` doesn't apply here.
+ */
+async function fetchChunk(
+	chunkPath: string,
+	chunkStart: number,
+	chunkEnd: number,
+	pageSize: number,
+	concurrency: number,
+	report?: (line: string) => void
+): Promise<{ recordCount: number; errors: number }> {
+	const pageRanges: Array<[number, number]> = []
+
+	for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
+		pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)])
+	}
+
+	// Run bounded-concurrency page fetches. Results indexed by page slot for in-order write.
+	const pageResults: Array<{ rows: unknown[]; error: Error | null }> = pageRanges.map(() => ({
+		rows: [],
+		error: null,
+	}))
+
+	let nextSlot = 0
+
+	const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
+		while (true) {
+			const slot = nextSlot++
+
+			if (slot >= pageRanges.length) return
+			const [s, e] = pageRanges[slot]!
+
+			try {
+				pageResults[slot]!.rows = await fetchPage(s, e, pageSize)
+			} catch (error) {
+				pageResults[slot]!.error = error as Error
+				report?.(`    ✗ page ${s}-${e}: ${(error as Error).message}`)
+			}
+		}
+	})
+
+	await Promise.all(workers)
+
+	// Single-writer phase — write all pages in OID order to keep NDJSON deterministic.
+	const lines: string[] = []
+	let errors = 0
+
+	for (const { rows, error } of pageResults) {
+		if (error) {
+			errors++
+
+			continue
+		}
+
+		for (const row of rows) {
+			lines.push(JSON.stringify(row))
+		}
+	}
+
+	await writeLocalTextFile(!lines.length ? "" : lines.join("\n") + "\n", chunkPath)
+
+	return { recordCount: lines.length, errors }
+}
+
+async function featureserverMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const chunkSize = options.chunkSize ?? 100_000
+	const pageSize = options.pageSize ?? 5000
+	const concurrency = options.concurrency ?? 4
+	const startOID = options.startOID ?? 1
+
+	const chunkDir = join(options.outRoot, SLUG, "featureserver")
+	await makeDirectories(chunkDir)
+
+	report?.(`=== ${SLUG} / featureserver`)
+	report?.(`  Discovering record count ...`)
+	const totalCount = await discoverTotalCount()
+	const endOID = options.endOID ?? totalCount
+	report?.(`  Total records: ${totalCount.toLocaleString()}`)
+	report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`)
+	report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`)
+
+	let fetched = 0
+	let skipped = 0
+	let totalRecords = 0
+	let totalErrors = 0
+	const failedCodes: string[] = []
+
+	for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
+		const chunkEnd = Math.min(cursor + chunkSize - 1, endOID)
+		const chunkName = `oids_${cursor}-${chunkEnd}`
+		const chunkPath = join(chunkDir, `${chunkName}.ndjson`)
+		const manifestPath = join(chunkDir, `${chunkName}.manifest.json`)
+
+		// Idempotency: skip a chunk only if it's marked complete (the bash version's bug was
+		// marking complete on partial-failure runs; we now only set complete after a clean fetch).
+		if (await pathExists(chunkPath)) {
+			const recorded = await readManifest<ChunkManifest>(manifestPath)
+
+			if (recorded?.complete) {
+				skipped++
+
+				continue
+			}
+		}
+
+		report?.(`  Fetching ${chunkName} ...`)
+		const t0 = Date.now()
+		const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report)
+		const elapsed = ((Date.now() - t0) / 1000).toFixed(1)
+		const bytes = (await statPath(chunkPath)).size
+		const sha = await sha256File(chunkPath)
+
+		const manifest: ChunkManifest = {
+			source_url: FEATURE_SERVICE_URL,
+			oid_range: [cursor, chunkEnd],
+			downloaded_at: new Date().toISOString(),
+			filename: `${chunkName}.ndjson`,
+			sha256: sha,
+			bytes,
+			record_count: recordCount,
+			page_errors: errors,
+			complete: errors === 0,
+		}
+
+		await writeManifest(manifestPath, manifest)
+
+		const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`
+
+		report?.(
+			`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`
+		)
+
+		fetched++
+		totalRecords += recordCount
+		totalErrors += errors
+
+		if (errors > 0) {
+			failedCodes.push(chunkName)
+		}
+	}
+
+	report?.(`=== featureserver summary ===`)
+	report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`)
+	report?.(`total records: ${totalRecords.toLocaleString()}`)
+	report?.(`page errors:   ${totalErrors}`)
+	report?.(`output:        ${chunkDir}`)
+
+	// `failed` counts page errors (the old exit-1 condition), `failedCodes` names the dirty chunks.
+	return { fetched, skipped, failed: totalErrors, failedCodes }
+}
+
+async function bulkMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	if (!options.nadURL) {
+		throw new Error(
+			`bulk mode requires --nad-url. The DOT page is Akamai-gated; ` +
+				`visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
+				`accept the disclaimer, and re-run with the pre-signed S3 URL.`
+		)
+	}
+
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+	const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip"
+	const destPath = join(destDir, filename)
+
+	report?.(`=== ${SLUG} / ${filename}`)
+	report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`)
+
+	const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report })
+
+	const sha = await sha256File(destPath)
+
+	await writeManifest(join(destDir, "MANIFEST.json"), {
+		source_url: options.nadURL,
+		downloaded_at: new Date().toISOString(),
+		filename,
+		sha256: sha,
+		bytes,
+	})
+
+	report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+export async function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const mode = options.mode ?? "featureserver"
+
+	if (mode === "featureserver") {
+		return featureserverMode(options, report)
+	}
+
+	if (mode === "bulk") {
+		return bulkMode(options, report)
+	}
+
+	throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`)
+}

package/lib/tools/fetch/nppes.ts

@@ -0,0 +1,165 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
+ *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
+ *   adapter. US Public Domain.
+ *
+ *   The file is published monthly by CMS. This module discovers the current filename by scraping the
+ *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
+ *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
+ *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
+ *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
+ *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
+ *   no resume, so a partial run re-downloads from the start.
+ *
+ *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
+ *   sha256 matches MANIFEST, skips download.
+ */
+
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { basename, join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html"
+const BASE_URL = "https://download.cms.gov/nppes"
+const SLUG = "usgov-nppes"
+
+export type FetchNPPESOptions = BaseFetchOptions
+
+/**
+ * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
+ * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
+ */
+async function discoverLatestZip(): Promise<string | undefined> {
+	// `responseType: "text"` — the index is HTML, scraped by regex below.
+	const html = await new APIClient({
+		displayName: "nppes-index",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
+	})
+		.fetch<string>({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
+
+	for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
+		const name = match[0]
+
+		if (name && !/\d{6}_\d{6}/.test(name)) return name
+	}
+
+	return undefined
+}
+
+/**
+ * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
+ * change file.
+ */
+async function findNpidataCSV(zipPath: string): Promise<string | undefined> {
+	const entries = await listZipEntries(zipPath)
+
+	return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
+}
+
+export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+	report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`)
+
+	const zipFilename = await discoverLatestZip()
+
+	if (!zipFilename) {
+		report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const zipURL = `${BASE_URL}/${zipFilename}`
+	const zipDest = join(destDir, zipFilename)
+	report?.(`  Latest full file: ${zipFilename}`)
+
+	// Idempotency check: if the main CSV already exists and sha matches,
+	// skip re-download.
+	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
+
+	if (recorded?.sha256 && recorded.filename) {
+		const recordedPath = join(destDir, recorded.filename)
+
+		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
+
+			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+		}
+	}
+
+	// MARK: Download ZIP (large; 60-minute timeout)
+
+	report?.(`  Downloading ${zipURL} ...`)
+
+	const { bytes: zipSize } = await downloadToFile({
+		url: zipURL,
+		dest: zipDest,
+		timeoutMs: 3_600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+
+	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
+
+	// MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
+
+	report?.("  Extracting npidata_pfile CSV from ZIP ...")
+	const csvName = await findNpidataCSV(zipDest)
+
+	if (!csvName) {
+		report?.("  ✗ Could not find npidata_pfile CSV inside ZIP")
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	report?.(`  Extracting: ${csvName}`)
+
+	const csvDest = join(destDir, basename(csvName))
+
+	await extractZipEntry(zipDest, csvName, csvDest)
+	const csvSize = (await statPath(csvDest)).size
+	const csvSha = await sha256File(csvDest)
+	report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`)
+
+	// MARK: Remove the ZIP to reclaim ~1 GB
+
+	await removePathIfPresent(zipDest)
+	report?.("  Removed ZIP (CSV kept)")
+
+	// MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
+
+	const manifest: SourceManifest = {
+		source_url: zipURL,
+		downloaded_at: new Date().toISOString(),
+		filename: csvName,
+		sha256: csvSha,
+		bytes: csvSize,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}