@mailwoman/corpus 9.1.0 → 9.2.0

package/src/tools/fetch/geonames-dump.ts

@@ -0,0 +1,267 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
+ *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
+ *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
+ *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
+ *
+ *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
+ *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
+ *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
+ *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
+ *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
+ *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
+ *   the basename) is reported as `wrong_format_present`, never counted as coverage.
+ */
+
+import { existsSync, mkdirSync } from "node:fs"
+import { open, readFile, rm } from "node:fs/promises"
+import { join } from "node:path"
+
+import { extractZipEntry } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, HTTPStatusError, writeManifest } from "./download.ts"
+
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404
+
+const SLUG = "geonames-dump"
+
+/**
+ * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
+ * bare text file.
+ */
+const BASE_URL = "https://download.geonames.org/export/dump"
+
+export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
+	/**
+	 * ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
+	 */
+	countries?: readonly string[]
+	/**
+	 * Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
+	 * against a local server.
+	 */
+	baseURL?: string
+	/**
+	 * Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
+	 */
+	force?: boolean
+}
+
+interface GeonamesDumpFileEntry {
+	country: string
+	filename: string
+	source_url: string
+	sha256: string
+	bytes: number
+}
+
+export interface GeonamesDumpManifest {
+	source: string
+	base_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: GeonamesDumpFileEntry[]
+	/**
+	 * `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
+	 */
+	skipped_present: string[]
+	/**
+	 * Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
+	 * recorded so a later reader does not spend the fetch to rediscover it.
+	 */
+	unavailable: string[]
+	/**
+	 * Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
+	 * and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
+	 * capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
+	 * own home and rerun.
+	 */
+	wrong_format_present: string[]
+}
+
+/**
+ * Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
+ * `<CC>.txt` basename.
+ */
+const GAZETTEER_DUMP_COLUMNS = 19
+
+/**
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
+ */
+export function looksLikeGazetteerDump(text: string): boolean {
+	let start = 0
+
+	while (start < text.length) {
+		const end = text.indexOf("\n", start)
+		const line = end === -1 ? text.slice(start) : text.slice(start, end)
+
+		if (line.trim()) {
+			let tabs = 0
+
+			for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
+				tabs++
+			}
+
+			return tabs === GAZETTEER_DUMP_COLUMNS - 1
+		}
+
+		if (end === -1) break
+
+		start = end + 1
+	}
+
+	return false
+}
+
+/**
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
+ * file's own commentary.
+ */
+export function parseCountryInfo(text: string): Array<{ country: string; capital: string }> {
+	const rows: Array<{ country: string; capital: string }> = []
+
+	// oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
+	for (const line of text.split("\n")) {
+		if (line.startsWith("#") || !line.trim()) continue
+
+		// oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
+		const cols = line.split("\t")
+		const country = cols[0]?.trim().toUpperCase()
+
+		if (country?.length === 2) {
+			rows.push({ country, capital: cols[5]?.trim() ?? "" })
+		}
+	}
+
+	return rows
+}
+
+/**
+ * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
+ * reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
+ */
+const FORMAT_SNIFF_BYTES = 65_536
+
+async function readFileHead(path: string): Promise<string> {
+	const handle = await open(path, "r")
+
+	try {
+		const buffer = Buffer.alloc(FORMAT_SNIFF_BYTES)
+		const { bytesRead } = await handle.read(buffer, 0, FORMAT_SNIFF_BYTES, 0)
+
+		return buffer.subarray(0, bytesRead).toString("utf8")
+	} finally {
+		await handle.close()
+	}
+}
+
+/**
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
+ */
+export async function fetchGeonamesDumps(
+	options: FetchGeonamesDumpOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary & { skippedPresent: string[] }> {
+	mkdirSync(options.outRoot, { recursive: true })
+
+	const baseURL = options.baseURL ?? BASE_URL
+	const countryInfoDest = join(options.outRoot, "countryInfo.txt")
+
+	await downloadToFile({
+		url: `${baseURL}/countryInfo.txt`,
+		dest: countryInfoDest,
+		timeoutMs: 120_000,
+		retries: 2,
+		report,
+	})
+
+	const catalog = parseCountryInfo(await readFile(countryInfoDest, "utf8"))
+	const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country)
+
+	const entries: GeonamesDumpFileEntry[] = []
+	const failedCodes: string[] = []
+	const unavailable: string[] = []
+	const skippedPresent: string[] = []
+	const wrongFormatPresent: string[] = []
+	let fetched = 0
+
+	for (const country of countries) {
+		const txtDest = join(options.outRoot, `${country}.txt`)
+
+		if (!options.force && existsSync(txtDest)) {
+			if (looksLikeGazetteerDump(await readFileHead(txtDest))) {
+				skippedPresent.push(country)
+			} else {
+				report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`)
+				wrongFormatPresent.push(country)
+			}
+
+			continue
+		}
+
+		const filename = `${country}.zip`
+		const url = `${baseURL}/${filename}`
+		const zipDest = join(options.outRoot, filename)
+
+		report?.(`=== ${SLUG} / ${country}`)
+
+		try {
+			await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report })
+			await extractZipEntry(zipDest, `${country}.txt`, txtDest)
+			await rm(zipDest, { force: true })
+
+			entries.push({
+				country,
+				filename: `${country}.txt`,
+				source_url: url,
+				sha256: await sha256File(txtDest),
+				bytes: (await readFile(txtDest)).byteLength,
+			})
+
+			fetched++
+		} catch (error) {
+			await rm(zipDest, { force: true })
+
+			const message = error instanceof Error ? error.message : String(error)
+
+			// Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
+			// can contain any substring.
+			if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+				report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`)
+				unavailable.push(country)
+			} else {
+				report?.(`✗ ${country}: ${message}`)
+			}
+
+			failedCodes.push(country)
+		}
+	}
+
+	const manifest: GeonamesDumpManifest = {
+		source: SLUG,
+		base_url: baseURL,
+		license: "CC-BY-4.0",
+		attribution: "GeoNames",
+		downloaded_at: new Date().toISOString(),
+		files: entries,
+		skipped_present: skippedPresent.toSorted(),
+		unavailable,
+		wrong_format_present: wrongFormatPresent.toSorted(),
+	}
+
+	await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest)
+
+	return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent }
+}

package/src/tools/fetch/geonames-postal.ts

@@ -0,0 +1,180 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
+ *   `(postcode, locality, region)` triple with the NAMES inline.
+ *
+ *   Source : https://download.geonames.org/export/zip/<CC>.zip
+ *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
+ *
+ *   ## Why this source and not a join
+ *
+ *   A postcode shard is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
+ *   only one of them works everywhere:
+ *
+ *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
+ *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
+ *     `locality`/`localadmin`). It is also the ONLY shard that does: `postalcode-geonames-intl.db` and every
+ *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
+ *     have none.
+ *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
+ *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
+ *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
+ *     own. Under 50% for four of the five, which is not a join.
+ *
+ *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
+ *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
+ *
+ *   ## Coverage is not universal, and the gap is the point
+ *
+ *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode shard cannot be built
+ *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
+ *   before assuming it is there; an absent country fails as one entry, never as the whole run.
+ *
+ *   ## Row counts from this source overstate, for some countries by exactly 2×
+ *
+ *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
+ *   on the shard built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
+ *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a shard from the row
+ *   count doubles its estimate for those countries.
+ *
+ *   ## Why `downloadToFile` and not `APIClient`
+ *
+ *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
+ *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
+ *
+ *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
+ */
+
+import { mkdirSync } from "node:fs"
+import { join } from "node:path"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, HTTPStatusError, writeManifest } from "./download.ts"
+
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404
+
+const SLUG = "geonames-postal"
+
+/**
+ * GeoNames' own export directory. One zip per ISO alpha-2 code, each holding `<CC>.txt` plus the shared `readme.txt`.
+ */
+const BASE_URL = "https://download.geonames.org/export/zip"
+
+/**
+ * Countries fetched when the caller names none.
+ *
+ * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
+ * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
+ */
+export const GEONAMES_POSTAL_DEFAULT_COUNTRIES = ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"] as const
+
+export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
+	/**
+	 * ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
+	 */
+	countries?: readonly string[]
+	/**
+	 * Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
+	 * exercised against a local server rather than by asking GeoNames for a country it does not have.
+	 */
+	baseURL?: string
+}
+
+interface GeonamesPostalFileEntry {
+	country: string
+	filename: string
+	source_url: string
+	sha256: string
+	bytes: number
+}
+
+interface GeonamesPostalManifest {
+	source: string
+	base_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: GeonamesPostalFileEntry[]
+	/**
+	 * Countries asked for and NOT published by GeoNames, recorded so a later reader does not spend the fetch again to
+	 * rediscover it. An absence here is a fact about the source, not about the run.
+	 */
+	unavailable: string[]
+}
+
+/**
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
+ * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
+ *
+ * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
+ */
+export async function fetchGeonamesPostal(
+	options: FetchGeonamesPostalOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+
+	const countries = (options.countries ?? GEONAMES_POSTAL_DEFAULT_COUNTRIES).map((code) => code.trim().toUpperCase())
+	const baseURL = options.baseURL ?? BASE_URL
+	const entries: GeonamesPostalFileEntry[] = []
+	const failedCodes: string[] = []
+	const unavailable: string[] = []
+	let fetched = 0
+	let failed = 0
+
+	for (const country of countries) {
+		const filename = `${country}.zip`
+		const url = `${baseURL}/${filename}`
+		const dest = join(destDir, filename)
+
+		report?.(`=== ${SLUG} / ${country}`)
+
+		try {
+			const { bytes } = await downloadToFile({ url, dest, timeoutMs: 300_000, retries: 2, report })
+
+			entries.push({ country, filename, source_url: url, sha256: await sha256File(dest), bytes })
+
+			fetched++
+		} catch (error) {
+			const message = error instanceof Error ? error.message : String(error)
+
+			// A 404 here means GeoNames does not publish the country at all, which is a different finding from a failed
+			// transfer and the one a caller planning a shard needs to see. Branch on the TYPED status: matching message
+			// prose classified a 500 as "unpublished" whenever the URL happened to contain the substring 404 — an
+			// ephemeral test-server port did exactly that in CI.
+			if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+				report?.(`✗ ${country}: GeoNames does not publish a postal export for this country`)
+				unavailable.push(country)
+			} else {
+				report?.(`✗ ${country}: ${message}`)
+			}
+
+			failedCodes.push(country)
+
+			failed++
+		}
+	}
+
+	const manifest: GeonamesPostalManifest = {
+		source: "GeoNames postal codes",
+		base_url: baseURL,
+		license: "CC-BY-4.0",
+		attribution: "GeoNames",
+		downloaded_at: new Date().toISOString(),
+		files: entries,
+		unavailable,
+	}
+
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	return { fetched, skipped: 0, failed, failedCodes }
+}

package/src/tools/fetch/imls-pls.ts

@@ -27,6 +27,7 @@ import { existsSync, mkdirSync, statSync } from "node:fs"
 import { rm } from "node:fs/promises"
 import { basename, join } from "node:path"
 
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
 import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/utils"
 
@@ -37,7 +38,6 @@ import { downloadToFile, readManifest, writeManifest } from "./download.ts"
  * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
  * than data.
  */
-const BYTES_PER_KIB = 1024
 
 /**
  * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
@@ -93,7 +93,7 @@ export async function fetchIMLSPLS(
 		report,
 	})
 
-	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
+	report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`)
 
 	if (zipSize < BYTES_PER_KIB) {
 		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)

package/src/tools/fetch/index.ts

@@ -33,6 +33,11 @@
  *       Ouverte 2.0).
  *   - `nad` — US DOT National Address Database (~97M address points, ArcGIS FeatureServer). Tier A
  *       (US PD).
+ *   - `geonames-postal` — GeoNames per-country postal exports (~80 countries). Tier B (CC-BY-4.0,
+ *       attribute "GeoNames"). The only source in this family carrying `(postcode, locality, region)`
+ *       with the names INLINE, which is why it exists: the `parent_id` join route covers exactly five
+ *       countries and the nearest-centroid fallback measured under 50% agreement on four of those five.
+ *       GeoNames does not publish every country — Venezuela 404s.
  *   - `hrsa` — HRSA Health Center Service Delivery Sites (federal). Tier A (US PD).
  *   - `imls-pls` — IMLS Public Libraries Survey, outlet-level (~17K library branches, FY 2023).
  *       Tier A (US PD).
@@ -93,6 +98,8 @@
  */
 
 import { fetchBan } from "./ban.ts"
+import { fetchGeonamesDumps } from "./geonames-dump.ts"
+import { fetchGeonamesPostal } from "./geonames-postal.ts"
 import { fetchHRSA } from "./hrsa.ts"
 import { fetchIMLSPLS } from "./imls-pls.ts"
 import { fetchNAD } from "./nad.ts"
@@ -105,6 +112,8 @@ import { fetchTigerFull } from "./tiger-full.ts"
 import { fetchWikidataSubVenue } from "./wikidata-subvenue.ts"
 
 export * from "./ban.ts"
+export * from "./geonames-dump.ts"
+export * from "./geonames-postal.ts"
 export * from "./hrsa.ts"
 export * from "./imls-pls.ts"
 export * from "./nad.ts"
@@ -122,6 +131,8 @@ export * from "./wikidata-subvenue.ts"
 export const FETCH_SOURCES = {
 	ban: fetchBan,
 	nad: fetchNAD,
+	"geonames-dump": fetchGeonamesDumps,
+	"geonames-postal": fetchGeonamesPostal,
 	hrsa: fetchHRSA,
 	"imls-pls": fetchIMLSPLS,
 	nppes: fetchNPPES,

package/src/tools/fetch/nad.ts

@@ -40,6 +40,7 @@ import { existsSync, mkdirSync, statSync } from "node:fs"
 import { writeFile } from "node:fs/promises"
 import { join } from "node:path"
 
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -93,6 +94,17 @@ interface ChunkManifest {
 	complete: boolean
 }
 
+/**
+ * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
+ * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
+ * records server-side.
+ */
+const nadClient = new APIClient({
+	displayName: "nad",
+	retry: true,
+	axios: { headers: { "Accept-Encoding": "gzip, br" } },
+})
+
 async function fetchPage(startOID: number, endOID: number, pageSize: number): Promise<unknown[]> {
 	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
 	url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`)
@@ -100,13 +112,12 @@ async function fetchPage(startOID: number, endOID: number, pageSize: number): Pr
 	url.searchParams.set("f", "json")
 	url.searchParams.set("resultRecordCount", String(pageSize))
 
-	const res = await fetch(url, {
-		headers: { "Accept-Encoding": "gzip, br" },
-		signal: AbortSignal.timeout(120_000),
-	})
-
-	if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} on OID ${startOID}-${endOID}`)
-	const data = (await res.json()) as { features?: Array<{ attributes: unknown }>; error?: { message: string } }
+	const data = await nadClient
+		.fetch<{ features?: Array<{ attributes: unknown }>; error?: { message: string } }>({
+			url: url.toString(),
+			timeout: 120_000,
+		})
+		.then(pluckResponseData)
 
 	if (data.error) throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`)
 
@@ -118,10 +129,10 @@ async function discoverTotalCount(): Promise<number> {
 	url.searchParams.set("where", "1=1")
 	url.searchParams.set("returnCountOnly", "true")
 	url.searchParams.set("f", "json")
-	const res = await fetch(url, { signal: AbortSignal.timeout(30_000) })
 
-	if (!res.ok) throw new Error(`Failed to discover NAD record count: HTTP ${res.status}`)
-	const data = (await res.json()) as { count?: number }
+	const data = await nadClient
+		.fetch<{ count?: number }>({ url: url.toString(), timeout: 30_000 })
+		.then(pluckResponseData)
 
 	if (typeof data.count !== "number") throw new Error("NAD count query returned no count field")
 

package/src/tools/fetch/nppes.ts

@@ -28,6 +28,7 @@ import { existsSync, mkdirSync, statSync } from "node:fs"
 import { rm } from "node:fs/promises"
 import { basename, join } from "node:path"
 
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
 import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
 import { sha256File } from "@mailwoman/core/utils"
 
@@ -53,13 +54,14 @@ interface SourceManifest {
  * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
  */
 async function discoverLatestZip(): Promise<string | undefined> {
-	const res = await fetch(INDEX_URL, {
-		headers: { "Accept-Encoding": "gzip, br" },
-		signal: AbortSignal.timeout(60_000),
+	// `responseType: "text"` — the index is HTML, scraped by regex below.
+	const html = await new APIClient({
+		displayName: "nppes-index",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
 	})
-
-	if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText} on ${INDEX_URL}`)
-	const html = await res.text()
+		.fetch<string>({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
 
 	for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
 		const name = match[0]

package/src/tools/fetch/openaddresses.ts

@@ -62,7 +62,9 @@ import { pipeline } from "node:stream/promises"
 import { setTimeout as sleep } from "node:timers/promises"
 import { promisify } from "node:util"
 
+import { APIClient, isSuccessStatus } from "@mailwoman/core/api"
 import { $private } from "@mailwoman/core/env"
+import { ByteFormatter } from "@mailwoman/core/fs/utils"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -82,8 +84,6 @@ const HTTP_OK = 200
  */
 const MIN_PLAUSIBLE_SHARD_BYTES = 10_240
 
-const BYTES_PER_KIB = 1024
-
 const execFileAsync = promisify(execFile)
 
 const OA_BASE = "https://batch.openaddresses.io"
@@ -132,20 +132,6 @@ async function countLines(path: string): Promise<number> {
 	return count
 }
 
-function humanBytes(bytes: number): string {
-	const units = ["B", "KiB", "MiB", "GiB", "TiB"]
-	let value = bytes
-	let unit = 0
-
-	while (value >= BYTES_PER_KIB && unit < units.length - 1) {
-		value /= 1024
-
-		unit++
-	}
-
-	return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`
-}
-
 interface StreamDownloadOpts {
 	headers?: Record<string, string>
 	timeoutMs: number
@@ -263,18 +249,28 @@ The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
 	if (collectionID === undefined) {
 		report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`)
 
-		const res = await fetch(`${OA_BASE}/api/collections`, {
-			headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" },
-			signal: AbortSignal.timeout(30_000),
+		// The collections API only. The COLLECTION ARCHIVES stay on raw `fetch` — they stream multi-gigabyte
+		// bodies straight to disk, where response caching is nonsense and axios would buffer them in memory.
+		const res = await new APIClient({
+			displayName: "openaddresses-api",
+			retry: true,
+			axios: { headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" } },
 		})
+			// `validateStatus` keeps a non-2xx as a RESPONSE rather than a throw: the caller reports the status and
+			// returns `fail(country)`, a graceful path this must not turn into an exception.
+			.fetch<OaCollection[]>({
+				url: `${OA_BASE}/api/collections`,
+				timeout: 30_000,
+				validateStatus: () => true,
+			})
 
-		if (!res.ok) {
+		if (!isSuccessStatus(res.status)) {
 			report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`)
 
 			return fail(country)
 		}
 
-		const collections = (await res.json()) as OaCollection[]
+		const collections = res.data
 		const match = collections.find((item) => item.name === country)
 
 		if (match?.id === undefined) {
@@ -395,7 +391,7 @@ URL tried: ${OA_BASE}/api/collections/${collectionID}/download
 
 	await writeManifest(manifestPath, manifest)
 
-	report?.(`  ✓ ${humanBytes(size)}  rows=${rowCount}  sha256=${sha}`)
+	report?.(`  ✓ ${ByteFormatter.formatIEC(size)}  rows=${rowCount}  sha256=${sha}`)
 	report?.(`  MANIFEST written to ${manifestPath}`)
 	report?.(`=== done`)
 	report?.(`Feed to the adapter:`)