@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/tools/fetch/hrsa.ts

@@ -1,59 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
- *   adapter. US Public Domain.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
- *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
- *   downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
- */
-
-import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "usgov-hrsa-fqhc"
-const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv"
-const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`
-
-export type FetchHRSAOptions = BaseFetchOptions
-
-export async function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const dest = join(destDir, FILENAME)
-
-	report?.(`=== ${SLUG} / ${FILENAME}`)
-
-	const { bytes } = await downloadToFile({
-		url: SOURCE_URL,
-		dest,
-		timeoutMs: 600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	const sha = await sha256File(dest)
-
-	const manifest: SourceManifest = {
-		source_url: SOURCE_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: FILENAME,
-		sha256: sha,
-		bytes,
-	}
-
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-	report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/imls-pls.ts

@@ -1,149 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
-import { sha256File } from "@mailwoman/core/hash"
-import { basename, join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-/**
- * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
- */
-const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip"
-const SLUG = "usgov-imls-pls"
-
-export type FetchIMLSPLSOptions = BaseFetchOptions
-
-export async function fetchIMLSPLS(
-	options: FetchIMLSPLSOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-
-	const zipDest = join(destDir, basename(ZIP_URL))
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-
-	// MARK: Idempotency check
-
-	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
-
-	if (recorded?.sha256 && recorded.filename) {
-		const recordedPath = join(destDir, recorded.filename)
-
-		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
-			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
-
-			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-		}
-	}
-
-	// MARK: Download ZIP
-
-	report?.(`  Downloading ${ZIP_URL} ...`)
-
-	const { bytes: zipSize } = await downloadToFile({
-		url: ZIP_URL,
-		dest: zipDest,
-		timeoutMs: 600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipSize)}`)
-
-	if (zipSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	// Discover the outlet-level CSV inside the ZIP.
-	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
-	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
-	report?.("  Inspecting ZIP contents ...")
-	const entries = (await listZipEntries(zipDest)).map((entry) => entry.name)
-
-	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
-
-	// Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
-	if (!csvName) {
-		csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name))
-	}
-
-	if (!csvName) {
-		report?.("  Available files in ZIP:")
-
-		for (const name of entries) {
-			report?.(`    ${name}`)
-		}
-
-		report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module")
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	report?.(`  Extracting outlet CSV: ${csvName}`)
-
-	const csvDest = join(destDir, basename(csvName))
-
-	await extractZipEntry(zipDest, csvName, csvDest)
-	const csvSize = (await statPath(csvDest)).size
-	const csvSha = await sha256File(csvDest)
-
-	// MARK: Remove ZIP (small, but keep destDir clean)
-
-	await removePathIfPresent(zipDest)
-	report?.("  Removed ZIP (CSV kept)")
-
-	// MARK: Write MANIFEST
-
-	const manifest: SourceManifest = {
-		source_url: ZIP_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: basename(csvName),
-		sha256: csvSha,
-		bytes: csvSize,
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/nad.ts

@@ -1,341 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-
-import { APIClient, pluckResponseData } from "@mailwoman/core/api"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "usgov-nad"
-
-const FEATURE_SERVICE_URL =
-	"https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0"
-
-export interface FetchNADOptions extends BaseFetchOptions {
-	/**
-	 * Fetch strategy. Default `featureserver`.
-	 */
-	mode?: "featureserver" | "bulk"
-	/**
-	 * Pre-signed S3 URL for bulk mode.
-	 */
-	nadURL?: string
-	/**
-	 * Records per output file. Default `100000`.
-	 */
-	chunkSize?: number
-	/**
-	 * Records per HTTP request. Default `5000`.
-	 */
-	pageSize?: number
-	/**
-	 * Parallel page fetches within a chunk. Default `4`.
-	 */
-	concurrency?: number
-	/**
-	 * Start OBJECTID. Default `1`.
-	 */
-	startOID?: number
-	/**
-	 * Stop before this OID. Default = total count.
-	 */
-	endOID?: number
-}
-
-interface ChunkManifest {
-	source_url: string
-	oid_range: [number, number]
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-	record_count: number
-	page_errors: number
-	complete: boolean
-}
-
-/**
- * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
- * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
- * records server-side.
- */
-const nadClient = new APIClient({
-	displayName: "nad",
-	retry: true,
-	axios: { headers: { "Accept-Encoding": "gzip, br" } },
-})
-
-async function fetchPage(startOID: number, endOID: number, pageSize: number): Promise<unknown[]> {
-	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
-	url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`)
-	url.searchParams.set("outFields", "*")
-	url.searchParams.set("f", "json")
-	url.searchParams.set("resultRecordCount", String(pageSize))
-
-	const data = await nadClient
-		.fetch<{ features?: Array<{ attributes: unknown }>; error?: { message: string } }>({
-			url: url.toString(),
-			timeout: 120_000,
-		})
-		.then(pluckResponseData)
-
-	if (data.error) throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`)
-
-	return (data.features ?? []).map((f) => f.attributes)
-}
-
-async function discoverTotalCount(): Promise<number> {
-	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
-	url.searchParams.set("where", "1=1")
-	url.searchParams.set("returnCountOnly", "true")
-	url.searchParams.set("f", "json")
-
-	const data = await nadClient
-		.fetch<{ count?: number }>({ url: url.toString(), timeout: 30_000 })
-		.then(pluckResponseData)
-
-	if (typeof data.count !== "number") throw new Error("NAD count query returned no count field")
-
-	return data.count
-}
-
-/**
- * Fetch a single chunk by paging through its OID range with bounded concurrency. Returns the count of records written
- * and the count of pages that errored. The caller decides whether to mark the chunk complete based on errors === 0.
- *
- * NOTE(phase1): this is a JSON API pager, not a file download — the shared `downloadToFile` doesn't apply here.
- */
-async function fetchChunk(
-	chunkPath: string,
-	chunkStart: number,
-	chunkEnd: number,
-	pageSize: number,
-	concurrency: number,
-	report?: (line: string) => void
-): Promise<{ recordCount: number; errors: number }> {
-	const pageRanges: Array<[number, number]> = []
-
-	for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
-		pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)])
-	}
-
-	// Run bounded-concurrency page fetches. Results indexed by page slot for in-order write.
-	const pageResults: Array<{ rows: unknown[]; error: Error | null }> = pageRanges.map(() => ({
-		rows: [],
-		error: null,
-	}))
-
-	let nextSlot = 0
-
-	const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
-		while (true) {
-			const slot = nextSlot++
-
-			if (slot >= pageRanges.length) return
-			const [s, e] = pageRanges[slot]!
-
-			try {
-				pageResults[slot]!.rows = await fetchPage(s, e, pageSize)
-			} catch (error) {
-				pageResults[slot]!.error = error as Error
-				report?.(`    ✗ page ${s}-${e}: ${(error as Error).message}`)
-			}
-		}
-	})
-
-	await Promise.all(workers)
-
-	// Single-writer phase — write all pages in OID order to keep NDJSON deterministic.
-	const lines: string[] = []
-	let errors = 0
-
-	for (const { rows, error } of pageResults) {
-		if (error) {
-			errors++
-
-			continue
-		}
-
-		for (const row of rows) {
-			lines.push(JSON.stringify(row))
-		}
-	}
-
-	await writeLocalTextFile(!lines.length ? "" : lines.join("\n") + "\n", chunkPath)
-
-	return { recordCount: lines.length, errors }
-}
-
-async function featureserverMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const chunkSize = options.chunkSize ?? 100_000
-	const pageSize = options.pageSize ?? 5000
-	const concurrency = options.concurrency ?? 4
-	const startOID = options.startOID ?? 1
-
-	const chunkDir = join(options.outRoot, SLUG, "featureserver")
-	await makeDirectories(chunkDir)
-
-	report?.(`=== ${SLUG} / featureserver`)
-	report?.(`  Discovering record count ...`)
-	const totalCount = await discoverTotalCount()
-	const endOID = options.endOID ?? totalCount
-	report?.(`  Total records: ${totalCount.toLocaleString()}`)
-	report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`)
-	report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`)
-
-	let fetched = 0
-	let skipped = 0
-	let totalRecords = 0
-	let totalErrors = 0
-	const failedCodes: string[] = []
-
-	for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
-		const chunkEnd = Math.min(cursor + chunkSize - 1, endOID)
-		const chunkName = `oids_${cursor}-${chunkEnd}`
-		const chunkPath = join(chunkDir, `${chunkName}.ndjson`)
-		const manifestPath = join(chunkDir, `${chunkName}.manifest.json`)
-
-		// Idempotency: skip a chunk only if it's marked complete (the bash version's bug was
-		// marking complete on partial-failure runs; we now only set complete after a clean fetch).
-		if (await pathExists(chunkPath)) {
-			const recorded = await readManifest<ChunkManifest>(manifestPath)
-
-			if (recorded?.complete) {
-				skipped++
-
-				continue
-			}
-		}
-
-		report?.(`  Fetching ${chunkName} ...`)
-		const t0 = Date.now()
-		const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report)
-		const elapsed = ((Date.now() - t0) / 1000).toFixed(1)
-		const bytes = (await statPath(chunkPath)).size
-		const sha = await sha256File(chunkPath)
-
-		const manifest: ChunkManifest = {
-			source_url: FEATURE_SERVICE_URL,
-			oid_range: [cursor, chunkEnd],
-			downloaded_at: new Date().toISOString(),
-			filename: `${chunkName}.ndjson`,
-			sha256: sha,
-			bytes,
-			record_count: recordCount,
-			page_errors: errors,
-			complete: errors === 0,
-		}
-
-		await writeManifest(manifestPath, manifest)
-
-		const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`
-
-		report?.(
-			`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`
-		)
-
-		fetched++
-		totalRecords += recordCount
-		totalErrors += errors
-
-		if (errors > 0) {
-			failedCodes.push(chunkName)
-		}
-	}
-
-	report?.(`=== featureserver summary ===`)
-	report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`)
-	report?.(`total records: ${totalRecords.toLocaleString()}`)
-	report?.(`page errors:   ${totalErrors}`)
-	report?.(`output:        ${chunkDir}`)
-
-	// `failed` counts page errors (the old exit-1 condition), `failedCodes` names the dirty chunks.
-	return { fetched, skipped, failed: totalErrors, failedCodes }
-}
-
-async function bulkMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	if (!options.nadURL) {
-		throw new Error(
-			`bulk mode requires --nad-url. The DOT page is Akamai-gated; ` +
-				`visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
-				`accept the disclaimer, and re-run with the pre-signed S3 URL.`
-		)
-	}
-
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip"
-	const destPath = join(destDir, filename)
-
-	report?.(`=== ${SLUG} / ${filename}`)
-	report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`)
-
-	const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report })
-
-	const sha = await sha256File(destPath)
-
-	await writeManifest(join(destDir, "MANIFEST.json"), {
-		source_url: options.nadURL,
-		downloaded_at: new Date().toISOString(),
-		filename,
-		sha256: sha,
-		bytes,
-	})
-
-	report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}
-
-export async function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const mode = options.mode ?? "featureserver"
-
-	if (mode === "featureserver") {
-		return featureserverMode(options, report)
-	}
-
-	if (mode === "bulk") {
-		return bulkMode(options, report)
-	}
-
-	throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`)
-}