@mailwoman/corpus 7.2.0 → 7.3.0

package/src/tools/fetch/ban.ts

@@ -0,0 +1,288 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+
+import { existsSync, mkdirSync, statSync } from "node:fs"
+import { readFile, unlink, writeFile } from "node:fs/promises"
+import { join } from "node:path"
+import { setTimeout as sleep } from "node:timers/promises"
+import { gunzipSync } from "node:zlib"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, loadManifestEntries, writeManifest } from "./download.ts"
+
+const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
+
+/**
+ * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
+ * change.
+ */
+const DEPT_CODES = [
+	"01",
+	"02",
+	"03",
+	"04",
+	"05",
+	"06",
+	"07",
+	"08",
+	"09",
+	"10",
+	"11",
+	"12",
+	"13",
+	"14",
+	"15",
+	"16",
+	"17",
+	"18",
+	"19",
+	"21",
+	"22",
+	"23",
+	"24",
+	"25",
+	"26",
+	"27",
+	"28",
+	"29",
+	"2A",
+	"2B",
+	"30",
+	"31",
+	"32",
+	"33",
+	"34",
+	"35",
+	"36",
+	"37",
+	"38",
+	"39",
+	"40",
+	"41",
+	"42",
+	"43",
+	"44",
+	"45",
+	"46",
+	"47",
+	"48",
+	"49",
+	"50",
+	"51",
+	"52",
+	"53",
+	"54",
+	"55",
+	"56",
+	"57",
+	"58",
+	"59",
+	"60",
+	"61",
+	"62",
+	"63",
+	"64",
+	"65",
+	"66",
+	"67",
+	"68",
+	"69",
+	"70",
+	"71",
+	"72",
+	"73",
+	"74",
+	"75",
+	"76",
+	"77",
+	"78",
+	"79",
+	"80",
+	"81",
+	"82",
+	"83",
+	"84",
+	"85",
+	"86",
+	"87",
+	"88",
+	"89",
+	"90",
+	"91",
+	"92",
+	"93",
+	"94",
+	"95",
+	"971",
+	"972",
+	"973",
+	"974",
+	"976",
+]
+
+export interface BanManifestEntry {
+	dept_code: string
+	filename: string
+	source_url: string
+	downloaded_at: string
+	sha256: string
+	bytes: number
+}
+
+export type FetchBanOptions = BaseFetchOptions
+
+/** Mimic `numfmt --to=iec` for a friendly byte-size log line. */
+function iec(bytes: number): string {
+	if (bytes < 1024) return String(bytes)
+	const units = ["K", "M", "G", "T", "P"]
+	let value = bytes / 1024
+	let i = 0
+
+	while (value >= 1024 && i < units.length - 1) {
+		value /= 1024
+		i++
+	}
+
+	const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString()
+
+	return `${rounded}${units[i] ?? ""}`
+}
+
+export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const banDir = join(options.outRoot, "ban")
+	const manifestPath = join(banDir, "MANIFEST.json")
+	mkdirSync(banDir, { recursive: true })
+
+	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
+	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
+
+	let fetched = 0
+	let skipped = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const code of DEPT_CODES) {
+		const filename = `adresses-${code}.csv`
+		const gzFile = join(banDir, `${filename}.gz`)
+		const csvFile = join(banDir, filename)
+		const url = `${BASE_URL}/adresses-${code}.csv.gz`
+
+		report?.(`=== dept ${code}`)
+
+		// If the CSV already exists, compare its sha256 against the manifest.
+		if (existsSync(csvFile)) {
+			const existingSha = await sha256File(csvFile)
+			const recordedSha = entries.get(code)?.sha256
+
+			if (recordedSha && existingSha === recordedSha) {
+				report?.(`  → already present + sha matches — skipping`)
+				skipped++
+				continue
+			}
+
+			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
+			await unlink(csvFile)
+		}
+
+		// Download the gzipped CSV.
+		try {
+			await downloadToFile({
+				url,
+				dest: gzFile,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			})
+		} catch (err) {
+			report?.(`  ✗ download failed: ${url} (${(err as Error).message})`)
+			failed++
+			failedCodes.push(code)
+			continue
+		}
+
+		// Guard against truncated 404/error pages.
+		const gzSize = statSync(gzFile).size
+
+		if (gzSize < 1024) {
+			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
+			await unlink(gzFile)
+			failed++
+			failedCodes.push(code)
+			continue
+		}
+
+		// Decompress in-place; delete the .gz.
+		try {
+			await writeFile(csvFile, gunzipSync(await readFile(gzFile)))
+		} catch (err) {
+			report?.(`  ✗ decompress failed: ${(err as Error).message}`)
+			await unlink(gzFile)
+			failed++
+			failedCodes.push(code)
+			continue
+		}
+		await unlink(gzFile)
+
+		if (!existsSync(csvFile)) {
+			report?.(`  ✗ decompressed file not found at ${csvFile}`)
+			failed++
+			failedCodes.push(code)
+			continue
+		}
+
+		const bytes = statSync(csvFile).size
+		const sha = await sha256File(csvFile)
+
+		entries.set(code, {
+			dept_code: code,
+			filename,
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			sha256: sha,
+			bytes,
+		})
+
+		report?.(`  ✓ ${iec(bytes)}  sha256=${sha}`)
+		fetched++
+
+		// Be a polite citizen — short pause between requests.
+		await sleep(200)
+	}
+
+	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
+	const sorted = [...entries.values()].sort((a, b) =>
+		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
+	)
+	await writeManifest(manifestPath, sorted)
+	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
+
+	report?.(`=== summary ===`)
+	report?.(`fetched:  ${fetched}`)
+	report?.(`skipped:  ${skipped} (already present + sha matched)`)
+	report?.(`failed:   ${failed}`)
+
+	if (failedCodes.length > 0) {
+		report?.(`failed codes: ${failedCodes.join(" ")}`)
+	}
+
+	return { fetched, skipped, failed, failedCodes }
+}

package/src/tools/fetch/download.ts

@@ -0,0 +1,119 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download + manifest plumbing for the `mailwoman corpus fetch <source>` family — one download-with-retry and
+ *   one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
+ *   (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
+ */
+
+import { existsSync } from "node:fs"
+import { readFile, writeFile } from "node:fs/promises"
+import { setTimeout as sleep } from "node:timers/promises"
+
+/** The option base every `mailwoman corpus fetch <source>` module extends. */
+export interface BaseFetchOptions {
+	/** Destination root for downloaded source data. Each source writes its own subdirectory. */
+	outRoot: string
+}
+
+/** The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1. */
+export interface FetchSummary {
+	fetched: number
+	skipped: number
+	failed: number
+	failedCodes: string[]
+}
+
+/** A status worth retrying: rate limiting or a server-side failure. */
+export function isTransientStatus(status: number): boolean {
+	return status === 429 || (status >= 500 && status <= 599)
+}
+
+export interface DownloadOptions {
+	url: string
+	dest: string
+	/** Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps. */
+	timeoutMs?: number
+	/** Extra attempts after the first, taken only on transient statuses or network errors. Default 0. */
+	retries?: number
+	/** Delay between attempts. Default 5s. */
+	retryDelayMs?: number
+	headers?: Record<string, string>
+	report?: (line: string) => void
+}
+
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export async function downloadToFile(options: DownloadOptions): Promise<{ bytes: number }> {
+	const { url, dest, timeoutMs = 600_000, retries = 0, retryDelayMs = 5_000, headers, report } = options
+	let lastError: unknown
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		if (attempt > 0) {
+			report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`)
+			await sleep(retryDelayMs)
+		}
+
+		let res: Response
+
+		try {
+			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
+		} catch (error) {
+			// AbortSignal timeouts and network-level failures are retryable.
+			lastError = error
+			continue
+		}
+
+		if (!res.ok) {
+			const error = new Error(`HTTP ${res.status} ${res.statusText} — ${url}`)
+
+			if (!isTransientStatus(res.status)) throw error
+			lastError = error
+			continue
+		}
+
+		try {
+			const buffer = Buffer.from(await res.arrayBuffer())
+			await writeFile(dest, buffer)
+
+			return { bytes: buffer.byteLength }
+		} catch (error) {
+			// A mid-stream abort while reading the body is retryable too.
+			lastError = error
+		}
+	}
+
+	throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+/** Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch). */
+export async function readManifest<T>(path: string): Promise<T | null> {
+	if (!existsSync(path)) return null
+
+	try {
+		return JSON.parse(await readFile(path, "utf8")) as T
+	} catch {
+		return null
+	}
+}
+
+/** Load manifest entries into a map so untouched keys survive a partial re-fetch. */
+export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
+	const entries = new Map<string, T>()
+	const parsed = await readManifest<T[]>(path)
+
+	for (const entry of parsed ?? []) {
+		entries.set(key(entry), entry)
+	}
+
+	return entries
+}
+
+/** Write a MANIFEST.json in the house shape: pretty-printed, trailing newline. */
+export async function writeManifest(path: string, manifest: unknown): Promise<void> {
+	await writeFile(path, JSON.stringify(manifest, null, 2) + "\n")
+}

package/src/tools/fetch/hrsa.ts

@@ -0,0 +1,65 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
+ *   adapter. US Public Domain.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
+ *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
+ *   downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
+ */
+
+import { mkdirSync } from "node:fs"
+import { join } from "node:path"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, writeManifest } from "./download.ts"
+
+const SLUG = "usgov-hrsa-fqhc"
+const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv"
+const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`
+
+export type FetchHRSAOptions = BaseFetchOptions
+
+interface SourceManifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+export async function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+	const dest = join(destDir, FILENAME)
+
+	report?.(`=== ${SLUG} / ${FILENAME}`)
+	const { bytes } = await downloadToFile({
+		url: SOURCE_URL,
+		dest,
+		timeoutMs: 600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+	const sha = await sha256File(dest)
+
+	const manifest: SourceManifest = {
+		source_url: SOURCE_URL,
+		downloaded_at: new Date().toISOString(),
+		filename: FILENAME,
+		sha256: sha,
+		bytes,
+	}
+	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+	report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/src/tools/fetch/imls-pls.ts

@@ -0,0 +1,162 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
+ *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
+ *   Public Domain (federal statistical survey).
+ *
+ *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
+ *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
+ *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
+ *   no per-branch address detail.
+ *
+ *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
+ *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
+ *   member listing + selective extraction).
+ *
+ *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
+ *   and sha matches MANIFEST, skips download.
+ */
+
+import { execFile } from "node:child_process"
+import { existsSync, mkdirSync, statSync } from "node:fs"
+import { rm } from "node:fs/promises"
+import { basename, join } from "node:path"
+import { promisify } from "node:util"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, readManifest, writeManifest } from "./download.ts"
+
+const execFileAsync = promisify(execFile)
+
+// The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05).
+// If IMLS publishes a newer year, update this URL.
+const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip"
+const SLUG = "usgov-imls-pls"
+
+export type FetchIMLSPLSOptions = BaseFetchOptions
+
+interface SourceManifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+/** Return the filenames listed inside a ZIP (the trailing column of each `unzip -l` row). */
+async function listZipEntries(zipPath: string): Promise<string[]> {
+	const listing = await execFileAsync("unzip", ["-l", zipPath])
+
+	return listing.stdout
+		.split("\n")
+		.map((line) => line.trim().split(/\s+/).pop() ?? "")
+		.filter((name) => name.length > 0)
+}
+
+export async function fetchIMLSPLS(
+	options: FetchIMLSPLSOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+
+	const zipDest = join(destDir, basename(ZIP_URL))
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+
+	// ------------------------------------------------------------------
+	// Idempotency check: if outlet CSV already exists and sha matches, skip.
+	// ------------------------------------------------------------------
+	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
+
+	if (recorded?.sha256 && recorded.filename) {
+		const recordedPath = join(destDir, recorded.filename)
+
+		if (existsSync(recordedPath) && (await sha256File(recordedPath)) === recorded.sha256) {
+			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
+
+			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+		}
+	}
+
+	// ------------------------------------------------------------------
+	// Download ZIP
+	// ------------------------------------------------------------------
+	report?.(`  Downloading ${ZIP_URL} ...`)
+	const { bytes: zipSize } = await downloadToFile({
+		url: ZIP_URL,
+		dest: zipDest,
+		timeoutMs: 600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
+
+	if (zipSize < 1024) {
+		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// ------------------------------------------------------------------
+	// Discover the outlet-level CSV inside the ZIP.
+	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
+	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
+	// ------------------------------------------------------------------
+	report?.("  Inspecting ZIP contents ...")
+	const entries = await listZipEntries(zipDest)
+
+	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
+
+	// Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
+	if (!csvName) {
+		csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name))
+	}
+
+	if (!csvName) {
+		report?.("  Available files in ZIP:")
+
+		for (const name of entries) {
+			report?.(`    ${name}`)
+		}
+		report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module")
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	report?.(`  Extracting outlet CSV: ${csvName}`)
+	await execFileAsync("unzip", ["-o", "-j", zipDest, csvName, "-d", destDir])
+
+	const csvDest = join(destDir, basename(csvName))
+	const csvSize = statSync(csvDest).size
+	const csvSha = await sha256File(csvDest)
+
+	// ------------------------------------------------------------------
+	// Remove ZIP (small, but keep destDir clean)
+	// ------------------------------------------------------------------
+	await rm(zipDest, { force: true })
+	report?.("  Removed ZIP (CSV kept)")
+
+	// ------------------------------------------------------------------
+	// Write MANIFEST
+	// ------------------------------------------------------------------
+	const manifest: SourceManifest = {
+		source_url: ZIP_URL,
+		downloaded_at: new Date().toISOString(),
+		filename: basename(csvName),
+		sha256: csvSha,
+		bytes: csvSize,
+	}
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}