@mailwoman/corpus 9.1.0 → 9.3.0

package/src/tools/fetch/ban.ts

@@ -1,310 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
- *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
- *
- *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
- *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
- *
- *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
- *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
- *   `<outRoot>/ban/MANIFEST.json` covers all codes.
- *
- *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
- *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
- *   Python.
- */
-
-import { existsSync, mkdirSync, statSync } from "node:fs"
-import { readFile, unlink, writeFile } from "node:fs/promises"
-import { join } from "node:path"
-import { setTimeout as sleep } from "node:timers/promises"
-import { gunzipSync } from "node:zlib"
-
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, loadManifestEntries, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const BYTES_PER_KIB = 1024
-
-const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
-
-/**
- * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
- * change.
- */
-const DEPT_CODES = [
-	"01",
-	"02",
-	"03",
-	"04",
-	"05",
-	"06",
-	"07",
-	"08",
-	"09",
-	"10",
-	"11",
-	"12",
-	"13",
-	"14",
-	"15",
-	"16",
-	"17",
-	"18",
-	"19",
-	"21",
-	"22",
-	"23",
-	"24",
-	"25",
-	"26",
-	"27",
-	"28",
-	"29",
-	"2A",
-	"2B",
-	"30",
-	"31",
-	"32",
-	"33",
-	"34",
-	"35",
-	"36",
-	"37",
-	"38",
-	"39",
-	"40",
-	"41",
-	"42",
-	"43",
-	"44",
-	"45",
-	"46",
-	"47",
-	"48",
-	"49",
-	"50",
-	"51",
-	"52",
-	"53",
-	"54",
-	"55",
-	"56",
-	"57",
-	"58",
-	"59",
-	"60",
-	"61",
-	"62",
-	"63",
-	"64",
-	"65",
-	"66",
-	"67",
-	"68",
-	"69",
-	"70",
-	"71",
-	"72",
-	"73",
-	"74",
-	"75",
-	"76",
-	"77",
-	"78",
-	"79",
-	"80",
-	"81",
-	"82",
-	"83",
-	"84",
-	"85",
-	"86",
-	"87",
-	"88",
-	"89",
-	"90",
-	"91",
-	"92",
-	"93",
-	"94",
-	"95",
-	"971",
-	"972",
-	"973",
-	"974",
-	"976",
-]
-
-export interface BanManifestEntry {
-	dept_code: string
-	filename: string
-	source_url: string
-	downloaded_at: string
-	sha256: string
-	bytes: number
-}
-
-export type FetchBanOptions = BaseFetchOptions
-
-/**
- * Mimic `numfmt --to=iec` for a friendly byte-size log line.
- */
-function iec(bytes: number): string {
-	if (bytes < BYTES_PER_KIB) return String(bytes)
-	const units = ["K", "M", "G", "T", "P"]
-	let value = bytes / 1024
-	let i = 0
-
-	while (value >= BYTES_PER_KIB && i < units.length - 1) {
-		value /= 1024
-
-		i++
-	}
-
-	const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString()
-
-	return `${rounded}${units[i] ?? ""}`
-}
-
-export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const banDir = join(options.outRoot, "ban")
-	const manifestPath = join(banDir, "MANIFEST.json")
-	mkdirSync(banDir, { recursive: true })
-
-	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
-	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
-
-	let fetched = 0
-	let skipped = 0
-	let failed = 0
-	const failedCodes: string[] = []
-
-	for (const code of DEPT_CODES) {
-		const filename = `adresses-${code}.csv`
-		const gzFile = join(banDir, `${filename}.gz`)
-		const csvFile = join(banDir, filename)
-		const url = `${BASE_URL}/adresses-${code}.csv.gz`
-
-		report?.(`=== dept ${code}`)
-
-		// If the CSV already exists, compare its sha256 against the manifest.
-		if (existsSync(csvFile)) {
-			const existingSha = await sha256File(csvFile)
-			const recordedSha = entries.get(code)?.sha256
-
-			if (recordedSha && existingSha === recordedSha) {
-				report?.(`  → already present + sha matches — skipping`)
-
-				skipped++
-
-				continue
-			}
-
-			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
-			await unlink(csvFile)
-		}
-
-		// Download the gzipped CSV.
-		try {
-			await downloadToFile({
-				url,
-				dest: gzFile,
-				timeoutMs: 600_000,
-				headers: { "Accept-Encoding": "gzip, br" },
-				report,
-			})
-		} catch (error) {
-			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Guard against truncated 404/error pages.
-		const gzSize = statSync(gzFile).size
-
-		if (gzSize < BYTES_PER_KIB) {
-			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
-			await unlink(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Decompress in-place; delete the .gz.
-		try {
-			await writeFile(csvFile, gunzipSync(await readFile(gzFile)))
-		} catch (error) {
-			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
-			await unlink(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		await unlink(gzFile)
-
-		if (!existsSync(csvFile)) {
-			report?.(`  ✗ decompressed file not found at ${csvFile}`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		const bytes = statSync(csvFile).size
-		const sha = await sha256File(csvFile)
-
-		entries.set(code, {
-			dept_code: code,
-			filename,
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			sha256: sha,
-			bytes,
-		})
-
-		report?.(`  ✓ ${iec(bytes)}  sha256=${sha}`)
-
-		fetched++
-
-		// Be a polite citizen — short pause between requests.
-		await sleep(200)
-	}
-
-	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
-	const sorted = [...entries.values()].toSorted((a, b) =>
-		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
-	)
-
-	await writeManifest(manifestPath, sorted)
-	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
-
-	report?.(`=== summary ===`)
-	report?.(`fetched:  ${fetched}`)
-	report?.(`skipped:  ${skipped} (already present + sha matched)`)
-	report?.(`failed:   ${failed}`)
-
-	if (failedCodes.length) {
-		report?.(`failed codes: ${failedCodes.join(" ")}`)
-	}
-
-	return { fetched, skipped, failed, failedCodes }
-}

package/src/tools/fetch/download.ts

@@ -1,158 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Download + manifest plumbing for the `mailwoman corpus fetch <source>` family — one download-with-retry and
- *   one MANIFEST.json idiom instead of the per-script clones the 2026-07-09 dedupe survey counted
- *   (6× `downloadToFile`, 2× `isTransientStatus`, 9× manifest writes).
- */
-
-import { existsSync } from "node:fs"
-import { readFile, writeFile } from "node:fs/promises"
-import { setTimeout as sleep } from "node:timers/promises"
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-
-/**
- * Rate limited — retryable, the server is asking us to back off.
- */
-const HTTP_TOO_MANY_REQUESTS = 429
-
-/**
- * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
- */
-const HTTP_SERVER_ERROR_MIN = 500
-
-/**
- * Highest 5xx status.
- */
-const HTTP_SERVER_ERROR_MAX = 599
-
-/**
- * The option base every `mailwoman corpus fetch <source>` module extends.
- */
-export interface BaseFetchOptions {
-	/**
-	 * Destination root for downloaded source data. Each source writes its own subdirectory.
-	 */
-	outRoot: string
-}
-
-/**
- * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
- */
-export interface FetchSummary {
-	fetched: number
-	skipped: number
-	failed: number
-	failedCodes: string[]
-}
-
-/**
- * A status worth retrying: rate limiting or a server-side failure.
- */
-export function isTransientStatus(status: number): boolean {
-	return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX)
-}
-
-export interface DownloadOptions {
-	url: string
-	dest: string
-	/**
-	 * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
-	 */
-	timeoutMs?: number
-	/**
-	 * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
-	 */
-	retries?: number
-	/**
-	 * Delay between attempts. Default 5s.
-	 */
-	retryDelayMs?: number
-	headers?: Record<string, string>
-	report?: (line: string) => void
-}
-
-/**
- * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
- * or once retries are exhausted. Returns the byte count written.
- */
-export async function downloadToFile(options: DownloadOptions): Promise<{ bytes: number }> {
-	const { url, dest, timeoutMs = 600_000, retries = 0, retryDelayMs = 5000, headers, report } = options
-	let lastError: unknown
-
-	for (let attempt = 0; attempt <= retries; attempt++) {
-		if (attempt > 0) {
-			report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`)
-			await sleep(retryDelayMs)
-		}
-
-		let res: Response
-
-		try {
-			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
-		} catch (error) {
-			// AbortSignal timeouts and network-level failures are retryable.
-			lastError = error
-
-			continue
-		}
-
-		if (!res.ok) {
-			const error = new Error(`HTTP ${res.status} ${res.statusText} — ${url}`)
-
-			if (!isTransientStatus(res.status)) throw error
-			lastError = error
-
-			continue
-		}
-
-		try {
-			const buffer = Buffer.from(await res.arrayBuffer())
-			await writeFile(dest, buffer)
-
-			return { bytes: buffer.byteLength }
-		} catch (error) {
-			// A mid-stream abort while reading the body is retryable too.
-			lastError = error
-		}
-	}
-
-	throw lastError instanceof Error ? lastError : new Error(String(lastError))
-}
-
-/**
- * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
- */
-export async function readManifest<T>(path: string): Promise<T | null> {
-	if (!existsSync(path)) return null
-
-	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
-	// JSON does; tryParsingJSON returns null for the non-string sentinel.
-	const text = await readFile(path, "utf8").catch(() => null)
-
-	return tryParsingJSON<T>(text)
-}
-
-/**
- * Load manifest entries into a map so untouched keys survive a partial re-fetch.
- */
-export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
-	const entries = new Map<string, T>()
-	const parsed = await readManifest<T[]>(path)
-
-	for (const entry of parsed ?? []) {
-		entries.set(key(entry), entry)
-	}
-
-	return entries
-}
-
-/**
- * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
- */
-export async function writeManifest(path: string, manifest: unknown): Promise<void> {
-	await writeFile(path, JSON.stringify(manifest, null, 2) + "\n")
-}

package/src/tools/fetch/hrsa.ts

@@ -1,68 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the HRSA Health Center Service Delivery Sites CSV. Source for the `usgov-hrsa-fqhc`
- *   adapter. US Public Domain.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum, and
- *   writes the same sibling `MANIFEST.json` (origin URL + fetch timestamp + byte count + sha256) so
- *   downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch hrsa --out-root <path>`.
- */
-
-import { mkdirSync } from "node:fs"
-import { join } from "node:path"
-
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, writeManifest } from "./download.ts"
-
-const SLUG = "usgov-hrsa-fqhc"
-const FILENAME = "Health_Center_Service_Delivery_and_LookAlike_Sites.csv"
-const SOURCE_URL = `https://data.hrsa.gov/DataDownload/DD_Files/${FILENAME}`
-
-export type FetchHRSAOptions = BaseFetchOptions
-
-interface SourceManifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-}
-
-export async function fetchHRSA(options: FetchHRSAOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	mkdirSync(destDir, { recursive: true })
-	const dest = join(destDir, FILENAME)
-
-	report?.(`=== ${SLUG} / ${FILENAME}`)
-
-	const { bytes } = await downloadToFile({
-		url: SOURCE_URL,
-		dest,
-		timeoutMs: 600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	const sha = await sha256File(dest)
-
-	const manifest: SourceManifest = {
-		source_url: SOURCE_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: FILENAME,
-		sha256: sha,
-		bytes,
-	}
-
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-	report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/src/tools/fetch/imls-pls.ts

@@ -1,158 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the IMLS Public Libraries Survey (PLS) outlet-level data. Each US public library branch
- *   (outlet) is one row, ~17K rows with address fields. Source for the `usgov-imls-pls` adapter. US
- *   Public Domain (federal statistical survey).
- *
- *   The FY 2023 release is the most current as of 2026-05. IMLS ships a single ZIP containing CSV,
- *   SAS, and SPSS variants. We extract the outlet-level CSV (pls_fy*_outlet*.csv or similar) and
- *   discard the rest. The administrative-entity (system-level) CSV is intentionally skipped — it has
- *   no per-branch address detail.
- *
- *   Uses Node's built-in fetch (gzip/brotli) and streaming sha256 instead of curl + sha256sum. The
- *   ZIP is unpacked with the `unzip` binary via `node:child_process` (no clean Node equivalent for
- *   member listing + selective extraction).
- *
- *   Invoke via `mailwoman corpus fetch imls-pls --out-root <path>`. Idempotent: if dest CSV exists
- *   and sha matches MANIFEST, skips download.
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { existsSync, mkdirSync, statSync } from "node:fs"
-import { rm } from "node:fs/promises"
-import { basename, join } from "node:path"
-
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, readManifest, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-const BYTES_PER_KIB = 1024
-
-/**
- * The PLS FY 2023 bulk CSV ZIP (most recent as of 2026-05). If IMLS publishes a newer year, update this URL.
- */
-const ZIP_URL = "https://www.imls.gov/sites/default/files/2025-08/pls_fy2023_csv.zip"
-const SLUG = "usgov-imls-pls"
-
-export type FetchIMLSPLSOptions = BaseFetchOptions
-
-interface SourceManifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-}
-
-export async function fetchIMLSPLS(
-	options: FetchIMLSPLSOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	mkdirSync(destDir, { recursive: true })
-
-	const zipDest = join(destDir, basename(ZIP_URL))
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-
-	// MARK: Idempotency check
-
-	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
-
-	if (recorded?.sha256 && recorded.filename) {
-		const recordedPath = join(destDir, recorded.filename)
-
-		if (existsSync(recordedPath) && (await sha256File(recordedPath)) === recorded.sha256) {
-			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
-
-			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-		}
-	}
-
-	// MARK: Download ZIP
-
-	report?.(`  Downloading ${ZIP_URL} ...`)
-
-	const { bytes: zipSize } = await downloadToFile({
-		url: ZIP_URL,
-		dest: zipDest,
-		timeoutMs: 600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
-
-	if (zipSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${zipSize} bytes) — probable error page`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	// Discover the outlet-level CSV inside the ZIP.
-	// Outlet files match: pls_fy*outlet*.csv (case-insensitive)
-	// Administrative-entity files match: pls_fy*ae*.csv — we skip those.
-	report?.("  Inspecting ZIP contents ...")
-	const entries = (await listZipEntries(zipDest)).map((entry) => entry.name)
-
-	let csvName = entries.find((name) => /pls_fy.*outlet.*\.csv/i.test(name))
-
-	// Fallback: if IMLS renames the file, grab any CSV that is NOT the ae file.
-	if (!csvName) {
-		csvName = entries.find((name) => /\.csv$/i.test(name) && !/system|state|_ae\b|_se\b/i.test(name))
-	}
-
-	if (!csvName) {
-		report?.("  Available files in ZIP:")
-
-		for (const name of entries) {
-			report?.(`    ${name}`)
-		}
-
-		report?.("  ✗ Could not identify outlet CSV — inspect above listing and update this module")
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	report?.(`  Extracting outlet CSV: ${csvName}`)
-
-	const csvDest = join(destDir, basename(csvName))
-
-	await extractZipEntry(zipDest, csvName, csvDest)
-	const csvSize = statSync(csvDest).size
-	const csvSha = await sha256File(csvDest)
-
-	// MARK: Remove ZIP (small, but keep destDir clean)
-
-	await rm(zipDest, { force: true })
-	report?.("  Removed ZIP (CSV kept)")
-
-	// MARK: Write MANIFEST
-
-	const manifest: SourceManifest = {
-		source_url: ZIP_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: basename(csvName),
-		sha256: csvSha,
-		bytes: csvSize,
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}