@mailwoman/corpus 9.2.0 → 9.4.0

package/lib/tools/fetch/state-hi-schools.ts

@@ -0,0 +1,161 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches MANIFEST, skips download.
+ */
+
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+import { XLSXSpliterator, type XLSXCellValue } from "spliterator"
+
+import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#adapters/state-hi-schools/workbook"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
+const SLUG = "state-hi-schools"
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
+
+export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
+	/**
+	 * Workbook URL. Defaults to the Hawaii DOE source; overridable for an isolated fetch test.
+	 */
+	sourceURL?: string
+}
+
+interface Manifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+	notes: string
+}
+
+async function validateWorkbook(path: string): Promise<void> {
+	for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+		const rows = await XLSXSpliterator.fromAsync<XLSXCellValue[]>(path, {
+			sheet,
+			header: false,
+			take: 2,
+		}).toArray()
+
+		const header = rows[0]
+
+		if (!header || rows.length < 2) {
+			throw new Error(`sheet ${sheet} must contain a header and at least one school row`)
+		}
+
+		// Exact casing is part of the adapter contract: object mode preserves the workbook's header names, so accepting
+		// `Code` here while the reader asks for `record.code` would validate a workbook whose every row is later skipped.
+		const columns = new Set(header.map((cell) => String(cell ?? "").trim()))
+
+		const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column))
+
+		if (missing.length) {
+			throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`)
+		}
+	}
+}
+
+export async function fetchStateHISchools(
+	options: FetchStateHISchoolsOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	await makeDirectories(destDir)
+
+	const xlsxDest = join(destDir, XLSX_FILENAME)
+	const manifestPath = join(destDir, "MANIFEST.json")
+	const sourceURL = options.sourceURL ?? SOURCE_URL
+
+	report?.(`=== ${SLUG}`)
+
+	// Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
+	if (await pathExists(xlsxDest)) {
+		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
+
+		if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
+			const actualSha = await sha256File(xlsxDest)
+
+			if (actualSha === recorded.sha256) {
+				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
+
+				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+			}
+		}
+	}
+
+	// Download XLSX.
+	report?.(`  Downloading ${sourceURL} ...`)
+
+	try {
+		await downloadToFile({
+			url: sourceURL,
+			dest: xlsxDest,
+			timeoutMs: 600_000,
+			headers: { "Accept-Encoding": "gzip, br" },
+			report,
+		})
+	} catch (error) {
+		report?.(`  ✗ Download failed (${(error as Error).message})`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const xlsxSize = (await statPath(xlsxDest)).size
+	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
+
+	if (xlsxSize < BYTES_PER_KIB) {
+		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
+		await removePathIfPresent(xlsxDest)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	try {
+		await validateWorkbook(xlsxDest)
+	} catch (error) {
+		report?.(`  ✗ Workbook validation failed (${(error as Error).message})`)
+		await removePathIfPresent(xlsxDest)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const xlsxSha = await sha256File(xlsxDest)
+
+	// Write MANIFEST.
+	const manifest: Manifest = {
+		source_url: sourceURL,
+		downloaded_at: new Date().toISOString(),
+		filename: XLSX_FILENAME,
+		sha256: xlsxSha,
+		bytes: xlsxSize,
+		notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(
+		`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`
+	)
+
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/lib/tools/fetch/state-sources.ts

@@ -0,0 +1,146 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
+ *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
+ *
+ *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
+ *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
+ *   before the adapter can consume it.
+ *
+ *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
+ *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
+ *
+ *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
+ *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
+ */
+
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, writeManifest } from "#tools/fetch/download"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+
+export type FetchStateSourcesOptions = BaseFetchOptions
+
+interface Source {
+	slug: string
+	filename: string
+	url: string
+}
+
+const SOURCES: readonly Source[] = [
+	{
+		slug: "state-ny-notaries",
+		filename: "NY_Commissioned_Notaries.csv",
+		url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-tx-notaries",
+		filename: "TX_Notary_Public_Commissions.csv",
+		url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-de-notaries",
+		filename: "DE_Notaries_Commissioned.csv",
+		url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-or-notaries",
+		filename: "OR_Active_Notaries.csv",
+		url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-ia-contractors",
+		filename: "IA_Active_Construction_Contractor_Registrations.csv",
+		url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-wa-health-providers",
+		filename: "WA_Health_Care_Provider_Credential_Data.csv",
+		url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
+	},
+	{
+		slug: "state-hi-lobbyists",
+		filename: "HI_Lobbyist_Registration_Statements.csv",
+		url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
+	},
+]
+
+export async function fetchStateSources(
+	options: FetchStateSourcesOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await makeDirectories(options.outRoot)
+
+	let fetched = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const { slug, filename, url } of SOURCES) {
+		const destDir = join(options.outRoot, slug)
+		await makeDirectories(destDir)
+		const dest = join(destDir, filename)
+
+		report?.(`=== ${slug} / ${filename}`)
+
+		let bytes: number
+
+		try {
+			;({ bytes } = await downloadToFile({
+				url,
+				dest,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			}))
+		} catch (error) {
+			report?.(`  ✗ download failed for ${url}: ${(error as Error).message}`)
+
+			failed++
+			failedCodes.push(slug)
+
+			continue
+		}
+
+		if (bytes < BYTES_PER_KIB) {
+			report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`)
+
+			failed++
+			failedCodes.push(slug)
+
+			continue
+		}
+
+		const sha = await sha256File(dest)
+
+		const manifest: SourceManifest = {
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			filename,
+			sha256: sha,
+			bytes,
+		}
+
+		await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+
+		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
+
+		fetched++
+	}
+
+	report?.(`=== summary ===`)
+	report?.(`fetched: ${fetched}`)
+	report?.(`failed:  ${failed}`)
+
+	return { fetched, skipped: 0, failed, failedCodes }
+}

package/lib/tools/fetch/tiger-full.ts

@@ -0,0 +1,308 @@
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
+ *
+ *   TIGER ADDRFEAT 2024 source:
+ *
+ *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
+ *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
+ *
+ *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
+ *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
+ *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
+ *   this module is download + provenance only.)
+ *
+ *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
+ *   county ZIP to disk (no curl subprocess).
+ */
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { isoSeconds } from "@mailwoman/core/utils"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { basename, join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+/**
+ * Lowest 2xx status; anything below is informational.
+ */
+const HTTP_OK = 200
+
+/**
+ * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
+ */
+const HTTP_REDIRECT = 300
+
+const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
+
+export interface FetchTigerFullOptions extends BaseFetchOptions {
+	/**
+	 * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
+	 * v0.1.1.
+	 */
+	skipStateFips?: string
+	/**
+	 * Seconds to sleep between downloads. Default `0.2`.
+	 */
+	rateSleep?: number
+	/**
+	 * Max concurrent download workers per state. Default `4`.
+	 */
+	maxParallel?: number
+	/**
+	 * Print planned downloads without fetching. Default `false`.
+	 */
+	dryRun?: boolean
+}
+
+interface CountyEntry {
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+/**
+ * Read a per-state MANIFEST.json into a filename → entry map.
+ */
+async function readCountyManifest(manifestPath: string): Promise<Map<string, CountyEntry>> {
+	const map = new Map<string, CountyEntry>()
+	const parsed = await readManifest<{ counties?: CountyEntry[] }>(manifestPath)
+
+	for (const c of parsed?.counties ?? []) {
+		if (c.filename) {
+			map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes })
+		}
+	}
+
+	return map
+}
+
+/**
+ * Check whether a file already matches a recorded sha256 and byte count.
+ */
+async function fileMatchesSha(path: string, expectedSha: string, expectedBytes: number): Promise<boolean> {
+	if (!(await pathExists(path))) return false
+
+	if ((await statPath(path)).size !== expectedBytes) return false
+
+	return (await sha256File(path)) === expectedSha
+}
+
+type CountyResult =
+	| { ok: true; filename: string; sha256: string; bytes: number }
+	| { ok: false; filename: string; reason: string }
+
+/**
+ * Download one county ZIP (size sanity check + sha256).
+ */
+async function downloadCounty(url: string, dest: string): Promise<CountyResult> {
+	const filename = basename(dest)
+	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 })
+
+	if (status < HTTP_OK || status >= HTTP_REDIRECT) {
+		return { ok: false, filename, reason: `HTTP ${status}` }
+	}
+
+	const bytes = (await statPath(dest)).size
+
+	if (bytes < BYTES_PER_KIB) {
+		await removePathIfPresent(dest)
+
+		return { ok: false, filename, reason: `too small (${bytes} bytes)` }
+	}
+
+	const sha256 = await sha256File(dest)
+
+	return { ok: true, filename, sha256, bytes }
+}
+
+export async function fetchTigerFull(
+	options: FetchTigerFullOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0)
+	const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000)
+	const maxParallel = options.maxParallel ?? 4
+	const dryRun = options.dryRun ?? false
+
+	const addrfeatDir = join(options.outRoot, "tiger", "addrfeat")
+	await makeDirectories(addrfeatDir)
+
+	// MARK: Step 1 — discover the county file list
+
+	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
+
+	// `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
+	// stay on raw `fetch` (they stream to disk; see `downloadOne`).
+	const listingRes = await new APIClient({
+		displayName: "tiger-listing",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
+	})
+		.fetch<string>({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
+
+	const html = listingRes
+	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted()
+	const totalCounties = allZips.length
+	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
+
+	// Build a map: state_fips -> list of filenames.
+	// tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
+	const stateFiles = new Map<string, string[]>()
+
+	for (const fname of allZips) {
+		const stateFips = fname.slice(8, 10)
+		const list = stateFiles.get(stateFips) ?? []
+		list.push(fname)
+		stateFiles.set(stateFips, list)
+	}
+
+	report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`)
+
+	// MARK: Step 2 — download the county ZIPs
+
+	let totalFetched = 0
+	let totalSkipped = 0
+	let totalSkippedState = 0
+	let totalFailed = 0
+	let totalBytesFetched = 0
+	const failedCodes: string[] = []
+
+	// Process states in sorted FIPS order for predictable output.
+	const sortedStates = [...stateFiles.keys()].toSorted()
+
+	for (const stateFips of sortedStates) {
+		const countyFiles = stateFiles.get(stateFips) ?? []
+
+		// --- Skip entire state if requested ------------------------------------------
+		if (skipStateFips.includes(stateFips)) {
+			report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`)
+			totalSkippedState += countyFiles.length
+
+			continue
+		}
+
+		const stateDir = join(addrfeatDir, `state-${stateFips}`)
+		await makeDirectories(stateDir)
+		const manifestPath = join(stateDir, "MANIFEST.json")
+
+		// Load existing manifest for O(1) verified-skip lookup.
+		const manifest = await readCountyManifest(manifestPath)
+
+		report?.(`--- State ${stateFips} — ${countyFiles.length} counties`)
+
+		// Build a list of URLs+dests that need fetching.
+		const pending: Array<{ url: string; dest: string }> = []
+
+		for (const fname of countyFiles) {
+			const dest = join(stateDir, fname)
+			const url = `${TIGER_BASE_URL}/${fname}`
+			const known = manifest.get(fname)
+
+			// Skip if already verified via MANIFEST.
+			if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
+				report?.(`  skip (verified) ${fname}`)
+
+				totalSkipped++
+
+				continue
+			}
+
+			if (dryRun) {
+				report?.(`  would fetch: ${url}`)
+
+				totalFetched++
+
+				continue
+			}
+
+			pending.push({ url, dest })
+		}
+
+		if (dryRun) continue
+
+		if (!pending.length) continue
+
+		// --- Download pending files with bounded parallelism + rate-limit spacing ---
+		const results: CountyResult[] = Array.from({ length: pending.length })
+		let cursor = 0
+
+		const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
+			while (true) {
+				const i = cursor++
+
+				if (i >= pending.length) return
+				const item = pending[i]!
+				// Rate-limit: polite spacing before each fetch.
+				await sleep(rateSleepMs)
+				results[i] = await downloadCounty(item.url, item.dest)
+			}
+		})
+
+		await Promise.all(workers)
+
+		// Collect results from this state.
+		for (const result of results) {
+			if (result.ok) {
+				report?.(
+					`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`
+				)
+
+				manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes })
+
+				totalFetched++
+				totalBytesFetched += result.bytes
+			} else {
+				report?.(`  FAIL ${result.filename} -- ${result.reason}`)
+
+				totalFailed++
+				failedCodes.push(result.filename)
+			}
+		}
+
+		// Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
+		const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename))
+
+		const manifestDoc = {
+			state_fips: stateFips,
+			updated_at: isoSeconds(),
+			tiger_base_url: TIGER_BASE_URL,
+			counties,
+		}
+
+		await writeManifest(manifestPath, manifestDoc)
+	}
+
+	// MARK: Summary
+
+	report?.(`=== Summary ===`)
+	report?.(`  Total counties in index   : ${totalCounties}`)
+	report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`)
+	report?.(`  Counties already present  : ${totalSkipped}`)
+	report?.(`  Counties fetched this run : ${totalFetched}`)
+	report?.(`  Counties failed           : ${totalFailed}`)
+
+	if (totalBytesFetched > 0) {
+		report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`)
+	}
+
+	if (totalFailed > 0) {
+		report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`)
+	}
+
+	return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes }
+}