@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/tools/fetch/nppes.ts

@@ -1,165 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
- *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
- *   adapter. US Public Domain.
- *
- *   The file is published monthly by CMS. This module discovers the current filename by scraping the
- *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
- *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
- *
- *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
- *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
- *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
- *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
- *   no resume, so a partial run re-downloads from the start.
- *
- *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
- *   sha256 matches MANIFEST, skips download.
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { APIClient, pluckResponseData } from "@mailwoman/core/api"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
-import { sha256File } from "@mailwoman/core/hash"
-import { basename, join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html"
-const BASE_URL = "https://download.cms.gov/nppes"
-const SLUG = "usgov-nppes"
-
-export type FetchNPPESOptions = BaseFetchOptions
-
-/**
- * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
- * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
- */
-async function discoverLatestZip(): Promise<string | undefined> {
-	// `responseType: "text"` — the index is HTML, scraped by regex below.
-	const html = await new APIClient({
-		displayName: "nppes-index",
-		retry: true,
-		axios: { headers: { "Accept-Encoding": "gzip, br" } },
-	})
-		.fetch<string>({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
-		.then(pluckResponseData)
-
-	for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
-		const name = match[0]
-
-		if (name && !/\d{6}_\d{6}/.test(name)) return name
-	}
-
-	return undefined
-}
-
-/**
- * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
- * change file.
- */
-async function findNpidataCSV(zipPath: string): Promise<string | undefined> {
-	const entries = await listZipEntries(zipPath)
-
-	return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
-}
-
-export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-	report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`)
-
-	const zipFilename = await discoverLatestZip()
-
-	if (!zipFilename) {
-		report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const zipURL = `${BASE_URL}/${zipFilename}`
-	const zipDest = join(destDir, zipFilename)
-	report?.(`  Latest full file: ${zipFilename}`)
-
-	// Idempotency check: if the main CSV already exists and sha matches,
-	// skip re-download.
-	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
-
-	if (recorded?.sha256 && recorded.filename) {
-		const recordedPath = join(destDir, recorded.filename)
-
-		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
-			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
-
-			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-		}
-	}
-
-	// MARK: Download ZIP (large; 60-minute timeout)
-
-	report?.(`  Downloading ${zipURL} ...`)
-
-	const { bytes: zipSize } = await downloadToFile({
-		url: zipURL,
-		dest: zipDest,
-		timeoutMs: 3_600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
-
-	// MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
-
-	report?.("  Extracting npidata_pfile CSV from ZIP ...")
-	const csvName = await findNpidataCSV(zipDest)
-
-	if (!csvName) {
-		report?.("  ✗ Could not find npidata_pfile CSV inside ZIP")
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	report?.(`  Extracting: ${csvName}`)
-
-	const csvDest = join(destDir, basename(csvName))
-
-	await extractZipEntry(zipDest, csvName, csvDest)
-	const csvSize = (await statPath(csvDest)).size
-	const csvSha = await sha256File(csvDest)
-	report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`)
-
-	// MARK: Remove the ZIP to reclaim ~1 GB
-
-	await removePathIfPresent(zipDest)
-	report?.("  Removed ZIP (CSV kept)")
-
-	// MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
-
-	const manifest: SourceManifest = {
-		source_url: zipURL,
-		downloaded_at: new Date().toISOString(),
-		filename: csvName,
-		sha256: csvSha,
-		bytes: csvSize,
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/state-hi-schools.ts

@@ -1,161 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
- *   `state-hi-schools` adapter reads both worksheets directly.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
- *   before the workbook is recorded as fetched.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   workbook exists and its sha matches MANIFEST, skips download.
- */
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-import { XLSXSpliterator, type XLSXCellValue } from "spliterator"
-
-import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#adapters/state-hi-schools/workbook"
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
-const SLUG = "state-hi-schools"
-const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
-
-export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
-	/**
-	 * Workbook URL. Defaults to the Hawaii DOE source; overridable for an isolated fetch test.
-	 */
-	sourceURL?: string
-}
-
-interface Manifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-	notes: string
-}
-
-async function validateWorkbook(path: string): Promise<void> {
-	for (const sheet of STATE_HI_SCHOOL_SHEETS) {
-		const rows = await XLSXSpliterator.fromAsync<XLSXCellValue[]>(path, {
-			sheet,
-			header: false,
-			take: 2,
-		}).toArray()
-
-		const header = rows[0]
-
-		if (!header || rows.length < 2) {
-			throw new Error(`sheet ${sheet} must contain a header and at least one school row`)
-		}
-
-		// Exact casing is part of the adapter contract: object mode preserves the workbook's header names, so accepting
-		// `Code` here while the reader asks for `record.code` would validate a workbook whose every row is later skipped.
-		const columns = new Set(header.map((cell) => String(cell ?? "").trim()))
-
-		const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column))
-
-		if (missing.length) {
-			throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`)
-		}
-	}
-}
-
-export async function fetchStateHISchools(
-	options: FetchStateHISchoolsOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-
-	const xlsxDest = join(destDir, XLSX_FILENAME)
-	const manifestPath = join(destDir, "MANIFEST.json")
-	const sourceURL = options.sourceURL ?? SOURCE_URL
-
-	report?.(`=== ${SLUG}`)
-
-	// Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
-	if (await pathExists(xlsxDest)) {
-		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
-
-		if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
-			const actualSha = await sha256File(xlsxDest)
-
-			if (actualSha === recorded.sha256) {
-				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
-
-				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-			}
-		}
-	}
-
-	// Download XLSX.
-	report?.(`  Downloading ${sourceURL} ...`)
-
-	try {
-		await downloadToFile({
-			url: sourceURL,
-			dest: xlsxDest,
-			timeoutMs: 600_000,
-			headers: { "Accept-Encoding": "gzip, br" },
-			report,
-		})
-	} catch (error) {
-		report?.(`  ✗ Download failed (${(error as Error).message})`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const xlsxSize = (await statPath(xlsxDest)).size
-	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
-
-	if (xlsxSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
-		await removePathIfPresent(xlsxDest)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	try {
-		await validateWorkbook(xlsxDest)
-	} catch (error) {
-		report?.(`  ✗ Workbook validation failed (${(error as Error).message})`)
-		await removePathIfPresent(xlsxDest)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const xlsxSha = await sha256File(xlsxDest)
-
-	// Write MANIFEST.
-	const manifest: Manifest = {
-		source_url: sourceURL,
-		downloaded_at: new Date().toISOString(),
-		filename: XLSX_FILENAME,
-		sha256: xlsxSha,
-		bytes: xlsxSize,
-		notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(
-		`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`
-	)
-
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/state-sources.ts

@@ -1,146 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
-import { makeDirectories } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, writeManifest } from "#tools/fetch/download"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-export type FetchStateSourcesOptions = BaseFetchOptions
-
-interface Source {
-	slug: string
-	filename: string
-	url: string
-}
-
-const SOURCES: readonly Source[] = [
-	{
-		slug: "state-ny-notaries",
-		filename: "NY_Commissioned_Notaries.csv",
-		url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-tx-notaries",
-		filename: "TX_Notary_Public_Commissions.csv",
-		url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-de-notaries",
-		filename: "DE_Notaries_Commissioned.csv",
-		url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-or-notaries",
-		filename: "OR_Active_Notaries.csv",
-		url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-ia-contractors",
-		filename: "IA_Active_Construction_Contractor_Registrations.csv",
-		url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-wa-health-providers",
-		filename: "WA_Health_Care_Provider_Credential_Data.csv",
-		url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-hi-lobbyists",
-		filename: "HI_Lobbyist_Registration_Statements.csv",
-		url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
-	},
-]
-
-export async function fetchStateSources(
-	options: FetchStateSourcesOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	await makeDirectories(options.outRoot)
-
-	let fetched = 0
-	let failed = 0
-	const failedCodes: string[] = []
-
-	for (const { slug, filename, url } of SOURCES) {
-		const destDir = join(options.outRoot, slug)
-		await makeDirectories(destDir)
-		const dest = join(destDir, filename)
-
-		report?.(`=== ${slug} / ${filename}`)
-
-		let bytes: number
-
-		try {
-			;({ bytes } = await downloadToFile({
-				url,
-				dest,
-				timeoutMs: 600_000,
-				headers: { "Accept-Encoding": "gzip, br" },
-				report,
-			}))
-		} catch (error) {
-			report?.(`  ✗ download failed for ${url}: ${(error as Error).message}`)
-
-			failed++
-			failedCodes.push(slug)
-
-			continue
-		}
-
-		if (bytes < BYTES_PER_KIB) {
-			report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`)
-
-			failed++
-			failedCodes.push(slug)
-
-			continue
-		}
-
-		const sha = await sha256File(dest)
-
-		const manifest: SourceManifest = {
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			filename,
-			sha256: sha,
-			bytes,
-		}
-
-		await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-		fetched++
-	}
-
-	report?.(`=== summary ===`)
-	report?.(`fetched: ${fetched}`)
-	report?.(`failed:  ${failed}`)
-
-	return { fetched, skipped: 0, failed, failedCodes }
-}