@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/fetch/nad.ts

@@ -1,341 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
- *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter (#30).
- *   US Public Domain (17 U.S.C. § 105).
- *
- *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
- *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
- *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
- *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
- *   - Per-chunk manifest with sha256 + record count + error count
- *
- *   ## Source layout
- *
- *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
- *
- *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
- *       `--nad-url <presigned>` from a browser visit to
- *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
- *   - **`featureserver`** mode (default) pages the live FeatureService via OBJECTID ranges, writing
- *       NDJSON chunks into `<outRoot>/usgov-nad/featureserver/`.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
- *
- *   # Resume from an OID
- *   mailwoman corpus fetch nad --start-oid 34400001
- *
- *   # Increase concurrency on a fast link
- *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
- *   ```
- */
-
-import { APIClient, pluckResponseData } from "@mailwoman/core/api"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const SLUG = "usgov-nad"
-
-const FEATURE_SERVICE_URL =
-	"https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0"
-
-export interface FetchNADOptions extends BaseFetchOptions {
-	/**
-	 * Fetch strategy. Default `featureserver`.
-	 */
-	mode?: "featureserver" | "bulk"
-	/**
-	 * Pre-signed S3 URL for bulk mode.
-	 */
-	nadURL?: string
-	/**
-	 * Records per output file. Default `100000`.
-	 */
-	chunkSize?: number
-	/**
-	 * Records per HTTP request. Default `5000`.
-	 */
-	pageSize?: number
-	/**
-	 * Parallel page fetches within a chunk. Default `4`.
-	 */
-	concurrency?: number
-	/**
-	 * Start OBJECTID. Default `1`.
-	 */
-	startOID?: number
-	/**
-	 * Stop before this OID. Default = total count.
-	 */
-	endOID?: number
-}
-
-interface ChunkManifest {
-	source_url: string
-	oid_range: [number, number]
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-	record_count: number
-	page_errors: number
-	complete: boolean
-}
-
-/**
- * ArcGIS paged reads. Retry is ON: the loop walks OBJECTID ranges to completion, so one throttled page previously ended
- * a multi-hour national download. No rate budget — pages are requested one at a time and each assembles thousands of
- * records server-side.
- */
-const nadClient = new APIClient({
-	displayName: "nad",
-	retry: true,
-	axios: { headers: { "Accept-Encoding": "gzip, br" } },
-})
-
-async function fetchPage(startOID: number, endOID: number, pageSize: number): Promise<unknown[]> {
-	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
-	url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`)
-	url.searchParams.set("outFields", "*")
-	url.searchParams.set("f", "json")
-	url.searchParams.set("resultRecordCount", String(pageSize))
-
-	const data = await nadClient
-		.fetch<{ features?: Array<{ attributes: unknown }>; error?: { message: string } }>({
-			url: url.toString(),
-			timeout: 120_000,
-		})
-		.then(pluckResponseData)
-
-	if (data.error) throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`)
-
-	return (data.features ?? []).map((f) => f.attributes)
-}
-
-async function discoverTotalCount(): Promise<number> {
-	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
-	url.searchParams.set("where", "1=1")
-	url.searchParams.set("returnCountOnly", "true")
-	url.searchParams.set("f", "json")
-
-	const data = await nadClient
-		.fetch<{ count?: number }>({ url: url.toString(), timeout: 30_000 })
-		.then(pluckResponseData)
-
-	if (typeof data.count !== "number") throw new Error("NAD count query returned no count field")
-
-	return data.count
-}
-
-/**
- * Fetch a single chunk by paging through its OID range with bounded concurrency. Returns the count of records written
- * and the count of pages that errored. The caller decides whether to mark the chunk complete based on errors === 0.
- *
- * NOTE(phase1): this is a JSON API pager, not a file download — the shared `downloadToFile` doesn't apply here.
- */
-async function fetchChunk(
-	chunkPath: string,
-	chunkStart: number,
-	chunkEnd: number,
-	pageSize: number,
-	concurrency: number,
-	report?: (line: string) => void
-): Promise<{ recordCount: number; errors: number }> {
-	const pageRanges: Array<[number, number]> = []
-
-	for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
-		pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)])
-	}
-
-	// Run bounded-concurrency page fetches. Results indexed by page slot for in-order write.
-	const pageResults: Array<{ rows: unknown[]; error: Error | null }> = pageRanges.map(() => ({
-		rows: [],
-		error: null,
-	}))
-
-	let nextSlot = 0
-
-	const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
-		while (true) {
-			const slot = nextSlot++
-
-			if (slot >= pageRanges.length) return
-			const [s, e] = pageRanges[slot]!
-
-			try {
-				pageResults[slot]!.rows = await fetchPage(s, e, pageSize)
-			} catch (error) {
-				pageResults[slot]!.error = error as Error
-				report?.(`    ✗ page ${s}-${e}: ${(error as Error).message}`)
-			}
-		}
-	})
-
-	await Promise.all(workers)
-
-	// Single-writer phase — write all pages in OID order to keep NDJSON deterministic.
-	const lines: string[] = []
-	let errors = 0
-
-	for (const { rows, error } of pageResults) {
-		if (error) {
-			errors++
-
-			continue
-		}
-
-		for (const row of rows) {
-			lines.push(JSON.stringify(row))
-		}
-	}
-
-	await writeLocalTextFile(!lines.length ? "" : lines.join("\n") + "\n", chunkPath)
-
-	return { recordCount: lines.length, errors }
-}
-
-async function featureserverMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const chunkSize = options.chunkSize ?? 100_000
-	const pageSize = options.pageSize ?? 5000
-	const concurrency = options.concurrency ?? 4
-	const startOID = options.startOID ?? 1
-
-	const chunkDir = join(options.outRoot, SLUG, "featureserver")
-	await makeDirectories(chunkDir)
-
-	report?.(`=== ${SLUG} / featureserver`)
-	report?.(`  Discovering record count ...`)
-	const totalCount = await discoverTotalCount()
-	const endOID = options.endOID ?? totalCount
-	report?.(`  Total records: ${totalCount.toLocaleString()}`)
-	report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`)
-	report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`)
-
-	let fetched = 0
-	let skipped = 0
-	let totalRecords = 0
-	let totalErrors = 0
-	const failedCodes: string[] = []
-
-	for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
-		const chunkEnd = Math.min(cursor + chunkSize - 1, endOID)
-		const chunkName = `oids_${cursor}-${chunkEnd}`
-		const chunkPath = join(chunkDir, `${chunkName}.ndjson`)
-		const manifestPath = join(chunkDir, `${chunkName}.manifest.json`)
-
-		// Idempotency: skip a chunk only if it's marked complete (the bash version's bug was
-		// marking complete on partial-failure runs; we now only set complete after a clean fetch).
-		if (await pathExists(chunkPath)) {
-			const recorded = await readManifest<ChunkManifest>(manifestPath)
-
-			if (recorded?.complete) {
-				skipped++
-
-				continue
-			}
-		}
-
-		report?.(`  Fetching ${chunkName} ...`)
-		const t0 = Date.now()
-		const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report)
-		const elapsed = ((Date.now() - t0) / 1000).toFixed(1)
-		const bytes = (await statPath(chunkPath)).size
-		const sha = await sha256File(chunkPath)
-
-		const manifest: ChunkManifest = {
-			source_url: FEATURE_SERVICE_URL,
-			oid_range: [cursor, chunkEnd],
-			downloaded_at: new Date().toISOString(),
-			filename: `${chunkName}.ndjson`,
-			sha256: sha,
-			bytes,
-			record_count: recordCount,
-			page_errors: errors,
-			complete: errors === 0,
-		}
-
-		await writeManifest(manifestPath, manifest)
-
-		const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`
-
-		report?.(
-			`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`
-		)
-
-		fetched++
-		totalRecords += recordCount
-		totalErrors += errors
-
-		if (errors > 0) {
-			failedCodes.push(chunkName)
-		}
-	}
-
-	report?.(`=== featureserver summary ===`)
-	report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`)
-	report?.(`total records: ${totalRecords.toLocaleString()}`)
-	report?.(`page errors:   ${totalErrors}`)
-	report?.(`output:        ${chunkDir}`)
-
-	// `failed` counts page errors (the old exit-1 condition), `failedCodes` names the dirty chunks.
-	return { fetched, skipped, failed: totalErrors, failedCodes }
-}
-
-async function bulkMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	if (!options.nadURL) {
-		throw new Error(
-			`bulk mode requires --nad-url. The DOT page is Akamai-gated; ` +
-				`visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
-				`accept the disclaimer, and re-run with the pre-signed S3 URL.`
-		)
-	}
-
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip"
-	const destPath = join(destDir, filename)
-
-	report?.(`=== ${SLUG} / ${filename}`)
-	report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`)
-
-	const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report })
-
-	const sha = await sha256File(destPath)
-
-	await writeManifest(join(destDir, "MANIFEST.json"), {
-		source_url: options.nadURL,
-		downloaded_at: new Date().toISOString(),
-		filename,
-		sha256: sha,
-		bytes,
-	})
-
-	report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}
-
-export async function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const mode = options.mode ?? "featureserver"
-
-	if (mode === "featureserver") {
-		return featureserverMode(options, report)
-	}
-
-	if (mode === "bulk") {
-		return bulkMode(options, report)
-	}
-
-	throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`)
-}

package/lib/tools/fetch/nppes.ts

@@ -1,165 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data
- *   dissemination file. ~7M provider rows with venue+address data. Source for the `usgov-nppes`
- *   adapter. US Public Domain.
- *
- *   The file is published monthly by CMS. This module discovers the current filename by scraping the
- *   NPI_Files.html index, then downloads the ZIP and extracts only the main registry CSV
- *   (npidata_pfile_*.csv). The smaller endpoint/othername/pl files stay zipped — we don't need them.
- *
- *   Uses Node's built-in fetch (gzip/brotli) to parse the HTML index and download the ZIP, and
- *   streaming sha256 instead of sha256sum. The ZIP is unpacked with the `unzip` binary via
- *   `node:child_process` (no clean Node equivalent for member listing + selective extraction). NOTE:
- *   the old bash fetcher used `curl --continue-at -` to resume a partial download; native fetch has
- *   no resume, so a partial run re-downloads from the start.
- *
- *   Invoke via `mailwoman corpus fetch nppes --out-root <path>`. Idempotent: if dest CSV exists and
- *   sha256 matches MANIFEST, skips download.
- */
-
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-
-import { APIClient, pluckResponseData } from "@mailwoman/core/api"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
-import { sha256File } from "@mailwoman/core/hash"
-import { basename, join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html"
-const BASE_URL = "https://download.cms.gov/nppes"
-const SLUG = "usgov-nppes"
-
-export type FetchNPPESOptions = BaseFetchOptions
-
-/**
- * Scrape the NPI_Files.html index for the latest full monthly ZIP. Full-replacement files match
- * `NPPES_Data_Dissemination_<Month>_<Year>*.zip`; weekly files carry a `MMDDYY_MMDDYY` date range, which we exclude.
- */
-async function discoverLatestZip(): Promise<string | undefined> {
-	// `responseType: "text"` — the index is HTML, scraped by regex below.
-	const html = await new APIClient({
-		displayName: "nppes-index",
-		retry: true,
-		axios: { headers: { "Accept-Encoding": "gzip, br" } },
-	})
-		.fetch<string>({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
-		.then(pluckResponseData)
-
-	for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
-		const name = match[0]
-
-		if (name && !/\d{6}_\d{6}/.test(name)) return name
-	}
-
-	return undefined
-}
-
-/**
- * The main registry CSV (npidata_pfile_*.csv), which the archive also carries alongside a header file and a per-month
- * change file.
- */
-async function findNpidataCSV(zipPath: string): Promise<string | undefined> {
-	const entries = await listZipEntries(zipPath)
-
-	return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
-}
-
-export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-	report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`)
-
-	const zipFilename = await discoverLatestZip()
-
-	if (!zipFilename) {
-		report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const zipURL = `${BASE_URL}/${zipFilename}`
-	const zipDest = join(destDir, zipFilename)
-	report?.(`  Latest full file: ${zipFilename}`)
-
-	// Idempotency check: if the main CSV already exists and sha matches,
-	// skip re-download.
-	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
-
-	if (recorded?.sha256 && recorded.filename) {
-		const recordedPath = join(destDir, recorded.filename)
-
-		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
-			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
-
-			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-		}
-	}
-
-	// MARK: Download ZIP (large; 60-minute timeout)
-
-	report?.(`  Downloading ${zipURL} ...`)
-
-	const { bytes: zipSize } = await downloadToFile({
-		url: zipURL,
-		dest: zipDest,
-		timeoutMs: 3_600_000,
-		headers: { "Accept-Encoding": "gzip, br" },
-		report,
-	})
-
-	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
-
-	// MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
-
-	report?.("  Extracting npidata_pfile CSV from ZIP ...")
-	const csvName = await findNpidataCSV(zipDest)
-
-	if (!csvName) {
-		report?.("  ✗ Could not find npidata_pfile CSV inside ZIP")
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	report?.(`  Extracting: ${csvName}`)
-
-	const csvDest = join(destDir, basename(csvName))
-
-	await extractZipEntry(zipDest, csvName, csvDest)
-	const csvSize = (await statPath(csvDest)).size
-	const csvSha = await sha256File(csvDest)
-	report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`)
-
-	// MARK: Remove the ZIP to reclaim ~1 GB
-
-	await removePathIfPresent(zipDest)
-	report?.("  Removed ZIP (CSV kept)")
-
-	// MARK: Write MANIFEST (records the extracted CSV, not the ZIP)
-
-	const manifest: SourceManifest = {
-		source_url: zipURL,
-		downloaded_at: new Date().toISOString(),
-		filename: csvName,
-		sha256: csvSha,
-		bytes: csvSize,
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/lib/tools/fetch/state-hi-schools.ts

@@ -1,161 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory as its original XLSX workbook. The
- *   `state-hi-schools` adapter reads both worksheets directly.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
- *   before the workbook is recorded as fetched.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   workbook exists and its sha matches MANIFEST, skips download.
- */
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
-import { XLSXSpliterator, type XLSXCellValue } from "spliterator"
-
-import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#adapters/state-hi-schools/workbook"
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
-
-const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
-const SLUG = "state-hi-schools"
-const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
-
-export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
-	/**
-	 * Workbook URL. Defaults to the Hawaii DOE source; overridable for an isolated fetch test.
-	 */
-	sourceURL?: string
-}
-
-interface Manifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-	notes: string
-}
-
-async function validateWorkbook(path: string): Promise<void> {
-	for (const sheet of STATE_HI_SCHOOL_SHEETS) {
-		const rows = await XLSXSpliterator.fromAsync<XLSXCellValue[]>(path, {
-			sheet,
-			header: false,
-			take: 2,
-		}).toArray()
-
-		const header = rows[0]
-
-		if (!header || rows.length < 2) {
-			throw new Error(`sheet ${sheet} must contain a header and at least one school row`)
-		}
-
-		// Exact casing is part of the adapter contract: object mode preserves the workbook's header names, so accepting
-		// `Code` here while the reader asks for `record.code` would validate a workbook whose every row is later skipped.
-		const columns = new Set(header.map((cell) => String(cell ?? "").trim()))
-
-		const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column))
-
-		if (missing.length) {
-			throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`)
-		}
-	}
-}
-
-export async function fetchStateHISchools(
-	options: FetchStateHISchoolsOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	await makeDirectories(destDir)
-
-	const xlsxDest = join(destDir, XLSX_FILENAME)
-	const manifestPath = join(destDir, "MANIFEST.json")
-	const sourceURL = options.sourceURL ?? SOURCE_URL
-
-	report?.(`=== ${SLUG}`)
-
-	// Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
-	if (await pathExists(xlsxDest)) {
-		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
-
-		if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
-			const actualSha = await sha256File(xlsxDest)
-
-			if (actualSha === recorded.sha256) {
-				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
-
-				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-			}
-		}
-	}
-
-	// Download XLSX.
-	report?.(`  Downloading ${sourceURL} ...`)
-
-	try {
-		await downloadToFile({
-			url: sourceURL,
-			dest: xlsxDest,
-			timeoutMs: 600_000,
-			headers: { "Accept-Encoding": "gzip, br" },
-			report,
-		})
-	} catch (error) {
-		report?.(`  ✗ Download failed (${(error as Error).message})`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const xlsxSize = (await statPath(xlsxDest)).size
-	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
-
-	if (xlsxSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
-		await removePathIfPresent(xlsxDest)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	try {
-		await validateWorkbook(xlsxDest)
-	} catch (error) {
-		report?.(`  ✗ Workbook validation failed (${(error as Error).message})`)
-		await removePathIfPresent(xlsxDest)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const xlsxSha = await sha256File(xlsxDest)
-
-	// Write MANIFEST.
-	const manifest: Manifest = {
-		source_url: sourceURL,
-		downloaded_at: new Date().toISOString(),
-		filename: XLSX_FILENAME,
-		sha256: xlsxSha,
-		bytes: xlsxSize,
-		notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(
-		`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`
-	)
-
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}