@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/us/tools/fetch/nad.ts

@@ -0,0 +1,359 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the US DOT National Address Database (NAD) — ~97 million structured US address-point
+ *   records aggregated from state and local authorities. Source for the `usgov-nad` adapter.
+ *   US Public Domain (17 U.S.C. § 105).
+ *
+ *   - Bounded per-chunk page concurrency (4× speedup at safe pressure)
+ *   - 5× larger page size (5 000 vs. the old bash fetcher's 2 000) — fewer round-trips per chunk
+ *   - Honest `complete: true` flag: only set when every page in the chunk fetched cleanly
+ *   - Built-in fetch with gzip/brotli decompression (no curl + Python subprocess tax)
+ *   - Per-chunk manifest with sha256 + record count + error count
+ *
+ *   ## Source layout
+ *
+ *   The ArcGIS FeatureServer is the only fully-automated path. As of 2026-05:
+ *
+ *   - **`bulk`** mode requires a pre-signed S3 URL (Akamai blocks scripted curl on the DOT page). Pass
+ *       `--nad-url <presigned>` from a browser visit to
+ *       [https://www.transportation.gov/gis/national-address-database](https://www.transportation.gov/gis/national-address-database).
+ *   - **`featureserver`** mode (default) pages the live FeatureService via objectid ranges, writing
+ *       ndjson chunks into `<outRoot>/usgov-nad/featureserver/`.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus fetch nad --out-root $MAILWOMAN_DATA_ROOT/corpus/sources
+ *
+ *   # Resume from an OID
+ *   mailwoman corpus fetch nad --start-oid 34400001
+ *
+ *   # Increase concurrency on a fast link
+ *   mailwoman corpus fetch nad --concurrency 8 --page-size 10000
+ *   ```
+ */
+
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { stringifyJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+const SLUG = "usgov-nad"
+
+const FEATURE_SERVICE_URL =
+	"https://services.arcgis.com/xOi1kZaI0eWDREZv/ArcGIS/rest/services/Address_Points_from_National_Address_Database_view/FeatureServer/0"
+
+export interface FetchNADOptions extends BaseFetchOptions {
+	/**
+	 * Fetch strategy.
+	 *
+	 * Default `featureserver`.
+	 */
+	mode?: "featureserver" | "bulk"
+	/**
+	 * Pre-signed S3 URL for bulk mode.
+	 */
+	nadURL?: string
+	/**
+	 * Records per output file.
+	 *
+	 * Default `100000`.
+	 */
+	chunkSize?: number
+	/**
+	 * Records per http request.
+	 *
+	 * Default `5000`.
+	 */
+	pageSize?: number
+	/**
+	 * Parallel page fetches within a chunk.
+	 *
+	 * Default `4`.
+	 */
+	concurrency?: number
+	/**
+	 * Start objectid.
+	 *
+	 * Default `1`.
+	 */
+	startOID?: number
+	/**
+	 * Stop before this OID.
+	 *
+	 * Default = total count.
+	 */
+	endOID?: number
+}
+
+interface ChunkManifest {
+	source_url: string
+	oid_range: [number, number]
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+	record_count: number
+	page_errors: number
+	complete: boolean
+}
+
+/**
+ * ArcGIS paged reads.
+ *
+ * Retry is on: the loop walks objectid ranges to completion, since one throttled
+ * page can otherwise end a multi-hour national download.
+ * No rate budget — pages are requested one at a time and each assembles thousands of records server-side.
+ */
+const nadClient = new APIClient({
+	displayName: "nad",
+	retry: true,
+	axios: { headers: { "Accept-Encoding": "gzip, br" } },
+})
+
+async function fetchPage(startOID: number, endOID: number, pageSize: number): Promise<unknown[]> {
+	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
+	url.searchParams.set("where", `OBJECTID BETWEEN ${startOID} AND ${endOID}`)
+	url.searchParams.set("outFields", "*")
+	url.searchParams.set("f", "json")
+	url.searchParams.set("resultRecordCount", String(pageSize))
+
+	const data = await nadClient
+		.fetch<{ features?: Array<{ attributes: unknown }>; error?: { message: string } }>({
+			url: url.toString(),
+			timeout: 120_000,
+		})
+		.then(pluckResponseData)
+
+	if (data.error) throw new Error(`ArcGIS error on OID ${startOID}-${endOID}: ${data.error.message}`)
+
+	return (data.features ?? []).map((f) => f.attributes)
+}
+
+async function discoverTotalCount(): Promise<number> {
+	const url = new URL(`${FEATURE_SERVICE_URL}/query`)
+	url.searchParams.set("where", "1=1")
+	url.searchParams.set("returnCountOnly", "true")
+	url.searchParams.set("f", "json")
+
+	const data = await nadClient
+		.fetch<{ count?: number }>({ url: url.toString(), timeout: 30_000 })
+		.then(pluckResponseData)
+
+	if (typeof data.count !== "number") throw new Error("NAD count query returned no count field")
+
+	return data.count
+}
+
+/**
+ * Fetch a single chunk by paging through its OID range with bounded concurrency.
+ *
+ * Returns the count of records written and the count of pages that errored.
+ * The caller decides whether to mark the chunk complete based on errors === 0.
+ *
+ * Note(phase1): this is a JSON API pager rather than a file download —
+ * the shared `downloadToFile` doesn't apply here.
+ */
+async function fetchChunk(
+	chunkPath: PathBuilderLike,
+	chunkStart: number,
+	chunkEnd: number,
+	pageSize: number,
+	concurrency: number,
+	report?: (line: string) => void
+): Promise<{ recordCount: number; errors: number }> {
+	const pageRanges: Array<[number, number]> = []
+
+	for (let cursor = chunkStart; cursor <= chunkEnd; cursor += pageSize) {
+		pageRanges.push([cursor, Math.min(cursor + pageSize - 1, chunkEnd)])
+	}
+
+	// Results indexed by page slot for in-order write.
+	const pageResults: Array<{ rows: unknown[]; error: Error | null }> = pageRanges.map(() => ({
+		rows: [],
+		error: null,
+	}))
+
+	let nextSlot = 0
+
+	const workers = Array.from({ length: Math.min(concurrency, pageRanges.length) }, async () => {
+		while (true) {
+			const slot = nextSlot++
+
+			if (slot >= pageRanges.length) return
+			const [s, e] = pageRanges[slot]!
+
+			try {
+				pageResults[slot]!.rows = await fetchPage(s, e, pageSize)
+			} catch (error) {
+				pageResults[slot]!.error = error as Error
+				report?.(`    ✗ page ${s}-${e}: ${(error as Error).message}`)
+			}
+		}
+	})
+
+	await Promise.all(workers)
+
+	// Single-writer phase — write all pages in OID order to keep ndjson deterministic.
+	const lines: string[] = []
+	let errors = 0
+
+	for (const { rows, error } of pageResults) {
+		if (error) {
+			errors++
+
+			continue
+		}
+
+		for (const row of rows) {
+			lines.push(stringifyJSON(row))
+		}
+	}
+
+	await writeLocalTextFile(lines, chunkPath)
+
+	return { recordCount: lines.length, errors }
+}
+
+async function featureserverMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const chunkSize = options.chunkSize ?? 100_000
+	const pageSize = options.pageSize ?? 5000
+	const concurrency = options.concurrency ?? 4
+	const startOID = options.startOID ?? 1
+
+	const chunkDir = options.outRoot(SLUG, "featureserver")
+	await makeDirectories(chunkDir)
+
+	report?.(`=== ${SLUG} / featureserver`)
+	report?.(`  Discovering record count ...`)
+	const totalCount = await discoverTotalCount()
+	const endOID = options.endOID ?? totalCount
+	report?.(`  Total records: ${totalCount.toLocaleString()}`)
+	report?.(`  OID range: ${startOID.toLocaleString()} .. ${endOID.toLocaleString()}`)
+	report?.(`  Chunk size: ${chunkSize}, page size: ${pageSize}, concurrency: ${concurrency}`)
+
+	let fetched = 0
+	let skipped = 0
+	let totalRecords = 0
+	let totalErrors = 0
+	const failedCodes: string[] = []
+
+	for (let cursor = startOID; cursor <= endOID; cursor += chunkSize) {
+		const chunkEnd = Math.min(cursor + chunkSize - 1, endOID)
+		const chunkName = `oids_${cursor}-${chunkEnd}`
+		const chunkPath = chunkDir(`${chunkName}.ndjson`)
+		const manifestPath = chunkDir(`${chunkName}.manifest.json`)
+
+		// Idempotency: skip a chunk only when its manifest is marked complete.
+		// Complete is set only after a clean fetch.
+		if (await pathExists(chunkPath)) {
+			const recorded = await readManifest<ChunkManifest>(manifestPath)
+
+			if (recorded?.complete) {
+				skipped++
+
+				continue
+			}
+		}
+
+		report?.(`  Fetching ${chunkName} ...`)
+		const t0 = Date.now()
+		const { recordCount, errors } = await fetchChunk(chunkPath, cursor, chunkEnd, pageSize, concurrency, report)
+		const elapsed = ((Date.now() - t0) / 1000).toFixed(1)
+		const bytes = (await statPath(chunkPath)).size
+		const sha = await sha256File(chunkPath)
+
+		const manifest: ChunkManifest = {
+			source_url: FEATURE_SERVICE_URL,
+			oid_range: [cursor, chunkEnd],
+			downloaded_at: new Date().toISOString(),
+			filename: `${chunkName}.ndjson`,
+			sha256: sha,
+			bytes,
+			record_count: recordCount,
+			page_errors: errors,
+			complete: errors === 0,
+		}
+
+		await writeManifest(manifestPath, manifest)
+
+		const status = errors === 0 ? "✓" : `⚠ ${errors} page errors`
+
+		report?.(
+			`    ${status}  ${recordCount.toLocaleString()} records in ${elapsed}s  (${(bytes / 1024 / 1024).toFixed(1)} MB)`
+		)
+
+		fetched++
+		totalRecords += recordCount
+		totalErrors += errors
+
+		if (errors > 0) {
+			failedCodes.push(chunkName)
+		}
+	}
+
+	report?.(`=== featureserver summary ===`)
+	report?.(`chunks fetched: ${fetched}  skipped: ${skipped}`)
+	report?.(`total records: ${totalRecords.toLocaleString()}`)
+	report?.(`page errors:   ${totalErrors}`)
+	report?.(`output:        ${chunkDir}`)
+
+	// `failed` counts page errors and `failedCodes` records the dirty chunks.
+	return { fetched, skipped, failed: totalErrors, failedCodes }
+}
+
+async function bulkMode(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	if (!options.nadURL) {
+		throw new Error(
+			`bulk mode requires --nad-url. Akamai protects the DOT page; ` +
+				`visit https://www.transportation.gov/gis/national-address-database in a browser, ` +
+				`accept the disclaimer, and re-run with the pre-signed S3 URL.`
+		)
+	}
+
+	const destDir = options.outRoot(SLUG)
+	await makeDirectories(destDir)
+	const filename = new URL(options.nadURL).pathname.split("/").pop() ?? "NAD.zip"
+	const destPath = destDir(filename)
+
+	report?.(`=== ${SLUG} / ${filename}`)
+	report?.(`  URL: ${options.nadURL.slice(0, 100)}${options.nadURL.length > 100 ? "…" : ""}`)
+
+	const { bytes } = await downloadToFile({ url: options.nadURL, dest: destPath, timeoutMs: 3 * 3600 * 1000, report })
+
+	const sha = await sha256File(destPath)
+
+	await writeManifest(destDir("MANIFEST.json"), {
+		source_url: options.nadURL,
+		downloaded_at: new Date().toISOString(),
+		filename,
+		sha256: sha,
+		bytes,
+	})
+
+	report?.(`  ✓ ${(bytes / 1024 / 1024 / 1024).toFixed(2)} GB  sha256=${sha}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+export async function fetchNAD(options: FetchNADOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const mode = options.mode ?? "featureserver"
+
+	if (mode === "featureserver") {
+		return featureserverMode(options, report)
+	}
+
+	if (mode === "bulk") {
+		return bulkMode(options, report)
+	}
+
+	throw new Error(`unknown mode "${String(mode)}" (expected featureserver|bulk)`)
+}

package/lib/us/tools/fetch/nppes.ts

@@ -0,0 +1,153 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Re-fetch the NPPES (National Plan and Provider Enumeration System) full monthly data dissemination
+ * file for the `usgov-nppes` adapter. The source is US Public Domain. Discovers the current filename
+ * by scraping the NPI_Files.html index, then extracts only the main registry CSV
+ * (`npidata_pfile_*.csv`) and leaves the endpoint/othername/pl files zipped.
+ *
+ * Native fetch has no resume, so a partial run re-downloads from the start.
+ */
+
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
+
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { basename, type PathBuilderLike } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+const INDEX_URL = "https://download.cms.gov/nppes/NPI_Files.html"
+const BASE_URL = "https://download.cms.gov/nppes"
+const SLUG = "usgov-nppes"
+
+export type FetchNPPESOptions = BaseFetchOptions
+
+/**
+ * Scrape the NPI_Files.html index for the latest full monthly ZIP: full-replacement
+ * files match `NPPES_Data_Dissemination_<Month>_<Year>*.zip`, while weekly files
+ * include a `MMDDYY_MMDDYY` date range and are excluded.
+ */
+async function discoverLatestZip(): Promise<string | undefined> {
+	const html = await new APIClient({
+		displayName: "nppes-index",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
+	})
+		.fetch<string>({ url: INDEX_URL, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
+
+	for (const match of html.matchAll(/NPPES_Data_Dissemination_[A-Za-z]+_\d{4}[^"]*\.zip/g)) {
+		const name = match[0]
+
+		if (name && !/\d{6}_\d{6}/.test(name)) return name
+	}
+
+	return undefined
+}
+
+/**
+ * The main registry CSV (`npidata_pfile_*.csv`); the archive also includes a
+ * header file and a per-month change file.
+ */
+async function findNpidataCSV(zipPath: PathBuilderLike): Promise<string | undefined> {
+	const entries = await listZipEntries(zipPath)
+
+	return entries.find((entry) => /npidata_pfile\S+\.csv/i.test(entry.name))?.name
+}
+
+export async function fetchNPPES(options: FetchNPPESOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const destDir = options.outRoot(SLUG)
+	await makeDirectories(destDir)
+	const manifestPath = destDir("MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+	report?.(`  Discovering latest full-replacement ZIP from ${INDEX_URL} ...`)
+
+	const zipFilename = await discoverLatestZip()
+
+	if (!zipFilename) {
+		report?.(`  ✗ Could not discover ZIP filename from ${INDEX_URL}`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const zipURL = `${BASE_URL}/${zipFilename}`
+	const zipDest = destDir(zipFilename)
+	report?.(`  Latest full file: ${zipFilename}`)
+
+	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
+
+	if (recorded?.sha256 && recorded.filename) {
+		const recordedPath = destDir(recorded.filename)
+
+		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
+
+			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+		}
+	}
+
+	// MARK: Download ZIP
+
+	report?.(`  Downloading ${zipURL} ...`)
+
+	const { bytes: zipSize } = await downloadToFile({
+		url: zipURL,
+		dest: zipDest,
+		timeoutMs: 3_600_000,
+		headers: { "Accept-Encoding": "gzip, br" },
+		report,
+	})
+
+	report?.(`  Downloaded: ${(zipSize / 1024 / 1024).toFixed(1)} MB`)
+
+	// MARK: Extract only the main registry CSV (npidata_pfile_*.csv)
+
+	report?.("  Extracting npidata_pfile CSV from ZIP ...")
+	const csvName = await findNpidataCSV(zipDest)
+
+	if (!csvName) {
+		report?.("  ✗ Could not find npidata_pfile CSV inside ZIP")
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	report?.(`  Extracting: ${csvName}`)
+
+	const csvDest = destDir(basename(csvName))
+
+	await extractZipEntry(zipDest, csvName, csvDest)
+	const csvSize = (await statPath(csvDest)).size
+	const csvSha = await sha256File(csvDest)
+	report?.(`  CSV size: ${(csvSize / 1024 / 1024).toFixed(1)} MB`)
+
+	// MARK: Remove the ZIP
+
+	await removePathIfPresent(zipDest)
+	report?.("  Removed ZIP (CSV kept)")
+
+	// MARK: Write manifest for extracted CSV
+
+	const manifest: SourceManifest = {
+		source_url: zipURL,
+		downloaded_at: new Date().toISOString(),
+		filename: csvName,
+		sha256: csvSha,
+		bytes: csvSize,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${(csvSize / 1024 / 1024).toFixed(1)} MB  sha256=${csvSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/lib/us/tools/fetch/state/hi-schools.ts

@@ -0,0 +1,167 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory as its original xlsx workbook. The
+ *   `state-hi-schools` adapter reads both worksheets directly.
+ *
+ *   Upstream is a single xlsx (~64 KB) with two sheets — `hidoe` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets are validated against the adapter's required columns
+ *   before the workbook is recorded as fetched.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   workbook exists and its sha matches manifest, skips download.
+ */
+
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { statPath, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import type { PathBuilderLike } from "path-ts"
+import { XLSXSpliterator, type XLSXCellValue } from "spliterator"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+import { STATE_HI_SCHOOL_REQUIRED_COLUMNS, STATE_HI_SCHOOL_SHEETS } from "#us/adapters/state/hi-schools/workbook"
+
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
+const SLUG = "state-hi-schools"
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
+
+export interface FetchStateHISchoolsOptions extends BaseFetchOptions {
+	/**
+	 * Workbook URL.
+	 *
+	 * Defaults to the Hawaii DOE source.
+	 * Overridable for an isolated fetch test.
+	 */
+	sourceURL?: string
+}
+
+interface Manifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+	notes: string
+}
+
+async function validateWorkbook(path: PathBuilderLike): Promise<void> {
+	for (const sheet of STATE_HI_SCHOOL_SHEETS) {
+		// XLSXSpliterator opens a string, URL, or byte source.
+		// A PathBuilder is none of those.
+		const rows = await XLSXSpliterator.fromAsync<XLSXCellValue[]>(path.toString(), {
+			sheet,
+			header: false,
+			take: 2,
+		}).toArray()
+
+		const header = rows[0]
+
+		if (!header || rows.length < 2) {
+			throw new Error(`sheet ${sheet} must contain a header and at least one school row`)
+		}
+
+		// Exact casing is part of the adapter interface: object mode preserves the workbook's
+		// header names, so accepting `Code` here while the reader asks for `record.code`
+		// would validate a workbook whose every row is later skipped.
+		const columns = new Set(header.map((cell) => String(cell ?? "").trim()))
+
+		const missing = STATE_HI_SCHOOL_REQUIRED_COLUMNS.filter((column) => !columns.has(column))
+
+		if (missing.length) {
+			throw new Error(`sheet ${sheet} is missing required columns: ${missing.join(", ")}`)
+		}
+	}
+}
+
+export async function fetchStateHISchools(
+	options: FetchStateHISchoolsOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = options.outRoot(SLUG)
+	await makeDirectories(destDir)
+
+	const xlsxDest = destDir(XLSX_FILENAME)
+	const manifestPath = destDir("MANIFEST.json")
+	const sourceURL = options.sourceURL ?? SOURCE_URL
+
+	report?.(`=== ${SLUG}`)
+
+	// Idempotency: skip if the source workbook exists and sha matches the recorded manifest.
+	if (await pathExists(xlsxDest)) {
+		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
+
+		if (recorded?.sha256 && recorded.filename === XLSX_FILENAME) {
+			const actualSha = await sha256File(xlsxDest)
+
+			if (actualSha === recorded.sha256) {
+				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
+
+				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+			}
+		}
+	}
+
+	// Download xlsx.
+	report?.(`  Downloading ${sourceURL} ...`)
+
+	try {
+		await downloadToFile({
+			url: sourceURL,
+			dest: xlsxDest,
+			timeoutMs: 600_000,
+			headers: { "Accept-Encoding": "gzip, br" },
+			report,
+		})
+	} catch (error) {
+		report?.(`  ✗ Download failed (${(error as Error).message})`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const xlsxSize = (await statPath(xlsxDest)).size
+	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
+
+	if (xlsxSize < BYTES_PER_KIB) {
+		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
+		await removePathIfPresent(xlsxDest)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	try {
+		await validateWorkbook(xlsxDest)
+	} catch (error) {
+		report?.(`  ✗ Workbook validation failed (${(error as Error).message})`)
+		await removePathIfPresent(xlsxDest)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const xlsxSha = await sha256File(xlsxDest)
+
+	// Write manifest.
+	const manifest: Manifest = {
+		source_url: sourceURL,
+		downloaded_at: new Date().toISOString(),
+		filename: XLSX_FILENAME,
+		sha256: xlsxSha,
+		bytes: xlsxSize,
+		notes: "Original XLSX; the adapter reads the HIDOE and PCS sheets directly.",
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(
+		`  ✓ ${STATE_HI_SCHOOL_SHEETS.join(" + ")} validated · ${ByteFormatter.formatIEC(xlsxSize)}  sha256=${xlsxSha}`
+	)
+
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}