@mailwoman/corpus 9.2.0 → 9.4.0

package/src/tools/fetch/ppd.ts

@@ -1,137 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   HM Land Registry Price Paid Data → OA-shaped GB tuples CSV for the `locale` shard recipe.
- *
- *   PPD is E&W-only, ALL-CAPS, and column-structured (no header row). We emit the exact OA header
- *   `readTuples` (`shard-recipes/locale.ts`) indexes by name, mapped for `districtAsLocality: true`:
- *   CITY = PPD locality (dependent locality; blank when it merely repeats the town — 1995-era rows
- *   pad locality=town), DISTRICT = PPD post town, REGION = county. SAON (flat/unit) rows and
- *   building-name PAONs are skipped in v1 and counted — `LocaleBaseTuple` has no unit field yet.
- *
- *   Snapshot provenance: `$MAILWOMAN_DATA_ROOT/ppd/<date>/pp-complete.csv` (md5 sibling).
- *   License: OGL v3 (attribution: HM Land Registry).
- */
-import { createReadStream, createWriteStream } from "node:fs"
-import { parseArgs } from "node:util"
-
-import { runIfScript } from "@mailwoman/core/scripting"
-import { dataRootPath } from "@mailwoman/core/utils"
-import { CSVSpliterator } from "spliterator"
-
-import { titleCaseGB } from "../gb-title-case.ts"
-
-const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/
-
-export interface PPDExtractStats {
-	kept: number
-	skippedSAON: number
-	skippedPAON: number
-	skippedNoStreet: number
-	skippedNoPostcode: number
-}
-
-const quote = (value: string): string => (value ? `"${value.replaceAll('"', '""')}"` : "")
-
-/**
- * Convert PPD rows (`id,price,date,postcode,type,new,tenure,PAON,SAON,street,locality,town,district,county,cat,status`
- * — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing. Accepts a plain array
- * of rows (tests) or a streamed async source (the real 31M-row extraction).
- */
-export async function extractPPDTuples(
-	input: AsyncIterable<string[]> | Iterable<string[]>,
-	write: (line: string) => void
-): Promise<PPDExtractStats> {
-	const stats: PPDExtractStats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 }
-	write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE")
-
-	for await (const cells of input) {
-		// HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
-		const postcode = cells[3]
-		const paon = cells[7]
-		const saon = cells[8]
-		const street = cells[9]
-		const locality = cells[10]
-		const town = cells[11]
-		const county = cells[13]
-
-		if (saon) {
-			stats.skippedSAON++
-
-			continue
-		}
-
-		if (!paon || !HOUSE_NUMBER_PATTERN.test(paon)) {
-			stats.skippedPAON++
-
-			continue
-		}
-
-		if (!street) {
-			stats.skippedNoStreet++
-
-			continue
-		}
-
-		if (!postcode) {
-			stats.skippedNoPostcode++
-
-			continue
-		}
-
-		const number = paon.replace(/\s*-\s*/, "-")
-		const city = locality && locality !== town ? titleCaseGB(locality) : ""
-
-		write(
-			[
-				number,
-				quote(titleCaseGB(street)),
-				quote(city),
-				quote(titleCaseGB(town ?? "")),
-				quote(titleCaseGB(county ?? "")),
-				postcode,
-			].join(",")
-		)
-
-		stats.kept++
-	}
-
-	return stats
-}
-
-/**
- * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
- */
-async function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats> {
-	// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
-	// shard-recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
-	const rows = CSVSpliterator.fromAsync<string[]>(createReadStream(inputPath), {
-		mode: "array",
-		header: false,
-		enableQuoteHandling: true,
-	})
-
-	const out = createWriteStream(outputPath, { encoding: "utf8" })
-	const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"))
-
-	await new Promise<void>((res) => {
-		out.end(res)
-	})
-
-	return stats
-}
-
-runIfScript(import.meta, async () => {
-	const { values } = parseArgs({
-		options: {
-			input: { type: "string", default: dataRootPath("ppd", "2026-07-22", "pp-complete.csv") },
-			output: { type: "string", default: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv") },
-		},
-	})
-
-	const stats = await runPPDExtract(values.input!, values.output!)
-
-	console.log(`[ppd] ${JSON.stringify(stats)}`)
-})

package/src/tools/fetch/state-hi-schools.ts

@@ -1,208 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
- *   `state-hi-schools` adapter can consume.
- *
- *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
- *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
- *   under one shared header so the adapter can stream a single CSV.
- *
- *   License: Hawaii state government open data (Tier A — state PD-equivalent).
- *
- *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
- *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
- *   adding a workbook-parsing dependency.
- *
- *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
- *   CSV exists and sha matches MANIFEST, skips download.
- */
-
-import { spawn, spawnSync } from "node:child_process"
-import { existsSync, mkdirSync, statSync } from "node:fs"
-import { unlink } from "node:fs/promises"
-import { join } from "node:path"
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, readManifest, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
-const SLUG = "state-hi-schools"
-const CSV_FILENAME = "HI_Public_Schools_List.csv"
-const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
-
-export type FetchStateHISchoolsOptions = BaseFetchOptions
-
-/**
- * The XLSX → CSV converter: concatenate every sheet under one shared header (the first sheet's). Runs as `python3 -c
- * <script> <xlsx-path> <csv-path>`, so `sys.argv[1]`/`sys.argv[2]` are the I/O paths. TODO: Get rid of this.
- */
-const PY_CONVERT = `
-import csv
-import sys
-from openpyxl import load_workbook
-
-xlsx_path, csv_path = sys.argv[1], sys.argv[2]
-wb = load_workbook(xlsx_path, data_only=True, read_only=True)
-
-with open(csv_path, "w", newline="", encoding="utf-8") as out:
-    writer = csv.writer(out)
-    shared_header = None
-    total_data_rows = 0
-    for sheet_name in wb.sheetnames:
-        ws = wb[sheet_name]
-        rows = ws.iter_rows(values_only=True)
-        try:
-            header = next(rows)
-        except StopIteration:
-            continue
-        norm_header = ["" if v is None else str(v).strip() for v in header]
-        if shared_header is None:
-            shared_header = norm_header
-            writer.writerow(shared_header)
-        elif norm_header != shared_header:
-            print(
-                f"  ! sheet '{sheet_name}' header diverges from shared header; concatenating anyway",
-                file=sys.stderr,
-            )
-        for row in rows:
-            if row is None:
-                continue
-            # Skip fully-empty rows (XLSX iter_rows can yield phantom trailing rows).
-            if all(v is None or (isinstance(v, str) and not v.strip()) for v in row):
-                continue
-            writer.writerow(["" if v is None else str(v).strip() for v in row])
-            total_data_rows += 1
-
-print(f"  converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
-`
-
-interface Manifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-	notes: string
-}
-
-/**
- * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
- * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
- */
-async function convertXLSXToCSV(xlsxPath: string, csvPath: string): Promise<void> {
-	const child = spawn("python3", ["-c", PY_CONVERT, xlsxPath, csvPath], {
-		stdio: ["ignore", "inherit", "inherit"],
-	})
-
-	await new Promise<void>((resolve, reject) => {
-		child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`python3 converter exited with ${code}`))))
-		child.on("error", reject)
-	})
-}
-
-export async function fetchStateHISchools(
-	options: FetchStateHISchoolsOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
-	mkdirSync(destDir, { recursive: true })
-
-	const xlsxDest = join(destDir, XLSX_FILENAME)
-	const csvDest = join(destDir, CSV_FILENAME)
-	const manifestPath = join(destDir, "MANIFEST.json")
-
-	report?.(`=== ${SLUG}`)
-
-	// Idempotency: skip if CSV exists and sha matches recorded MANIFEST.
-	if (existsSync(csvDest)) {
-		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
-
-		if (recorded?.sha256 && recorded.filename === CSV_FILENAME) {
-			const actualSha = await sha256File(csvDest)
-
-			if (actualSha === recorded.sha256) {
-				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
-
-				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
-			}
-		}
-	}
-
-	// Preflight: openpyxl must be importable.
-	const preflight = spawnSync("python3", ["-c", "import openpyxl"], { stdio: "ignore" })
-
-	if (preflight.status !== 0) {
-		report?.(
-			`  ✗ python3 with the \`openpyxl\` package is required to convert the HIDOE XLSX.\n` +
-				`    Debian/Ubuntu:  sudo apt-get install -y python3-openpyxl\n` +
-				`    macOS Homebrew: brew install python && pip3 install openpyxl`
-		)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	// Download XLSX.
-	report?.(`  Downloading ${SOURCE_URL} ...`)
-
-	try {
-		await downloadToFile({
-			url: SOURCE_URL,
-			dest: xlsxDest,
-			timeoutMs: 600_000,
-			headers: { "Accept-Encoding": "gzip, br" },
-			report,
-		})
-	} catch (error) {
-		report?.(`  ✗ Download failed (${(error as Error).message})`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	const xlsxSize = statSync(xlsxDest).size
-	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
-
-	if (xlsxSize < BYTES_PER_KIB) {
-		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
-
-		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
-	}
-
-	// Convert XLSX → CSV (concatenate both sheets under one shared header).
-	report?.(`  Converting XLSX → CSV (concatenating sheets) ...`)
-	await convertXLSXToCSV(xlsxDest, csvDest)
-
-	const csvSize = statSync(csvDest).size
-	const csvSha = await sha256File(csvDest)
-
-	// Remove XLSX (CSV is the canonical artifact the adapter consumes).
-	await unlink(xlsxDest)
-	report?.(`  Removed XLSX (CSV kept)`)
-
-	// Write MANIFEST.
-	const manifest: Manifest = {
-		source_url: SOURCE_URL,
-		downloaded_at: new Date().toISOString(),
-		filename: CSV_FILENAME,
-		sha256: csvSha,
-		bytes: csvSize,
-		notes: "Converted from XLSX (sheets HIDOE + PCS concatenated under shared header).",
-	}
-
-	await writeManifest(manifestPath, manifest)
-
-	report?.(`  ✓ ${ByteFormatter.formatIEC(csvSize)}  sha256=${csvSha}`)
-	report?.(`  MANIFEST written to ${manifestPath}`)
-
-	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
-}

package/src/tools/fetch/state-sources.ts

@@ -1,155 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Re-fetch the state-level open-data sources (NY/TX/DE/OR notaries, IA contractors, WA health
- *   providers, HI lobbyists). Reproducible recovery if `$MAILWOMAN_DATA_ROOT` is lost.
- *
- *   HI public schools is fetched separately by `mailwoman corpus fetch state-hi-schools` — its
- *   upstream is an XLSX workbook that requires an openpyxl-driven sheet-concatenation pre-step
- *   before the adapter can consume it.
- *
- *   Each source lands in its own subdirectory of `<outRoot>/<slug>/` along with a `MANIFEST.json`
- *   recording origin URL + download timestamp + sha256 so downstream adapters can verify provenance.
- *
- *   Invoke via `mailwoman corpus fetch state-sources --out-root <path>`. Uses Node's built-in fetch
- *   (gzip/brotli) and streaming sha256 instead of curl + sha256sum.
- */
-
-import { mkdirSync } from "node:fs"
-import { join } from "node:path"
-
-import { BYTES_PER_KIB } from "@mailwoman/core/fs/utils"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-export type FetchStateSourcesOptions = BaseFetchOptions
-
-interface Source {
-	slug: string
-	filename: string
-	url: string
-}
-
-const SOURCES: readonly Source[] = [
-	{
-		slug: "state-ny-notaries",
-		filename: "NY_Commissioned_Notaries.csv",
-		url: "https://data.ny.gov/api/views/rwbv-mz6z/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-tx-notaries",
-		filename: "TX_Notary_Public_Commissions.csv",
-		url: "https://data.texas.gov/api/views/gmd3-bnrd/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-de-notaries",
-		filename: "DE_Notaries_Commissioned.csv",
-		url: "https://data.delaware.gov/api/views/q8dr-mj6p/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-or-notaries",
-		filename: "OR_Active_Notaries.csv",
-		url: "https://data.oregon.gov/api/views/j2pk-zk6z/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-ia-contractors",
-		filename: "IA_Active_Construction_Contractor_Registrations.csv",
-		url: "https://data.iowa.gov/api/views/dpf3-iz94/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-wa-health-providers",
-		filename: "WA_Health_Care_Provider_Credential_Data.csv",
-		url: "https://data.wa.gov/api/views/qxh8-f4bd/rows.csv?accessType=DOWNLOAD",
-	},
-	{
-		slug: "state-hi-lobbyists",
-		filename: "HI_Lobbyist_Registration_Statements.csv",
-		url: "https://data.hawaii.gov/api/views/cm7c-skav/rows.csv?accessType=DOWNLOAD",
-	},
-]
-
-interface SourceManifest {
-	source_url: string
-	downloaded_at: string
-	filename: string
-	sha256: string
-	bytes: number
-}
-
-export async function fetchStateSources(
-	options: FetchStateSourcesOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	mkdirSync(options.outRoot, { recursive: true })
-
-	let fetched = 0
-	let failed = 0
-	const failedCodes: string[] = []
-
-	for (const { slug, filename, url } of SOURCES) {
-		const destDir = join(options.outRoot, slug)
-		mkdirSync(destDir, { recursive: true })
-		const dest = join(destDir, filename)
-
-		report?.(`=== ${slug} / ${filename}`)
-
-		let bytes: number
-
-		try {
-			;({ bytes } = await downloadToFile({
-				url,
-				dest,
-				timeoutMs: 600_000,
-				headers: { "Accept-Encoding": "gzip, br" },
-				report,
-			}))
-		} catch (error) {
-			report?.(`  ✗ download failed for ${url}: ${(error as Error).message}`)
-
-			failed++
-			failedCodes.push(slug)
-
-			continue
-		}
-
-		if (bytes < BYTES_PER_KIB) {
-			report?.(`  ✗ response too small (${bytes} bytes) — probable 404 / error page`)
-
-			failed++
-			failedCodes.push(slug)
-
-			continue
-		}
-
-		const sha = await sha256File(dest)
-
-		const manifest: SourceManifest = {
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			filename,
-			sha256: sha,
-			bytes,
-		}
-
-		await writeManifest(join(destDir, "MANIFEST.json"), manifest)
-
-		report?.(`  ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB  sha256=${sha}`)
-
-		fetched++
-	}
-
-	report?.(`=== summary ===`)
-	report?.(`fetched: ${fetched}`)
-	report?.(`failed:  ${failed}`)
-
-	return { fetched, skipped: 0, failed, failedCodes }
-}