@mailwoman/corpus 7.2.0 → 7.2.1

package/src/tools/fetch/openaddresses.ts

@@ -0,0 +1,380 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch an OpenAddresses country collection from batch.openaddresses.io.
+ *
+ *   Source: https://batch.openaddresses.io
+ *   License: MIXED — OpenAddresses aggregates hundreds of upstream sources with per-source licenses
+ *   (CC-BY, CC0, PDDL, ODbL, CC-BY-SA, and proprietary attribution-only). The per-row LICENSE filter
+ *   in the openaddresses adapter is essential for proprietary-weights training: Tier-C rows (ODbL,
+ *   CC-BY-SA, CC-SA) are dropped at ingest by default. This module downloads the raw collection; the
+ *   adapter does the license gating.
+ *
+ *   Native `fetch` streams the download to disk (no curl + Python subprocess tax);
+ *   `node:child_process` keeps the genuine shell ops it still needs (`file` magic detection +
+ *   `gunzip` decompression, both nice/ionice-deprioritized).
+ *
+ *   ## Authentication note (2026-05-18)
+ *
+ *   The batch.openaddresses.io download endpoint now requires a registered account. Downloads are
+ *   still free at the "basic" tier (GeoJSON+LD output).
+ *
+ *   1. Register at https://batch.openaddresses.io/register
+ *   2. Log in and go to Profile → "Create Token"
+ *   3. Export the token: `export OA_BATCH_TOKEN=<your-token>`
+ *   4. Re-run the command.
+ *
+ *   The collection URL pattern (verified 2026-05-18):
+ *
+ *   - `POST /api/login {username, password}` → `{token}`
+ *   - `GET  /api/job/{job_id}/output/source.geojson.gz?token={token}`
+ *
+ *   Collections are downloaded as a combined GeoJSON.gz via:
+ *
+ *   - `GET  /api/collections/{collection_id}/download` (returns a redirect to S3)
+ *
+ *   Collection IDs discovered from `/api/collections`:
+ *
+ *   - `id=6  name="ca"  size=2044467556` (~1.9 GiB uncompressed, verified 2026-05-18)
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   # With token (preferred). Default country: ca. Supports any OA country code (us-west, fr, …)
+ *   OA_BATCH_TOKEN=<token> mailwoman corpus fetch openaddresses --country ca \
+ *     --out-root /mnt/playpen/mailwoman-data/corpus/sources
+ *
+ *   # Without token (will detect + print instructions, then report the failure):
+ *   mailwoman corpus fetch openaddresses --country ca
+ *   ```
+ */
+
+import { execFile, spawn } from "node:child_process"
+import { createReadStream, createWriteStream, existsSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs"
+import { join } from "node:path"
+import { Readable } from "node:stream"
+import { pipeline } from "node:stream/promises"
+import { setTimeout as sleep } from "node:timers/promises"
+import { promisify } from "node:util"
+
+import { $private } from "@mailwoman/core/env"
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { isTransientStatus, writeManifest } from "./download.ts"
+
+const execFileAsync = promisify(execFile)
+
+const OA_BASE = "https://batch.openaddresses.io"
+
+/**
+ * Collection IDs known as of 2026-05-18 (discovered via `GET /api/collections`). OA assigns stable integer IDs to each
+ * country collection; re-check `GET /api/collections` if a new country is needed and the ID is unknown.
+ */
+const OA_COLLECTION_IDS: Record<string, number> = {
+	ca: 6,
+	"us-west": 4,
+	"us-south": 3,
+	"us-northeast": 2,
+	"us-midwest": 5,
+	global: 1,
+}
+
+export interface FetchOpenAddressesOptions extends BaseFetchOptions {
+	/** OA country collection code. Default `ca`. */
+	country?: string
+}
+
+interface OaCollection {
+	name?: string
+	id?: number
+	human?: string
+	size?: number
+}
+
+/** Stream-count newlines, matching `wc -l` (memory-safe for the multi-GB collection). */
+async function countLines(path: string): Promise<number> {
+	let count = 0
+
+	for await (const chunk of createReadStream(path) as AsyncIterable<Buffer>) {
+		for (let i = 0; i < chunk.length; i++) {
+			if (chunk[i] === 0x0a) {
+				count++
+			}
+		}
+	}
+
+	return count
+}
+
+function humanBytes(bytes: number): string {
+	const units = ["B", "KiB", "MiB", "GiB", "TiB"]
+	let value = bytes
+	let unit = 0
+
+	while (value >= 1024 && unit < units.length - 1) {
+		value /= 1024
+		unit++
+	}
+
+	return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`
+}
+
+interface StreamDownloadOpts {
+	headers?: Record<string, string>
+	timeoutMs: number
+	retries: number
+	retryDelayMs: number
+}
+
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the OA download endpoint 302s to a pre-signed S3 URL).
+ *
+ * NOTE(phase1): kept local instead of the shared `downloadToFile` — this one streams a multi-GB body to disk (the
+ * shared util buffers via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the caller needs for
+ * its two-URL fallback ladder.
+ */
+async function streamDownload(url: string, dest: string, opts: StreamDownloadOpts): Promise<number> {
+	for (let attempt = 0; attempt <= opts.retries; attempt++) {
+		try {
+			const res = await fetch(url, {
+				headers: opts.headers ?? {},
+				redirect: "follow",
+				signal: AbortSignal.timeout(opts.timeoutMs),
+			})
+
+			if (res.ok && res.body) {
+				await pipeline(Readable.fromWeb(res.body), createWriteStream(dest))
+
+				return res.status
+			}
+
+			if (attempt < opts.retries && isTransientStatus(res.status)) {
+				await sleep(opts.retryDelayMs)
+				continue
+			}
+
+			return res.status
+		} catch {
+			if (attempt < opts.retries) {
+				await sleep(opts.retryDelayMs)
+				continue
+			}
+
+			return 0
+		}
+	}
+
+	return 0
+}
+
+/** Decompress `src` → `dest` with the same deprioritized subprocess the old fetcher used. */
+async function gunzipToFile(src: string, dest: string): Promise<void> {
+	const child = spawn("nice", ["-n", "15", "ionice", "-c", "3", "gunzip", "-c", src], {
+		stdio: ["ignore", "pipe", "inherit"],
+	})
+	await pipeline(child.stdout!, createWriteStream(dest))
+	await new Promise<void>((resolve, reject) => {
+		child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`gunzip exited with code ${code}`))))
+		child.on("error", reject)
+	})
+}
+
+export async function fetchOpenAddresses(
+	options: FetchOpenAddressesOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const country = options.country ?? "ca"
+	const token = $private.OA_BATCH_TOKEN
+
+	const destDir = join(options.outRoot, "openaddresses", country)
+	const manifestPath = join(destDir, "MANIFEST.json")
+	const outputFile = join(destDir, "collection.geojsonl")
+
+	const fail = (code: string): FetchSummary => ({ fetched: 0, skipped: 0, failed: 1, failedCodes: [code] })
+
+	report?.(`=== fetch openaddresses: country=${country}`)
+	report?.(`    dest: ${destDir}`)
+
+	mkdirSync(destDir, { recursive: true })
+
+	// -------------------------------------------------------------------------
+	// Authentication check
+	// -------------------------------------------------------------------------
+	if (!token) {
+		report?.(`
+ERROR: OA_BATCH_TOKEN is not set.
+
+As of 2026-05-18, batch.openaddresses.io requires a registered (free) account
+to download collection files.  Data remains openly licensed — the auth gate
+is there to prevent CDN abuse, not to restrict access.
+
+Steps to get a token:
+  1. Register at: https://batch.openaddresses.io/register
+  2. Verify your email and log in.
+  3. Go to Profile → "Create Token" → copy the token.
+  4. Export it in this shell:
+       export OA_BATCH_TOKEN=<your-token>
+  5. Re-run this command.
+
+The Canada collection (ca) is ~2 GiB compressed / ~7 GiB uncompressed
+(estimated), so budget ~20–45 minutes at typical cloud-to-host bandwidth.
+`)
+
+		return fail("OA_BATCH_TOKEN")
+	}
+
+	// -------------------------------------------------------------------------
+	// Determine collection ID
+	// -------------------------------------------------------------------------
+	let collectionID = OA_COLLECTION_IDS[country]
+
+	if (collectionID === undefined) {
+		report?.(`Unknown country code '${country}'. Fetching collection list to find ID...`)
+		const res = await fetch(`${OA_BASE}/api/collections`, {
+			headers: { Authorization: `Bearer ${token}`, "Accept-Encoding": "gzip, br" },
+			signal: AbortSignal.timeout(30_000),
+		})
+
+		if (!res.ok) {
+			report?.(`ERROR: GET /api/collections returned HTTP ${res.status}.`)
+
+			return fail(country)
+		}
+
+		const collections = (await res.json()) as OaCollection[]
+		const match = collections.find((item) => item.name === country)
+
+		if (match?.id === undefined) {
+			report?.(`ERROR: Could not find a collection named '${country}' in GET /api/collections.`)
+			report?.(`Available collections:`)
+
+			for (const item of collections) {
+				const size = (item.size ?? 0).toLocaleString()
+				report?.(`  ${(item.name ?? "").padEnd(20)}  id=${item.id}  ${item.human ?? ""}  size=${size} bytes`)
+			}
+
+			return fail(country)
+		}
+
+		collectionID = match.id
+		report?.(`  Found collection id=${collectionID} for '${country}'`)
+	}
+
+	// -------------------------------------------------------------------------
+	// Download via the collections download endpoint (302s to S3)
+	// -------------------------------------------------------------------------
+	report?.(`  Resolving download URL for collection id=${collectionID}...`)
+	report?.(`  Attempting authenticated download...`)
+
+	const tmpGz = join(destDir, "collection.geojsonl.gz.tmp")
+	const tmpRaw = join(destDir, "collection.geojsonl.tmp")
+	const sourceURL = `${OA_BASE}/api/collections/${collectionID}/download`
+
+	let httpStatus = await streamDownload(sourceURL, tmpGz, {
+		headers: { Authorization: `Bearer ${token}` },
+		timeoutMs: 7_200_000,
+		retries: 3,
+		retryDelayMs: 30_000,
+	})
+
+	if (httpStatus !== 200) {
+		// Try the geojsonl.gz directly with token as query param (alternate URL shape).
+		httpStatus = await streamDownload(`${OA_BASE}/api/collections/${collectionID}/geojsonl.gz?token=${token}`, tmpGz, {
+			timeoutMs: 7_200_000,
+			retries: 3,
+			retryDelayMs: 30_000,
+		})
+	}
+
+	if (httpStatus !== 200) {
+		rmSync(tmpGz, { force: true })
+		report?.(`
+ERROR: Download returned HTTP ${httpStatus}.
+
+Likely causes:
+  1. OA_BATCH_TOKEN is invalid or expired — re-create it at Profile → Tokens.
+  2. The collection download endpoint URL has changed (this module was written
+     against the 2026-05-18 batch.openaddresses.io API; it may need updating).
+  3. Network error or CDN outage.
+
+Manual download (after logging in to batch.openaddresses.io):
+  - Navigate to https://batch.openaddresses.io/collection/${collectionID}
+  - Click "GeoJSON+LD" to download the collection.
+  - Save as: ${outputFile}
+
+URL tried: ${OA_BASE}/api/collections/${collectionID}/download
+`)
+
+		return fail(country)
+	}
+
+	// -------------------------------------------------------------------------
+	// Decompress if the downloaded file is gzipped
+	// -------------------------------------------------------------------------
+	const fileMagic = (await execFileAsync("file", ["--brief", tmpGz]).catch(() => ({ stdout: "" }))).stdout
+
+	if (/gzip|compressed/i.test(fileMagic)) {
+		report?.(`  Decompressing gzip archive...`)
+		await gunzipToFile(tmpGz, tmpRaw)
+		rmSync(tmpGz, { force: true })
+		renameSync(tmpRaw, outputFile)
+	} else if (/JSON|ASCII|UTF-8/i.test(fileMagic)) {
+		// Already line-delimited GeoJSON.
+		renameSync(tmpGz, outputFile)
+		rmSync(tmpRaw, { force: true })
+	} else {
+		// Unknown type — keep as-is and let the operator inspect.
+		renameSync(tmpGz, outputFile)
+		report?.(`  WARNING: Downloaded file type is '${fileMagic.trim()}' — may need manual decompression.`)
+	}
+
+	// -------------------------------------------------------------------------
+	// Verify + write MANIFEST
+	// -------------------------------------------------------------------------
+	if (!existsSync(outputFile)) {
+		report?.(`ERROR: Output file not found at ${outputFile} after download.`)
+
+		return fail(country)
+	}
+
+	const size = statSync(outputFile).size
+
+	if (size < 10240) {
+		report?.(`ERROR: File is suspiciously small (${size} bytes) — likely an error response.`)
+
+		return fail(country)
+	}
+
+	const sha = await sha256File(outputFile)
+	const rowCount = await countLines(outputFile)
+	const downloadedAt = new Date().toISOString().replace(/\.\d{3}Z$/, "Z")
+
+	const manifest = {
+		source_url: sourceURL,
+		collection_id: collectionID,
+		country,
+		filename: "collection.geojsonl",
+		downloaded_at: downloadedAt,
+		sha256: sha,
+		bytes: size,
+		row_count: rowCount,
+		notes:
+			"batch.openaddresses.io requires a free registered account for downloads. License is mixed per-row; use the openaddresses adapter with allowShareAlike=false (default) to filter Tier-C rows.",
+	}
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${humanBytes(size)}  rows=${rowCount}  sha256=${sha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+	report?.(`=== done`)
+	report?.(`Feed to the adapter:`)
+	report?.(`  mailwoman corpus run openaddresses \\`)
+	report?.(`    --input ${outputFile} \\`)
+	report?.(`    --country ${country.toUpperCase()} \\`)
+	report?.(`    --output ${options.outRoot}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/src/tools/fetch/state-hi-schools.ts

@@ -0,0 +1,217 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Re-fetch the Hawaii State DOE school directory and convert the XLSX workbook to a flat CSV the
+ *   `state-hi-schools` adapter can consume.
+ *
+ *   Upstream is a single XLSX (~64 KB) with two sheets — `HIDOE` (~258 district schools) and `PCS`
+ *   (~38 public charter schools). Both sheets share the same header. This module concatenates them
+ *   under one shared header so the adapter can stream a single CSV.
+ *
+ *   License: Hawaii state government open data (Tier A — state PD-equivalent).
+ *
+ *   Built-in `fetch` (gzip/brotli) replaces curl for the download; the XLSX → CSV step still rides
+ *   `python3` + `openpyxl` via `node:child_process` — there is no clean node equivalent without
+ *   adding a workbook-parsing dependency.
+ *
+ *   Invoke via `mailwoman corpus fetch state-hi-schools --out-root <path>`. Idempotent: if the dest
+ *   CSV exists and sha matches MANIFEST, skips download.
+ */
+
+import { spawn, spawnSync } from "node:child_process"
+import { existsSync, mkdirSync, statSync } from "node:fs"
+import { unlink } from "node:fs/promises"
+import { join } from "node:path"
+
+import { sha256File } from "@mailwoman/core/utils"
+
+import type { BaseFetchOptions, FetchSummary } from "./download.ts"
+import { downloadToFile, readManifest, writeManifest } from "./download.ts"
+
+const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
+const SLUG = "state-hi-schools"
+const CSV_FILENAME = "HI_Public_Schools_List.csv"
+const XLSX_FILENAME = "HI_Public_Schools_List.xlsx"
+
+export type FetchStateHISchoolsOptions = BaseFetchOptions
+
+/**
+ * The XLSX → CSV converter: concatenate every sheet under one shared header (the first sheet's). Runs as `python3 -c
+ * <script> <xlsx-path> <csv-path>`, so `sys.argv[1]`/`sys.argv[2]` are the I/O paths. TODO: Get rid of this.
+ */
+const PY_CONVERT = `
+import csv
+import sys
+from openpyxl import load_workbook
+
+xlsx_path, csv_path = sys.argv[1], sys.argv[2]
+wb = load_workbook(xlsx_path, data_only=True, read_only=True)
+
+with open(csv_path, "w", newline="", encoding="utf-8") as out:
+    writer = csv.writer(out)
+    shared_header = None
+    total_data_rows = 0
+    for sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        rows = ws.iter_rows(values_only=True)
+        try:
+            header = next(rows)
+        except StopIteration:
+            continue
+        norm_header = ["" if v is None else str(v).strip() for v in header]
+        if shared_header is None:
+            shared_header = norm_header
+            writer.writerow(shared_header)
+        elif norm_header != shared_header:
+            print(
+                f"  ! sheet '{sheet_name}' header diverges from shared header; concatenating anyway",
+                file=sys.stderr,
+            )
+        for row in rows:
+            if row is None:
+                continue
+            # Skip fully-empty rows (XLSX iter_rows can yield phantom trailing rows).
+            if all(v is None or (isinstance(v, str) and not v.strip()) for v in row):
+                continue
+            writer.writerow(["" if v is None else str(v).strip() for v in row])
+            total_data_rows += 1
+
+print(f"  converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
+`
+
+interface Manifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+	notes: string
+}
+
+/** Mimic `numfmt --to=iec` for a friendly byte-size log line. */
+function iec(bytes: number): string {
+	if (bytes < 1024) return String(bytes)
+	const units = ["K", "M", "G", "T", "P"]
+	let value = bytes / 1024
+	let i = 0
+
+	while (value >= 1024 && i < units.length - 1) {
+		value /= 1024
+		i++
+	}
+
+	const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString()
+
+	return `${rounded}${units[i] ?? ""}`
+}
+
+/**
+ * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
+ * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
+ */
+async function convertXLSXToCSV(xlsxPath: string, csvPath: string): Promise<void> {
+	const child = spawn("python3", ["-c", PY_CONVERT, xlsxPath, csvPath], {
+		stdio: ["ignore", "inherit", "inherit"],
+	})
+	await new Promise<void>((resolve, reject) => {
+		child.on("close", (code) => (code === 0 ? resolve() : reject(new Error(`python3 converter exited with ${code}`))))
+		child.on("error", reject)
+	})
+}
+
+export async function fetchStateHISchools(
+	options: FetchStateHISchoolsOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = join(options.outRoot, SLUG)
+	mkdirSync(destDir, { recursive: true })
+
+	const xlsxDest = join(destDir, XLSX_FILENAME)
+	const csvDest = join(destDir, CSV_FILENAME)
+	const manifestPath = join(destDir, "MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+
+	// Idempotency: skip if CSV exists and sha matches recorded MANIFEST.
+	if (existsSync(csvDest)) {
+		const recorded = await readManifest<Partial<Manifest>>(manifestPath)
+
+		if (recorded?.sha256 && recorded.filename === CSV_FILENAME) {
+			const actualSha = await sha256File(csvDest)
+
+			if (actualSha === recorded.sha256) {
+				report?.(`  ✓ Already current (sha256 matches MANIFEST) — skipping download.`)
+
+				return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+			}
+		}
+	}
+
+	// Preflight: openpyxl must be importable.
+	const preflight = spawnSync("python3", ["-c", "import openpyxl"], { stdio: "ignore" })
+
+	if (preflight.status !== 0) {
+		report?.(
+			`  ✗ python3 with the \`openpyxl\` package is required to convert the HIDOE XLSX.\n` +
+				`    Debian/Ubuntu:  sudo apt-get install -y python3-openpyxl\n` +
+				`    macOS Homebrew: brew install python && pip3 install openpyxl`
+		)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// Download XLSX.
+	report?.(`  Downloading ${SOURCE_URL} ...`)
+
+	try {
+		await downloadToFile({
+			url: SOURCE_URL,
+			dest: xlsxDest,
+			timeoutMs: 600_000,
+			headers: { "Accept-Encoding": "gzip, br" },
+			report,
+		})
+	} catch (err) {
+		report?.(`  ✗ Download failed (${(err as Error).message})`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const xlsxSize = statSync(xlsxDest).size
+	report?.(`  Downloaded XLSX: ${iec(xlsxSize)}`)
+
+	if (xlsxSize < 1024) {
+		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// Convert XLSX → CSV (concatenate both sheets under one shared header).
+	report?.(`  Converting XLSX → CSV (concatenating sheets) ...`)
+	await convertXLSXToCSV(xlsxDest, csvDest)
+
+	const csvSize = statSync(csvDest).size
+	const csvSha = await sha256File(csvDest)
+
+	// Remove XLSX (CSV is the canonical artifact the adapter consumes).
+	await unlink(xlsxDest)
+	report?.(`  Removed XLSX (CSV kept)`)
+
+	// Write MANIFEST.
+	const manifest: Manifest = {
+		source_url: SOURCE_URL,
+		downloaded_at: new Date().toISOString(),
+		filename: CSV_FILENAME,
+		sha256: csvSha,
+		bytes: csvSize,
+		notes: "Converted from XLSX (sheets HIDOE + PCS concatenated under shared header).",
+	}
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${iec(csvSize)}  sha256=${csvSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}