@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/tools/fetch/tiger-full.ts

@@ -1,308 +0,0 @@
-import { APIClient, pluckResponseData } from "@mailwoman/core/api"
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full TIGER 2024 ADDRFEAT dataset — all US counties.
- *
- *   TIGER ADDRFEAT 2024 source:
- *
- *   - https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT/
- *   - Files: `tl_2024_<statefips><countyfips>_addrfeat.zip`
- *
- *   Each state's ZIPs land in `<outRoot>/tiger/addrfeat/state-<statefips>/` with a per-state
- *   `MANIFEST.json` recording filename, sha256, and bytes for every county ZIP so re-runs can skip
- *   already-verified files. (Extraction + ogr2ogr ingestion happen later, in the `tiger` adapter;
- *   this module is download + provenance only.)
- *
- *   Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
- *   county ZIP to disk (no curl subprocess).
- */
-/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
-   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
-import { statPath, pathExists } from "@mailwoman/core/fs/readers"
-import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { isoSeconds } from "@mailwoman/core/utils"
-import { sleep } from "@mailwoman/core/utils/sleep"
-import { basename, join } from "path-ts"
-
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
-import { readManifest, streamDownload, writeManifest } from "#tools/fetch/download"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-/**
- * Lowest 2xx status; anything below is informational.
- */
-const HTTP_OK = 200
-
-/**
- * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
- */
-const HTTP_REDIRECT = 300
-
-const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
-
-export interface FetchTigerFullOptions extends BaseFetchOptions {
-	/**
-	 * Space-separated list of 2-digit state FIPS codes to skip entirely. Default `"50"` — Vermont, already fetched in
-	 * v0.1.1.
-	 */
-	skipStateFips?: string
-	/**
-	 * Seconds to sleep between downloads. Default `0.2`.
-	 */
-	rateSleep?: number
-	/**
-	 * Max concurrent download workers per state. Default `4`.
-	 */
-	maxParallel?: number
-	/**
-	 * Print planned downloads without fetching. Default `false`.
-	 */
-	dryRun?: boolean
-}
-
-interface CountyEntry {
-	filename: string
-	sha256: string
-	bytes: number
-}
-
-/**
- * Read a per-state MANIFEST.json into a filename → entry map.
- */
-async function readCountyManifest(manifestPath: string): Promise<Map<string, CountyEntry>> {
-	const map = new Map<string, CountyEntry>()
-	const parsed = await readManifest<{ counties?: CountyEntry[] }>(manifestPath)
-
-	for (const c of parsed?.counties ?? []) {
-		if (c.filename) {
-			map.set(c.filename, { filename: c.filename, sha256: c.sha256, bytes: c.bytes })
-		}
-	}
-
-	return map
-}
-
-/**
- * Check whether a file already matches a recorded sha256 and byte count.
- */
-async function fileMatchesSha(path: string, expectedSha: string, expectedBytes: number): Promise<boolean> {
-	if (!(await pathExists(path))) return false
-
-	if ((await statPath(path)).size !== expectedBytes) return false
-
-	return (await sha256File(path)) === expectedSha
-}
-
-type CountyResult =
-	| { ok: true; filename: string; sha256: string; bytes: number }
-	| { ok: false; filename: string; reason: string }
-
-/**
- * Download one county ZIP (size sanity check + sha256).
- */
-async function downloadCounty(url: string, dest: string): Promise<CountyResult> {
-	const filename = basename(dest)
-	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 })
-
-	if (status < HTTP_OK || status >= HTTP_REDIRECT) {
-		return { ok: false, filename, reason: `HTTP ${status}` }
-	}
-
-	const bytes = (await statPath(dest)).size
-
-	if (bytes < BYTES_PER_KIB) {
-		await removePathIfPresent(dest)
-
-		return { ok: false, filename, reason: `too small (${bytes} bytes)` }
-	}
-
-	const sha256 = await sha256File(dest)
-
-	return { ok: true, filename, sha256, bytes }
-}
-
-export async function fetchTigerFull(
-	options: FetchTigerFullOptions,
-	report?: (line: string) => void
-): Promise<FetchSummary> {
-	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0)
-	const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000)
-	const maxParallel = options.maxParallel ?? 4
-	const dryRun = options.dryRun ?? false
-
-	const addrfeatDir = join(options.outRoot, "tiger", "addrfeat")
-	await makeDirectories(addrfeatDir)
-
-	// MARK: Step 1 — discover the county file list
-
-	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
-
-	// `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
-	// stay on raw `fetch` (they stream to disk; see `downloadOne`).
-	const listingRes = await new APIClient({
-		displayName: "tiger-listing",
-		retry: true,
-		axios: { headers: { "Accept-Encoding": "gzip, br" } },
-	})
-		.fetch<string>({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
-		.then(pluckResponseData)
-
-	const html = listingRes
-	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted()
-	const totalCounties = allZips.length
-	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
-
-	// Build a map: state_fips -> list of filenames.
-	// tl_2024_SSCCC_addrfeat.zip — SS = 2-digit state FIPS (chars 8-9), CCC = county FIPS.
-	const stateFiles = new Map<string, string[]>()
-
-	for (const fname of allZips) {
-		const stateFips = fname.slice(8, 10)
-		const list = stateFiles.get(stateFips) ?? []
-		list.push(fname)
-		stateFiles.set(stateFips, list)
-	}
-
-	report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`)
-
-	// MARK: Step 2 — download the county ZIPs
-
-	let totalFetched = 0
-	let totalSkipped = 0
-	let totalSkippedState = 0
-	let totalFailed = 0
-	let totalBytesFetched = 0
-	const failedCodes: string[] = []
-
-	// Process states in sorted FIPS order for predictable output.
-	const sortedStates = [...stateFiles.keys()].toSorted()
-
-	for (const stateFips of sortedStates) {
-		const countyFiles = stateFiles.get(stateFips) ?? []
-
-		// --- Skip entire state if requested ------------------------------------------
-		if (skipStateFips.includes(stateFips)) {
-			report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`)
-			totalSkippedState += countyFiles.length
-
-			continue
-		}
-
-		const stateDir = join(addrfeatDir, `state-${stateFips}`)
-		await makeDirectories(stateDir)
-		const manifestPath = join(stateDir, "MANIFEST.json")
-
-		// Load existing manifest for O(1) verified-skip lookup.
-		const manifest = await readCountyManifest(manifestPath)
-
-		report?.(`--- State ${stateFips} — ${countyFiles.length} counties`)
-
-		// Build a list of URLs+dests that need fetching.
-		const pending: Array<{ url: string; dest: string }> = []
-
-		for (const fname of countyFiles) {
-			const dest = join(stateDir, fname)
-			const url = `${TIGER_BASE_URL}/${fname}`
-			const known = manifest.get(fname)
-
-			// Skip if already verified via MANIFEST.
-			if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
-				report?.(`  skip (verified) ${fname}`)
-
-				totalSkipped++
-
-				continue
-			}
-
-			if (dryRun) {
-				report?.(`  would fetch: ${url}`)
-
-				totalFetched++
-
-				continue
-			}
-
-			pending.push({ url, dest })
-		}
-
-		if (dryRun) continue
-
-		if (!pending.length) continue
-
-		// --- Download pending files with bounded parallelism + rate-limit spacing ---
-		const results: CountyResult[] = Array.from({ length: pending.length })
-		let cursor = 0
-
-		const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
-			while (true) {
-				const i = cursor++
-
-				if (i >= pending.length) return
-				const item = pending[i]!
-				// Rate-limit: polite spacing before each fetch.
-				await sleep(rateSleepMs)
-				results[i] = await downloadCounty(item.url, item.dest)
-			}
-		})
-
-		await Promise.all(workers)
-
-		// Collect results from this state.
-		for (const result of results) {
-			if (result.ok) {
-				report?.(
-					`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`
-				)
-
-				manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes })
-
-				totalFetched++
-				totalBytesFetched += result.bytes
-			} else {
-				report?.(`  FAIL ${result.filename} -- ${result.reason}`)
-
-				totalFailed++
-				failedCodes.push(result.filename)
-			}
-		}
-
-		// Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
-		const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename))
-
-		const manifestDoc = {
-			state_fips: stateFips,
-			updated_at: isoSeconds(),
-			tiger_base_url: TIGER_BASE_URL,
-			counties,
-		}
-
-		await writeManifest(manifestPath, manifestDoc)
-	}
-
-	// MARK: Summary
-
-	report?.(`=== Summary ===`)
-	report?.(`  Total counties in index   : ${totalCounties}`)
-	report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`)
-	report?.(`  Counties already present  : ${totalSkipped}`)
-	report?.(`  Counties fetched this run : ${totalFetched}`)
-	report?.(`  Counties failed           : ${totalFailed}`)
-
-	if (totalBytesFetched > 0) {
-		report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`)
-	}
-
-	if (totalFailed > 0) {
-		report?.(`WARNING: ${totalFailed} download(s) failed. Re-run to retry.`)
-	}
-
-	return { fetched: totalFetched, skipped: totalSkipped + totalSkippedState, failed: totalFailed, failedCodes }
-}

package/lib/tools/golden-promote.ts

@@ -1,337 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
- *   human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
- *
- *   ## What it does
- *
- *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
- *   2. Reads the previous-version golden dir for forward-copy + dedup base
- *   3. Applies filters that drop candidates unlikely to be human-typed:
- *
- *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
- *                 before locality only)
- *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
- *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
- *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
- *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
- *   7. Writes MANIFEST.json with sha256 of each output file
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus golden promote \
- *   --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
- *   --bump-to v0.1.1 \
- *   --prior v0.1.0
- *   ```
- */
-
-import { readDirectory, pathExists } from "@mailwoman/core/fs/readers"
-import { copyFileTo, makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
-import { sha256File } from "@mailwoman/core/hash"
-import { foldCaseWhitespace } from "@mailwoman/normalize/fold"
-import { join } from "path-ts"
-import { createNewlineWriter, JSONSpliterator } from "spliterator"
-
-import type { GoldenCandidateEntry as GoldenEntry } from "#utils/golden"
-
-// ── Types ──────────────────────────────────────────────────────────────────
-
-/**
- * Components a candidate needs before it is rich enough to be worth a golden-set slot.
- */
-const MIN_GOLDEN_COMPONENTS = 5
-
-/**
- * Components an existing entry must still carry to survive promotion.
- */
-const MIN_PROMOTABLE_COMPONENTS = 4
-
-/**
- * Quote-delimited segments above which the raw line is malformed rather than merely quoted.
- */
-const MAX_QUOTE_SEGMENTS = 3
-
-export interface PromoteStats {
-	candidatesIn: number
-	filteredOut: { glued: number; postcodeLeading: number; suspicious: number; duplicate: number; forwardDup: number }
-	kept: number
-	perCountry: Record<string, number>
-}
-
-export interface PromoteGoldenOptions {
-	/**
-	 * Candidates JSONL (required).
-	 */
-	input: string
-	/**
-	 * Target golden version dir (required, e.g. `v0.1.1`).
-	 */
-	bumpTo: string
-	/**
-	 * Previous version to forward-copy + dedup against. Default `v0.1.0`.
-	 */
-	prior?: string
-	/**
-	 * Golden dir root. Default `data/eval/golden`.
-	 */
-	goldenRoot?: string
-	/**
-	 * Skip the human-typed-likelihood filters (keep everything that passed expand-golden's validator).
-	 */
-	noFilters?: boolean
-	/**
-	 * Report what would be written but don't touch disk.
-	 */
-	dryRun?: boolean
-}
-
-// ── Filters ───────────────────────────────────────────────────────────────
-
-/**
- * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
- * together rather than human-typed.
- */
-function isComponentsGlued(entry: GoldenEntry): boolean {
-	const componentCount = Object.keys(entry.components).length
-
-	if (componentCount < MIN_GOLDEN_COMPONENTS) return false
-	const separators = (entry.raw.match(/[,\n;]/g) ?? []).length
-
-	return separators < 2
-}
-
-/**
- * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
- * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
- * often precedes locality there).
- */
-function isPostcodeBadlyLeading(entry: GoldenEntry): boolean {
-	if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS) return false
-
-	if (entry.country === "FR" || entry.country === "France") return false
-	const postcode = entry.components.postcode
-
-	if (!postcode) return false
-	const postcodeIdx = entry.raw.indexOf(postcode)
-
-	if (postcodeIdx === -1) return false
-
-	return postcodeIdx < entry.raw.length / 3
-}
-
-/**
- * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
- * suggests the LLM emitted markup instead of an address.
- */
-function isSuspicious(entry: GoldenEntry): boolean {
-	const raw = entry.raw
-
-	// eslint-disable-next-line no-control-regex
-	if (/[\u0000-\u001F\u007F]/.test(raw)) return true
-	const openBrackets = (raw.match(/[[({<]/g) ?? []).length
-	const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length
-
-	if (openBrackets !== closeBrackets) return true
-
-	if (raw.split('"').length > MAX_QUOTE_SEGMENTS) return true
-
-	// too many quote marks
-	return false
-}
-
-// ── Main ───────────────────────────────────────────────────────────────────
-
-export async function promoteGolden(
-	options: PromoteGoldenOptions,
-	report?: (line: string) => void
-): Promise<PromoteStats> {
-	const prior = options.prior ?? "v0.1.0"
-	const goldenRoot = options.goldenRoot ?? "data/eval/golden"
-	const applyFilters = !options.noFilters
-	const dryRun = options.dryRun ?? false
-
-	report?.(`reading candidates: ${options.input}`)
-
-	// The candidates file is the REQUIRED input, so its absence must not read as "zero candidates" —
-	// that promotes the prior version unchanged and reports success. `JSONSpliterator.fromAsync`
-	// reports a missing path as a bare `TypeError` naming its own internals, so name the path here.
-	if (!(await pathExists(options.input))) {
-		throw new Error(`Candidates file not found: ${options.input}`)
-	}
-
-	const candidates = await Array.fromAsync(JSONSpliterator.fromAsync<GoldenEntry>(options.input))
-	report?.(`  ${candidates.length} candidates loaded`)
-
-	// Forward-copy base: existing entries from the prior golden version go forward verbatim,
-	// and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
-	const priorDir = join(goldenRoot, prior)
-	const priorEntries: { country: string; entries: GoldenEntry[] }[] = []
-	const seenNormalized = new Set<string>()
-
-	if (await pathExists(priorDir)) {
-		for (const f of (await readDirectory(priorDir)).filter((n) => n.endsWith(".jsonl"))) {
-			const country = f.replace(".jsonl", "").toUpperCase()
-			const entries = await Array.fromAsync(JSONSpliterator.fromAsync<GoldenEntry>(join(priorDir, f)))
-			priorEntries.push({ country, entries })
-
-			for (const e of entries) {
-				seenNormalized.add(foldCaseWhitespace(e.raw))
-			}
-
-			report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`)
-		}
-	} else {
-		report?.(`  ⚠ prior dir ${priorDir} not found — starting fresh`)
-	}
-
-	// Filter pass
-	const stats: PromoteStats = {
-		candidatesIn: candidates.length,
-		filteredOut: { glued: 0, postcodeLeading: 0, suspicious: 0, duplicate: 0, forwardDup: 0 },
-		kept: 0,
-		perCountry: {},
-	}
-
-	const accepted: GoldenEntry[] = []
-	const seenInBatch = new Set<string>()
-
-	for (const cand of candidates) {
-		const norm = foldCaseWhitespace(cand.raw)
-
-		// Dedup pass 1: against prior versioned golden
-		if (seenNormalized.has(norm)) {
-			stats.filteredOut.forwardDup++
-
-			continue
-		}
-
-		// Dedup pass 2: against this batch
-		if (seenInBatch.has(norm)) {
-			stats.filteredOut.duplicate++
-
-			continue
-		}
-
-		if (applyFilters) {
-			if (isComponentsGlued(cand)) {
-				stats.filteredOut.glued++
-
-				continue
-			}
-
-			if (isPostcodeBadlyLeading(cand)) {
-				stats.filteredOut.postcodeLeading++
-
-				continue
-			}
-
-			if (isSuspicious(cand)) {
-				stats.filteredOut.suspicious++
-
-				continue
-			}
-		}
-
-		accepted.push(cand)
-		seenInBatch.add(norm)
-
-		stats.kept++
-		const country = cand.country || "OTHER"
-		stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1
-	}
-
-	// Bucket per country, including forward-copied entries
-	const buckets = new Map<string, GoldenEntry[]>()
-
-	for (const { country, entries } of priorEntries) {
-		// Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
-		buckets.set(country, [...entries])
-	}
-
-	for (const cand of accepted) {
-		const key = (cand.country || "OTHER").toUpperCase()
-
-		if (!buckets.has(key)) {
-			buckets.set(key, [])
-		}
-
-		buckets.get(key)!.push(cand)
-	}
-
-	// Output
-	const outDir = join(goldenRoot, options.bumpTo)
-	report?.(`=== plan ===`)
-	report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`)
-
-	for (const [key, entries] of buckets) {
-		report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`)
-	}
-
-	report?.(`=== filter stats ===`)
-	report?.(`candidates in:        ${stats.candidatesIn}`)
-	report?.(`  filtered (glued):           ${stats.filteredOut.glued}`)
-	report?.(`  filtered (postcode-lead):   ${stats.filteredOut.postcodeLeading}`)
-	report?.(`  filtered (suspicious):      ${stats.filteredOut.suspicious}`)
-	report?.(`  filtered (dup-in-batch):    ${stats.filteredOut.duplicate}`)
-	report?.(`  filtered (dup-vs-prior):    ${stats.filteredOut.forwardDup}`)
-	report?.(`  kept:                       ${stats.kept}`)
-	report?.(`per-country (kept):`)
-
-	for (const [c, n] of Object.entries(stats.perCountry).toSorted((a, b) => b[1] - a[1])) {
-		report?.(`  ${c}: ${n}`)
-	}
-
-	if (dryRun) {
-		report?.(`(dry-run: no files written)`)
-
-		return stats
-	}
-
-	await makeDirectories(outDir)
-
-	const manifest: {
-		promoted_at: string
-		from: string
-		from_sha256: string
-		files: Record<string, { entries: number; sha256: string }>
-	} = {
-		promoted_at: new Date().toISOString(),
-		from: options.input,
-		from_sha256: await sha256File(options.input),
-		files: {},
-	}
-
-	for (const [key, entries] of buckets) {
-		const filename = `${key.toLowerCase()}.jsonl`
-		const path = join(outDir, filename)
-
-		{
-			await using out = createNewlineWriter(path)
-
-			for (const entry of entries) {
-				await out.write(JSON.stringify(entry))
-			}
-		}
-
-		manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) }
-	}
-
-	// Forward-copy non-.jsonl files (README.md, etc.) from prior
-	if (await pathExists(priorDir)) {
-		for (const f of (await readDirectory(priorDir)).filter((n) => !n.endsWith(".jsonl"))) {
-			await copyFileTo(join(priorDir, f), join(outDir, f))
-			report?.(`  forward-copied: ${f}`)
-		}
-	}
-
-	await writeLocalJSONFile(manifest, join(outDir, "MANIFEST.json"))
-	report?.(`✓ promoted to ${outDir}`)
-
-	return stats
-}