@mailwoman/corpus 8.0.0 → 8.2.0

package/src/tools/fetch/tiger-full.ts

@@ -19,6 +19,9 @@
  *   county ZIP to disk (no curl subprocess).
  */
 
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
+
 import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs"
 import { basename, join } from "node:path"
 import { Readable } from "node:stream"
@@ -30,6 +33,22 @@ import { sha256File } from "@mailwoman/core/utils"
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
 import { isTransientStatus, readManifest, writeManifest } from "./download.ts"
 
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+/**
+ * Lowest 2xx status; anything below is informational.
+ */
+const HTTP_OK = 200
+
+/**
+ * Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
+ */
+const HTTP_REDIRECT = 300
+
+const BYTES_PER_KIB = 1024
+
 const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
 
 export interface FetchTigerFullOptions extends BaseFetchOptions {
@@ -38,11 +57,17 @@ export interface FetchTigerFullOptions extends BaseFetchOptions {
 	 * v0.1.1.
 	 */
 	skipStateFips?: string
-	/** Seconds to sleep between downloads. Default `0.2`. */
+	/**
+	 * Seconds to sleep between downloads. Default `0.2`.
+	 */
 	rateSleep?: number
-	/** Max concurrent download workers per state. Default `4`. */
+	/**
+	 * Max concurrent download workers per state. Default `4`.
+	 */
 	maxParallel?: number
-	/** Print planned downloads without fetching. Default `false`. */
+	/**
+	 * Print planned downloads without fetching. Default `false`.
+	 */
 	dryRun?: boolean
 }
 
@@ -57,8 +82,9 @@ function humanBytes(bytes: number): string {
 	let value = bytes
 	let unit = 0
 
-	while (value >= 1024 && unit < units.length - 1) {
+	while (value >= BYTES_PER_KIB && unit < units.length - 1) {
 		value /= 1024
+
 		unit++
 	}
 
@@ -89,6 +115,7 @@ async function streamDownload(
 
 			if (attempt < opts.retries && isTransientStatus(res.status)) {
 				await sleep(opts.retryDelayMs)
+
 				continue
 			}
 
@@ -96,6 +123,7 @@ async function streamDownload(
 		} catch {
 			if (attempt < opts.retries) {
 				await sleep(opts.retryDelayMs)
+
 				continue
 			}
 
@@ -106,7 +134,9 @@ async function streamDownload(
 	return 0
 }
 
-/** Read a per-state MANIFEST.json into a filename → entry map. */
+/**
+ * Read a per-state MANIFEST.json into a filename → entry map.
+ */
 async function readCountyManifest(manifestPath: string): Promise<Map<string, CountyEntry>> {
 	const map = new Map<string, CountyEntry>()
 	const parsed = await readManifest<{ counties?: CountyEntry[] }>(manifestPath)
@@ -120,7 +150,9 @@ async function readCountyManifest(manifestPath: string): Promise<Map<string, Cou
 	return map
 }
 
-/** Check whether a file already matches a recorded sha256 and byte count. */
+/**
+ * Check whether a file already matches a recorded sha256 and byte count.
+ */
 async function fileMatchesSha(path: string, expectedSha: string, expectedBytes: number): Promise<boolean> {
 	if (!existsSync(path)) return false
 
@@ -133,18 +165,20 @@ type CountyResult =
 	| { ok: true; filename: string; sha256: string; bytes: number }
 	| { ok: false; filename: string; reason: string }
 
-/** Download one county ZIP (size sanity check + sha256). */
+/**
+ * Download one county ZIP (size sanity check + sha256).
+ */
 async function downloadCounty(url: string, dest: string): Promise<CountyResult> {
 	const filename = basename(dest)
-	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5_000 })
+	const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 })
 
-	if (status < 200 || status >= 300) {
+	if (status < HTTP_OK || status >= HTTP_REDIRECT) {
 		return { ok: false, filename, reason: `HTTP ${status}` }
 	}
 
 	const bytes = statSync(dest).size
 
-	if (bytes < 1024) {
+	if (bytes < BYTES_PER_KIB) {
 		rmSync(dest, { force: true })
 
 		return { ok: false, filename, reason: `too small (${bytes} bytes)` }
@@ -167,10 +201,10 @@ export async function fetchTigerFull(
 	const addrfeatDir = join(options.outRoot, "tiger", "addrfeat")
 	mkdirSync(addrfeatDir, { recursive: true })
 
-	// -------------------------------------------------------------------------
-	// Step 1: Discover the full county file list from the TIGER directory listing.
-	// -------------------------------------------------------------------------
+	// MARK: Step 1 — discover the county file list
+
 	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
+
 	const listingRes = await fetch(`${TIGER_BASE_URL}/`, {
 		headers: { "Accept-Encoding": "gzip, br" },
 		signal: AbortSignal.timeout(60_000),
@@ -178,7 +212,7 @@ export async function fetchTigerFull(
 
 	if (!listingRes.ok) throw new Error(`Failed to fetch TIGER directory listing: HTTP ${listingRes.status}`)
 	const html = await listingRes.text()
-	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g) ?? [])].sort()
+	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted()
 	const totalCounties = allZips.length
 	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
 
@@ -195,9 +229,8 @@ export async function fetchTigerFull(
 
 	report?.(`  Spans ${stateFiles.size} state/territory FIPS codes.`)
 
-	// -------------------------------------------------------------------------
-	// Step 2: For each state, download missing/unverified county ZIPs.
-	// -------------------------------------------------------------------------
+	// MARK: Step 2 — download the county ZIPs
+
 	let totalFetched = 0
 	let totalSkipped = 0
 	let totalSkippedState = 0
@@ -206,7 +239,7 @@ export async function fetchTigerFull(
 	const failedCodes: string[] = []
 
 	// Process states in sorted FIPS order for predictable output.
-	const sortedStates = [...stateFiles.keys()].sort()
+	const sortedStates = [...stateFiles.keys()].toSorted()
 
 	for (const stateFips of sortedStates) {
 		const countyFiles = stateFiles.get(stateFips) ?? []
@@ -215,6 +248,7 @@ export async function fetchTigerFull(
 		if (skipStateFips.includes(stateFips)) {
 			report?.(`--- State ${stateFips} — SKIPPED (in --skip-state-fips, ${countyFiles.length} counties)`)
 			totalSkippedState += countyFiles.length
+
 			continue
 		}
 
@@ -238,13 +272,17 @@ export async function fetchTigerFull(
 			// Skip if already verified via MANIFEST.
 			if (known && (await fileMatchesSha(dest, known.sha256, known.bytes))) {
 				report?.(`  skip (verified) ${fname}`)
+
 				totalSkipped++
+
 				continue
 			}
 
 			if (dryRun) {
 				report?.(`  would fetch: ${url}`)
+
 				totalFetched++
+
 				continue
 			}
 
@@ -253,11 +291,12 @@ export async function fetchTigerFull(
 
 		if (dryRun) continue
 
-		if (pending.length === 0) continue
+		if (!pending.length) continue
 
 		// --- Download pending files with bounded parallelism + rate-limit spacing ---
 		const results: CountyResult[] = new Array(pending.length)
 		let cursor = 0
+
 		const workers = Array.from({ length: Math.min(maxParallel, pending.length) }, async () => {
 			while (true) {
 				const i = cursor++
@@ -269,6 +308,7 @@ export async function fetchTigerFull(
 				results[i] = await downloadCounty(item.url, item.dest)
 			}
 		})
+
 		await Promise.all(workers)
 
 		// Collect results from this state.
@@ -276,29 +316,32 @@ export async function fetchTigerFull(
 			if (result.ok) {
 				report?.(`  ok ${result.filename}  ${humanBytes(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`)
 				manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes })
+
 				totalFetched++
 				totalBytesFetched += result.bytes
 			} else {
 				report?.(`  FAIL ${result.filename} -- ${result.reason}`)
+
 				totalFailed++
 				failedCodes.push(result.filename)
 			}
 		}
 
 		// Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
-		const counties = [...manifest.values()].sort((a, b) => a.filename.localeCompare(b.filename))
+		const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename))
+
 		const manifestDoc = {
 			state_fips: stateFips,
 			updated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
 			tiger_base_url: TIGER_BASE_URL,
 			counties,
 		}
+
 		await writeManifest(manifestPath, manifestDoc)
 	}
 
-	// -------------------------------------------------------------------------
-	// Summary
-	// -------------------------------------------------------------------------
+	// MARK: Summary
+
 	report?.(`=== Summary ===`)
 	report?.(`  Total counties in index   : ${totalCounties}`)
 	report?.(`  State(s) fully skipped    : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`)

package/src/tools/gb-title-case.ts

@@ -59,6 +59,7 @@ export function titleCaseGB(value: string): string {
 				.split("-")
 				.map((seg, segIndex) => caseWord(seg, wordIndex === 0 && segIndex === 0))
 				.join("-")
+
 			wordIndex += 1
 
 			return cased

package/src/tools/golden-expand.ts

@@ -55,6 +55,11 @@ import { dataRootPath, writeJSONL } from "@mailwoman/core/utils"
 
 // ── Types ─────────────────────────────────────────────────────────────────
 
+/**
+ * Longest raw line still plausibly a single address rather than a concatenated record.
+ */
+const MAX_CANDIDATE_LENGTH = 500
+
 interface CorpusRow {
 	raw: string
 	tokens: string[]
@@ -90,21 +95,37 @@ interface GoldenCandidate {
 }
 
 export interface ExpandGoldenOptions {
-	/** Corpus test shard path(s), comma-separated. Default: the v0.2.0 test shard under the data root. */
+	/**
+	 * Corpus test shard path(s), comma-separated. Default: the v0.2.0 test shard under the data root.
+	 */
 	corpus?: string
-	/** Total seeds to process. Default `100` (pilot). */
+	/**
+	 * Total seeds to process. Default `100` (pilot).
+	 */
 	count?: number
-	/** Variants requested per seed. Default `5`. */
+	/**
+	 * Variants requested per seed. Default `5`.
+	 */
 	variants?: number
-	/** JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`. */
+	/**
+	 * JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`.
+	 */
 	output?: string
-	/** LLM provider. Default `deepseek`. */
+	/**
+	 * LLM provider. Default `deepseek`.
+	 */
 	provider?: "deepseek" | "anthropic"
-	/** Model id. Default depends on provider. */
+	/**
+	 * Model id. Default depends on provider.
+	 */
 	model?: string
-	/** Parallel LLM calls. Default `4`. */
+	/**
+	 * Parallel LLM calls. Default `4`.
+	 */
 	concurrency?: number
-	/** Comma-separated source allow-list. */
+	/**
+	 * Comma-separated source allow-list.
+	 */
 	includeSources?: string
 }
 
@@ -127,10 +148,12 @@ function decodeComponents(tokens: string[], labels: string[]): Record<string, st
 	const out: Record<string, string> = {}
 	let currentTag: string | null = null
 	let currentTokens: string[] = []
+
 	const flush = () => {
-		if (currentTag && currentTokens.length > 0 && !(currentTag in out)) {
+		if (currentTag && currentTokens.length && !(currentTag in out)) {
 			out[currentTag] = currentTokens.join(" ").trim()
 		}
+
 		currentTag = null
 		currentTokens = []
 	}
@@ -141,8 +164,10 @@ function decodeComponents(tokens: string[], labels: string[]): Record<string, st
 
 		if (label === "O") {
 			flush()
+
 			continue
 		}
+
 		const [prefix, tag] = label.split("-", 2)
 
 		if (prefix === "B" || currentTag !== tag) {
@@ -153,6 +178,7 @@ function decodeComponents(tokens: string[], labels: string[]): Record<string, st
 			currentTokens.push(tok)
 		}
 	}
+
 	flush()
 
 	return out
@@ -168,6 +194,7 @@ async function loadSeeds(
 		.split(",")
 		.map((p) => p.trim())
 		.filter(Boolean)
+
 	report?.(`reading seeds from ${paths.length} shard(s) (target: ${count}, stratified)`)
 
 	if (includeSources) {
@@ -190,6 +217,7 @@ async function loadSeeds(
 			const row = (await cursor.next()) as CorpusRow | null
 
 			if (!row) break
+
 			scanned++
 
 			// Source allow-list (--include-sources) — applied early to skip parsing rows we won't use
@@ -199,8 +227,10 @@ async function loadSeeds(
 			// Skip rows with too few components — single-name wof-admin entries don't make useful seeds
 			if (Object.keys(components).length < 2) {
 				skippedThinComponents++
+
 				continue
 			}
+
 			const seed: Seed = {
 				raw: row.raw,
 				components,
@@ -208,6 +238,7 @@ async function loadSeeds(
 				source: row.source,
 				source_id: row.source_id,
 			}
+
 			let bucket = bySource.get(row.source)
 
 			if (!bucket) {
@@ -219,12 +250,14 @@ async function loadSeeds(
 				bucket.push(seed)
 			}
 		}
+
 		await reader.close()
 	}
 
 	report?.(
 		`  scanned ${scanned} rows across ${paths.length} shard(s); thin-components dropped: ${skippedThinComponents}`
 	)
+
 	report?.(`  per-source pool sizes:`)
 
 	for (const [src, pool] of bySource) {
@@ -233,7 +266,7 @@ async function loadSeeds(
 
 	// Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
 	// to sources in alphabetical order. If a pool is smaller than its target, take all of it.
-	const sources = Array.from(bySource.keys()).sort()
+	const sources = Array.from(bySource.keys()).toSorted()
 	const perSource = Math.floor(count / sources.length)
 	const remainder = count - perSource * sources.length
 	const picked: Seed[] = []
@@ -248,6 +281,7 @@ async function loadSeeds(
 			const k = Math.floor(Math.random() * (j + 1))
 			;[pool[j], pool[k]] = [pool[k]!, pool[j]!]
 		}
+
 		const take = Math.min(target, pool.length)
 		picked.push(...pool.slice(0, take))
 
@@ -255,6 +289,7 @@ async function loadSeeds(
 			report?.(`    ⚠ ${src}: requested ${target}, pool had ${pool.length}`)
 		}
 	}
+
 	report?.(`  → loaded ${picked.length} seeds across ${sources.length} sources`)
 
 	return picked
@@ -369,7 +404,7 @@ function makeAnthropicProvider(model: string): LlmProvider {
 
 function parseCandidates(text: string): Candidate[] {
 	// Strip markdown fences the model sometimes wraps around JSON
-	const cleaned = text.replace(/^```(?:json)?\n?|\n?```$/g, "").trim()
+	const cleaned = text.replaceAll(/^```(?:json)?\n?|\n?```$/g, "").trim()
 
 	try {
 		const parsed = JSON.parse(cleaned) as unknown
@@ -394,20 +429,22 @@ function parseCandidates(text: string): Candidate[] {
 // ── Validator ─────────────────────────────────────────────────────────────
 
 function normalize(s: string): string {
-	return s.toLowerCase().replace(/\s+/g, " ").trim()
+	return s.toLowerCase().replaceAll(/\s+/g, " ").trim()
 }
 
-// Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
+/**
+ * Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
+ */
 const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"])
 
 function validate(seed: Seed, candidate: Candidate): boolean {
 	if (!candidate.raw || typeof candidate.raw !== "string") return false
 
-	if (candidate.raw.length > 500) return false
+	if (candidate.raw.length > MAX_CANDIDATE_LENGTH) return false
 
 	if (/```|<\/?\w+>|^\s*\{/.test(candidate.raw)) return false
 	const normRaw = normalize(candidate.raw)
-	const dropped = new Set(candidate.dropped ?? [])
+	const dropped = new Set(candidate.dropped)
 
 	// LLM cannot drop required components present in the seed.
 	for (const tag of dropped) {
@@ -423,6 +460,7 @@ function validate(seed: Seed, candidate: Candidate): boolean {
 		if (!value) continue
 
 		if (!normRaw.includes(normalize(value))) return false
+
 		keptCount++
 	}
 
@@ -440,13 +478,14 @@ export async function expandGolden(
 ): Promise<ExpandGoldenSummary> {
 	const corpusPath =
 		options.corpus ?? dataRootPath("corpus", "versioned", "v0.2.0", "corpus-v0.2.0", "test", "part-0000.parquet")
+
 	const count = options.count ?? 100
 	const variants = options.variants ?? 5
 	const providerName = options.provider ?? "deepseek"
 	const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat")
 	const concurrencyLimit = options.concurrency ?? 4
 	const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null
-	const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)
+	const ts = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19)
 	const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`
 
 	const provider = providerName === "anthropic" ? makeAnthropicProvider(model) : makeDeepseekProvider(model)
@@ -454,7 +493,7 @@ export async function expandGolden(
 
 	const seeds = await loadSeeds(corpusPath, count, includeSources, report)
 
-	if (seeds.length === 0) {
+	if (!seeds.length) {
 		throw new Error("no seeds loaded — corpus path or filter is wrong")
 	}
 
@@ -466,6 +505,7 @@ export async function expandGolden(
 
 	// Bounded-concurrency worker pool
 	let cursor = 0
+
 	const workers = Array.from({ length: Math.min(concurrencyLimit, seeds.length) }, async () => {
 		while (true) {
 			const i = cursor++
@@ -491,17 +531,20 @@ export async function expandGolden(
 
 						// Remove dropped components from the components map
 						for (const tag of goldenCandidate.dropped_components) {
+							// oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record; the object is not on a hot path
 							delete goldenCandidate.components[tag]
 						}
+
 						outRows.push(goldenCandidate)
+
 						kept++
 					} else {
 						dropped++
 					}
 				}
-			} catch (err) {
+			} catch (error) {
 				errored++
-				report?.(`  ✗ seed ${seed.source_id}: ${(err as Error).message}`)
+				report?.(`  ✗ seed ${seed.source_id}: ${(error as Error).message}`)
 			}
 
 			if ((i + 1) % 10 === 0) {
@@ -509,6 +552,7 @@ export async function expandGolden(
 			}
 		}
 	})
+
 	await Promise.all(workers)
 
 	writeJSONL(outputPath, outRows)
@@ -517,7 +561,7 @@ export async function expandGolden(
 	report?.(`candidates kept:  ${kept}`)
 	report?.(`candidates dropped (validator): ${dropped}`)
 	report?.(`seeds with errors: ${errored}`)
-	report?.(`yield: ${seeds.length > 0 ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`)
+	report?.(`yield: ${seeds.length ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`)
 	report?.(`output:           ${outputPath}`)
 
 	return { seedsProcessed: seeds.length, kept, dropped, errored, outputPath }