@mailwoman/corpus 9.1.0 → 9.2.0

package/src/tools/fetch/ourairports.ts

@@ -39,6 +39,7 @@
 import { mkdirSync } from "node:fs"
 import { join } from "node:path"
 
+import { APIClient } from "@mailwoman/core/api"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -92,9 +93,13 @@ interface OurAirportsManifest {
  */
 async function readLastModified(url: string): Promise<string | null> {
 	try {
-		const res = await fetch(url, { method: "HEAD", signal: AbortSignal.timeout(30_000) })
+		const res = await new APIClient({ displayName: "ourairports", retry: true }).fetch({
+			url,
+			method: "head",
+			timeout: 30_000,
+		})
 
-		return res.ok ? res.headers.get("last-modified") : null
+		return (res.headers["last-modified"] as string | undefined) ?? null
 	} catch {
 		return null
 	}

package/src/tools/fetch/ppd.ts

@@ -104,7 +104,7 @@ export async function extractPPDTuples(
 /**
  * Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
  */
-export async function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats> {
+async function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats> {
 	// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
 	// shard-recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
 	const rows = CSVSpliterator.fromAsync<string[]>(createReadStream(inputPath), {

package/src/tools/fetch/state-hi-schools.ts

@@ -25,6 +25,7 @@ import { existsSync, mkdirSync, statSync } from "node:fs"
 import { unlink } from "node:fs/promises"
 import { join } from "node:path"
 
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -34,7 +35,6 @@ import { downloadToFile, readManifest, writeManifest } from "./download.ts"
  * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
  * than data.
  */
-const BYTES_PER_KIB = 1024
 
 const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx"
 const SLUG = "state-hi-schools"
@@ -96,26 +96,6 @@ interface Manifest {
 	notes: string
 }
 
-/**
- * Mimic `numfmt --to=iec` for a friendly byte-size log line.
- */
-function iec(bytes: number): string {
-	if (bytes < BYTES_PER_KIB) return String(bytes)
-	const units = ["K", "M", "G", "T", "P"]
-	let value = bytes / 1024
-	let i = 0
-
-	while (value >= BYTES_PER_KIB && i < units.length - 1) {
-		value /= 1024
-
-		i++
-	}
-
-	const rounded = value < 10 ? value.toFixed(1) : Math.round(value).toString()
-
-	return `${rounded}${units[i] ?? ""}`
-}
-
 /**
  * Run the openpyxl converter. Its stderr narration streams straight through to the process stderr (matching the old
  * `stdio: inherit` behavior) rather than routing through `report` — the python child owns those lines.
@@ -190,7 +170,7 @@ export async function fetchStateHISchools(
 	}
 
 	const xlsxSize = statSync(xlsxDest).size
-	report?.(`  Downloaded XLSX: ${iec(xlsxSize)}`)
+	report?.(`  Downloaded XLSX: ${ByteFormatter.formatIEC(xlsxSize)}`)
 
 	if (xlsxSize < BYTES_PER_KIB) {
 		report?.(`  ✗ Response too small (${xlsxSize} bytes) — probable error page`)
@@ -221,7 +201,7 @@ export async function fetchStateHISchools(
 
 	await writeManifest(manifestPath, manifest)
 
-	report?.(`  ✓ ${iec(csvSize)}  sha256=${csvSha}`)
+	report?.(`  ✓ ${ByteFormatter.formatIEC(csvSize)}  sha256=${csvSha}`)
 	report?.(`  MANIFEST written to ${manifestPath}`)
 
 	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }

package/src/tools/fetch/state-sources.ts

@@ -20,6 +20,7 @@
 import { mkdirSync } from "node:fs"
 import { join } from "node:path"
 
+import { BYTES_PER_KIB } from "@mailwoman/core/fs/utils"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -29,7 +30,6 @@ import { downloadToFile, writeManifest } from "./download.ts"
  * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
  * than data.
  */
-const BYTES_PER_KIB = 1024
 
 export type FetchStateSourcesOptions = BaseFetchOptions
 

package/src/tools/fetch/tiger-full.ts

@@ -28,6 +28,8 @@ import { Readable } from "node:stream"
 import { pipeline } from "node:stream/promises"
 import { setTimeout as sleep } from "node:timers/promises"
 
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
 import { sha256File } from "@mailwoman/core/utils"
 
 import type { BaseFetchOptions, FetchSummary } from "./download.ts"
@@ -47,8 +49,6 @@ const HTTP_OK = 200
  */
 const HTTP_REDIRECT = 300
 
-const BYTES_PER_KIB = 1024
-
 const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT"
 
 export interface FetchTigerFullOptions extends BaseFetchOptions {
@@ -77,20 +77,6 @@ interface CountyEntry {
 	bytes: number
 }
 
-function humanBytes(bytes: number): string {
-	const units = ["B", "KiB", "MiB", "GiB", "TiB"]
-	let value = bytes
-	let unit = 0
-
-	while (value >= BYTES_PER_KIB && unit < units.length - 1) {
-		value /= 1024
-
-		unit++
-	}
-
-	return `${value.toFixed(unit === 0 ? 0 : 1)}${units[unit]}`
-}
-
 /**
  * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries).
  *
@@ -193,7 +179,7 @@ export async function fetchTigerFull(
 	options: FetchTigerFullOptions,
 	report?: (line: string) => void
 ): Promise<FetchSummary> {
-	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter(Boolean)
+	const skipStateFips = (options.skipStateFips ?? "50").split(/\s+/).filter((value) => value.length > 0)
 	const rateSleepMs = Math.round((options.rateSleep ?? 0.2) * 1000)
 	const maxParallel = options.maxParallel ?? 4
 	const dryRun = options.dryRun ?? false
@@ -205,13 +191,17 @@ export async function fetchTigerFull(
 
 	report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`)
 
-	const listingRes = await fetch(`${TIGER_BASE_URL}/`, {
-		headers: { "Accept-Encoding": "gzip, br" },
-		signal: AbortSignal.timeout(60_000),
+	// `responseType: "text"` — an Apache directory index, scraped below. The per-county ARCHIVE downloads
+	// stay on raw `fetch` (they stream to disk; see `downloadOne`).
+	const listingRes = await new APIClient({
+		displayName: "tiger-listing",
+		retry: true,
+		axios: { headers: { "Accept-Encoding": "gzip, br" } },
 	})
+		.fetch<string>({ url: `${TIGER_BASE_URL}/`, responseType: "text", timeout: 60_000 })
+		.then(pluckResponseData)
 
-	if (!listingRes.ok) throw new Error(`Failed to fetch TIGER directory listing: HTTP ${listingRes.status}`)
-	const html = await listingRes.text()
+	const html = listingRes
 	const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted()
 	const totalCounties = allZips.length
 	report?.(`  Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`)
@@ -314,7 +304,10 @@ export async function fetchTigerFull(
 		// Collect results from this state.
 		for (const result of results) {
 			if (result.ok) {
-				report?.(`  ok ${result.filename}  ${humanBytes(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`)
+				report?.(
+					`  ok ${result.filename}  ${ByteFormatter.formatIEC(result.bytes)}  sha256=${result.sha256.slice(0, 12)}...`
+				)
+
 				manifest.set(result.filename, { filename: result.filename, sha256: result.sha256, bytes: result.bytes })
 
 				totalFetched++
@@ -350,7 +343,7 @@ export async function fetchTigerFull(
 	report?.(`  Counties failed           : ${totalFailed}`)
 
 	if (totalBytesFetched > 0) {
-		report?.(`  Bytes fetched this run    : ${humanBytes(totalBytesFetched)} (${totalBytesFetched})`)
+		report?.(`  Bytes fetched this run    : ${ByteFormatter.formatIEC(totalBytesFetched)} (${totalBytesFetched})`)
 	}
 
 	if (totalFailed > 0) {

package/src/tools/golden-expand.ts

@@ -50,8 +50,9 @@ import { mkdir } from "node:fs/promises"
 import { dirname } from "node:path"
 
 import { ParquetReader } from "@dsnp/parquetjs"
+import { APIClient, pluckResponseData } from "@mailwoman/core/api"
 import { $private } from "@mailwoman/core/env"
-import { tryParsingJSON } from "@mailwoman/core/objects"
+import { isPresent, tryParsingJSON } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
 import { createNewlineWriter } from "spliterator"
 
@@ -195,7 +196,7 @@ async function loadSeeds(
 	const paths = corpusPath
 		.split(",")
 		.map((p) => p.trim())
-		.filter(Boolean)
+		.filter(isPresent)
 
 	report?.(`reading seeds from ${paths.length} shard(s) (target: ${count}, stratified)`)
 
@@ -345,24 +346,28 @@ function makeDeepseekProvider(model: string): LlmProvider {
 		name: "deepseek",
 		model,
 		async generateVariants(seed, n) {
-			const res = await fetch("https://api.deepseek.com/chat/completions", {
-				method: "POST",
-				headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
-				body: JSON.stringify({
-					model,
-					messages: [
-						{ role: "system", content: SYSTEM_PROMPT },
-						{ role: "user", content: buildUserPrompt(seed, n) },
-					],
-					response_format: { type: "json_object" },
-					max_tokens: 800,
-					temperature: 0.7,
-				}),
-				signal: AbortSignal.timeout(60_000),
+			const data = await new APIClient({
+				displayName: "deepseek",
+				retry: true,
+				axios: { headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" } },
 			})
+				.fetch<{ choices?: Array<{ message: { content: string } }> }>({
+					url: "https://api.deepseek.com/chat/completions",
+					method: "post",
+					timeout: 60_000,
+					data: {
+						model,
+						messages: [
+							{ role: "system", content: SYSTEM_PROMPT },
+							{ role: "user", content: buildUserPrompt(seed, n) },
+						],
+						response_format: { type: "json_object" },
+						max_tokens: 800,
+						temperature: 0.7,
+					},
+				})
+				.then(pluckResponseData)
 
-			if (!res.ok) throw new Error(`DeepSeek HTTP ${res.status}: ${await res.text()}`)
-			const data = (await res.json()) as { choices?: Array<{ message: { content: string } }> }
 			const content = data.choices?.[0]?.message.content ?? "{}"
 
 			return parseCandidates(content)
@@ -379,24 +384,30 @@ function makeAnthropicProvider(model: string): LlmProvider {
 		name: "anthropic",
 		model,
 		async generateVariants(seed, n) {
-			const res = await fetch("https://api.anthropic.com/v1/messages", {
-				method: "POST",
-				headers: {
-					"x-api-key": apiKey,
-					"anthropic-version": "2023-06-01",
-					"Content-Type": "application/json",
+			const data = await new APIClient({
+				displayName: "anthropic",
+				retry: true,
+				axios: {
+					headers: {
+						"x-api-key": apiKey,
+						"anthropic-version": "2023-06-01",
+						"Content-Type": "application/json",
+					},
 				},
-				body: JSON.stringify({
-					model,
-					max_tokens: 800,
-					system: SYSTEM_PROMPT,
-					messages: [{ role: "user", content: buildUserPrompt(seed, n) }],
-				}),
-				signal: AbortSignal.timeout(60_000),
 			})
+				.fetch<{ content?: Array<{ type: string; text: string }> }>({
+					url: "https://api.anthropic.com/v1/messages",
+					method: "post",
+					timeout: 60_000,
+					data: {
+						model,
+						max_tokens: 800,
+						system: SYSTEM_PROMPT,
+						messages: [{ role: "user", content: buildUserPrompt(seed, n) }],
+					},
+				})
+				.then(pluckResponseData)
 
-			if (!res.ok) throw new Error(`Anthropic HTTP ${res.status}: ${await res.text()}`)
-			const data = (await res.json()) as { content?: Array<{ type: string; text: string }> }
 			const text = data.content?.find((c) => c.type === "text")?.text ?? "[]"
 
 			return parseCandidates(text)

package/src/tools/golden-relabel-street.ts

@@ -60,7 +60,7 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from
 import { basename, join } from "node:path"
 
 import { isStreetDirectionalToken, matchTrailingSuffix, type USStreetSuffix } from "@mailwoman/codex/us"
-import { parseJSONStrict, tryParsingJSON } from "@mailwoman/core/objects"
+import { isPresent, parseJSONStrict, tryParsingJSON } from "@mailwoman/core/objects"
 import { sha256File } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
@@ -616,7 +616,7 @@ export async function relabelGoldenDirectory(
 function renderDeckMarkdown(deck: GoldenRelabelDeckEntry[], parent: string, version: string): string {
 	const span = (components: Record<string, string>): string =>
 		[components.street_prefix, components.street, components.street_suffix]
-			.filter(Boolean)
+			.filter(isPresent)
 			.map((s) => JSON.stringify(s))
 			.join(" + ")
 

package/src/tools/index.ts

@@ -4,8 +4,7 @@
  * @author Teffen Ellis, et al.
  *
  *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
- *   no `process.exit`: commands own parsing, rendering, and exit codes (see the 2026-07-09
- *   scripts→Pastel spec).
+ *   no `process.exit`: commands own parsing, rendering, and exit codes.
  */
 
 export * from "./align-shard.ts"
@@ -23,6 +22,7 @@ export * from "./lint-shard.ts"
 export * from "./lint-shard-vocab.ts"
 export * from "./shard-kryptonite.ts"
 export * from "./overture-subvenue.ts"
+export * from "./postcode-triples.ts"
 export * from "./sub-venue-lexicon.ts"
 export * from "./sub-venue-promotions.ts"
 export * from "./shard-translit.ts"

package/src/tools/ingest-csv.ts

@@ -241,7 +241,6 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`
 
 	const tempCols = columns.map((c) => `"${c.name}"`).join(", ")
-	const insertSQL = `INSERT INTO "${opts.tableName}" (${tempCols})\nSELECT ${tempCols} FROM temp."${opts.tableName}_source";`
 
 	process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`)
 	process.stderr.write(`${createTableSQL}\n\n`)
@@ -264,17 +263,6 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 	// Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
 	// NULL normalization and type coercion.
-	const csvBasename = basename(opts.inputPath)
-
-	const importSQL = [
-		`CREATE TEMP TABLE "${opts.tableName}_source" (${colDefs});`,
-		`.mode csv`,
-		`.separator "${opts.separator}"`,
-		`.import "${csvBasename}" --skip ${opts.skipLines + (opts.hasHeader ? 1 : 0)} --schema temp ${opts.tableName}_source`,
-		insertSQL,
-		`DROP TABLE temp."${opts.tableName}_source";`,
-	]
-
 	// better-sqlite3 doesn't support .import natively, so we use a different approach:
 	// Read the CSV line-by-line and INSERT in a transaction.
 	process.stderr.write(`Importing rows...\n`)
@@ -373,7 +361,6 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	db.close()
 
 	// Write MANIFEST
-	const fileSize = (await import("node:fs/promises")).stat
 	const stat = await (await import("node:fs/promises")).stat(opts.outputPath)
 
 	const manifest = {

package/src/tools/overlay-manifest.ts

@@ -24,7 +24,7 @@
  */
 
 import { readFileSync, statSync, writeFileSync } from "node:fs"
-import { join } from "node:path"
+import { basename, dirname, join } from "node:path"
 
 import { parseJSONStrict } from "@mailwoman/core/objects"
 import { sha256Hex } from "@mailwoman/core/utils"
@@ -96,6 +96,24 @@ export interface OverlayManifestOptions {
 	note: string
 }
 
+/**
+ * Resolve a base manifest's shard path to the mounted corpus tree used by Modal.
+ */
+export function rerootBaseShardPath(path: string, baseManifestPath: string): string {
+	const versionedIndex = path.indexOf("/corpus/versioned/")
+
+	if (versionedIndex !== -1) return "/data" + path.slice(versionedIndex)
+
+	if (/^\/data\/(?:train|val|test)\//u.test(path)) {
+		const localBaseDir = dirname(baseManifestPath)
+		const baseModalRoot = `/data/corpus/versioned/${basename(dirname(localBaseDir))}/${basename(localBaseDir)}`
+
+		return baseModalRoot + path.slice("/data".length)
+	}
+
+	return path
+}
+
 export async function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void> {
 	const base = parseJSONStrict<BaseManifest>(readFileSync(args.base, "utf8"))
 
@@ -103,13 +121,7 @@ export async function assembleOverlayManifest(args: OverlayManifestOptions): Pro
 		console.log(`WARN: base already contains source '${args.source}' — is this the right base?`)
 	}
 
-	const reroot = (p: string): string => {
-		const i = p.indexOf("/corpus/versioned/")
-
-		return i !== -1 ? "/data" + p.slice(i) : p
-	}
-
-	const kept = base.shards.map((s) => ({ ...s, path: reroot(s.path) }))
+	const kept = base.shards.map((s) => ({ ...s, path: rerootBaseShardPath(s.path, args.base) }))
 
 	const newTrain = await descriptor(
 		join(args.newDir, "train", args.shardParquet),