@mailwoman/corpus 9.2.0 → 9.3.0

package/lib/tools/corpus-stats.ts

@@ -0,0 +1,136 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
+ *
+ *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
+ *   as the baseline against which a new slice is compared.
+ *
+ *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
+ *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
+ *   mainline slice added, a source-pool re-weighted, etc.).
+ *
+ *   Output schema:
+ *
+ *   ```ts
+ *   interface CorpusStats {
+ *     row_count: number
+ *     slice_paths: string[]
+ *     tokens: { [token: string]: { [label: string]: number } }
+ *     bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
+ *     // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
+ *     // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
+ *   }
+ * ```
+ *
+ *   Usage: node scripts/build-corpus-stats.ts\
+ *   --slices <glob-pattern-or-dir>\
+ *   --output <stats.json>
+ *
+ *   For a quick local-corpus baseline (limited but useful for linter testing): node
+ *   scripts/build-corpus-stats.ts\
+ *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --output /tmp/corpus-stats-local.json
+ */
+
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { readDirectory, statPath } from "@mailwoman/core/fs/readers"
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { join } from "path-ts"
+
+import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats"
+
+const MIN_BIGRAM_COUNT = 2
+
+export interface CorpusStatsOptions {
+	slicesArg: string
+	outputPath: string
+	limitPerSlice?: number
+}
+
+async function discoverSlices(slicesArg: string): Promise<string[]> {
+	const stat = await statPath(slicesArg)
+
+	if (stat.isDirectory()) {
+		return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f))
+	}
+
+	if (stat.isFile() && slicesArg.endsWith(".parquet")) return [slicesArg]
+
+	// Otherwise treat as a literal path list (one per line if it's stdin-friendly).
+	return [slicesArg]
+}
+
+export async function buildCorpusStats(args: CorpusStatsOptions): Promise<void> {
+	const slicePaths = await discoverSlices(args.slicesArg)
+
+	console.error(`Discovered ${slicePaths.length} parquet slice(s)`)
+
+	const stats = createCooccurrenceStats()
+	const { tokens: tokenStats, bigrams: bigramStats } = stats
+	let totalRows = 0
+
+	for (const path of slicePaths) {
+		console.error(`Reading ${path}...`)
+
+		const before = totalRows
+
+		for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerSlice)) {
+			totalRows++
+
+			if (tokens.length !== labels.length) continue // skip malformed
+
+			accumulateCooccurrences(stats, tokens, labels)
+		}
+
+		console.error(
+			`  ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`
+		)
+	}
+
+	// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
+	// stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
+	// detection.
+	let prunedBigrams = 0
+
+	for (const [k, labelMap] of bigramStats) {
+		let total = 0
+
+		for (const v of labelMap.values()) {
+			total += v
+		}
+
+		if (total < MIN_BIGRAM_COUNT) {
+			bigramStats.delete(k)
+
+			prunedBigrams++
+		}
+	}
+
+	console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`)
+
+	const out = {
+		row_count: totalRows,
+		slice_paths: slicePaths,
+		tokens: {} as Record<string, Record<string, number>>,
+		bigrams: {} as Record<string, Record<string, number>>,
+	}
+
+	for (const [tk, labelMap] of tokenStats) {
+		out.tokens[tk] = Object.fromEntries(labelMap)
+	}
+
+	for (const [k, labelMap] of bigramStats) {
+		out.bigrams[k] = Object.fromEntries(labelMap)
+	}
+
+	await writeLocalJSONFile(out, args.outputPath)
+	const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(JSON.stringify(out)))
+
+	console.error(
+		`Wrote ${args.outputPath} (${formattedSize}) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`
+	)
+}

package/lib/tools/fetch/ban.ts

@@ -0,0 +1,289 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
+ *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
+ *
+ *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
+ *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
+ *
+ *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
+ *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
+ *   `<outRoot>/ban/MANIFEST.json` covers all codes.
+ *
+ *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
+ *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
+ *   Python.
+ */
+
+import { gunzip } from "@mailwoman/core/fs/compression"
+import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { isFile, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePath, writeLocalFile } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, loadManifestEntries, writeManifest } from "#tools/fetch/download"
+
+/**
+ * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
+ * than data.
+ */
+
+const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
+
+/**
+ * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
+ * change.
+ */
+const DEPT_CODES = [
+	"01",
+	"02",
+	"03",
+	"04",
+	"05",
+	"06",
+	"07",
+	"08",
+	"09",
+	"10",
+	"11",
+	"12",
+	"13",
+	"14",
+	"15",
+	"16",
+	"17",
+	"18",
+	"19",
+	"21",
+	"22",
+	"23",
+	"24",
+	"25",
+	"26",
+	"27",
+	"28",
+	"29",
+	"2A",
+	"2B",
+	"30",
+	"31",
+	"32",
+	"33",
+	"34",
+	"35",
+	"36",
+	"37",
+	"38",
+	"39",
+	"40",
+	"41",
+	"42",
+	"43",
+	"44",
+	"45",
+	"46",
+	"47",
+	"48",
+	"49",
+	"50",
+	"51",
+	"52",
+	"53",
+	"54",
+	"55",
+	"56",
+	"57",
+	"58",
+	"59",
+	"60",
+	"61",
+	"62",
+	"63",
+	"64",
+	"65",
+	"66",
+	"67",
+	"68",
+	"69",
+	"70",
+	"71",
+	"72",
+	"73",
+	"74",
+	"75",
+	"76",
+	"77",
+	"78",
+	"79",
+	"80",
+	"81",
+	"82",
+	"83",
+	"84",
+	"85",
+	"86",
+	"87",
+	"88",
+	"89",
+	"90",
+	"91",
+	"92",
+	"93",
+	"94",
+	"95",
+	"971",
+	"972",
+	"973",
+	"974",
+	"976",
+]
+
+export interface BanManifestEntry {
+	dept_code: string
+	filename: string
+	source_url: string
+	downloaded_at: string
+	sha256: string
+	bytes: number
+}
+
+export type FetchBanOptions = BaseFetchOptions
+
+export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	const banDir = join(options.outRoot, "ban")
+	const manifestPath = join(banDir, "MANIFEST.json")
+	await makeDirectories(banDir)
+
+	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
+	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
+
+	let fetched = 0
+	let skipped = 0
+	let failed = 0
+	const failedCodes: string[] = []
+
+	for (const code of DEPT_CODES) {
+		const filename = `adresses-${code}.csv`
+		const gzFile = join(banDir, `${filename}.gz`)
+		const csvFile = join(banDir, filename)
+		const url = `${BASE_URL}/adresses-${code}.csv.gz`
+
+		report?.(`=== dept ${code}`)
+
+		// If the CSV already exists, compare its sha256 against the manifest.
+		if (await isFile(csvFile)) {
+			const existingSha = await sha256File(csvFile)
+			const recordedSha = entries.get(code)?.sha256
+
+			if (recordedSha && existingSha === recordedSha) {
+				report?.(`  → already present + sha matches — skipping`)
+
+				skipped++
+
+				continue
+			}
+
+			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
+			await removePath(csvFile)
+		}
+
+		// Download the gzipped CSV.
+		try {
+			await downloadToFile({
+				url,
+				dest: gzFile,
+				timeoutMs: 600_000,
+				headers: { "Accept-Encoding": "gzip, br" },
+				report,
+			})
+		} catch (error) {
+			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Guard against truncated 404/error pages.
+		const gzSize = (await tryStat(gzFile))?.size ?? 0
+
+		if (gzSize < BYTES_PER_KIB) {
+			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		// Decompress in-place; delete the .gz.
+		try {
+			await writeLocalFile(await gunzip(await readLocalBuffer(gzFile)), csvFile)
+		} catch (error) {
+			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
+			await removePath(gzFile)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		await removePath(gzFile)
+
+		if (!(await isFile(csvFile))) {
+			report?.(`  ✗ decompressed file not found at ${csvFile}`)
+
+			failed++
+			failedCodes.push(code)
+
+			continue
+		}
+
+		const bytes = (await tryStat(csvFile))?.size ?? 0
+		const sha = await sha256File(csvFile)
+
+		entries.set(code, {
+			dept_code: code,
+			filename,
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			sha256: sha,
+			bytes,
+		})
+
+		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
+
+		fetched++
+
+		// Be a polite citizen — short pause between requests.
+		await sleep(200)
+	}
+
+	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
+	const sorted = [...entries.values()].toSorted((a, b) =>
+		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
+	)
+
+	await writeManifest(manifestPath, sorted)
+	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
+
+	report?.(`=== summary ===`)
+	report?.(`fetched:  ${fetched}`)
+	report?.(`skipped:  ${skipped} (already present + sha matched)`)
+	report?.(`failed:   ${failed}`)
+
+	if (failedCodes.length) {
+		report?.(`failed codes: ${failedCodes.join(" ")}`)
+	}
+
+	return { fetched, skipped, failed, failedCodes }
+}

package/lib/tools/fetch/download.ts

@@ -0,0 +1,262 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
+ */
+
+import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { writeLocalFile, writeLocalTextFile } from "@mailwoman/core/fs/writers"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import type { PathBuilderLike } from "path-ts"
+
+/**
+ * Rate limited — retryable, the server is asking us to back off.
+ */
+const HTTP_TOO_MANY_REQUESTS = 429
+
+/**
+ * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
+ */
+const HTTP_SERVER_ERROR_MIN = 500
+
+/**
+ * Highest 5xx status.
+ */
+const HTTP_SERVER_ERROR_MAX = 599
+
+/**
+ * The option base every `mailwoman corpus fetch <source>` module extends.
+ */
+/**
+ * How long a failed transfer waits before the next attempt.
+ */
+export const DEFAULT_RETRY_DELAY_MS = 5000
+
+export interface BaseFetchOptions {
+	/**
+	 * Destination root for downloaded source data. Each source writes its own subdirectory.
+	 */
+	outRoot: PathBuilderLike
+	/**
+	 * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
+	 *
+	 * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two
+	 * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the
+	 * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten.
+	 */
+	retryDelayMs?: number
+}
+
+/**
+ * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
+ */
+export interface FetchSummary {
+	fetched: number
+	skipped: number
+	failed: number
+	failedCodes: string[]
+}
+
+/**
+ * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count +
+ * sha256, so downstream adapters can verify provenance.
+ */
+export interface SourceManifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+/**
+ * A status worth retrying: rate limiting or a server-side failure.
+ */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
+ * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
+ * run and vanishes locally.
+ */
+export class HTTPStatusError extends Error {
+	readonly status: number
+
+	constructor(status: number, message: string) {
+		super(message)
+		this.name = "HTTPStatusError"
+		this.status = status
+	}
+}
+
+export function isTransientStatus(status: number): boolean {
+	return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX)
+}
+
+export interface DownloadOptions {
+	url: string
+	dest: string
+	/**
+	 * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
+	 */
+	timeoutMs?: number
+	/**
+	 * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
+	 */
+	retries?: number
+	/**
+	 * Delay between attempts. Default 5s.
+	 */
+	retryDelayMs?: number
+	headers?: Record<string, string>
+	report?: (line: string) => void
+}
+
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export async function downloadToFile(options: DownloadOptions): Promise<{ bytes: number }> {
+	const {
+		url,
+		dest,
+		timeoutMs = 600_000,
+		retries = 0,
+		retryDelayMs = DEFAULT_RETRY_DELAY_MS,
+		headers,
+		report,
+	} = options
+
+	let lastError: unknown
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		if (attempt > 0) {
+			report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`)
+			await sleep(retryDelayMs)
+		}
+
+		let res: Response
+
+		try {
+			// Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
+			// `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
+			// non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
+			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
+		} catch (error) {
+			// AbortSignal timeouts and network-level failures are retryable.
+			lastError = error
+
+			continue
+		}
+
+		if (!res.ok) {
+			const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`)
+
+			if (!isTransientStatus(res.status)) throw error
+			lastError = error
+
+			continue
+		}
+
+		try {
+			const buffer = Buffer.from(await res.arrayBuffer())
+			await writeLocalFile(buffer, dest)
+
+			return { bytes: buffer.byteLength }
+		} catch (error) {
+			// A mid-stream abort while reading the body is retryable too.
+			lastError = error
+		}
+	}
+
+	throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+export interface StreamDownloadOptions {
+	headers?: Record<string, string>
+	timeoutMs: number
+	retries: number
+	retryDelayMs: number
+}
+
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
+ *
+ * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
+ * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
+ * two-URL fallback ladders consume.
+ */
+export async function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise<number> {
+	for (let attempt = 0; attempt <= opts.retries; attempt++) {
+		try {
+			const res = await fetch(url, {
+				headers: opts.headers ?? {},
+				redirect: "follow",
+				signal: AbortSignal.timeout(opts.timeoutMs),
+			})
+
+			if (res.ok && res.body) {
+				await pipeline(Readable.fromWeb(res.body), openWriteStream(dest))
+
+				return res.status
+			}
+
+			if (attempt < opts.retries && isTransientStatus(res.status)) {
+				await sleep(opts.retryDelayMs)
+
+				continue
+			}
+
+			return res.status
+		} catch {
+			if (attempt < opts.retries) {
+				await sleep(opts.retryDelayMs)
+
+				continue
+			}
+
+			return 0
+		}
+	}
+
+	return 0
+}
+
+/**
+ * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
+ */
+export async function readManifest<T>(path: string): Promise<T | null> {
+	if (!(await pathExists(path))) return null
+
+	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
+	// JSON does; tryParsingJSON returns null for the non-string sentinel.
+	const text = await readLocalTextFile(path).catch(() => null)
+
+	return tryParsingJSON<T>(text)
+}
+
+/**
+ * Load manifest entries into a map so untouched keys survive a partial re-fetch.
+ */
+export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
+	const entries = new Map<string, T>()
+	const parsed = await readManifest<T[]>(path)
+
+	for (const entry of parsed ?? []) {
+		entries.set(key(entry), entry)
+	}
+
+	return entries
+}
+
+/**
+ * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
+ */
+export async function writeManifest(path: string, manifest: unknown): Promise<void> {
+	await writeLocalTextFile(JSON.stringify(manifest, null, 2) + "\n", path)
+}