@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/tools/fetch/download/network.ts

@@ -0,0 +1,226 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
+ */
+
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { writeLocalFile } from "@mailwoman/core/fs/writers"
+import { sleep } from "@mailwoman/core/utils/sleep"
+import type { PathBuilderLike } from "path-ts"
+
+/**
+ * Rate limited — retryable, the server is asking us to back off.
+ */
+const HTTP_TOO_MANY_REQUESTS = 429
+
+/**
+ * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
+ */
+const HTTP_SERVER_ERROR_MIN = 500
+
+/**
+ * Highest 5xx status.
+ */
+const HTTP_SERVER_ERROR_MAX = 599
+
+/**
+ * The option base every `mailwoman corpus fetch <source>` module extends.
+ */
+/**
+ * How long a failed transfer waits before the next attempt.
+ */
+export const DEFAULT_RETRY_DELAY_MS = 5000
+
+export interface BaseFetchOptions {
+	/**
+	 * Destination root for downloaded source data. Each source writes its own subdirectory.
+	 */
+	outRoot: PathBuilderLike
+	/**
+	 * Pause between transfer retries, in milliseconds. Defaults to {@linkcode DEFAULT_RETRY_DELAY_MS}.
+	 *
+	 * A test that exercises the failure path pays this delay once per retry in real time — measured at 20.1 s for the two
+	 * failing-transfer cases in `geonames-postal.test.ts`, which is the whole cost of that file. The retry COUNT is the
+	 * behaviour under test there; the pause between attempts is not, so it is a caller's to shorten.
+	 */
+	retryDelayMs?: number
+}
+
+/**
+ * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
+ */
+export interface FetchSummary {
+	fetched: number
+	skipped: number
+	failed: number
+	failedCodes: string[]
+}
+
+/**
+ * The sibling `MANIFEST.json` shape the single-file fetch modules write: origin URL + fetch timestamp + byte count +
+ * sha256, so downstream adapters can verify provenance.
+ */
+export interface SourceManifest {
+	source_url: string
+	downloaded_at: string
+	filename: string
+	sha256: string
+	bytes: number
+}
+
+/**
+ * A status worth retrying: rate limiting or a server-side failure.
+ */
+/**
+ * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
+ * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
+ * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is 500.
+ * Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns a CI
+ * run and vanishes locally.
+ */
+export class HTTPStatusError extends Error {
+	readonly status: number
+
+	constructor(status: number, message: string) {
+		super(message)
+		this.name = "HTTPStatusError"
+		this.status = status
+	}
+}
+
+export function isTransientStatus(status: number): boolean {
+	return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX)
+}
+
+export interface DownloadOptions {
+	url: string
+	dest: string
+	/**
+	 * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
+	 */
+	timeoutMs?: number
+	/**
+	 * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
+	 */
+	retries?: number
+	/**
+	 * Delay between attempts. Default 5s.
+	 */
+	retryDelayMs?: number
+	headers?: Record<string, string>
+	report?: (line: string) => void
+}
+
+/**
+ * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
+ * or once retries are exhausted. Returns the byte count written.
+ */
+export async function downloadToFile(options: DownloadOptions): Promise<{ bytes: number }> {
+	const {
+		url,
+		dest,
+		timeoutMs = 600_000,
+		retries = 0,
+		retryDelayMs = DEFAULT_RETRY_DELAY_MS,
+		headers,
+		report,
+	} = options
+
+	let lastError: unknown
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		if (attempt > 0) {
+			report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`)
+			await sleep(retryDelayMs)
+		}
+
+		let res: Response
+
+		try {
+			// Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
+			// `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
+			// non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
+			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
+		} catch (error) {
+			// AbortSignal timeouts and network-level failures are retryable.
+			lastError = error
+
+			continue
+		}
+
+		if (!res.ok) {
+			const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`)
+
+			if (!isTransientStatus(res.status)) throw error
+			lastError = error
+
+			continue
+		}
+
+		try {
+			const buffer = Buffer.from(await res.arrayBuffer())
+			await writeLocalFile(buffer, dest)
+
+			return { bytes: buffer.byteLength }
+		} catch (error) {
+			// A mid-stream abort while reading the body is retryable too.
+			lastError = error
+		}
+	}
+
+	throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+export interface StreamDownloadOptions {
+	headers?: Record<string, string>
+	timeoutMs: number
+	retries: number
+	retryDelayMs: number
+}
+
+/**
+ * Stream an HTTP download to disk, returning the final HTTP status (0 on network error after retries). Follows
+ * redirects (the Census and OpenAddresses endpoints both 302 to their real hosts).
+ *
+ * Kept separate from {@link downloadToFile} on purpose: this one streams a multi-GB body to disk (the buffered helper
+ * reads via `arrayBuffer()`) and returns the HTTP status instead of throwing, which the per-file result collectors and
+ * two-URL fallback ladders consume.
+ */
+export async function streamDownload(url: string, dest: string, opts: StreamDownloadOptions): Promise<number> {
+	for (let attempt = 0; attempt <= opts.retries; attempt++) {
+		try {
+			const res = await fetch(url, {
+				headers: opts.headers ?? {},
+				redirect: "follow",
+				signal: AbortSignal.timeout(opts.timeoutMs),
+			})
+
+			if (res.ok && res.body) {
+				await pipeline(Readable.fromWeb(res.body), openWriteStream(dest))
+
+				return res.status
+			}
+
+			if (attempt < opts.retries && isTransientStatus(res.status)) {
+				await sleep(opts.retryDelayMs)
+
+				continue
+			}
+
+			return res.status
+		} catch {
+			if (attempt < opts.retries) {
+				await sleep(opts.retryDelayMs)
+
+				continue
+			}
+
+			return 0
+		}
+	}
+
+	return 0
+}

package/lib/tools/fetch/download/retry.ts

@@ -0,0 +1,173 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Retries and resumes interrupted corpus downloads.
+ */
+
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { movePath } from "@mailwoman/core/fs/writers"
+import { sleep } from "@mailwoman/core/utils/sleep"
+
+import { DEFAULT_RETRY_DELAY_MS, HTTPStatusError } from "#tools/fetch/download/network"
+
+/**
+ * The answer to a range that starts at or past the end of the body: the resume point is already the whole file.
+ */
+const HTTP_RANGE_NOT_SATISFIABLE = 416
+
+/**
+ * The answer to a satisfiable `Range` request; anything else means the host ignored the range and would send the whole
+ * body again.
+ */
+const HTTP_PARTIAL_CONTENT = 206
+
+/**
+ * Run one transfer up to `1 + retries` times, pausing `retryDelayMs` between attempts. A government portal drops a
+ * multi-hundred-megabyte connection often enough that a collection fetch which lets the first `TypeError: terminated`
+ * propagate loses the whole run to one file; the per-file loop calls the transfer through this and records the failure
+ * only once the attempts are spent. The last error is rethrown so the caller can name the file it lost.
+ */
+export async function withRetries<T>(
+	transfer: () => Promise<T>,
+	options: { retries?: number; retryDelayMs?: number; report?: (line: string) => void; label?: string } = {}
+): Promise<T> {
+	const { retries = 3, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report, label = "transfer" } = options
+	let lastError: unknown
+
+	for (let attempt = 0; attempt <= retries; attempt++) {
+		if (attempt > 0) {
+			report?.(`  retry ${attempt}/${retries} after ${retryDelayMs}ms — ${label}`)
+			await sleep(retryDelayMs)
+		}
+
+		try {
+			return await transfer()
+		} catch (error) {
+			lastError = error
+		}
+	}
+
+	throw lastError instanceof Error ? lastError : new Error(String(lastError))
+}
+
+/**
+ * The byte count already at `path`, or 0 when nothing is there — the resume point of an interrupted range download.
+ */
+async function bytesOnDisk(path: string): Promise<number> {
+	const stat = await tryStat(path)
+
+	return stat?.size ?? 0
+}
+
+/**
+ * Download `url` to `dest` in ranges, resuming from whatever the `.tmp` sibling already holds. For a host that drops a
+ * long connection every few megabytes but answers `Range` with 206 (the Korean address portal does both, measured at
+ * 0.9–12 MB per connection against a 181 MB file), a whole-body transfer never finishes and a plain retry starts over;
+ * this one asks for the remainder each time and keeps what landed. The total comes from the first `Content-Range`, and
+ * the loop gives up after `maxConnections` connections so a host that keeps answering 206 with no bytes cannot spin.
+ * Answers the byte count written; throws when the host answers anything but 206 for a range.
+ */
+export async function resumableDownload(options: {
+	url: string
+	dest: string
+	headers?: Record<string, string>
+	maxConnections?: number
+	retryDelayMs?: number
+	report?: (line: string) => void
+}): Promise<number> {
+	const { url, dest, headers = {}, maxConnections = 400, retryDelayMs = DEFAULT_RETRY_DELAY_MS, report } = options
+	const tmp = dest + ".tmp"
+	let have = await bytesOnDisk(tmp)
+	let total: number | undefined
+
+	for (let connection = 0; connection < maxConnections; connection++) {
+		if (total !== undefined && have >= total) break
+
+		let res: Response
+
+		try {
+			res = await fetch(url, { headers: { ...headers, range: `bytes=${have}-` }, signal: AbortSignal.timeout(600_000) })
+		} catch {
+			await sleep(retryDelayMs)
+
+			continue
+		}
+
+		if (res.status === HTTP_RANGE_NOT_SATISFIABLE && have > 0) {
+			// Nothing past `have`: the file on disk is already the whole body (a parallel filler or an earlier run
+			// landed it), and `Content-Range: bytes */<total>` says how long it is.
+			const whole = /\*\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1]
+			total = whole ? Number(whole) : have
+			await res.body?.cancel()
+
+			break
+		}
+
+		if (res.status !== HTTP_PARTIAL_CONTENT) {
+			throw new HTTPStatusError(res.status, `HTTP ${res.status} for a range request — ${url}`)
+		}
+
+		const range = /bytes \d+-\d+\/(\d+)/.exec(res.headers.get("content-range") ?? "")?.[1]
+
+		if (range) {
+			total = Number(range)
+		}
+
+		const before = have
+
+		try {
+			// Append: the stream opens the file for appending so a partial body extends what earlier connections left.
+			await pipeline(
+				Readable.fromWeb(res.body as Parameters<typeof Readable.fromWeb>[0]),
+				openWriteStream(tmp, { flags: "a" })
+			)
+		} catch {
+			// The connection dropped mid-body; the bytes that landed are on disk, and the next range starts after them.
+		}
+
+		have = await bytesOnDisk(tmp)
+
+		if (have === before) {
+			await sleep(retryDelayMs)
+		} else {
+			report?.(`  ${(have / 1024 / 1024).toFixed(1)} MB${total ? ` of ${(total / 1024 / 1024).toFixed(1)} MB` : ""}`)
+		}
+	}
+
+	if (total === undefined || have < total) {
+		throw new Error(`resumable download stalled at ${have} of ${total ?? "?"} bytes — ${url}`)
+	}
+
+	await movePath(tmp, dest)
+
+	return have
+}
+
+/**
+ * The `Set-Cookie` values of a response folded into one `Cookie` header value, so a second request to the same portal
+ * carries the session the first one opened.
+ */
+export function cookieHeader(res: Response): string {
+	return res.headers
+		.getSetCookie()
+		.map((cookie) => cookie.split(";")[0])
+		.join("; ")
+}
+
+/**
+ * The file name a `Content-Disposition: attachment` header names, RFC 5987 form (`filename*=utf-8''…`) first, plain
+ * `filename="…"` second, or `fallback` when the header carries neither.
+ */
+export function attachmentFilename(res: Response, fallback: string): string {
+	const disposition = res.headers.get("content-disposition") ?? ""
+	const extended = /filename\*=(?:utf-8|UTF-8)'[^']*'([^;]+)/.exec(disposition)?.[1]
+
+	if (extended) return decodeURIComponent(extended.trim())
+	const plain = /filename="?([^";]+)"?/.exec(disposition)?.[1]
+
+	if (plain) return decodeURIComponent(plain.trim())
+
+	return fallback
+}

package/lib/tools/fetch/geonames/dump.ts

@@ -0,0 +1,257 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Fetch GeoNames per-country GAZETTEER dumps — the 19-column `<CC>.txt` files under
+ *   `https://download.geonames.org/export/dump/` (NOT the postal exports; those are `export/zip/` and
+ *   `geonames-postal.ts`'s job). The dumps carry feature classes and codes (column 8: `PPLC` national capital,
+ *   `PPLA` first-order administrative seat), which is what the capitals reference build consumes (#1880).
+ *
+ *   The catalog question is answered by the SOURCE, not by an ISO list: `countryInfo.txt` in the same directory
+ *   enumerates every country GeoNames publishes, one row per ISO alpha-2 code, and also names each country's
+ *   capital — the cross-check the capitals build grades its `PPLC` extraction against. Fetch that first; derive
+ *   the country set from it; then a dump absent from disk is a measured gap against the source's own catalog
+ *   rather than a silent hole. The dump directory may hold files this tool did not fetch: present files are never
+ *   overwritten, and a present `<CC>.txt` that is NOT a 19-column gazetteer dump (GeoNames' postal exports share
+ *   the basename) is reported as `wrong_format_present`, never counted as coverage.
+ */
+
+import { pathExists, readFileHead, readLocalBuffer, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { extractZipEntry } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { join } from "path-ts"
+
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download/index"
+
+/**
+ * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ */
+const HTTP_NOT_FOUND = 404
+
+const SLUG = "geonames-dump"
+
+/**
+ * GeoNames' gazetteer-dump directory — one zip per ISO alpha-2 code holding `<CC>.txt`, plus `countryInfo.txt` as a
+ * bare text file.
+ */
+const BASE_URL = "https://download.geonames.org/export/dump"
+
+export interface FetchGeonamesDumpOptions extends BaseFetchOptions {
+	/**
+	 * ISO alpha-2 codes, any casing. Absent → every country `countryInfo.txt` enumerates.
+	 */
+	countries?: readonly string[]
+	/**
+	 * Dump directory to read from. Defaults to GeoNames' own; exists so the 404 and coverage behaviour can be exercised
+	 * against a local server.
+	 */
+	baseURL?: string
+	/**
+	 * Refetch a dump whose `<CC>.txt` already exists. Default false — the tool fills gaps.
+	 */
+	force?: boolean
+}
+
+interface GeonamesDumpFileEntry {
+	country: string
+	filename: string
+	source_url: string
+	sha256: string
+	bytes: number
+}
+
+export interface GeonamesDumpManifest {
+	source: string
+	base_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: GeonamesDumpFileEntry[]
+	/**
+	 * `<CC>.txt` files already on disk and left alone — the hand-fetched population this tool extends.
+	 */
+	skipped_present: string[]
+	/**
+	 * Countries in the source catalog that the source's dump directory nonetheless 404s — a fact about the source,
+	 * recorded so a later reader does not spend the fetch to rediscover it.
+	 */
+	unavailable: string[]
+	/**
+	 * Present `<CC>.txt` files that are NOT 19-column gazetteer dumps — GeoNames' postal exports share the same basename,
+	 * and seven tier-1 postal files sat at these paths reading as "present" until the capitals build found them
+	 * capital-less. Left in place (this tool never overwrites data it did not fetch); the fix is to move the file to its
+	 * own home and rerun.
+	 */
+	wrong_format_present: string[]
+}
+
+/**
+ * Column count of a gazetteer dump row — the discriminator against GeoNames' 12-column postal exports, which share the
+ * `<CC>.txt` basename.
+ */
+const GAZETTEER_DUMP_COLUMNS = 19
+
+/**
+ * True when the first non-empty line carries the gazetteer dump's 19 tab-separated columns. Accepts a partial head read
+ * — the first line is the whole question, so callers need not hand it a resident 350 MB dump.
+ *
+ * Walk the string directly rather than constructing a spliterator: the capitals builder already holds whole country
+ * dumps as strings, and a byte-oriented spliterator would UTF-8 encode that input — allocating up to another 350 MB —
+ * merely to inspect its first non-empty line.
+ */
+export function looksLikeGazetteerDump(text: string): boolean {
+	let start = 0
+
+	while (start < text.length) {
+		const end = text.indexOf("\n", start)
+		const line = end === -1 ? text.slice(start) : text.slice(start, end)
+
+		if (line.trim()) {
+			let tabs = 0
+
+			for (let i = line.indexOf("\t"); i !== -1; i = line.indexOf("\t", i + 1)) {
+				tabs++
+			}
+
+			return tabs === GAZETTEER_DUMP_COLUMNS - 1
+		}
+
+		if (end === -1) break
+
+		start = end + 1
+	}
+
+	return false
+}
+
+/**
+ * Parse the ISO codes (column 1) and capital names (column 6) out of `countryInfo.txt` — `#`-prefixed lines are the
+ * file's own commentary.
+ */
+export function parseCountryInfo(text: string): Array<{ country: string; capital: string }> {
+	const rows: Array<{ country: string; capital: string }> = []
+
+	// oxlint-disable-next-line mailwoman/prefer-spliterator -- countryInfo.txt is ~40 KB, one row per country on Earth; it cannot grow past that
+	for (const line of text.split("\n")) {
+		if (line.startsWith("#") || !line.trim()) continue
+
+		// oxlint-disable-next-line mailwoman/prefer-spliterator -- one bounded 19-column catalog row
+		const cols = line.split("\t")
+		const country = cols[0]?.trim().toUpperCase()
+
+		if (country?.length === 2) {
+			rows.push({ country, capital: cols[5]?.trim() ?? "" })
+		}
+	}
+
+	return rows
+}
+
+/**
+ * Bytes read to classify a present file: enough to cover a first dump row whose alternate-names column runs long (they
+ * reach several KB), a fraction of the largest dumps (US.txt is ~350 MB).
+ */
+const FORMAT_SNIFF_BYTES = 65_536
+
+/**
+ * Download `countryInfo.txt` plus every missing `<CC>.zip`, extracting each to `<outRoot>/<CC>.txt` beside the
+ * hand-fetched dumps, with a `MANIFEST.json` naming fetched, skipped-present, and source-unavailable countries.
+ */
+export async function fetchGeonamesDumps(
+	options: FetchGeonamesDumpOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary & { skippedPresent: string[] }> {
+	await makeDirectories(options.outRoot)
+
+	const baseURL = options.baseURL ?? BASE_URL
+	const countryInfoDest = join(options.outRoot, "countryInfo.txt")
+
+	await downloadToFile({
+		url: `${baseURL}/countryInfo.txt`,
+		dest: countryInfoDest,
+		timeoutMs: 120_000,
+		retries: 2,
+		report,
+	})
+
+	const catalog = parseCountryInfo(await readLocalTextFile(countryInfoDest))
+	const countries = options.countries?.map((code) => code.trim().toUpperCase()) ?? catalog.map((row) => row.country)
+
+	const entries: GeonamesDumpFileEntry[] = []
+	const failedCodes: string[] = []
+	const unavailable: string[] = []
+	const skippedPresent: string[] = []
+	const wrongFormatPresent: string[] = []
+	let fetched = 0
+
+	for (const country of countries) {
+		const txtDest = join(options.outRoot, `${country}.txt`)
+
+		if (!options.force && (await pathExists(txtDest))) {
+			if (looksLikeGazetteerDump(await readFileHead(txtDest, FORMAT_SNIFF_BYTES))) {
+				skippedPresent.push(country)
+			} else {
+				report?.(`✗ ${country}.txt is present but is not a 19-column gazetteer dump — move it aside and rerun`)
+				wrongFormatPresent.push(country)
+			}
+
+			continue
+		}
+
+		const filename = `${country}.zip`
+		const url = `${baseURL}/${filename}`
+		const zipDest = join(options.outRoot, filename)
+
+		report?.(`=== ${SLUG} / ${country}`)
+
+		try {
+			await downloadToFile({ url, dest: zipDest, timeoutMs: 300_000, retries: 2, report })
+			await extractZipEntry(zipDest, `${country}.txt`, txtDest)
+			await removePathIfPresent(zipDest)
+
+			entries.push({
+				country,
+				filename: `${country}.txt`,
+				source_url: url,
+				sha256: await sha256File(txtDest),
+				bytes: (await readLocalBuffer(txtDest)).byteLength,
+			})
+
+			fetched++
+		} catch (error) {
+			await removePathIfPresent(zipDest)
+
+			const message = error instanceof Error ? error.message : String(error)
+
+			// Branch on the TYPED status (the geonames-postal lesson): message prose contains the URL, and a URL
+			// can contain any substring.
+			if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
+				report?.(`✗ ${country}: GeoNames publishes no gazetteer dump for this country`)
+				unavailable.push(country)
+			} else {
+				report?.(`✗ ${country}: ${message}`)
+			}
+
+			failedCodes.push(country)
+		}
+	}
+
+	const manifest: GeonamesDumpManifest = {
+		source: SLUG,
+		base_url: baseURL,
+		license: "CC-BY-4.0",
+		attribution: "GeoNames",
+		downloaded_at: new Date().toISOString(),
+		files: entries,
+		skipped_present: skippedPresent.toSorted(),
+		unavailable,
+		wrong_format_present: wrongFormatPresent.toSorted(),
+	}
+
+	await writeManifest(join(options.outRoot, "MANIFEST.json"), manifest)
+
+	return { fetched, skipped: skippedPresent.length, failed: failedCodes.length, failedCodes, skippedPresent }
+}