@mailwoman/corpus 9.2.0 → 9.3.0

package/src/tools/fetch/ban.ts

@@ -1,290 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Fetch the full French BAN (Base Adresse Nationale) — all metropolitan départements (01-95, 2A,
- *   2B) plus 5 overseas DOM/TOM (971-976 excl. 975).
- *
- *   Source: https://adresse.data.gouv.fr/data/ban/adresses/latest/csv/
- *   Licence: Licence Ouverte 2.0 (attribution required — Tier B).
- *
- *   Files already present with matching sha256 are skipped (re-runnable). Downloads `.csv.gz`,
- *   decompresses to `.csv`, deletes the `.gz` artifact. One shared `MANIFEST.json` at
- *   `<outRoot>/ban/MANIFEST.json` covers all codes.
- *
- *   Invoke via `mailwoman corpus fetch ban --out-root <path>`. Built-in `fetch` with gzip/brotli
- *   decompression replaces curl; native `node:zlib` gunzip replaces the `gunzip` subprocess; no
- *   Python.
- */
-
-import { existsSync, mkdirSync, statSync } from "node:fs"
-import { readFile, unlink, writeFile } from "node:fs/promises"
-import { join } from "node:path"
-import { setTimeout as sleep } from "node:timers/promises"
-import { gunzipSync } from "node:zlib"
-
-import { BYTES_PER_KIB, ByteFormatter } from "@mailwoman/core/fs/utils"
-import { sha256File } from "@mailwoman/core/utils"
-
-import type { BaseFetchOptions, FetchSummary } from "./download.ts"
-import { downloadToFile, loadManifestEntries, writeManifest } from "./download.ts"
-
-/**
- * Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
- * than data.
- */
-
-const BASE_URL = "https://adresse.data.gouv.fr/data/ban/adresses/latest/csv"
-
-/**
- * All département codes — metropolitan 01-95 (with 2A/2B for Corsica instead of 20) plus overseas DOM/TOM. Codes do not
- * change.
- */
-const DEPT_CODES = [
-	"01",
-	"02",
-	"03",
-	"04",
-	"05",
-	"06",
-	"07",
-	"08",
-	"09",
-	"10",
-	"11",
-	"12",
-	"13",
-	"14",
-	"15",
-	"16",
-	"17",
-	"18",
-	"19",
-	"21",
-	"22",
-	"23",
-	"24",
-	"25",
-	"26",
-	"27",
-	"28",
-	"29",
-	"2A",
-	"2B",
-	"30",
-	"31",
-	"32",
-	"33",
-	"34",
-	"35",
-	"36",
-	"37",
-	"38",
-	"39",
-	"40",
-	"41",
-	"42",
-	"43",
-	"44",
-	"45",
-	"46",
-	"47",
-	"48",
-	"49",
-	"50",
-	"51",
-	"52",
-	"53",
-	"54",
-	"55",
-	"56",
-	"57",
-	"58",
-	"59",
-	"60",
-	"61",
-	"62",
-	"63",
-	"64",
-	"65",
-	"66",
-	"67",
-	"68",
-	"69",
-	"70",
-	"71",
-	"72",
-	"73",
-	"74",
-	"75",
-	"76",
-	"77",
-	"78",
-	"79",
-	"80",
-	"81",
-	"82",
-	"83",
-	"84",
-	"85",
-	"86",
-	"87",
-	"88",
-	"89",
-	"90",
-	"91",
-	"92",
-	"93",
-	"94",
-	"95",
-	"971",
-	"972",
-	"973",
-	"974",
-	"976",
-]
-
-export interface BanManifestEntry {
-	dept_code: string
-	filename: string
-	source_url: string
-	downloaded_at: string
-	sha256: string
-	bytes: number
-}
-
-export type FetchBanOptions = BaseFetchOptions
-
-export async function fetchBan(options: FetchBanOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const banDir = join(options.outRoot, "ban")
-	const manifestPath = join(banDir, "MANIFEST.json")
-	mkdirSync(banDir, { recursive: true })
-
-	// Load existing entries (code -> entry): skip detection + preservation of untouched codes.
-	const entries = await loadManifestEntries<BanManifestEntry>(manifestPath, (entry) => entry.dept_code)
-
-	let fetched = 0
-	let skipped = 0
-	let failed = 0
-	const failedCodes: string[] = []
-
-	for (const code of DEPT_CODES) {
-		const filename = `adresses-${code}.csv`
-		const gzFile = join(banDir, `${filename}.gz`)
-		const csvFile = join(banDir, filename)
-		const url = `${BASE_URL}/adresses-${code}.csv.gz`
-
-		report?.(`=== dept ${code}`)
-
-		// If the CSV already exists, compare its sha256 against the manifest.
-		if (existsSync(csvFile)) {
-			const existingSha = await sha256File(csvFile)
-			const recordedSha = entries.get(code)?.sha256
-
-			if (recordedSha && existingSha === recordedSha) {
-				report?.(`  → already present + sha matches — skipping`)
-
-				skipped++
-
-				continue
-			}
-
-			report?.(`  → present but sha mismatch or no manifest entry — re-fetching`)
-			await unlink(csvFile)
-		}
-
-		// Download the gzipped CSV.
-		try {
-			await downloadToFile({
-				url,
-				dest: gzFile,
-				timeoutMs: 600_000,
-				headers: { "Accept-Encoding": "gzip, br" },
-				report,
-			})
-		} catch (error) {
-			report?.(`  ✗ download failed: ${url} (${(error as Error).message})`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Guard against truncated 404/error pages.
-		const gzSize = statSync(gzFile).size
-
-		if (gzSize < BYTES_PER_KIB) {
-			report?.(`  ✗ response too small (${gzSize} bytes) — probable 404 / error page`)
-			await unlink(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		// Decompress in-place; delete the .gz.
-		try {
-			await writeFile(csvFile, gunzipSync(await readFile(gzFile)))
-		} catch (error) {
-			report?.(`  ✗ decompress failed: ${(error as Error).message}`)
-			await unlink(gzFile)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		await unlink(gzFile)
-
-		if (!existsSync(csvFile)) {
-			report?.(`  ✗ decompressed file not found at ${csvFile}`)
-
-			failed++
-			failedCodes.push(code)
-
-			continue
-		}
-
-		const bytes = statSync(csvFile).size
-		const sha = await sha256File(csvFile)
-
-		entries.set(code, {
-			dept_code: code,
-			filename,
-			source_url: url,
-			downloaded_at: new Date().toISOString(),
-			sha256: sha,
-			bytes,
-		})
-
-		report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha}`)
-
-		fetched++
-
-		// Be a polite citizen — short pause between requests.
-		await sleep(200)
-	}
-
-	// Write the consolidated MANIFEST.json (entries sorted by dept_code, codepoint order).
-	const sorted = [...entries.values()].toSorted((a, b) =>
-		a.dept_code < b.dept_code ? -1 : a.dept_code > b.dept_code ? 1 : 0
-	)
-
-	await writeManifest(manifestPath, sorted)
-	report?.(`Wrote ${manifestPath} with ${sorted.length} entries.`)
-
-	report?.(`=== summary ===`)
-	report?.(`fetched:  ${fetched}`)
-	report?.(`skipped:  ${skipped} (already present + sha matched)`)
-	report?.(`failed:   ${failed}`)
-
-	if (failedCodes.length) {
-		report?.(`failed codes: ${failedCodes.join(" ")}`)
-	}
-
-	return { fetched, skipped, failed, failedCodes }
-}

package/src/tools/fetch/download.ts

@@ -1,176 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Download and manifest utilities for `mailwoman corpus fetch <source>`.
- */
-
-import { existsSync } from "node:fs"
-import { readFile, writeFile } from "node:fs/promises"
-import { setTimeout as sleep } from "node:timers/promises"
-
-import { tryParsingJSON } from "@mailwoman/core/objects"
-
-/**
- * Rate limited — retryable, the server is asking us to back off.
- */
-const HTTP_TOO_MANY_REQUESTS = 429
-
-/**
- * Lowest 5xx status. Server-side failures are retryable; 4xx are not.
- */
-const HTTP_SERVER_ERROR_MIN = 500
-
-/**
- * Highest 5xx status.
- */
-const HTTP_SERVER_ERROR_MAX = 599
-
-/**
- * The option base every `mailwoman corpus fetch <source>` module extends.
- */
-export interface BaseFetchOptions {
-	/**
-	 * Destination root for downloaded source data. Each source writes its own subdirectory.
-	 */
-	outRoot: string
-}
-
-/**
- * The per-run result every fetch module returns; the command maps `failed > 0` to exit code 1.
- */
-export interface FetchSummary {
-	fetched: number
-	skipped: number
-	failed: number
-	failedCodes: string[]
-}
-
-/**
- * A status worth retrying: rate limiting or a server-side failure.
- */
-/**
- * An HTTP failure that CARRIES its status, so callers branch on `error.status` rather than on message prose. The prose
- * route shipped a real flake: a caller classified "not published upstream" with `message.includes("404")`, and the
- * message contains the URL — an ephemeral test-server port such as `:40453` satisfies it while the actual status is
- * 500. Roughly 1–2% of ephemeral ports contain the substring, which is exactly the kind of sometimes-failure that burns
- * a CI run and vanishes locally.
- */
-export class HTTPStatusError extends Error {
-	readonly status: number
-
-	constructor(status: number, message: string) {
-		super(message)
-		this.name = "HTTPStatusError"
-		this.status = status
-	}
-}
-
-export function isTransientStatus(status: number): boolean {
-	return status === HTTP_TOO_MANY_REQUESTS || (status >= HTTP_SERVER_ERROR_MIN && status <= HTTP_SERVER_ERROR_MAX)
-}
-
-export interface DownloadOptions {
-	url: string
-	dest: string
-	/**
-	 * Per-attempt timeout. Default 10 minutes — these are multi-GB government dumps.
-	 */
-	timeoutMs?: number
-	/**
-	 * Extra attempts after the first, taken only on transient statuses or network errors. Default 0.
-	 */
-	retries?: number
-	/**
-	 * Delay between attempts. Default 5s.
-	 */
-	retryDelayMs?: number
-	headers?: Record<string, string>
-	report?: (line: string) => void
-}
-
-/**
- * Download `url` to `dest` with per-attempt timeout and transient-status retry. Throws on a non-transient HTTP status
- * or once retries are exhausted. Returns the byte count written.
- */
-export async function downloadToFile(options: DownloadOptions): Promise<{ bytes: number }> {
-	const { url, dest, timeoutMs = 600_000, retries = 0, retryDelayMs = 5000, headers, report } = options
-	let lastError: unknown
-
-	for (let attempt = 0; attempt <= retries; attempt++) {
-		if (attempt > 0) {
-			report?.(`retry ${attempt}/${retries} after ${retryDelayMs}ms — ${url}`)
-			await sleep(retryDelayMs)
-		}
-
-		let res: Response
-
-		try {
-			// Raw `fetch`, deliberately: this is the shared FILE downloader and the body is piped to disk below.
-			// `APIClient` is the repo default for API requests — small bodies, repeated calls — and buffers a
-			// non-stream response in memory, which is the one thing a multi-gigabyte transfer must not do.
-			res = await fetch(url, { headers, signal: AbortSignal.timeout(timeoutMs) })
-		} catch (error) {
-			// AbortSignal timeouts and network-level failures are retryable.
-			lastError = error
-
-			continue
-		}
-
-		if (!res.ok) {
-			const error = new HTTPStatusError(res.status, `HTTP ${res.status} ${res.statusText} — ${url}`)
-
-			if (!isTransientStatus(res.status)) throw error
-			lastError = error
-
-			continue
-		}
-
-		try {
-			const buffer = Buffer.from(await res.arrayBuffer())
-			await writeFile(dest, buffer)
-
-			return { bytes: buffer.byteLength }
-		} catch (error) {
-			// A mid-stream abort while reading the body is retryable too.
-			lastError = error
-		}
-	}
-
-	throw lastError instanceof Error ? lastError : new Error(String(lastError))
-}
-
-/**
- * Read a MANIFEST.json; `null` when missing or corrupt (callers re-fetch from scratch).
- */
-export async function readManifest<T>(path: string): Promise<T | null> {
-	if (!existsSync(path)) return null
-
-	// A read failure (e.g. the file vanished after the existsSync probe) maps to null like corrupt
-	// JSON does; tryParsingJSON returns null for the non-string sentinel.
-	const text = await readFile(path, "utf8").catch(() => null)
-
-	return tryParsingJSON<T>(text)
-}
-
-/**
- * Load manifest entries into a map so untouched keys survive a partial re-fetch.
- */
-export async function loadManifestEntries<T>(path: string, key: (entry: T) => string): Promise<Map<string, T>> {
-	const entries = new Map<string, T>()
-	const parsed = await readManifest<T[]>(path)
-
-	for (const entry of parsed ?? []) {
-		entries.set(key(entry), entry)
-	}
-
-	return entries
-}
-
-/**
- * Write a MANIFEST.json in the house shape: pretty-printed, trailing newline.
- */
-export async function writeManifest(path: string, manifest: unknown): Promise<void> {
-	await writeFile(path, JSON.stringify(manifest, null, 2) + "\n")
-}