@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/fetch/geonames/postal.ts

@@ -3,86 +3,52 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch the GeoNames per-country postal-code exports — the only source in this family that carries a
- *   `(postcode, locality, region)` triple with the NAMES inline.
+ *   Fetch the GeoNames per-country postal-code exports — the only source in this family that includes a
+ *   `(postcode, locality, region)` triple with the names inline.
  *
  *   Source : https://download.geonames.org/export/zip/<CC>.zip
  *   License: CC-BY-4.0, attribute "GeoNames". Tier B.
  *
- *   ## Why this source and not a join
- *
- *   A postcode slice is only useful to the corpus if a postcode reaches a locality and a region. Two routes exist and
- *   only one of them works everywhere:
- *
- *   - **`parent_id`** — `postalcode-intl.db` carries a real parent that resolves in the admin gazetteer (measured: NL
- *     97.5% of rows linked, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%, and 93.8–100% of those resolve to a
- *     `locality`/`localadmin`). It is also the ONLY slice that does: `postalcode-geonames-intl.db` and every
- *     `postalcode-<cc>-overture.db` carry `parent_id = 0` on every row, so five countries have this route and the rest
- *     have none.
- *   - **Nearest locality centroid** — the obvious fallback, and it does not work. Scored against the `parent_id` truth
- *     on 1,000–1,500 postcodes per country: NL 81.1%, DE 44.7%, ES 35.1%, FR 29.0%, IT 13.5%. A locality's centroid
- *     sits at its middle, so a postcode near the edge is routinely closer to a neighbouring town's centroid than to its
- *     own. Under 50% for four of the five, which is not a join.
- *
- *   The GeoNames export sidesteps both: columns 3 and 4 ARE the place and admin1 names, so there is nothing to join and
- *   nothing to approximate. `@mailwoman/corpus`'s `geonames-postal` adapter consumes it directly.
- *
- *   ## Coverage is not universal, and the gap is the point
- *
- *   GeoNames publishes ~80 countries, NOT all of them. Venezuela returns 404 — so a VE postcode slice cannot be built
- *   from this source at any effort, and that is an acquisition question rather than a build one. Ask for a country
- *   before assuming it is there; an absent country fails as one entry, never as the whole run.
- *
- *   ## Row counts from this source overstate, for some countries by exactly 2×
- *
- *   Countries whose postcode format contains a hyphen are published TWICE — once `3750-000`, once `3750000`. Measured
- *   on the slice built from this source: PT 395,544 rows over 197,772 distinct codes and PL 40,598 over 20,299, both
- *   exactly 2.00×, while AU, CZ and AT (no hyphen in the format) are 1.00×. A consumer sizing a slice from the row
- *   count doubles its estimate for those countries.
- *
- *   ## Why `downloadToFile` and not `APIClient`
- *
- *   Same split `AGENTS.md` draws and the `ourairports` sibling explains: these are static file transfers from a plain
- *   file host, run once per refresh. The pacing, retry and caching `APIClient` exists for have nothing to act on here.
- *
- *   Invoke via `mailwoman corpus fetch geonames-postal --countries pt,au,nz`.
+ *   GeoNames publishes roughly 80 countries. An absent country fails as one entry, never as the whole run.
  */
 
 import { makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
-import { join } from "path-ts"
 
-import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download/index"
-import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { downloadToFile, HTTPStatusError, writeManifest } from "#tools/fetch/download"
 
 /**
- * The one status that means "the source does not publish this country" rather than "the transfer failed".
+ * The status that means the source does not publish this country rather than that the transfer failed.
  */
 const HTTP_NOT_FOUND = 404
 
 const SLUG = "geonames-postal"
 
 /**
- * GeoNames' own export directory. One zip per ISO alpha-2 code, each holding `<CC>.txt` plus the shared `readme.txt`.
+ * GeoNames' own export directory, one zip per ISO alpha-2 code holding `<CC>.txt`
+ * plus the shared `readme.txt`.
  */
 const BASE_URL = "https://download.geonames.org/export/zip"
 
 /**
  * Countries fetched when the caller names none.
  *
- * These are the ones the corpus wants and cannot get from `postalcode-intl.db`'s `parent_id` route — see the header for
- * why that route covers exactly five countries. Venezuela is deliberately absent because GeoNames does not publish it.
+ * Venezuela is deliberately absent because GeoNames does not publish it.
  */
 export const GEONAMES_POSTAL_DEFAULT_COUNTRIES = ["PT", "AU", "NZ", "IE", "BR", "ZA", "MX"] as const
 
 export interface FetchGeonamesPostalOptions extends BaseFetchOptions {
 	/**
-	 * ISO alpha-2 codes, in any casing. Defaults to {@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}.
+	 * ISO alpha-2 codes in any casing.
+	 *
+	 * Defaults to `{@linkcode GEONAMES_POSTAL_DEFAULT_COUNTRIES}`.
 	 */
 	countries?: readonly string[]
 	/**
-	 * Export directory to read from. Defaults to GeoNames' own. Exists so the 404-is-a-coverage-finding behaviour can be
-	 * exercised against a local server rather than by asking GeoNames for a country it does not have.
+	 * Export directory to read from.
+	 *
+	 * Defaults to GeoNames' own, so the 404-is-coverage behavior can be exercised against a local server.
 	 */
 	baseURL?: string
 }
@@ -103,23 +69,23 @@ interface GeonamesPostalManifest {
 	downloaded_at: string
 	files: GeonamesPostalFileEntry[]
 	/**
-	 * Countries asked for and NOT published by GeoNames, recorded so a later reader does not spend the fetch again to
-	 * rediscover it. An absence here is a fact about the source, not about the run.
+	 * Countries asked for and not published by GeoNames.
+	 *
+	 * An absence here is a fact about the source rather than about the run.
 	 */
 	unavailable: string[]
 }
 
 /**
- * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/`, with a sibling `MANIFEST.json`
- * carrying each file's origin URL, sha256 and byte count, plus the countries the source does not publish.
- *
- * A country the source does not carry is counted as failed and named in `failedCodes` — it does not stop the rest.
+ * Download the requested GeoNames postal zips into `<outRoot>/geonames-postal/` with
+ * a sibling `manifest.json`; a country the source does not cover is counted as failed
+ * and recorded in `failedCodes` without stopping the rest.
  */
 export async function fetchGeonamesPostal(
 	options: FetchGeonamesPostalOptions,
 	report?: (line: string) => void
 ): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
+	const destDir = options.outRoot(SLUG)
 	await makeDirectories(destDir)
 
 	const countries = (options.countries ?? GEONAMES_POSTAL_DEFAULT_COUNTRIES).map((code) => code.trim().toUpperCase())
@@ -134,7 +100,7 @@ export async function fetchGeonamesPostal(
 	for (const country of countries) {
 		const filename = `${country}.zip`
 		const url = `${baseURL}/${filename}`
-		const dest = join(destDir, filename)
+		const dest = destDir(filename)
 
 		report?.(`=== ${SLUG} / ${country}`)
 
@@ -147,10 +113,9 @@ export async function fetchGeonamesPostal(
 		} catch (error) {
 			const message = error instanceof Error ? error.message : String(error)
 
-			// A 404 here means GeoNames does not publish the country at all, which is a different finding from a failed
-			// transfer and the one a caller planning a slice needs to see. Branch on the TYPED status: matching message
-			// prose classified a 500 as "unpublished" whenever the URL happened to contain the substring 404 — an
-			// ephemeral test-server port did exactly that in CI.
+			// A 404 means GeoNames does not publish the country at all, a different
+			// finding from a failed transfer.
+			// Branch on the typed status rather than message prose.
 			if (error instanceof HTTPStatusError && error.status === HTTP_NOT_FOUND) {
 				report?.(`✗ ${country}: GeoNames does not publish a postal export for this country`)
 				unavailable.push(country)
@@ -174,7 +139,7 @@ export async function fetchGeonamesPostal(
 		unavailable,
 	}
 
-	await writeManifest(join(destDir, "MANIFEST.json"), manifest)
+	await writeManifest(destDir("MANIFEST.json"), manifest)
 
 	return { fetched, skipped: 0, failed, failedCodes }
 }

package/lib/tools/fetch/gleif.ts

@@ -0,0 +1,360 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter.
+ *
+ * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at
+ * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the
+ * CDF version it follows, so this module reads that answer first and downloads the file it names.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the
+ * `2026-10-03 08:00:00` publish:
+ *
+ * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
+ *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    publish replaced mid-transfer is refused rather than recorded.
+ * 2. **The host honors `Range`.** An interrupted transfer resumes through
+ *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
+ *    compresses small responses on the fly and a range must count the archive's own bytes.
+ * 3. **Each publish has its own file name**, date-prefixed as
+ *    `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that
+ *    name, so a partial transfer of one publish never resumes into another, and the adapter reads the
+ *    newest name in the directory.
+ *
+ * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter
+ * streams the member out of the archive rather than reading an extracted copy.
+ *
+ * The receipt written beside the archive records the URL, the publish date, the byte count and the
+ * sha256 of the archive, which identify the bytes a corpus row was read from.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import {
+	GLEIF_ADAPTER_ID,
+	GLEIF_ARCHIVE_PATTERN,
+	GLEIF_ATTRIBUTION,
+	GLEIF_LICENSE,
+	GLEIF_LICENSE_URL,
+} from "#adapters/gleif/adapter"
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download"
+import { readManifest, resumableDownload, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The publish API's answer for the newest golden copy and its delta files.
+ */
+export const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest"
+
+/**
+ * The directory the archive and its receipt are written under, which is the adapter's `inputPath`.
+ */
+const SLUG = "gleif"
+
+/**
+ * The file the publish API names for the Level 1 golden copy in CSV.
+ */
+export interface GLEIFPublishedFile {
+	/**
+	 * The publish timestamp the API states, as `2026-10-03 08:00:00`.
+	 */
+	publishDate: string
+	url: string
+	/**
+	 * The archive's length in bytes, as the API states it.
+	 */
+	bytes: number
+	recordCount: number
+	cdfVersion: string
+}
+
+/**
+ * The receipt written beside the archive.
+ */
+export interface GLEIFGoldenCopyReceipt {
+	source: string
+	source_url: string
+	api_url: string
+	publish_date: string
+	cdf_version: string
+	/**
+	 * The record count the API states for this publish.
+	 */
+	record_count: number
+	downloaded_at: string
+	filename: string
+	bytes: number
+	sha256: string
+	/**
+	 * The CSV member inside the archive, and its decompressed length as the central directory states it.
+	 */
+	csv_member: string
+	csv_member_bytes: number
+	license: string
+	license_url: string
+	attribution: string
+}
+
+/**
+ * The fields this module reads from the publish API's answer.
+ */
+interface PublishResponse {
+	data?: {
+		lei2?: {
+			publish_date?: string
+			full_file?: {
+				csv?: { url?: string; size?: number; record_count?: number; cdf_version?: string }
+			}
+		}
+	}
+}
+
+/**
+ * Read which file the publish API names as the current Level 1 golden copy in CSV.
+ *
+ * @throws When the answer omits any of the five fields, naming the one it lacks,
+ * rather than downloading a file whose length or date cannot be checked.
+ */
+export async function readGLEIFLatestPublish(client: Pick<APIClient, "fetch">): Promise<GLEIFPublishedFile> {
+	const response = await client.fetch<PublishResponse>({ method: "GET", url: GLEIF_LATEST_PUBLISH_URL })
+	const lei2 = response.data?.data?.lei2
+	const csv = lei2?.full_file?.csv
+
+	const fields = {
+		"data.lei2.publish_date": lei2?.publish_date,
+		"data.lei2.full_file.csv.url": csv?.url,
+		"data.lei2.full_file.csv.size": csv?.size,
+		"data.lei2.full_file.csv.record_count": csv?.record_count,
+		"data.lei2.full_file.csv.cdf_version": csv?.cdf_version,
+	}
+
+	const missing = Object.entries(fields)
+		.filter(([, value]) => value === undefined || value === null || value === "")
+		.map(([name]) => name)
+
+	if (missing.length) {
+		throw new Error(`${GLEIF_LATEST_PUBLISH_URL} answered without ${missing.join(", ")}`)
+	}
+
+	return {
+		publishDate: lei2!.publish_date!,
+		url: csv!.url!,
+		bytes: Number(csv!.size),
+		recordCount: Number(csv!.record_count),
+		cdfVersion: csv!.cdf_version!,
+	}
+}
+
+/**
+ * The archive's file name, which is the last path segment of the URL the API names.
+ */
+export function gleifArchiveFilename(url: string): string {
+	const name = new URL(url).pathname.split("/").at(-1) ?? ""
+
+	if (!GLEIF_ARCHIVE_PATTERN.test(name)) {
+		throw new Error(`${url} does not name a golden-copy archive matching ${GLEIF_ARCHIVE_PATTERN}`)
+	}
+
+	return name
+}
+
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export function gleifInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG)
+}
+
+/**
+ * Whether the archive on disk is the publish the API names and the length the receipt records.
+ *
+ * Exported so a test can exercise the decision without a transfer.
+ */
+export async function isGLEIFArchiveCurrent(
+	recorded: GLEIFGoldenCopyReceipt | null,
+	published: GLEIFPublishedFile,
+	path: PathBuilderLike,
+	verifyDigest: boolean
+): Promise<boolean> {
+	if (!recorded) return false
+
+	if (recorded.source_url !== published.url || recorded.publish_date !== published.publishDate) return false
+
+	const stat = await tryStat(path)
+
+	if (!stat || stat.size !== recorded.bytes) return false
+
+	if (!verifyDigest) return true
+
+	return (await sha256File(path)) === recorded.sha256
+}
+
+export interface DownloadGLEIFOptions {
+	/**
+	 * Where the archive and the receipt are written.
+	 * The directory itself is the adapter's `inputPath`.
+	 */
+	outputDir: PathBuilderLike
+	/**
+	 * Re-read the archive's sha256 on a re-run instead of comparing its byte count.
+	 */
+	verifyDigest?: boolean
+	/**
+	 * Download even where the receipt already records the current publish.
+	 */
+	force?: boolean
+	retryDelayMs?: number
+	report?: (line: string) => void
+}
+
+export type FetchGLEIFOptions = BaseFetchOptions & Omit<DownloadGLEIFOptions, "outputDir" | "report">
+
+/**
+ * Download the current Level 1 golden copy into `options.outputDir` and write its receipt.
+ *
+ * Re-runnable: a receipt naming the current publish, with the archive still on disk
+ * at the recorded length, makes no request for the body.
+ * Archives of earlier publishes are removed once the new one is verified,
+ * so the directory holds one publish.
+ */
+export async function downloadGLEIF(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadGLEIFOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const summary: FetchSummary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] }
+
+	await makeDirectories(destDir)
+
+	report?.(`=== ${SLUG}`)
+
+	const published = await readGLEIFLatestPublish(client)
+	const filename = gleifArchiveFilename(published.url)
+	const archivePath = destDir(filename)
+	const receiptPath = destDir("MANIFEST.json")
+
+	report?.(
+		`  publish ${published.publishDate}: ${published.recordCount.toLocaleString()} records, ` +
+			`${ByteFormatter.formatIEC(published.bytes)}, ${published.cdfVersion}`
+	)
+
+	const recorded = await readManifest<GLEIFGoldenCopyReceipt>(receiptPath)
+
+	if (
+		!options.force &&
+		(await isGLEIFArchiveCurrent(recorded, published, archivePath, options.verifyDigest ?? false))
+	) {
+		report?.(`  ✓ ${filename} already current — no download.`)
+
+		summary.skipped++
+
+		return summary
+	}
+
+	let bytes: number
+
+	try {
+		bytes = await resumableDownload({
+			url: published.url,
+			dest: archivePath,
+			headers: { "accept-encoding": "identity", accept: "application/zip" },
+			retryDelayMs: options.retryDelayMs,
+			report,
+		})
+	} catch (error) {
+		report?.(`  ✗ download failed: ${error instanceof Error ? error.message : String(error)}`)
+
+		summary.failed++
+		summary.failedCodes.push(filename)
+
+		return summary
+	}
+
+	if (bytes !== published.bytes) {
+		await removePathIfPresent(archivePath)
+
+		report?.(`  ✗ the API stated ${published.bytes} bytes and ${bytes} arrived, so the archive is not recorded`)
+
+		summary.failed++
+		summary.failedCodes.push(filename)
+
+		return summary
+	}
+
+	const members = (await listZipEntries(archivePath)).filter((entry) => entry.name.toLowerCase().endsWith(".csv"))
+
+	if (members.length !== 1) {
+		report?.(`  ✗ ${filename} holds ${members.length} CSV members, where the golden copy publishes one`)
+
+		summary.failed++
+		summary.failedCodes.push(filename)
+
+		return summary
+	}
+
+	const receipt: GLEIFGoldenCopyReceipt = {
+		source: GLEIF_ADAPTER_ID,
+		source_url: published.url,
+		api_url: GLEIF_LATEST_PUBLISH_URL,
+		publish_date: published.publishDate,
+		cdf_version: published.cdfVersion,
+		record_count: published.recordCount,
+		downloaded_at: new Date().toISOString(),
+		filename,
+		bytes,
+		sha256: await sha256File(archivePath),
+		csv_member: members[0]!.name,
+		csv_member_bytes: members[0]!.uncompressedSize,
+		license: GLEIF_LICENSE,
+		license_url: GLEIF_LICENSE_URL,
+		attribution: GLEIF_ATTRIBUTION,
+	}
+
+	await writeManifest(receiptPath, receipt)
+
+	// The adapter reads the newest archive in the directory, so an earlier publish
+	// is removed only after this one is verified and recorded.
+	const earlier = await Globerator.from("*.csv.zip", { cwd: destDir.toString(), absolute: false }).toArray()
+
+	for (const name of earlier) {
+		if (name !== filename && GLEIF_ARCHIVE_PATTERN.test(name)) {
+			await removePathIfPresent(destDir(name))
+		}
+	}
+
+	report?.(`  ✓ ${filename}: ${ByteFormatter.formatIEC(bytes)}, sha256 ${receipt.sha256}`)
+	report?.(`  MANIFEST written to ${receiptPath.toString()}`)
+
+	summary.fetched++
+
+	return summary
+}
+
+/**
+ * Download the current golden copy into `<outRoot>/gleif/`.
+ *
+ * The registry entry point.
+ * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself.
+ */
+export async function fetchGLEIF(options: FetchGLEIFOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits.
+	return await downloadGLEIF(client, {
+		outputDir: options.outRoot(SLUG),
+		verifyDigest: options.verifyDigest,
+		force: options.force,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+}

package/lib/tools/fetch/header.ts

@@ -0,0 +1,69 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * indexes.
+ *
+ * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * because a renamed column reaches the adapter as an empty string on every row rather than as an
+ * error. The check belongs here rather than in each publisher's module: every caller needs the same
+ * two steps, and each one that wrote them again would also be a second answer to how the file is
+ * parsed.
+ */
+
+import type { PathBuilderLike } from "path-ts"
+import { CSVSpliterator } from "spliterator"
+
+/**
+ * The header line's column names.
+ *
+ * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
+ * a split of a fetch module's own: which columns a publisher quotes is a property of
+ * the edition, and a second reader would be a second answer to that question.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file,
+ * which the default `"auto"` does even for one row.
+ *
+ * @param path The written file.
+ * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export async function readDelimitedHeader(path: PathBuilderLike, columnDelimiter?: string): Promise<readonly string[]> {
+	const [header] = await Array.fromAsync(
+		CSVSpliterator.fromAsync<string[]>(path, {
+			header: false,
+			mode: "array",
+			columnScan: "rows",
+			take: 1,
+			...(columnDelimiter === undefined ? {} : { columnDelimiter }),
+		})
+	)
+
+	if (!header) {
+		throw new Error(`${path.toString()}: the file holds no row, so its header could not be read`)
+	}
+
+	return header
+}
+
+/**
+ * Refuse a header that does not name every column a caller reads.
+ *
+ * @param columns The header {@linkcode readDelimitedHeader} returned.
+ * @param required The columns the adapter indexes by name.
+ * @param context What is being checked, which opens the message.
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export function assertHeaderColumns(columns: readonly string[], required: readonly string[], context: string): void {
+	const present = new Set(columns)
+	const absent = required.filter((column) => !present.has(column))
+
+	if (absent.length) {
+		throw new Error(
+			`${context}: the header names ${columns.length} columns and not ${absent.join(", ")}, which the adapter reads by name`
+		)
+	}
+}