@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/be/tools/fetch/brussels.ts

@@ -0,0 +1,239 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Downloads the INSPIRE Addresses theme Paradigm publishes for the Brussels-Capital Region.
+ *
+ *   ## What the publisher offers
+ *
+ *   One archive at a bare HTTPS URL, `URBIS_ADM_Adresses.zip`, holding a single member
+ *   `UrbAdm_Adresses.gml`. Paradigm publishes no INSPIRE ATOM feed for this theme, so there is no
+ *   `<updated>` element to poll and no dataset feed to resolve: the URL is the whole acquisition
+ *   path. `last-modified` is the one freshness signal the service offers, and this fetcher records it
+ *   in the manifest so a later run can tell a refreshed archive from the one it already holds.
+ *
+ *   ## Why the archive stays compressed
+ *
+ *   `#be/adapters/brussels/adapter` reads the member through `inspireGMLChunks`, which inflates it
+ *   through the zip reader rather than from disk. The member is 474,245,978 bytes against the
+ *   archive's 11,521,182, so unpacking it here would cost 41 times the disk for no reader that wants
+ *   it.
+ *
+ *   ## Two hostnames that do not serve it
+ *
+ *   `urbis.brussels` and `urbisonline.brussels` both fail TLS with `ERR_TLS_CERT_ALTNAME_INVALID`,
+ *   because the certificate's altnames are `*.irisnet.be` and `irisnet.be`. The services live on
+ *   `irisnet.be` and the download lives on `datastore.brussels`.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { BRUSSELS_ADAPTER_ID } from "#be/adapters/brussels/adapter"
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The archive's URL, which is the whole acquisition path for this theme.
+ */
+export const BE_BRUSSELS_ARCHIVE_URL = "https://urbisdownload.datastore.brussels/INSPIRE/URBIS_ADM_Adresses.zip"
+
+/**
+ * The name the archive is written under, which is what the adapter's `inputPath` states.
+ */
+export const BE_BRUSSELS_ARCHIVE_FILENAME = "URBIS_ADM_Adresses.zip"
+
+/**
+ * The attribution CC BY 4.0 §3(a)(1) requires on a publication derived from these rows.
+ */
+export const BE_BRUSSELS_ATTRIBUTION = "Paradigm"
+
+const SLUG = BRUSSELS_ADAPTER_ID
+
+/**
+ * What one run recorded, so a later run can tell a refreshed archive from the one on disk.
+ *
+ * `lastModified` is the publisher's own header.
+ * With no ATOM feed for this theme it is the only freshness signal, and a changed
+ * value is the one reason to download again.
+ */
+export interface BrusselsManifest extends SourceManifest {
+	last_modified: string | null
+	attribution: string
+}
+
+/**
+ * Per-invocation options.
+ */
+export interface DownloadBrusselsOptions {
+	outputDir: PathBuilderLike
+	/**
+	 * Downloads the archive even where the manifest's `lastModified` and byte count still match.
+	 */
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+/**
+ * What the publisher's HEAD response states about the archive it holds.
+ *
+ * Both fields read `null` where the header is absent, rather than an empty string or zero.
+ * An absent `last-modified` is the service declining to state a version, which is a
+ * different fact from a version that happens to match the one on disk.
+ */
+export interface BrusselsPublication {
+	lastModified: string | null
+	reportedBytes: number | null
+}
+
+/**
+ * Reads the publisher's HEAD response.
+ *
+ * Separate from {@linkcode downloadBrussels} because this one request carries the whole freshness
+ * decision, and the download itself runs on global `fetch`, which a unit test cannot intercept.
+ */
+export async function readBrusselsPublication(
+	client: Pick<APIClient, "fetch">,
+	options: { signal?: AbortSignal } = {}
+): Promise<BrusselsPublication> {
+	const head = await client.fetch<unknown>({
+		method: "HEAD",
+		url: BE_BRUSSELS_ARCHIVE_URL,
+		timeout: 120_000,
+		signal: options.signal,
+	})
+
+	const reported = Number(head.headers?.["content-length"] ?? Number.NaN)
+
+	return {
+		lastModified: String(head.headers?.["last-modified"] ?? "") || null,
+		reportedBytes: Number.isFinite(reported) ? reported : null,
+	}
+}
+
+/**
+ * Whether the archive on disk is the one the publisher currently serves.
+ *
+ * A skip requires the publisher to state a `last-modified` value.
+ * Where it states none, both sides read `null` and an equality test would hold,
+ * which would keep an archive of unknown age for as long as the service stayed silent.
+ * Downloading 11 MiB again is the cheaper error.
+ */
+export function brusselsPublicationIsRecorded(
+	recorded: BrusselsManifest,
+	bytesOnDisk: number,
+	publication: BrusselsPublication
+): boolean {
+	if (publication.lastModified === null) return false
+
+	return recorded.last_modified === publication.lastModified && recorded.bytes === bytesOnDisk
+}
+
+/**
+ * Downloads the archive unless the manifest and the file on disk already agree with the publisher.
+ */
+export async function downloadBrussels(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadBrusselsOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const archivePath = destDir(BE_BRUSSELS_ARCHIVE_FILENAME)
+	const manifestPath = destDir("MANIFEST.json")
+
+	await makeDirectories(destDir)
+
+	report?.(`=== ${SLUG} / ${BE_BRUSSELS_ARCHIVE_FILENAME}`)
+
+	const publication = await readBrusselsPublication(client, { signal: options.signal })
+
+	report?.(
+		`  HEAD: ${publication.reportedBytes === null ? "no content-length" : ByteFormatter.formatIEC(publication.reportedBytes)}` +
+			`, last-modified ${publication.lastModified ?? "unstated"}`
+	)
+
+	const recorded = await readManifest<BrusselsManifest>(manifestPath)
+	const stat = await tryStat(archivePath)
+
+	if (!options.force && recorded && stat && brusselsPublicationIsRecorded(recorded, stat.size, publication)) {
+		report?.(`  present, and the publisher's last-modified is unchanged`)
+
+		return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+	}
+
+	const lastModified = publication.lastModified
+
+	const { bytes } = await downloadToFile({
+		url: BE_BRUSSELS_ARCHIVE_URL,
+		dest: archivePath,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+
+	const sha256 = await sha256File(archivePath)
+
+	report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha256}`)
+
+	const manifest: BrusselsManifest = {
+		source_url: BE_BRUSSELS_ARCHIVE_URL,
+		filename: BE_BRUSSELS_ARCHIVE_FILENAME,
+		bytes,
+		sha256,
+		last_modified: lastModified,
+		attribution: BE_BRUSSELS_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+/**
+ * The path `#be/adapters/brussels/adapter` reads, given the root a fetch wrote under.
+ */
+export function brusselsInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG, BE_BRUSSELS_ARCHIVE_FILENAME)
+}
+
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchBrusselsOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+}
+
+/**
+ * The registry entry.
+ */
+export async function fetchBrussels(
+	options: FetchBrusselsOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits,
+	// and a disposed `APIClient` refuses every later request.
+	return await downloadBrussels(client, {
+		outputDir: options.outRoot(SLUG),
+		force: options.force,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}

package/lib/be/tools/fetch/wallonie.ts

@@ -0,0 +1,274 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Downloads the INSPIRE Addresses theme the Service public de Wallonie publishes for the Walloon
+ *   Region.
+ *
+ *   ## Two feeds and one archive
+ *
+ *   The service document lists one dataset feed, and that feed offers one archive for the whole
+ *   region rather than one per municipality. Czechia's ČÚZK publishes 6,258 archives through the same
+ *   INSPIRE pattern, so the resolution here is the same shape with a list of one:
+ *
+ *       AD_Service.xml  →  AD_Dataset_<uuid>.xml  →  AD.Addresses.gml.zip
+ *
+ *   The service document's `<updated>` is the freshness signal this fetcher records. A dataset feed
+ *   that reports the same value as the manifest means the archive on disk is the one the publisher
+ *   serves, and the download is skipped.
+ *
+ *   ## Why the archive stays compressed
+ *
+ *   `#be/adapters/wallonie/adapter` reads the member through `inspireGMLChunks`, which inflates from
+ *   the archive. The member is 5,163,917,131 bytes against the archive's 106,665,832, so unpacking it
+ *   here would cost 48 times the disk for no reader that wants it.
+ */
+
+import { APIClient, assertNoOGCServiceException } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { WALLONIE_ADAPTER_ID } from "#be/adapters/wallonie/adapter"
+import { feedChunks, linkWithRel, readAtomFeed } from "#tools/fetch/atom"
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The ATOM service document, which lists the region's one dataset feed.
+ */
+export const BE_WALLONIE_SERVICE_URL = "https://geoservices.wallonie.be/inspire/atom/AD_Service.xml"
+
+/**
+ * The name the archive is written under, which is what the adapter's `inputPath` states.
+ */
+export const BE_WALLONIE_ARCHIVE_FILENAME = "AD.Addresses.gml.zip"
+
+/**
+ * The attribution the publisher's own license statement requires, quoted as it writes it.
+ */
+export const BE_WALLONIE_ATTRIBUTION = "Service public de Wallonie (SPW)"
+
+const SLUG = WALLONIE_ADAPTER_ID
+
+/**
+ * What one run recorded.
+ *
+ * `feed_updated` is the dataset feed's own `<updated>`.
+ * A later run downloads again when that value changes, which is what the INSPIRE
+ * pattern offers in place of an HTTP validator.
+ */
+export interface WallonieManifest extends SourceManifest {
+	service_url: string
+	dataset_feed_url: string
+	feed_updated: string | null
+	attribution: string
+}
+
+/**
+ * Per-invocation options.
+ */
+export interface DownloadWallonieOptions {
+	outputDir: PathBuilderLike
+	/**
+	 * Downloads the archive even where the dataset feed reports the recorded `<updated>`.
+	 */
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+/**
+ * Where the archive is, and what the publisher states about its freshness.
+ */
+export interface WallonieArchiveReference {
+	archiveURL: string
+	datasetFeedURL: string
+	/**
+	 * The archive entry's own `<updated>`, or `null` where the feed states none.
+	 *
+	 * `updated` sits on an Atom entry rather than on the feed, so this reads the
+	 * entry that carried the archive link.
+	 * Wallonia's feed reported `2025-12-11` when this reader was written.
+	 */
+	feedUpdated: string | null
+}
+
+/**
+ * Resolves the service document to the one archive it leads to, through the dataset feed it lists.
+ *
+ * Raises where the service document lists no dataset feed, or where that feed offers no archive.
+ * The alternative would return `{fetched: 0, skipped: 0, failed: 0}`, which a caller
+ * reads as a fetch that completed and found the publisher empty.
+ *
+ * Separate from {@linkcode downloadWallonie} because the resolution is the part a test can drive.
+ * The download runs on global `fetch`, which a unit test cannot intercept.
+ */
+export async function resolveWallonieArchive(
+	client: Pick<APIClient, "fetch">,
+	options: { signal?: AbortSignal; report?: (line: string) => void } = {}
+): Promise<WallonieArchiveReference> {
+	const { report } = options
+
+	report?.(`=== ${SLUG}: reading ${BE_WALLONIE_SERVICE_URL}`)
+
+	const { data: serviceXML } = await client.fetch<string>({
+		url: BE_WALLONIE_SERVICE_URL,
+		timeout: 120_000,
+		signal: options.signal,
+	})
+
+	assertNoOGCServiceException(String(serviceXML), `${SLUG} service document`)
+
+	const service = await readAtomFeed(feedChunks(String(serviceXML)))
+
+	const datasetLink = service.entries.flatMap((entry) => {
+		const link = linkWithRel(entry.links, "alternate")
+
+		return link ? [link] : []
+	})[0]
+
+	if (!datasetLink) {
+		throw new Error(
+			`${SLUG}: the service document lists no dataset feed through a rel="alternate" link, ` +
+				`so no archive can be resolved. It listed ${service.entries.length} entries.`
+		)
+	}
+
+	report?.(`  dataset feed: ${datasetLink.href}`)
+
+	const { data: datasetXML } = await client.fetch<string>({
+		url: datasetLink.href,
+		timeout: 120_000,
+		signal: options.signal,
+	})
+
+	assertNoOGCServiceException(String(datasetXML), `${SLUG} dataset feed`)
+
+	const dataset = await readAtomFeed(feedChunks(String(datasetXML)))
+
+	const offered = dataset.entries.flatMap((entry) => {
+		const link = linkWithRel(entry.links, "enclosure") ?? linkWithRel(entry.links, "alternate")
+
+		return link ? [{ entry, link }] : []
+	})[0]
+
+	if (!offered) {
+		throw new Error(
+			`${SLUG}: the dataset feed offers no archive through a rel="enclosure" or rel="alternate" ` +
+				`link. It listed ${dataset.entries.length} entries.`
+		)
+	}
+
+	return {
+		archiveURL: offered.link.href,
+		datasetFeedURL: datasetLink.href,
+		feedUpdated: offered.entry.updated || null,
+	}
+}
+
+/**
+ * Resolves the service document to its one archive and downloads it.
+ */
+export async function downloadWallonie(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadWallonieOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const archivePath = destDir(BE_WALLONIE_ARCHIVE_FILENAME)
+	const manifestPath = destDir("MANIFEST.json")
+
+	// Resolved before the directory is made, so a publisher that answers with an
+	// exception report leaves no empty source directory behind.
+	const { archiveURL, datasetFeedURL, feedUpdated } = await resolveWallonieArchive(client, {
+		signal: options.signal,
+		report,
+	})
+
+	await makeDirectories(destDir)
+
+	const recorded = await readManifest<WallonieManifest>(manifestPath)
+	const stat = await tryStat(archivePath)
+
+	if (!options.force && recorded && stat && feedUpdated !== null && recorded.feed_updated === feedUpdated) {
+		report?.(`  present, and the dataset feed reports the recorded updated ${feedUpdated}`)
+
+		return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+	}
+
+	report?.(`  archive: ${archiveURL}`)
+
+	const { bytes } = await downloadToFile({
+		url: archiveURL,
+		dest: archivePath,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+
+	const sha256 = await sha256File(archivePath)
+
+	report?.(`  ✓ ${ByteFormatter.formatIEC(bytes)}  sha256=${sha256}`)
+
+	const manifest: WallonieManifest = {
+		source_url: archiveURL,
+		service_url: BE_WALLONIE_SERVICE_URL,
+		dataset_feed_url: datasetFeedURL,
+		filename: BE_WALLONIE_ARCHIVE_FILENAME,
+		bytes,
+		sha256,
+		feed_updated: feedUpdated,
+		attribution: BE_WALLONIE_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}
+
+/**
+ * The path `#be/adapters/wallonie/adapter` reads, given the root a fetch wrote under.
+ */
+export function wallonieInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG, BE_WALLONIE_ARCHIVE_FILENAME)
+}
+
+/**
+ * Per-invocation options for the registry entry.
+ */
+export interface FetchWallonieOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+}
+
+/**
+ * The registry entry.
+ */
+export async function fetchWallonie(
+	options: FetchWallonieOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits,
+	// and a disposed `APIClient` refuses every later request.
+	return await downloadWallonie(client, {
+		outputDir: options.outRoot(SLUG),
+		force: options.force,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}

package/lib/build/adapters.ts

@@ -0,0 +1,110 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The adapter phase: drive each configured adapter to its own `canonical.jsonl`, or reuse the run a
+ *   previous build finished.
+ */
+
+import { pathExists, readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers"
+import type { PathBuilder } from "path-ts"
+
+import type { BuildStage } from "#build/types"
+import { $public } from "#env"
+import { runAdapter, type AdapterRunManifest } from "#runner"
+import type { AdapterOptions, CorpusAdapter } from "#types"
+
+export interface AdapterPhaseOptions {
+	adapters: readonly CorpusAdapter[]
+	adapterInputs: Record<string, AdapterOptions>
+	corpusVersion: string
+	intermediateDir: PathBuilder
+	onProgress?: (stage: BuildStage, message: string) => void
+}
+
+export interface AdapterPhaseResult {
+	/**
+	 * One manifest per adapter that ran or was reused, in the order they were driven.
+	 */
+	runs: AdapterRunManifest[]
+
+	/**
+	 * Adapter ids with no entry in `adapterInputs`.
+	 */
+	skipped: string[]
+}
+
+/**
+ * Reuse a finished adapter run, or return `null` when this build has to drive the adapter.
+ *
+ * @throws When the cached manifest and the file on disk disagree on the byte count.
+ * The writer flushes the canonical file before writing the manifest.
+ * Manifest presence establishes that the writer reached the end of its run.
+ * The manifest describes the file as of that moment.
+ * A truncated copy, an interrupted move or a partial transfer leaves the
+ * manifest intact beside a shorter file.
+ * The recorded byte count distinguishes those cases.
+ * A resumed build that reuses a short file can produce a corpus whose row count later readers accept.
+ */
+async function reusableRun(adapterID: string, adapterDir: PathBuilder): Promise<AdapterRunManifest | null> {
+	if ($public.MAILWOMAN_RESUME !== "1") return null
+
+	const cachedManifest = adapterDir("MANIFEST.json")
+	const canonicalPath = adapterDir("canonical.jsonl")
+
+	if (!(await pathExists(cachedManifest)) || !(await pathExists(canonicalPath))) return null
+
+	const cached = await readLocalJSONFile<AdapterRunManifest>(cachedManifest)
+	const onDisk = (await statPath(canonicalPath)).size
+
+	if (onDisk !== cached.bytes) {
+		throw new Error(
+			`resume refused for ${adapterID}: ${canonicalPath} is ${onDisk.toLocaleString()} bytes and its ` +
+				`manifest records ${cached.bytes.toLocaleString()}. Delete ${cachedManifest} to ` +
+				`re-run the adapter, or restore the file this manifest describes.`
+		)
+	}
+
+	return cached
+}
+
+/**
+ * Drive every configured adapter, reusing whatever a previous build already finished.
+ */
+export async function runAdapterPhase(opts: AdapterPhaseOptions): Promise<AdapterPhaseResult> {
+	const runs: AdapterRunManifest[] = []
+	const skipped: string[] = []
+
+	for (const adapter of opts.adapters) {
+		const adapterOptions = opts.adapterInputs[adapter.id]
+
+		if (!adapterOptions) {
+			skipped.push(adapter.id)
+			opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`)
+
+			continue
+		}
+
+		const cached = await reusableRun(adapter.id, opts.intermediateDir(adapter.id))
+
+		if (cached) {
+			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
+			runs.push(cached)
+
+			continue
+		}
+
+		opts.onProgress?.("adapter-run", `running ${adapter.id}`)
+
+		runs.push(
+			await runAdapter({
+				adapter,
+				adapterOptions,
+				outputDir: opts.intermediateDir,
+				corpusVersion: opts.corpusVersion,
+			})
+		)
+	}
+
+	return { runs, skipped }
+}