@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/tools/fetch/wfs-harvest.ts

@@ -0,0 +1,954 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * The paging, manifest and resumption machinery a paged WFS harvest shares.
+ *
+ * `./inspire-addresses.ts` owns what an INSPIRE Addresses service exposes — its capabilities, its
+ * feature-type names and its GeoJSON pages. This module owns what a harvest of any paged WFS does
+ * with those pages: it appends each one to the single file an adapter's `inputPath` points at,
+ * records the page in a manifest beside it, and resumes from that manifest after an interrupted run.
+ *
+ * One file rather than one file per page, because the adapters read one. `#ee/adapters/ads/adapter`
+ * opens `opts.inputPath` as newline-delimited JSON and `#pl/adapters/emuia/adapter` opens it as one
+ * markup stream, so a directory of pages reaches neither. The manifest therefore records each page's
+ * byte range inside that file: `offset`, `bytes` and the sha256 of exactly those bytes. A range is
+ * what makes a page verifiable after it has been concatenated into its neighbors, and what lets an
+ * interrupted append be truncated back to a page boundary rather than discarding the harvest.
+ *
+ * Three properties of a WFS make this harder than reading pages until they run out, and each is
+ * handled here rather than by each country's module:
+ *
+ * 1. A service may ignore `startIndex` and answer every page with the first. `countWFSFeaturesByPaging`
+ *    in `@mailwoman/core/api` refuses such a service by comparing two pages' leading feature. A
+ *    harvest sees the same defect as two consecutive pages with equal bytes. It refuses that pair.
+ * 2. A service's own `numberMatched` may be a per-request cap rather than a count. The caller states
+ *    what its count is and where it came from, and {@linkcode PagedWFSHarvestOptions.featureCount}
+ *    holds both. No count is written into this module, and a harvest with no usable count ends on
+ *    the first page that returns no features.
+ * 3. A page may return fewer features than `count` asks for. The next `startIndex` therefore advances
+ *    by the features the service returned rather than by the page size, so a service that caps a page
+ *    below the requested size is read whole instead of being read in strides that skip features.
+ */
+
+import { APIClient, assertNoOGCServiceException, type CheckedWFSFeatureCount } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { readFileRange } from "@mailwoman/core/fs/readers"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
+import { appendLocalTextFile, makeDirectories, removePathIfPresent, truncateFile } from "@mailwoman/core/fs/writers"
+import { sha256File, sha256Hex } from "@mailwoman/core/hash"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { isoSecondsUTC } from "@mailwoman/core/utils"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { readManifest, writeManifest, type FetchSummary } from "#tools/fetch/download"
+
+/**
+ * The WFS version this repository's harvesters speak.
+ *
+ * 2.0.0 defines `startIndex` paging and the `hits` result type, and it is what
+ * every service measured for these modules advertises.
+ */
+export const WFS_VERSION = "2.0.0"
+
+/**
+ * The file name a harvest writes its manifest under, beside the data file.
+ */
+export const HARVEST_MANIFEST_FILE = "MANIFEST.json"
+
+/**
+ * The first element of a response, as the service wrote it.
+ */
+export interface MarkupRoot {
+	/**
+	 * The opening tag verbatim, `<wfs:FeatureCollection …>`.
+	 */
+	tag: string
+	/**
+	 * The element's qualified name, prefix included.
+	 */
+	name: string
+	/**
+	 * The text between the name and the tag's terminator.
+	 * It holds the attributes.
+	 */
+	attributes: string
+	/**
+	 * The offset just past the opening tag.
+	 */
+	contentStart: number
+}
+
+/**
+ * The first element tag in a document.
+ *
+ * An index scan over the tag rather than a parse of the document: a harvest needs the
+ * opening tag verbatim to open the file it assembles, and parsing a 1.6 MB page per request
+ * to recover a tag that the first 400 bytes already hold costs a document parse per page.
+ *
+ * The leading `[A-Za-z_]` skips the XML declaration.
+ * That declaration opens `<?xml`.
+ */
+const ROOT_OPENING_TAG = /<([A-Za-z_][\w.:-]*)([^>]*)>/u
+
+/**
+ * One attribute of an opening tag, by local name.
+ */
+function tagAttribute(root: MarkupRoot, attribute: string): string | null {
+	const pattern = new RegExp(`(?:^|\\s)(?:[\\w.-]+:)?${attribute}\\s*=\\s*"([^"]*)"`, "iu")
+
+	return pattern.exec(root.attributes)?.[1] ?? null
+}
+
+/**
+ * The count an opening tag states, or `null` where it states none or declines to state one.
+ *
+ * WFS 2.0 permits `numberMatched="unknown"`, which declines to count rather than counting none.
+ */
+export function rootCount(root: MarkupRoot, attribute: string): number | null {
+	const value = tagAttribute(root, attribute)
+
+	return value && /^\d+$/u.test(value) ? Number(value) : null
+}
+
+/**
+ * The document's first element tag.
+ *
+ * @throws When the body holds no element tag, so a response that is not markup
+ * reads as unreadable rather than as a page of no features.
+ */
+export function readMarkupRoot(body: string, context: string): MarkupRoot {
+	const match = ROOT_OPENING_TAG.exec(body)
+
+	if (!match) {
+		throw new TypeError(
+			`${context}: the response carried no element tag, so its root could not be read, beginning ${stringifyJSON(body.slice(0, 120))}`
+		)
+	}
+
+	return {
+		tag: match[0],
+		name: match[1] ?? "",
+		attributes: match[2] ?? "",
+		contentStart: match.index + match[0].length,
+	}
+}
+
+/**
+ * The content between a document's root tags.
+ * For a `wfs:FeatureCollection` those are its members.
+ *
+ * @throws When the root element is not closed, so a truncated transfer reads as
+ * unreadable rather than as a page of no features.
+ */
+export function rootContent(body: string, root: MarkupRoot, context: string): string {
+	const closing = `</${root.name}>`
+	const end = body.lastIndexOf(closing)
+
+	if (end < root.contentStart) {
+		throw new TypeError(
+			`${context}: the response's ${root.name} element is not closed, so the members it holds could not be separated from the document that carries them`
+		)
+	}
+
+	return body.slice(root.contentStart, end)
+}
+
+/**
+ * The attributes a page states about itself rather than about the feature type.
+ *
+ * A harvest assembles many pages under one root element, so the first page's counts
+ * and timestamp would describe the assembled file wrongly.
+ * They are dropped and the manifest records them per page.
+ */
+const PAGE_ATTRIBUTES = /\s(?:[\w.-]+:)?(?:numberMatched|numberReturned|timeStamp|next|previous)\s*=\s*"[^"]*"/giu
+
+/**
+ * A page's root tag with its namespace declarations kept and its per-page attributes
+ * dropped, so it can open a file holding every page.
+ */
+export function collectionOpeningTag(root: MarkupRoot): string {
+	return `<${root.name}${root.attributes.replace(PAGE_ATTRIBUTES, "")}>`
+}
+
+/**
+ * One `GetFeature` response in a markup format, with what its root states about itself.
+ */
+export interface WFSMarkupPage {
+	body: string
+	root: MarkupRoot
+	/**
+	 * The `numberReturned` the root states, or `null` where it states none.
+	 */
+	numberReturned: number | null
+	/**
+	 * The `numberMatched` the root states, or `null` where it answers `unknown`.
+	 */
+	numberMatched: number | null
+	/**
+	 * The `timeStamp` the root states, or `null` where it states none.
+	 */
+	timeStamp: string | null
+}
+
+/**
+ * Issues one `GetFeature` and reads the counts off the root element.
+ *
+ * @throws When the service answers an OGC exception report, so an HTTP 400 carrying
+ * `ows:ExceptionCode` raises rather than being written into the harvest as a page.
+ */
+export async function readWFSMarkupPage(
+	client: Pick<APIClient, "fetch">,
+	options: { wfsURL: string; context: string; params: Readonly<Record<string, string>> }
+): Promise<WFSMarkupPage> {
+	const { data } = await client.fetch<string>({
+		method: "GET",
+		url: options.wfsURL,
+		responseType: "text",
+		params: { service: "WFS", version: WFS_VERSION, request: "GetFeature", ...options.params },
+	})
+
+	assertNoOGCServiceException(data, options.context)
+
+	const root = readMarkupRoot(data, options.context)
+
+	return {
+		body: data,
+		root,
+		numberReturned: rootCount(root, "numberReturned"),
+		numberMatched: rootCount(root, "numberMatched"),
+		timeStamp: tagAttribute(root, "timeStamp"),
+	}
+}
+
+/**
+ * What a harvest asked the service for.
+ *
+ * Recorded in the manifest and compared against a resumed run's request: a harvest whose page
+ * size, ordering or format changed cannot be continued from pages taken under the old one.
+ */
+export interface WFSHarvestRequest {
+	wfsURL: string
+	typeName: string
+	outputFormat: string
+	/**
+	 * The property the service was asked to order by, or `null` where it grants no ordering.
+	 *
+	 * Without one, a resumed harvest rests on the service returning the same
+	 * features in the same order as the earlier run.
+	 * No WFS guarantees that order.
+	 */
+	sortBy: string | null
+}
+
+/**
+ * A feature count and how it was obtained.
+ *
+ * `count` is `null` where the service states no usable count.
+ * The harvest then ends on the first page that returns no features, and
+ * `because` records what the service said instead.
+ */
+export interface FeatureCountStatement {
+	count: number | null
+	because: string
+}
+
+/**
+ * The count a service states, or no count and the reason there is none.
+ *
+ * `readCheckedWFSFeatureCount` proves a reported `numberMatched` against a page of the same type.
+ * That comparison catches a service whose count is smaller than one of its own pages.
+ *
+ * It cannot catch a count that equals the service's page cap: no page can return more
+ * features than the cap, so the page it asks for agrees with the cap every time.
+ *
+ * Poland's service is that case.
+ * Measured 2026-10-02, `resultType=hits` at `startIndex=1` reports `numberMatched="1000"`, its
+ * capabilities document advertises `CountDefault` as 1000, and a page returns at most 1,000 features.
+ *
+ * The reported number is the cap, and a harvest that read it as a count would store 1,000
+ * of the 8.6 million features the service holds and record the type as read whole.
+ * The harvest therefore records no count when the reported number equals the cap.
+ *
+ * @param pageCap The largest page the service will serve, from its advertised `CountDefault`.
+ */
+export function statedFeatureCount(checked: CheckedWFSFeatureCount, pageCap: number | null): FeatureCountStatement {
+	if (!checked.usable || checked.reported === null) {
+		return { count: null, because: `the service stated no usable count: ${checked.because}` }
+	}
+
+	if (pageCap !== null && checked.reported === pageCap) {
+		return {
+			count: null,
+			because: `the service reported numberMatched=${checked.reported}, which is the ${pageCap}-feature cap it advertises as CountDefault, so the number describes the largest page it will serve rather than what the type holds`,
+		}
+	}
+
+	return {
+		count: checked.reported,
+		because: `the service reported numberMatched=${checked.reported} past index 0, and ${checked.because}`,
+	}
+}
+
+/**
+ * One page's contribution to the harvest file.
+ */
+export interface HarvestedPayload {
+	/**
+	 * The text appended to the harvest file for this page.
+	 */
+	payload: string
+	/**
+	 * Text written once, before the first page's payload — a document prolog
+	 * and root tag, where the format needs one.
+	 *
+	 * Read from the first page a run writes and recorded in the manifest.
+	 */
+	header?: string
+	/**
+	 * Text written once the harvest reaches the end of the feature type, such as a closing root tag.
+	 */
+	footer?: string
+	/**
+	 * The features the service returned.
+	 * The next `startIndex` advances by this number.
+	 *
+	 * `null` where the service stated none.
+	 * The harvest refuses that rather than reading it as zero.
+	 */
+	numberReturned: number | null
+	numberMatched: number | null
+	retrievedAt: string | null
+}
+
+/**
+ * One page as the manifest records it.
+ *
+ * `offset` and `bytes` locate the page inside the harvest file and `sha256` is the digest of
+ * exactly those bytes, so a page stays verifiable after the pages around it were appended.
+ */
+export interface WFSHarvestPageRecord {
+	start_index: number
+	/**
+	 * The page size requested.
+	 * A service may answer with fewer features.
+	 */
+	count: number
+	offset: number
+	bytes: number
+	sha256: string
+	number_returned: number
+	/**
+	 * The `numberMatched` this page stated, where the service states a real one.
+	 */
+	number_matched: number | null
+	retrieved_at: string | null
+}
+
+/**
+ * The manifest a paged harvest writes beside its data file.
+ *
+ * Written after every page, so an interrupted run resumes from the last page that reached disk.
+ */
+export interface WFSHarvestManifest {
+	source: string
+	source_url: string
+	wfs_version: string
+	type_name: string
+	output_format: string
+	sort_by: string | null
+	license: string
+	attribution: string
+	downloaded_at: string
+	/**
+	 * The file an adapter's `inputPath` points at, beside this manifest.
+	 */
+	filename: string
+	page_size: number
+	/**
+	 * The service's own count where it states a usable one, and `null` where it does not.
+	 */
+	feature_count: number | null
+	feature_count_source: string
+	features_written: number
+	header_bytes: number
+	/**
+	 * The text that closes the data file, appended once the harvest reaches the end of the type.
+	 */
+	footer: string
+	/**
+	 * The data file's byte length as this manifest describes it, footer included once complete.
+	 */
+	bytes: number
+	/**
+	 * The whole data file's digest, once the harvest is complete.
+	 *
+	 * `null` on a partial harvest, where the file is still being appended to
+	 * and a digest of it would describe a prefix rather than the harvest.
+	 */
+	sha256: string | null
+	complete: boolean
+	pages: WFSHarvestPageRecord[]
+}
+
+export interface PagedWFSHarvestOptions {
+	/**
+	 * Names the service in every refusal.
+	 */
+	context: string
+	/**
+	 * The adapter slug this harvest feeds, stamped into the manifest.
+	 */
+	source: string
+	/**
+	 * Where the harvest is written.
+	 *
+	 * The adapter reads {@linkcode PagedWFSHarvestOptions.filename} inside this directory.
+	 */
+	outputDir: PathBuilderLike
+	filename: string
+	request: WFSHarvestRequest
+	/**
+	 * The license the address-source register elected for this publisher.
+	 */
+	license: string
+	/**
+	 * The attribution the elected terms require, or an empty string where they require none.
+	 */
+	attribution: string
+	featureCount: FeatureCountStatement
+	pageSize: number
+	/**
+	 * Stop once the manifest holds this many pages, for a probe rather than a full harvest.
+	 *
+	 * A cap on the manifest rather than on the run, so a second run under the same cap makes no request.
+	 */
+	maxPages?: number
+	signal?: AbortSignal
+	/**
+	 * Reads one page and states the bytes the harvest file receives for it.
+	 */
+	readPage: (request: { startIndex: number; count: number }) => Promise<HarvestedPayload>
+	report?: (line: string) => void
+}
+
+/**
+ * Does a recorded manifest describe the request being made now?
+ *
+ * A harvest taken at another page size, ordering, format or type cannot be continued:
+ * its pages were cut at different boundaries, and appending to them would leave
+ * a file whose manifest describes neither half.
+ */
+function describesSameRequest(manifest: WFSHarvestManifest, options: PagedWFSHarvestOptions): boolean {
+	return (
+		manifest.source_url === options.request.wfsURL &&
+		manifest.type_name === options.request.typeName &&
+		manifest.output_format === options.request.outputFormat &&
+		manifest.sort_by === options.request.sortBy &&
+		manifest.page_size === options.pageSize &&
+		manifest.filename === options.filename
+	)
+}
+
+function emptyManifest(options: PagedWFSHarvestOptions): WFSHarvestManifest {
+	return {
+		source: options.source,
+		source_url: options.request.wfsURL,
+		wfs_version: WFS_VERSION,
+		type_name: options.request.typeName,
+		output_format: options.request.outputFormat,
+		sort_by: options.request.sortBy,
+		license: options.license,
+		attribution: options.attribution,
+		downloaded_at: isoSecondsUTC(),
+		filename: options.filename,
+		page_size: options.pageSize,
+		feature_count: options.featureCount.count,
+		feature_count_source: options.featureCount.because,
+		features_written: 0,
+		header_bytes: 0,
+		footer: "",
+		bytes: 0,
+		sha256: null,
+		complete: false,
+		pages: [],
+	}
+}
+
+/**
+ * The offset the next page is appended at: the header plus every recorded page.
+ */
+function dataEndOf(manifest: WFSHarvestManifest): number {
+	let end = manifest.header_bytes
+
+	for (const page of manifest.pages) {
+		end += page.bytes
+	}
+
+	return end
+}
+
+/**
+ * The features the recorded pages hold between them.
+ */
+function featuresIn(pages: readonly WFSHarvestPageRecord[]): number {
+	let features = 0
+
+	for (const page of pages) {
+		features += page.number_returned
+	}
+
+	return features
+}
+
+/**
+ * The index the next page starts at: one past the last feature any recorded page reached.
+ */
+function nextStartIndexOf(pages: readonly WFSHarvestPageRecord[]): number {
+	let next = 0
+
+	for (const page of pages) {
+		next = Math.max(next, page.start_index + page.number_returned)
+	}
+
+	return next
+}
+
+/**
+ * The recorded pages present on disk, with the file truncated back to the last of them.
+ *
+ * An interrupted run can leave a partial page at the end of the file,
+ * or a footer the next page has to be appended past.
+ * Both are resolved by truncating the file at the last page boundary it reaches.
+ *
+ * @throws When the last surviving page's bytes no longer hash to what the manifest recorded.
+ * A mismatch means the file was altered after the harvest recorded it.
+ */
+async function reconcileHarvestFile(
+	manifest: WFSHarvestManifest,
+	dataPath: PathBuilderLike,
+	context: string,
+	report?: (line: string) => void
+): Promise<WFSHarvestManifest> {
+	const stat = await tryStat(dataPath)
+	const size = stat?.size ?? 0
+
+	const kept: WFSHarvestPageRecord[] = []
+
+	for (const page of manifest.pages) {
+		if (page.offset + page.bytes > size) break
+
+		kept.push(page)
+	}
+
+	if (kept.length < manifest.pages.length) {
+		report?.(
+			`  ${manifest.pages.length - kept.length} of ${manifest.pages.length} recorded pages are not on disk — resuming from page ${kept.length}`
+		)
+	}
+
+	const reconciled: WFSHarvestManifest = {
+		...manifest,
+		pages: kept,
+		features_written: featuresIn(kept),
+		complete: manifest.complete && kept.length === manifest.pages.length,
+	}
+
+	if (!kept.length) {
+		// A header with no page behind it describes an empty document, so the file starts over.
+		await removePathIfPresent(dataPath)
+
+		return { ...reconciled, header_bytes: 0, footer: "", bytes: 0, sha256: null, complete: false }
+	}
+
+	const last = kept.at(-1)!
+	const recordedBytes = await readFileRange(dataPath, last.offset, last.bytes)
+	const digest = sha256Hex(recordedBytes)
+
+	if (digest !== last.sha256) {
+		throw new Error(
+			`${context}: the ${last.bytes} bytes the manifest records for the page at startIndex ${last.start_index} hash to ${digest} rather than ${last.sha256}, so ${dataPath} holds something other than the harvest it describes`
+		)
+	}
+
+	const dataEnd = dataEndOf(reconciled)
+
+	if (reconciled.complete) {
+		const expected = dataEnd + Buffer.byteLength(reconciled.footer)
+
+		if (size === expected) return { ...reconciled, bytes: expected }
+
+		report?.(`  The complete harvest records ${expected} bytes and ${size} are on disk — closing it again`)
+	}
+
+	if (size !== dataEnd) {
+		await truncateFile(dataPath, dataEnd)
+	}
+
+	return { ...reconciled, bytes: dataEnd, sha256: null, complete: false }
+}
+
+/**
+ * What a caller knows about the request it is about to make before it has asked the service.
+ *
+ * The type name and the output format come from a capabilities document, so they are not here:
+ * the point of this check is to answer whether a harvest is already current without sending a request.
+ */
+export interface HarvestCurrencyExpectation {
+	wfsURL: string
+	sortBy: string | null
+	pageSize: number
+}
+
+/**
+ * The manifest of a harvest that needs no further request, or `null` where one does.
+ *
+ * A caller reads this before it reads a capabilities document, so a current harvest costs no request at all.
+ * Two harvests need none:
+ *
+ * - A complete one whose data file still hashes to the digest the manifest records.
+ * - A partial one that already holds `maxPages` pages, the cap a bounded probe requests.
+ *
+ * @throws When a page's recorded byte range no longer hashes to its recorded digest,
+ * so a file that was altered under its manifest is reported rather than used.
+ */
+export async function readCurrentHarvest(options: {
+	outputDir: PathBuilderLike
+	filename: string
+	context: string
+	expect: HarvestCurrencyExpectation
+	maxPages?: number
+}): Promise<WFSHarvestManifest | null> {
+	const root = PathBuilder.from(options.outputDir)
+	const manifest = await readManifest<WFSHarvestManifest>(root(HARVEST_MANIFEST_FILE))
+
+	if (!manifest || !Array.isArray(manifest.pages)) return null
+
+	const describesThisRequest =
+		manifest.filename === options.filename &&
+		manifest.source_url === options.expect.wfsURL &&
+		manifest.sort_by === options.expect.sortBy &&
+		manifest.page_size === options.expect.pageSize
+
+	if (!describesThisRequest) return null
+
+	const dataPath = root(options.filename)
+
+	if (manifest.complete) {
+		const stat = await tryStat(dataPath)
+
+		if (!stat || stat.size !== manifest.bytes) return null
+
+		return (await sha256File(dataPath)) === manifest.sha256 ? manifest : null
+	}
+
+	if (options.maxPages === undefined || manifest.pages.length < options.maxPages) return null
+
+	const last = manifest.pages.at(-1)
+
+	if (!last) return null
+
+	const stat = await tryStat(dataPath)
+
+	if (!stat || stat.size < last.offset + last.bytes) return null
+
+	const digest = sha256Hex(await readFileRange(dataPath, last.offset, last.bytes))
+
+	if (digest !== last.sha256) {
+		throw new Error(
+			`${options.context}: the ${last.bytes} bytes the manifest records for the page at startIndex ${last.start_index} hash to ${digest} rather than ${last.sha256}, so ${dataPath} holds something other than the harvest it describes`
+		)
+	}
+
+	return manifest
+}
+
+/**
+ * Harvests one WFS feature type into a single file, recording each page in a manifest beside it.
+ *
+ * Re-runnable in three ways, each of which makes no request:
+ *
+ * - A complete harvest whose file still hashes to the manifest's `sha256`.
+ * - A partial harvest that already holds {@linkcode PagedWFSHarvestOptions.maxPages} pages.
+ * - A partial harvest resumes at the page after the last one on disk,
+ *   so the pages already taken are not taken again.
+ *
+ * @returns The manifest as written.
+ */
+export async function harvestPagedWFS(options: PagedWFSHarvestOptions): Promise<WFSHarvestManifest> {
+	const root = PathBuilder.from(options.outputDir)
+
+	await makeDirectories(root)
+
+	const dataPath = root(options.filename)
+	const manifestPath = root(HARVEST_MANIFEST_FILE)
+	const recorded = await readManifest<WFSHarvestManifest>(manifestPath)
+	const resumable = recorded !== null && Array.isArray(recorded.pages) && describesSameRequest(recorded, options)
+
+	if (recorded && !resumable) {
+		options.report?.("  The manifest describes a different request — starting the harvest over")
+
+		await removePathIfPresent(dataPath)
+	}
+
+	let manifest = resumable
+		? await reconcileHarvestFile(recorded, dataPath, options.context, options.report)
+		: emptyManifest(options)
+
+	// The count may have become readable, or stopped being readable, since the recorded run.
+	manifest = {
+		...manifest,
+		feature_count: options.featureCount.count,
+		feature_count_source: options.featureCount.because,
+	}
+
+	if (manifest.complete) {
+		const digest = await sha256File(dataPath)
+
+		if (digest === manifest.sha256) {
+			options.report?.(`  ✓ Already harvested (sha256 matches ${HARVEST_MANIFEST_FILE}) — no request made.`)
+
+			return manifest
+		}
+
+		options.report?.(`  ${dataPath} hashes to ${digest} rather than ${manifest.sha256} — harvesting it again`)
+
+		manifest = { ...manifest, complete: false, sha256: null }
+	}
+
+	const atPageCap = (): boolean => options.maxPages !== undefined && manifest.pages.length >= options.maxPages
+
+	if (atPageCap()) {
+		options.report?.(`  ✓ The manifest already holds ${manifest.pages.length} pages — no request made.`)
+
+		return manifest
+	}
+
+	let startIndex = nextStartIndexOf(manifest.pages)
+	let complete = false
+
+	for (;;) {
+		if (options.signal?.aborted) break
+
+		if (atPageCap()) break
+
+		if (manifest.feature_count !== null && manifest.features_written >= manifest.feature_count) {
+			complete = true
+
+			break
+		}
+
+		const page = await options.readPage({ startIndex, count: options.pageSize })
+
+		if (page.numberReturned === null) {
+			throw new Error(
+				`${options.context}: the page at startIndex ${startIndex} stated no numberReturned, so what it holds is unknown rather than empty`
+			)
+		}
+
+		if (page.numberReturned === 0) {
+			complete = true
+
+			break
+		}
+
+		const payloadBytes = Buffer.byteLength(page.payload)
+		const digest = sha256Hex(page.payload)
+		const previous = manifest.pages.at(-1)
+
+		// A service that ignores `startIndex` answers every page with the first,
+		// and a harvest of one would hold the same features repeated.
+		// `countWFSFeaturesByPaging` refuses such a service by comparing two pages' leading feature.
+		// Identical bytes is the same defect seen here.
+		if (previous && previous.bytes === payloadBytes && previous.sha256 === digest) {
+			throw new Error(
+				`${options.context}: the page at startIndex ${startIndex} is byte-identical to the page at startIndex ${previous.start_index}, so the service is answering a new startIndex with the previous page rather than paging`
+			)
+		}
+
+		if (!manifest.pages.length && page.header) {
+			await appendLocalTextFile(page.header, dataPath)
+
+			manifest = { ...manifest, header_bytes: Buffer.byteLength(page.header) }
+		}
+
+		const offset = dataEndOf(manifest)
+
+		await appendLocalTextFile(page.payload, dataPath)
+
+		manifest = {
+			...manifest,
+			downloaded_at: isoSecondsUTC(),
+			footer: page.footer ?? manifest.footer,
+			features_written: manifest.features_written + page.numberReturned,
+			bytes: offset + payloadBytes,
+			pages: [
+				...manifest.pages,
+				{
+					start_index: startIndex,
+					count: options.pageSize,
+					offset,
+					bytes: payloadBytes,
+					sha256: digest,
+					number_returned: page.numberReturned,
+					number_matched: page.numberMatched,
+					retrieved_at: page.retrievedAt,
+				},
+			],
+		}
+
+		await writeManifest(manifestPath, manifest)
+
+		options.report?.(
+			`  page ${manifest.pages.length} at startIndex ${startIndex}: ${page.numberReturned} features, ${ByteFormatter.formatIEC(payloadBytes)} (${manifest.features_written} written)`
+		)
+
+		// The service's own answer rather than the page size: a service that caps a page below the
+		// requested size would otherwise be read in strides that skip the features it did not return.
+		startIndex += page.numberReturned
+	}
+
+	if (complete) {
+		if (manifest.footer) {
+			await appendLocalTextFile(manifest.footer, dataPath)
+		}
+
+		// A feature type of no features still needs the file the adapter opens, and an empty append creates it.
+		if (!(await tryStat(dataPath))) {
+			await appendLocalTextFile("", dataPath)
+		}
+
+		const stat = await tryStat(dataPath)
+
+		manifest = {
+			...manifest,
+			downloaded_at: isoSecondsUTC(),
+			bytes: stat?.size ?? manifest.bytes,
+			sha256: await sha256File(dataPath),
+			complete: true,
+		}
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	return manifest
+}
+
+/**
+ * What a registered `mailwoman corpus fetch` entry needs to run one service's harvest.
+ */
+export interface RunWFSHarvestOptions {
+	/**
+	 * The slug the harvest is written under and reported by.
+	 * It matches the adapter's own id.
+	 */
+	slug: string
+
+	/**
+	 * Where the harvest directory sits, as the fetch registry hands it over.
+	 */
+	outRoot: (segment: string) => PathBuilderLike
+
+	/**
+	 * The data file inside the harvest directory.
+	 * The adapter's `inputPath` points here.
+	 */
+	filename: string
+
+	/**
+	 * What a stored manifest has to agree with before its harvest counts as current.
+	 */
+	expect: HarvestCurrencyExpectation
+
+	/**
+	 * The name the client reports itself under.
+	 */
+	displayName: string
+
+	/**
+	 * The floor between two requests to one government host.
+	 */
+	minRequestIntervalMs: number
+
+	/**
+	 * Runs the harvest against a client the caller does not own.
+	 */
+	harvest: (client: Pick<APIClient, "fetch">) => Promise<WFSHarvestManifest>
+
+	/**
+	 * A client to use rather than constructing one.
+	 * A test supplies this.
+	 */
+	client?: Pick<APIClient, "fetch">
+
+	/**
+	 * A page cap.
+	 * A capped run treats an incomplete harvest as a success.
+	 */
+	maxPages?: number
+}
+
+/**
+ * Runs one service's harvest and reports it as a fetch registry entry does.
+ *
+ * Estonia and Poland wrote this sequence twice: ask whether the harvest on disk is
+ * already current, construct a paced client unless the caller supplied one, run,
+ * report, and decide whether an incomplete harvest is a failure.
+ * A third WFS source would have written it a third time.
+ *
+ * The currency check runs before any request, so a complete harvest or a bounded
+ * probe that already holds its pages makes no request at all.
+ *
+ * A bounded run stops short by instruction rather than by failure, so a page cap the caller
+ * asked for reads as a success while an incomplete harvest without one reads as a failure.
+ */
+export async function runWFSHarvest(
+	options: RunWFSHarvestOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	report?.(`=== ${options.slug}`)
+
+	const destination = options.outRoot(options.slug)
+
+	const current = await readCurrentHarvest({
+		outputDir: destination,
+		filename: options.filename,
+		context: options.slug,
+		expect: options.expect,
+		maxPages: options.maxPages,
+	})
+
+	if (current) {
+		report?.(
+			`  ✓ Already harvested: ${current.features_written} features over ${current.pages.length} pages — no request made.`
+		)
+
+		return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+	}
+
+	let manifest: WFSHarvestManifest
+
+	if (options.client) {
+		manifest = await options.harvest(options.client)
+	} else {
+		// Bounded retry, because a harvest is many requests and one 5xx would otherwise end the run.
+		await using client = new APIClient({
+			displayName: options.displayName,
+			minRequestIntervalMs: options.minRequestIntervalMs,
+			retry: true,
+		})
+
+		manifest = await options.harvest(client)
+	}
+
+	report?.(
+		`  ${manifest.complete ? "✓" : "partial:"} ${manifest.features_written} features over ${manifest.pages.length} pages, ${manifest.bytes} bytes`
+	)
+
+	const bounded = options.maxPages !== undefined && manifest.pages.length >= options.maxPages
+
+	return manifest.complete || bounded
+		? { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+		: { fetched: 0, skipped: 0, failed: 1, failedCodes: [options.slug] }
+}