@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/fetch/atom.ts

@@ -0,0 +1,164 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reads an INSPIRE predefined-dataset ATOM feed into its links and entries.
+ *
+ *   Several member states serve the Addresses (AD) theme as an ATOM feed rather than as a WFS, so
+ *   this is the shape a fetcher reads before it downloads anything. The elements are the same at
+ *   both levels of a two-level service: a service document's entry points at a dataset feed, and
+ *   that feed's entries point at files, and both are `<entry>` with `<link>` children.
+ *
+ *   Parsed with `@mailwoman/core/html/elements` rather than matched with a pattern, because a feed
+ *   needs the two things a pattern handles badly. An entry's open tag carries attributes
+ *   (`xml:lang`), and an href escapes the ampersand between its query parameters, so a reader has to
+ *   keep attributes and decode entities. XML mode is required: a feed's links self-close, and HTML
+ *   mode leaves them open.
+ *
+ *   Feed-level links are told from entry-level links by the parser rather than by position. Asking
+ *   for `link` and `entry` together yields a `link` only where it is not already inside an `entry`,
+ *   because capture runs to the outermost requested element's close.
+ */
+
+import { childElement, childElements, type MarkupElement, streamMarkupElements } from "@mailwoman/core/html/elements"
+
+/**
+ * One `<link>` of a feed or of one of its entries.
+ *
+ * `rel` is lower-cased, since it is a controlled value, while `href`, `type`
+ * and `title` are kept as the publisher wrote them.
+ */
+export interface AtomLink {
+	href: string
+	/**
+	 * The relation, lower-cased, or an empty string where the link states none.
+	 */
+	rel: string
+	/**
+	 * The media type the link advertises, or `null` where it advertises none.
+	 *
+	 * A publisher may advertise the type of the data inside the file rather than of the file:
+	 * ČÚZK writes `application/gml+xml` on a link that serves `application/zip`.
+	 */
+	type: string | null
+	title: string | null
+	/**
+	 * The byte count the link claims, or `null` where it claims none or claims a non-numeric one.
+	 *
+	 * Never taken as the size of a download.
+	 * Denmark's feed understates its file by 23 times, so every byte count a fetcher
+	 * records is counted off the delivered body instead.
+	 */
+	length: number | null
+}
+
+/**
+ * One `<entry>` of a feed.
+ */
+export interface AtomEntry {
+	/**
+	 * The entry's `<id>`, which is a URI rather than a name.
+	 */
+	id: string
+	title: string
+	/**
+	 * The entry's `<updated>`, as the feed spells it, or an empty string where it states none.
+	 *
+	 * Whether this describes the data or the feed is a property of the publisher: Denmark's is
+	 * older than the file it serves, while ČÚZK's equals the file's `Last-Modified` to the second.
+	 * A fetcher that reads it for freshness has to have measured that agreement first.
+	 */
+	updated: string
+	/**
+	 * The entry's `<rights>`, or `null` where it states none.
+	 */
+	rights: string | null
+	links: readonly AtomLink[]
+	/**
+	 * The entry's `inspire_dls:spatial_dataset_identifier_code`, or `null` for a
+	 * feed that is not an INSPIRE predefined-dataset feed.
+	 */
+	identifierCode: string | null
+}
+
+/**
+ * A feed's own links and its entries.
+ */
+export interface AtomFeed {
+	/**
+	 * The feed's own links, which are the ones not inside an entry.
+	 */
+	links: readonly AtomLink[]
+	entries: readonly AtomEntry[]
+}
+
+function atomLinkOf(element: MarkupElement): AtomLink {
+	const length = element.attributes.length
+
+	return {
+		href: element.attributes.href ?? "",
+		rel: (element.attributes.rel ?? "").toLowerCase(),
+		type: element.attributes.type ?? null,
+		title: element.attributes.title ?? null,
+		length: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+	}
+}
+
+function atomEntryOf(element: MarkupElement): AtomEntry {
+	return {
+		id: childElement(element, "id")?.text ?? "",
+		title: childElement(element, "title")?.text ?? "",
+		updated: childElement(element, "updated")?.text ?? "",
+		rights: childElement(element, "rights")?.text ?? null,
+		links: childElements(element, "link").map(atomLinkOf),
+		identifierCode: childElement(element, "inspire_dls:spatial_dataset_identifier_code")?.text ?? null,
+	}
+}
+
+/**
+ * Read a feed's links and entries out of its markup.
+ *
+ * Streaming rather than whole-document, because a service document may be large:
+ * ČÚZK's lists one entry per municipality in 8,373,817 bytes, measured 2026-10-02.
+ *
+ * @param chunks The feed's bytes or text, in order.
+ * A Node `Readable` satisfies this.
+ * @throws When the markup is malformed, through the parser, so an unreadable feed raises
+ * rather than reading as a feed with fewer entries.
+ */
+export async function readAtomFeed(chunks: AsyncIterable<string | Uint8Array>): Promise<AtomFeed> {
+	const links: AtomLink[] = []
+	const entries: AtomEntry[] = []
+
+	for await (const element of streamMarkupElements(chunks, ["link", "entry"], { xml: true })) {
+		if (element.name === "entry") {
+			entries.push(atomEntryOf(element))
+
+			continue
+		}
+
+		links.push(atomLinkOf(element))
+	}
+
+	return { links, entries }
+}
+
+/**
+ * The feed's text as the one chunk {@linkcode readAtomFeed} takes.
+ *
+ * For a caller that already holds the whole document, such as a test
+ * or a feed read through a buffering client.
+ */
+export async function* feedChunks(text: string): AsyncIterable<string> {
+	yield text
+}
+
+/**
+ * The first link carrying `rel`, or `undefined` where the collection holds none.
+ *
+ * A feed's entry commonly carries several: ČÚZK writes a `describedby` link to its ISO 19139
+ * record beside the `alternate` link that is the data, and either would download as a dataset.
+ */
+export function linkWithRel(links: readonly AtomLink[], rel: string): AtomLink | undefined {
+	return links.find((link) => link.rel === rel.toLowerCase())
+}

package/lib/tools/fetch/cordis.ts

@@ -0,0 +1,318 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the CORDIS project archives for the `cordis` adapter.
+ *
+ * CORDIS serves one CSV archive per framework programme at
+ * `https://cordis.europa.eu/data/cordis-<programme>projects-csv.zip`, where the programme is `fp7`,
+ * `h2020` or `HORIZON` (Horizon Europe), the spelling the host serves. These are the distributions the
+ * data.europa.eu records `cordisfp7projects`, `cordish2020projects` and
+ * `cordis-eu-research-projects-under-horizon-europe-2021-2027` link. Each archive holds an
+ * `organization.csv` beside the project, topic and link tables, and the adapter reads that member
+ * straight out of the archive, so the archive is what this module keeps.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`.
+ *
+ * Measured on 2026-10-03, the host answers `HEAD` with `accept-ranges: bytes`, a `content-length`
+ * and a `last-modified` for each archive: `fp7` 32,953,397 bytes last modified `Thu, 02 Jan 2025
+ * 10:27:58 GMT`, `h2020` 55,219,250 bytes and `HORIZON` 36,908,998 bytes, both last modified on
+ * 22 September 2026. The re-run check compares that pair against the manifest, so a rerun makes
+ * one small request per archive, and an interrupted transfer resumes through
+ * {@linkcode resumableDownload}.
+ *
+ * A delivered archive is checked for its `organization.csv` member before the manifest records it,
+ * so an error page served under the archive's name never reaches the adapter.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import {
+	CORDIS_ADAPTER_ID,
+	CORDIS_ATTRIBUTION,
+	CORDIS_LICENSE,
+	CORDIS_ORGANIZATION_MEMBER,
+} from "#adapters/cordis/adapter"
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { loadCollectionFiles, resumableDownload, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The programmes CORDIS publishes an archive for, in the spelling the host serves.
+ */
+export const CORDIS_PROGRAMMES: readonly string[] = ["fp7", "h2020", "HORIZON"]
+
+/**
+ * The download URL for one programme's archive.
+ */
+export function cordisArchiveURL(programme: string): string {
+	return `https://cordis.europa.eu/data/cordis-${programme}projects-csv.zip`
+}
+
+/**
+ * The file name one programme's archive is kept under, which is the URL's own last segment.
+ */
+export function cordisArchiveFilename(programme: string): string {
+	return `cordis-${programme}projects-csv.zip`
+}
+
+/**
+ * The page whose license statement the adapter quotes.
+ */
+export const CORDIS_LEGAL_NOTICE_URL = "https://cordis.europa.eu/about/legal"
+
+/**
+ * The directory the archives are written under, which is the adapter's `inputPath`.
+ */
+const SLUG = CORDIS_ADAPTER_ID
+
+/**
+ * What the manifest records per archive.
+ *
+ * {@linkcode SourceManifest}'s five fields describe the archive as it arrived.
+ * `last_modified` is what the re-run check compares before it downloads the body again.
+ */
+export interface CORDISArchiveManifest extends SourceManifest {
+	/**
+	 * The `last-modified` the host served the archive under, or `null` where it served none.
+	 */
+	last_modified: string | null
+}
+
+/**
+ * What the host states about one archive without sending it.
+ */
+export interface CORDISArchiveHead {
+	lastModified: string | null
+	contentLength: number | null
+}
+
+/**
+ * Read what the host states about one archive.
+ *
+ * A header the host does not send is reported as `null` rather than raised on,
+ * and the caller then downloads rather than reading the absence as a match.
+ */
+export async function readCORDISArchiveHead(
+	client: Pick<APIClient, "fetch">,
+	programme: string
+): Promise<CORDISArchiveHead> {
+	const response = await client.fetch<unknown>({ method: "head", url: cordisArchiveURL(programme) })
+	const headers = response.headers as Record<string, string> | undefined
+	const length = headers?.["content-length"]
+
+	return {
+		lastModified: headers?.["last-modified"] ?? null,
+		contentLength: length !== undefined && /^\d+$/u.test(length) ? Number(length) : null,
+	}
+}
+
+/**
+ * Whether an archive on disk is still the one the manifest describes and the host still serves.
+ *
+ * The `HEAD` decides first, then the file's own length, then, when asked, its digest.
+ */
+export async function isCORDISArchiveCurrent(
+	recorded: CORDISArchiveManifest | undefined,
+	head: CORDISArchiveHead,
+	path: PathBuilderLike,
+	verifyDigest: boolean
+): Promise<boolean> {
+	if (!recorded) return false
+
+	if (head.lastModified === null && head.contentLength === null) return false
+
+	if (head.lastModified !== null && head.lastModified !== recorded.last_modified) return false
+
+	if (head.contentLength !== null && head.contentLength !== recorded.bytes) return false
+
+	const stat = await tryStat(path)
+
+	if (!stat || stat.size !== recorded.bytes) return false
+
+	if (!verifyDigest) return true
+
+	return (await sha256File(path)) === recorded.sha256
+}
+
+export interface DownloadCORDISOptions {
+	/**
+	 * Where the archives and the manifest are written.
+	 * The directory itself is the adapter's `inputPath`.
+	 */
+	outputDir: PathBuilderLike
+	/**
+	 * Which programmes to fetch.
+	 * Defaults to {@linkcode CORDIS_PROGRAMMES}.
+	 */
+	programmes?: readonly string[]
+	/**
+	 * Re-read each archive's sha256 on a re-run instead of comparing its byte count.
+	 */
+	verifyDigest?: boolean
+	/**
+	 * Download even where the `HEAD` agrees with the manifest.
+	 */
+	force?: boolean
+	/**
+	 * Pause between range attempts, in milliseconds.
+	 */
+	retryDelayMs?: number
+	report?: (line: string) => void
+}
+
+export type FetchCORDISOptions = BaseFetchOptions & Omit<DownloadCORDISOptions, "outputDir" | "report">
+
+/**
+ * The directory `#adapters/cordis/adapter` reads under a fetch root.
+ */
+export function cordisInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG)
+}
+
+/**
+ * Download each requested programme's archive into `options.outputDir` beside one manifest.
+ *
+ * Re-runnable per archive: one whose `HEAD` matches the manifest and whose file is
+ * still on disk at the recorded length makes no request for the body.
+ */
+export async function downloadCORDIS(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadCORDISOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const programmes = options.programmes?.length ? options.programmes : CORDIS_PROGRAMMES
+
+	await makeDirectories(destDir)
+
+	const manifestPath = destDir("MANIFEST.json")
+	const recorded = (await loadCollectionFiles(manifestPath)) as Map<string, CORDISArchiveManifest>
+	const written = new Map<string, CORDISArchiveManifest>(recorded)
+	const summary: FetchSummary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] }
+
+	report?.(`=== ${SLUG}`)
+
+	for (const programme of programmes) {
+		const filename = cordisArchiveFilename(programme)
+		const archivePath = destDir(filename)
+		const url = cordisArchiveURL(programme)
+
+		let head: CORDISArchiveHead
+
+		try {
+			head = await readCORDISArchiveHead(client, programme)
+		} catch (error) {
+			report?.(`  ✗ ${programme}: ${error instanceof Error ? error.message : String(error)}`)
+
+			summary.failed++
+			summary.failedCodes.push(programme)
+
+			continue
+		}
+
+		report?.(
+			`  ${programme}: ${head.contentLength === null ? "no content-length" : ByteFormatter.formatIEC(head.contentLength)}` +
+				`, last-modified ${head.lastModified ?? "absent"}`
+		)
+
+		const current =
+			!options.force &&
+			(await isCORDISArchiveCurrent(recorded.get(filename), head, archivePath, options.verifyDigest ?? false))
+
+		if (current) {
+			report?.(`  ✓ ${programme} already current — no download.`)
+
+			summary.skipped++
+
+			continue
+		}
+
+		let bytes: number
+
+		try {
+			bytes = await resumableDownload({
+				url,
+				dest: archivePath,
+				headers: { accept: "application/zip, */*" },
+				retryDelayMs: options.retryDelayMs,
+				report,
+			})
+
+			const members = await listZipEntries(archivePath)
+
+			if (!members.some((member) => member.name === CORDIS_ORGANIZATION_MEMBER)) {
+				throw new Error(`the archive holds no ${CORDIS_ORGANIZATION_MEMBER}`)
+			}
+		} catch (error) {
+			report?.(`  ✗ ${programme}: ${error instanceof Error ? error.message : String(error)}`)
+			await removePathIfPresent(archivePath)
+
+			summary.failed++
+			summary.failedCodes.push(programme)
+
+			continue
+		}
+
+		if (head.contentLength !== null && head.contentLength !== bytes) {
+			report?.(
+				`  ! ${programme}: the HEAD stated ${head.contentLength} bytes and ${bytes} arrived, ` +
+					`so the archive was republished between the two requests`
+			)
+		}
+
+		written.set(filename, {
+			source_url: url,
+			downloaded_at: new Date().toISOString(),
+			filename,
+			sha256: await sha256File(archivePath),
+			bytes,
+			last_modified: head.lastModified,
+		})
+
+		report?.(`  ✓ ${programme}: ${ByteFormatter.formatIEC(bytes)}`)
+
+		summary.fetched++
+	}
+
+	const manifest: SourceCollectionManifest = {
+		source: SLUG,
+		source_url: CORDIS_LEGAL_NOTICE_URL,
+		license: CORDIS_LICENSE,
+		attribution: CORDIS_ATTRIBUTION,
+		downloaded_at: new Date().toISOString(),
+		files: [...written.values()].toSorted((left, right) => left.filename.localeCompare(right.filename)),
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  MANIFEST written to ${manifestPath.toString()}`)
+
+	return summary
+}
+
+/**
+ * Download the CORDIS archives into `<outRoot>/cordis/`.
+ *
+ * The registry entry point.
+ */
+export async function fetchCORDIS(options: FetchCORDISOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned: `await using` disposes the client when this scope exits.
+	return await downloadCORDIS(client, {
+		outputDir: options.outRoot(SLUG),
+		programmes: options.programmes,
+		verifyDigest: options.verifyDigest,
+		force: options.force,
+		retryDelayMs: options.retryDelayMs,
+		report,
+	})
+}

package/lib/tools/fetch/download/manifest.ts

@@ -0,0 +1,114 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reads and writes corpus-source manifests.
+ */
+
+import { pathExists, readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { openWriteStream, pipeline, Readable } from "@mailwoman/core/fs/streams"
+import { movePath, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
+
+import { HTTPStatusError, type SourceManifest } from "#tools/fetch/download/network"
+
+/**
+ * Read a manifest.json; `null` when missing or corrupt (callers re-fetch from scratch).
+ */
+export async function readManifest<T>(path: PathBuilderLike): Promise<T | null> {
+	if (!(await pathExists(path))) return null
+
+	// A read failure (e.g. The file vanished after the existsSync probe) maps to null like
+	// corrupt JSON does. tryParsingJSON returns null for the non-string sentinel.
+	const text = await readLocalTextFile(path).catch(() => null)
+
+	return tryParsingJSON<T>(text)
+}
+
+/**
+ * Load manifest entries into a map so untouched keys survive a partial re-fetch.
+ */
+export async function loadManifestEntries<T>(
+	path: PathBuilderLike,
+	key: (entry: T) => string
+): Promise<Map<string, T>> {
+	const entries = new Map<string, T>()
+	const parsed = await readManifest<T[]>(path)
+
+	for (const entry of parsed ?? []) {
+		entries.set(key(entry), entry)
+	}
+
+	return entries
+}
+
+/**
+ * Write a manifest.json in the house shape: pretty-printed, trailing newline.
+ */
+export async function writeManifest(path: PathBuilderLike, manifest: unknown): Promise<void> {
+	await writeLocalJSONFile(manifest, path)
+}
+
+/**
+ * The sibling `manifest.json` shape for a source that is a collection of files behind one
+ * portal (a monthly register published per region, per industry, or per first letter).
+ *
+ * It records what a trained artifact has to be able to cite later: the license the portal labels
+ * the data with, the attribution wording it requires and one {@link SourceManifest} per file.
+ */
+export interface SourceCollectionManifest {
+	source: string
+	source_url: string
+	license: string
+	attribution: string
+	downloaded_at: string
+	files: SourceManifest[]
+}
+
+/**
+ * Pipe a response body to `dest` and answer the byte count.
+ *
+ * The one primitive the portal fetchers share when the request is not a bare GET.
+ * A session cookie, a csrf header, or a form post stands between the listing
+ * and the file, so {@link streamDownload}'s URL-only interface does not fit
+ * and each module builds its own `Response` first.
+ *
+ * Writes a `.tmp` sibling and renames, so an interrupted transfer never lands
+ * at the final path looking complete.
+ */
+export async function streamBodyToFile(res: Response, dest: PathBuilderLike): Promise<number> {
+	if (!res.body) throw new HTTPStatusError(res.status, `HTTP ${res.status} with no body — ${res.url}`)
+	let bytes = 0
+
+	const counter = new TransformStream<Uint8Array, Uint8Array>({
+		transform(chunk, controller) {
+			bytes += chunk.byteLength
+			controller.enqueue(chunk)
+		},
+	})
+
+	const tmp = dest + ".tmp"
+	await pipeline(Readable.fromWeb(res.body.pipeThrough(counter)), openWriteStream(tmp))
+	await movePath(tmp, dest)
+
+	return bytes
+}
+
+/**
+ * The per-file entries of a {@link SourceCollectionManifest} keyed by file name,
+ * or an empty map when the manifest is missing or not in the collection shape,
+ * so a re-run after an interruption fetches only what is missing.
+ *
+ * The single-file modules' `loadManifestEntries` reads a bare array and is not this.
+ */
+export async function loadCollectionFiles(path: PathBuilderLike): Promise<Map<string, SourceManifest>> {
+	const parsed = await readManifest<SourceCollectionManifest>(path)
+	const entries = new Map<string, SourceManifest>()
+
+	for (const entry of Array.isArray(parsed?.files) ? parsed.files : []) {
+		entries.set(entry.filename, entry)
+	}
+
+	return entries
+}