@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/fetch/gleif.js

@@ -0,0 +1,220 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter.
+ *
+ * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at
+ * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the
+ * CDF version it follows, so this module reads that answer first and downloads the file it names.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the
+ * `2026-10-03 08:00:00` publish:
+ *
+ * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
+ *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    publish replaced mid-transfer is refused rather than recorded.
+ * 2. **The host honors `Range`.** An interrupted transfer resumes through
+ *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
+ *    compresses small responses on the fly and a range must count the archive's own bytes.
+ * 3. **Each publish has its own file name**, date-prefixed as
+ *    `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that
+ *    name, so a partial transfer of one publish never resumes into another, and the adapter reads the
+ *    newest name in the directory.
+ *
+ * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter
+ * streams the member out of the archive rather than reading an extracted copy.
+ *
+ * The receipt written beside the archive records the URL, the publish date, the byte count and the
+ * sha256 of the archive, which identify the bytes a corpus row was read from.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { listZipEntries } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { GLEIF_ADAPTER_ID, GLEIF_ARCHIVE_PATTERN, GLEIF_ATTRIBUTION, GLEIF_LICENSE, GLEIF_LICENSE_URL, } from "#adapters/gleif/adapter";
+import { readManifest, resumableDownload, writeManifest } from "#tools/fetch/download";
+/**
+ * The publish API's answer for the newest golden copy and its delta files.
+ */
+export const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest";
+/**
+ * The directory the archive and its receipt are written under, which is the adapter's `inputPath`.
+ */
+const SLUG = "gleif";
+/**
+ * Read which file the publish API names as the current Level 1 golden copy in CSV.
+ *
+ * @throws When the answer omits any of the five fields, naming the one it lacks,
+ * rather than downloading a file whose length or date cannot be checked.
+ */
+export async function readGLEIFLatestPublish(client) {
+    const response = await client.fetch({ method: "GET", url: GLEIF_LATEST_PUBLISH_URL });
+    const lei2 = response.data?.data?.lei2;
+    const csv = lei2?.full_file?.csv;
+    const fields = {
+        "data.lei2.publish_date": lei2?.publish_date,
+        "data.lei2.full_file.csv.url": csv?.url,
+        "data.lei2.full_file.csv.size": csv?.size,
+        "data.lei2.full_file.csv.record_count": csv?.record_count,
+        "data.lei2.full_file.csv.cdf_version": csv?.cdf_version,
+    };
+    const missing = Object.entries(fields)
+        .filter(([, value]) => value === undefined || value === null || value === "")
+        .map(([name]) => name);
+    if (missing.length) {
+        throw new Error(`${GLEIF_LATEST_PUBLISH_URL} answered without ${missing.join(", ")}`);
+    }
+    return {
+        publishDate: lei2.publish_date,
+        url: csv.url,
+        bytes: Number(csv.size),
+        recordCount: Number(csv.record_count),
+        cdfVersion: csv.cdf_version,
+    };
+}
+/**
+ * The archive's file name, which is the last path segment of the URL the API names.
+ */
+export function gleifArchiveFilename(url) {
+    const name = new URL(url).pathname.split("/").at(-1) ?? "";
+    if (!GLEIF_ARCHIVE_PATTERN.test(name)) {
+        throw new Error(`${url} does not name a golden-copy archive matching ${GLEIF_ARCHIVE_PATTERN}`);
+    }
+    return name;
+}
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export function gleifInputPath(outRoot) {
+    return outRoot(SLUG);
+}
+/**
+ * Whether the archive on disk is the publish the API names and the length the receipt records.
+ *
+ * Exported so a test can exercise the decision without a transfer.
+ */
+export async function isGLEIFArchiveCurrent(recorded, published, path, verifyDigest) {
+    if (!recorded)
+        return false;
+    if (recorded.source_url !== published.url || recorded.publish_date !== published.publishDate)
+        return false;
+    const stat = await tryStat(path);
+    if (!stat || stat.size !== recorded.bytes)
+        return false;
+    if (!verifyDigest)
+        return true;
+    return (await sha256File(path)) === recorded.sha256;
+}
+/**
+ * Download the current Level 1 golden copy into `options.outputDir` and write its receipt.
+ *
+ * Re-runnable: a receipt naming the current publish, with the archive still on disk
+ * at the recorded length, makes no request for the body.
+ * Archives of earlier publishes are removed once the new one is verified,
+ * so the directory holds one publish.
+ */
+export async function downloadGLEIF(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const summary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] };
+    await makeDirectories(destDir);
+    report?.(`=== ${SLUG}`);
+    const published = await readGLEIFLatestPublish(client);
+    const filename = gleifArchiveFilename(published.url);
+    const archivePath = destDir(filename);
+    const receiptPath = destDir("MANIFEST.json");
+    report?.(`  publish ${published.publishDate}: ${published.recordCount.toLocaleString()} records, ` +
+        `${ByteFormatter.formatIEC(published.bytes)}, ${published.cdfVersion}`);
+    const recorded = await readManifest(receiptPath);
+    if (!options.force &&
+        (await isGLEIFArchiveCurrent(recorded, published, archivePath, options.verifyDigest ?? false))) {
+        report?.(`  ✓ ${filename} already current — no download.`);
+        summary.skipped++;
+        return summary;
+    }
+    let bytes;
+    try {
+        bytes = await resumableDownload({
+            url: published.url,
+            dest: archivePath,
+            headers: { "accept-encoding": "identity", accept: "application/zip" },
+            retryDelayMs: options.retryDelayMs,
+            report,
+        });
+    }
+    catch (error) {
+        report?.(`  ✗ download failed: ${error instanceof Error ? error.message : String(error)}`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    if (bytes !== published.bytes) {
+        await removePathIfPresent(archivePath);
+        report?.(`  ✗ the API stated ${published.bytes} bytes and ${bytes} arrived, so the archive is not recorded`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    const members = (await listZipEntries(archivePath)).filter((entry) => entry.name.toLowerCase().endsWith(".csv"));
+    if (members.length !== 1) {
+        report?.(`  ✗ ${filename} holds ${members.length} CSV members, where the golden copy publishes one`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    const receipt = {
+        source: GLEIF_ADAPTER_ID,
+        source_url: published.url,
+        api_url: GLEIF_LATEST_PUBLISH_URL,
+        publish_date: published.publishDate,
+        cdf_version: published.cdfVersion,
+        record_count: published.recordCount,
+        downloaded_at: new Date().toISOString(),
+        filename,
+        bytes,
+        sha256: await sha256File(archivePath),
+        csv_member: members[0].name,
+        csv_member_bytes: members[0].uncompressedSize,
+        license: GLEIF_LICENSE,
+        license_url: GLEIF_LICENSE_URL,
+        attribution: GLEIF_ATTRIBUTION,
+    };
+    await writeManifest(receiptPath, receipt);
+    // The adapter reads the newest archive in the directory, so an earlier publish
+    // is removed only after this one is verified and recorded.
+    const earlier = await Globerator.from("*.csv.zip", { cwd: destDir.toString(), absolute: false }).toArray();
+    for (const name of earlier) {
+        if (name !== filename && GLEIF_ARCHIVE_PATTERN.test(name)) {
+            await removePathIfPresent(destDir(name));
+        }
+    }
+    report?.(`  ✓ ${filename}: ${ByteFormatter.formatIEC(bytes)}, sha256 ${receipt.sha256}`);
+    report?.(`  MANIFEST written to ${receiptPath.toString()}`);
+    summary.fetched++;
+    return summary;
+}
+/**
+ * Download the current golden copy into `<outRoot>/gleif/`.
+ *
+ * The registry entry point.
+ * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself.
+ */
+export async function fetchGLEIF(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits.
+    return await downloadGLEIF(client, {
+        outputDir: options.outRoot(SLUG),
+        verifyDigest: options.verifyDigest,
+        force: options.force,
+        retryDelayMs: options.retryDelayMs,
+        report,
+    });
+}
+//# sourceMappingURL=gleif.js.map

package/out/tools/fetch/gleif.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gleif.js","sourceRoot":"","sources":["../../../lib/tools/fetch/gleif.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EACN,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,EACb,iBAAiB,GACjB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,oEAAoE,CAAA;AAE5G;;GAEG;AACH,MAAM,IAAI,GAAG,OAAO,CAAA;AA4DpB;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,MAAgC;IAC5E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,wBAAwB,EAAE,CAAC,CAAA;IACtG,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAA;IACtC,MAAM,GAAG,GAAG,IAAI,EAAE,SAAS,EAAE,GAAG,CAAA;IAEhC,MAAM,MAAM,GAAG;QACd,wBAAwB,EAAE,IAAI,EAAE,YAAY;QAC5C,6BAA6B,EAAE,GAAG,EAAE,GAAG;QACvC,8BAA8B,EAAE,GAAG,EAAE,IAAI;QACzC,sCAAsC,EAAE,GAAG,EAAE,YAAY;QACzD,qCAAqC,EAAE,GAAG,EAAE,WAAW;KACvD,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;SAC5E,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IAEvB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,GAAG,wBAAwB,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,OAAO;QACN,WAAW,EAAE,IAAK,CAAC,YAAa;QAChC,GAAG,EAAE,GAAI,CAAC,GAAI;QACd,KAAK,EAAE,MAAM,CAAC,GAAI,CAAC,IAAI,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC,GAAI,CAAC,YAAY,CAAC;QACtC,UAAU,EAAE,GAAI,CAAC,WAAY;KAC7B,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAE1D,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,iDAAiD,qBAAqB,EAAE,CAAC,CAAA;IAChG,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAoC;IAClE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,QAAuC,EACvC,SAA6B,EAC7B,IAAqB,EACrB,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3B,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,GAAG,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,WAAW;QAAE,OAAO,KAAK,CAAA;IAE1G,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK;QAAE,OAAO,KAAK,CAAA;IAEvD,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAE9B,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAA;AACpD,CAAC;AAsBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC,EAChC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,OAAO,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAEpF,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACpD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE5C,MAAM,EAAE,CACP,aAAa,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,YAAY;QACxF,GAAG,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,UAAU,EAAE,CACvE,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAyB,WAAW,CAAC,CAAA;IAExE,IACC,CAAC,OAAO,CAAC,KAAK;QACd,CAAC,MAAM,qBAAqB,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,EAC7F,CAAC;QACF,MAAM,EAAE,CAAC,OAAO,QAAQ,iCAAiC,CAAC,CAAA;QAE1D,OAAO,CAAC,OAAO,EAAE,CAAA;QAEjB,OAAO,OAAO,CAAA;IACf,CAAC;IAED,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,iBAAiB,CAAC;YAC/B,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,EAAE,iBAAiB,EAAE;YACrE,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1F,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAEtC,MAAM,EAAE,CAAC,sBAAsB,SAAS,CAAC,KAAK,cAAc,KAAK,0CAA0C,CAAC,CAAA;QAE5G,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IAEhH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,OAAO,QAAQ,UAAU,OAAO,CAAC,MAAM,mDAAmD,CAAC,CAAA;QAEpG,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,OAAO,GAA2B;QACvC,MAAM,EAAE,gBAAgB;QACxB,UAAU,EAAE,SAAS,CAAC,GAAG;QACzB,OAAO,EAAE,wBAAwB;QACjC,YAAY,EAAE,SAAS,CAAC,WAAW;QACnC,WAAW,EAAE,SAAS,CAAC,UAAU;QACjC,YAAY,EAAE,SAAS,CAAC,WAAW;QACnC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ;QACR,KAAK;QACL,MAAM,EAAE,MAAM,UAAU,CAAC,WAAW,CAAC;QACrC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI;QAC5B,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,gBAAgB;QAC9C,OAAO,EAAE,aAAa;QACtB,WAAW,EAAE,iBAAiB;QAC9B,WAAW,EAAE,iBAAiB;KAC9B,CAAA;IAED,MAAM,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAEzC,+EAA+E;IAC/E,2DAA2D;IAC3D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IAE1G,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;QACzC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,OAAO,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACxF,MAAM,EAAE,CAAC,yBAAyB,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE3D,OAAO,CAAC,OAAO,EAAE,CAAA;IAEjB,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,yFAAyF;IACzF,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE;QAClC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM;KACN,CAAC,CAAA;AACH,CAAC"}

package/out/tools/fetch/header.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * indexes.
+ *
+ * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * because a renamed column reaches the adapter as an empty string on every row rather than as an
+ * error. The check belongs here rather than in each publisher's module: every caller needs the same
+ * two steps, and each one that wrote them again would also be a second answer to how the file is
+ * parsed.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The header line's column names.
+ *
+ * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
+ * a split of a fetch module's own: which columns a publisher quotes is a property of
+ * the edition, and a second reader would be a second answer to that question.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file,
+ * which the default `"auto"` does even for one row.
+ *
+ * @param path The written file.
+ * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export declare function readDelimitedHeader(path: PathBuilderLike, columnDelimiter?: string): Promise<readonly string[]>;
+/**
+ * Refuse a header that does not name every column a caller reads.
+ *
+ * @param columns The header {@linkcode readDelimitedHeader} returned.
+ * @param required The columns the adapter indexes by name.
+ * @param context What is being checked, which opens the message.
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export declare function assertHeaderColumns(columns: readonly string[], required: readonly string[], context: string): void;
+//# sourceMappingURL=header.d.ts.map

package/out/tools/fetch/header.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"header.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9C;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAgBrH;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CASlH"}

package/out/tools/fetch/header.js

@@ -0,0 +1,59 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * indexes.
+ *
+ * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * because a renamed column reaches the adapter as an empty string on every row rather than as an
+ * error. The check belongs here rather than in each publisher's module: every caller needs the same
+ * two steps, and each one that wrote them again would also be a second answer to how the file is
+ * parsed.
+ */
+import { CSVSpliterator } from "spliterator";
+/**
+ * The header line's column names.
+ *
+ * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
+ * a split of a fetch module's own: which columns a publisher quotes is a property of
+ * the edition, and a second reader would be a second answer to that question.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file,
+ * which the default `"auto"` does even for one row.
+ *
+ * @param path The written file.
+ * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export async function readDelimitedHeader(path, columnDelimiter) {
+    const [header] = await Array.fromAsync(CSVSpliterator.fromAsync(path, {
+        header: false,
+        mode: "array",
+        columnScan: "rows",
+        take: 1,
+        ...(columnDelimiter === undefined ? {} : { columnDelimiter }),
+    }));
+    if (!header) {
+        throw new Error(`${path.toString()}: the file holds no row, so its header could not be read`);
+    }
+    return header;
+}
+/**
+ * Refuse a header that does not name every column a caller reads.
+ *
+ * @param columns The header {@linkcode readDelimitedHeader} returned.
+ * @param required The columns the adapter indexes by name.
+ * @param context What is being checked, which opens the message.
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export function assertHeaderColumns(columns, required, context) {
+    const present = new Set(columns);
+    const absent = required.filter((column) => !present.has(column));
+    if (absent.length) {
+        throw new Error(`${context}: the header names ${columns.length} columns and not ${absent.join(", ")}, which the adapter reads by name`);
+    }
+}
+//# sourceMappingURL=header.js.map

package/out/tools/fetch/header.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"header.js","sourceRoot":"","sources":["../../../lib/tools/fetch/header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAqB,EAAE,eAAwB;IACxF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CACrC,cAAc,CAAC,SAAS,CAAW,IAAI,EAAE;QACxC,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,MAAM;QAClB,IAAI,EAAE,CAAC;QACP,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC;KAC7D,CAAC,CACF,CAAA;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,0DAA0D,CAAC,CAAA;IAC9F,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAA0B,EAAE,QAA2B,EAAE,OAAe;IAC3G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAEhE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,sBAAsB,OAAO,CAAC,MAAM,oBAAoB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAmC,CACtH,CAAA;IACF,CAAC;AACF,CAAC"}

package/out/tools/fetch/inspire-addresses.d.ts

@@ -0,0 +1,249 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Read an INSPIRE Addresses (AD) theme from a member state's WFS download service.
+ *
+ * The INSPIRE Directive makes Addresses a mandatory Annex I theme and accepts two kinds of download
+ * service: a predefined-dataset ATOM feed and a Web Feature Service. Most member states publish the
+ * second, so a file download reaches only a minority of them.
+ *
+ * An `ad:Address` feature carries its point, its lifecycle dates and its locator. It does not carry
+ * its street name, its postcode or its administrative units: those are separate feature types, and
+ * the address references them through `component`. A complete address is therefore assembled from
+ * several feature types rather than read from one.
+ *
+ * Measured on Slovakia's service, one address holds four component references that resolve to three
+ * `ad:AdminUnitName` features and one `ad:PostalDescriptor`. Resolving each reference with its own
+ * request would cost roughly four requests per address, which is 6.8 million for that service's
+ * 1,704,196 addresses. This module therefore pages each feature type in bulk and joins locally.
+ *
+ * A service's feature count is read with `readCheckedWFSFeatureCount` from `@mailwoman/core/api`,
+ * which owns every WFS count this repository takes.
+ */
+import { type APIClient } from "@mailwoman/core/api";
+/**
+ * The feature types an INSPIRE Addresses service publishes.
+ *
+ * `Address` is the subject.
+ * The other four are the components an address references, and each has to be
+ * paged separately to build the join.
+ */
+export declare const AD_FEATURE_TYPES: readonly ["Address", "ThoroughfareName", "PostalDescriptor", "AdminUnitName", "AddressAreaName"];
+/**
+ * One of the {@link AD_FEATURE_TYPES}.
+ */
+export type ADFeatureType = (typeof AD_FEATURE_TYPES)[number];
+/**
+ * The AD feature type a service's qualified name denotes, or `null` where it denotes none.
+ *
+ * Services name the same type four ways, measured across the four this module was written from.
+ * Slovakia and Flanders publish `ad:Address`.
+ *
+ * Estonia publishes `AD_Address:AD.Address` and `AD_Address:AD.Address_ThoroughfareName`.
+ * Poland publishes `ms:AD.Address`.
+ *
+ * A reader that compares the part after the colon against `Address` finds a type
+ * on two services and none on the other two, which reads as a service publishing
+ * no addresses rather than as a naming difference.
+ */
+export declare function adFeatureTypeOf(qualifiedName: string): ADFeatureType | null;
+/**
+ * What a service's capabilities document states about itself.
+ */
+export interface WFSCapabilities {
+    /**
+     * Every `outputFormat` value the service advertises for `GetFeature`.
+     */
+    outputFormats: readonly string[];
+    /**
+     * The advertised format this module would ask for, or `null` when the service offers no JSON.
+     *
+     * A service without one is readable through its GML, which this module leaves
+     * to its caller rather than reporting as unreadable.
+     *
+     * `application/json` is preferred over `application/geo+json` because a service
+     * may advertise the second and reject it.
+     * Estonia advertises both and answers a `GetFeature` for `application/geo+json`
+     * with `InvalidParameterValue: Failed to find response for output format`,
+     * while the same request under `application/json` returns the features.
+     */
+    jsonFormat: string | null;
+    /**
+     * Every advertised JSON format, in the order this module would try them.
+     *
+     * A caller whose first request is rejected tries the next rather than concluding that the
+     * service serves no JSON, because an advertised format is a claim rather than a guarantee.
+     */
+    jsonFormats: readonly string[];
+    /**
+     * Whether the service advertises `ImplementsResultPaging`.
+     *
+     * A service that does not cannot be paged, so a caller has to take the whole
+     * type in one request or decline it.
+     * Reading past the first page of such a service silently repeats page one.
+     */
+    supportsPaging: boolean;
+    /**
+     * The `CountDefault` the service advertises, or `null` where it advertises none.
+     *
+     * The largest page the service will serve, which is also the number its own
+     * `numberMatched` reports when that number is a cap rather than a count.
+     * Estonia advertises 1000000 and Poland 1000, measured 2026-10-02.
+     */
+    countDefault: number | null;
+    /**
+     * The qualified type name for each AD feature type the service publishes, keyed by the local name.
+     *
+     * The prefix differs per service: Slovakia and Flanders publish `ad:Address`,
+     * and Estonia publishes under its own prefix.
+     * A caller that assumes `ad:` receives HTTP 400 from the rest.
+     */
+    typeNames: Readonly<Partial<Record<ADFeatureType, string>>>;
+}
+/**
+ * A value a GeoServer JSON response uses for a voidable property that holds no value.
+ *
+ * INSPIRE marks many attributes voidable, which requires the property to be present
+ * carrying either a value or a void with a reason.
+ * GeoServer encodes that void as an object rather than as `null`, so `String(value)`
+ * on one yields `[object Object]` and stores it as though it were data.
+ */
+export interface VoidedValue {
+    "@nil": string | boolean;
+    "@nilReason"?: string;
+}
+/**
+ * Is this property value a void rather than a value?
+ */
+export declare function isVoided(value: unknown): value is VoidedValue;
+/**
+ * The value of a property, or `null` where the service marked it void or omitted it.
+ *
+ * `null` for a void keeps the void from reaching a caller as the text `[object Object]`.
+ *
+ * INSPIRE separates a void from an absence: an absent property states that no value exists,
+ * and a void states that whether one exists is unknown.
+ * This collapses both to `null`, so a caller that needs the two apart reads the
+ * property's presence on `properties` itself.
+ */
+export declare function readVoidable<T>(value: T | VoidedValue | undefined): T | null;
+/**
+ * Reads a service's capabilities, so a caller asks only for what the service offers.
+ */
+export declare function readWFSCapabilities(client: Pick<APIClient, "fetch">, options: {
+    wfsURL: string;
+    context: string;
+}): Promise<WFSCapabilities>;
+/**
+ * One page of features, as the service returned them.
+ */
+export interface FeaturePage<Feature> {
+    features: readonly Feature[];
+    /**
+     * The count the service reported for the whole query, or `null` where it declined to state one.
+     *
+     * A service may answer `unknown`, which is a refusal to count rather than a count of zero.
+     */
+    numberMatched: number | null;
+    numberReturned: number;
+    /**
+     * The `timeStamp` the service dated the page with, or `null` where it stated none.
+     *
+     * The service's own clock rather than the caller's, which is what a manifest
+     * records to date a page against the service that served it.
+     */
+    timeStamp: string | null;
+}
+/**
+ * A GeoJSON feature, as far as this module reads one.
+ */
+export interface GeoJSONFeature {
+    type: "Feature";
+    id?: string;
+    geometry: {
+        type: string;
+        coordinates: unknown;
+    } | null;
+    properties: Record<string, unknown>;
+}
+/**
+ * Reads one page of a feature type.
+ *
+ * `startIndex` past the first page needs {@linkcode WFSCapabilities.supportsPaging}.
+ * A service that declines paging answers page two with page one, so this refuses
+ * rather than returning the repeat.
+ */
+export declare function readFeaturePage(client: Pick<APIClient, "fetch">, options: {
+    wfsURL: string;
+    typeName: string;
+    outputFormat: string;
+    count: number;
+    startIndex: number;
+    supportsPaging: boolean;
+    context: string;
+    /**
+     * The property the service orders the results by, where it honors one.
+     *
+     * Paging without an ordering rests on the service returning the same features in
+     * the same order for every request, which no WFS guarantees.
+     * Estonia honors `sortBy=gml_id` and Flanders answers HTTP 504 for its identifier,
+     * so the parameter is the caller's to supply or leave out.
+     */
+    sortBy?: string | null;
+}): Promise<FeaturePage<GeoJSONFeature>>;
+/**
+ * Every `component` reference an address carries, as the service wrote them.
+ *
+ * Two encodings appear across the services measured.
+ * Slovakia and Flanders write a `component` array of objects carrying `@href`.
+ *
+ * Estonia writes one flat property per slot, `component1_xlink_href` through `component6_xlink_href`,
+ * and fills an unused slot with the string `unpopulated` rather than with a void object.
+ * Reading only the array reported 0 references for every Estonian address, where each carries four.
+ *
+ * A reference may leave the service: Estonia's first three slots address its Administrative
+ * Units theme at `AU_haldusyksused` rather than its Addresses theme.
+ * That is a reference this module reads and {@linkcode resolveComponents} reports as unjoined
+ * against Addresses features, which is the honest answer rather than a dropped reference.
+ */
+export declare function componentReferences(feature: GeoJSONFeature): readonly string[];
+/**
+ * The key a component feature publishes for an address to reference it by.
+ *
+ * `identifier.value` is the INSPIRE external object identifier, which is what
+ * Flanders writes on both sides of the join.
+ * `gml_id` is the fallback for a service that publishes no identifier,
+ * and a stored-query reference addresses that id.
+ */
+export declare function componentIdentifier(feature: GeoJSONFeature): string | null;
+/**
+ * What a pass over one service established about its references.
+ *
+ * `joined` and `unjoined` are counted against the component features the caller supplied,
+ * so a caller that paged part of a component type reads `unjoined` as "not among
+ * the features read" rather than as "absent from the service".
+ * `unjoinedExample` is there to be looked at for that reason: a reference into another register
+ * and a reference past the end of a page look identical in the counts and different in the URL.
+ */
+export interface ComponentResolution {
+    /**
+     * References whose key equals an identifier among the component features supplied.
+     */
+    joined: number;
+    /**
+     * References whose key matched no supplied component feature, with one example.
+     */
+    unjoined: number;
+    unjoinedExample: string | null;
+    /**
+     * References this module could not read a key from at all.
+     */
+    unreadable: number;
+}
+/**
+ * Joins an address page's component references against the component features supplied.
+ */
+export declare function resolveComponents(addresses: readonly GeoJSONFeature[], components: readonly GeoJSONFeature[]): ComponentResolution;
+//# sourceMappingURL=inspire-addresses.d.ts.map

package/out/tools/fetch/inspire-addresses.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inspire-addresses.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/inspire-addresses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAA+B,KAAK,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAMjF;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,YAC5B,SAAS,EACT,kBAAkB,EAClB,kBAAkB,EAClB,eAAe,EACf,iBAAiB,CACR,CAAA;AAEV;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE7D;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAU3E;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,aAAa,EAAE,SAAS,MAAM,EAAE,CAAA;IAChC;;;;;;;;;;;OAWG;IACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB;;;;;OAKG;IACH,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B;;;;;;OAMG;IACH,cAAc,EAAE,OAAO,CAAA;IACvB;;;;;;OAMG;IACH,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B;;;;;;OAMG;IACH,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;CAC3D;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IAC3B,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;IACxB,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,WAAW,CAI7D;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,WAAW,GAAG,SAAS,GAAG,CAAC,GAAG,IAAI,CAI5E;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACxC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC1C,OAAO,CAAC,eAAe,CAAC,CA0D1B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW,CAAC,OAAO;IACnC,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAA;IAC5B;;;;OAIG;IACH,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,cAAc,EAAE,MAAM,CAAA;IACtB;;;;;OAKG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,SAAS,CAAA;IACf,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,QAAQ,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,CAAA;IACvD,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CACpC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE;IACR,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,EAAE,MAAM,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,OAAO,CAAA;IACvB,OAAO,EAAE,MAAM,CAAA;IACf;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACtB,GACC,OAAO,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAoDtC;AAmBD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,MAAM,EAAE,CAqB9E;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,cAAc,GAAG,MAAM,GAAG,IAAI,CAc1E;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAChC,SAAS,EAAE,SAAS,cAAc,EAAE,EACpC,UAAU,EAAE,SAAS,cAAc,EAAE,GACnC,mBAAmB,CAsCrB"}