@mailwoman/corpus 10.0.0 → 10.1.0

package/out/tools/fetch/gleif.d.ts

@@ -0,0 +1,137 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter.
+ *
+ * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at
+ * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the
+ * CDF version it follows, so this module reads that answer first and downloads the file it names.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the
+ * `2026-10-03 08:00:00` publish:
+ *
+ * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
+ *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    publish replaced mid-transfer is refused rather than recorded.
+ * 2. **The host honors `Range`.** An interrupted transfer resumes through
+ *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
+ *    compresses small responses on the fly and a range must count the archive's own bytes.
+ * 3. **Each publish has its own file name**, date-prefixed as
+ *    `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that
+ *    name, so a partial transfer of one publish never resumes into another, and the adapter reads the
+ *    newest name in the directory.
+ *
+ * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter
+ * streams the member out of the archive rather than reading an extracted copy.
+ *
+ * The receipt written beside the archive records the URL, the publish date, the byte count and the
+ * sha256 of the archive, which identify the bytes a corpus row was read from.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { type PathBuilderLike } from "path-ts";
+import type { BaseFetchOptions, FetchSummary } from "#tools/fetch/download";
+/**
+ * The publish API's answer for the newest golden copy and its delta files.
+ */
+export declare const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest";
+/**
+ * The file the publish API names for the Level 1 golden copy in CSV.
+ */
+export interface GLEIFPublishedFile {
+    /**
+     * The publish timestamp the API states, as `2026-10-03 08:00:00`.
+     */
+    publishDate: string;
+    url: string;
+    /**
+     * The archive's length in bytes, as the API states it.
+     */
+    bytes: number;
+    recordCount: number;
+    cdfVersion: string;
+}
+/**
+ * The receipt written beside the archive.
+ */
+export interface GLEIFGoldenCopyReceipt {
+    source: string;
+    source_url: string;
+    api_url: string;
+    publish_date: string;
+    cdf_version: string;
+    /**
+     * The record count the API states for this publish.
+     */
+    record_count: number;
+    downloaded_at: string;
+    filename: string;
+    bytes: number;
+    sha256: string;
+    /**
+     * The CSV member inside the archive, and its decompressed length as the central directory states it.
+     */
+    csv_member: string;
+    csv_member_bytes: number;
+    license: string;
+    license_url: string;
+    attribution: string;
+}
+/**
+ * Read which file the publish API names as the current Level 1 golden copy in CSV.
+ *
+ * @throws When the answer omits any of the five fields, naming the one it lacks,
+ * rather than downloading a file whose length or date cannot be checked.
+ */
+export declare function readGLEIFLatestPublish(client: Pick<APIClient, "fetch">): Promise<GLEIFPublishedFile>;
+/**
+ * The archive's file name, which is the last path segment of the URL the API names.
+ */
+export declare function gleifArchiveFilename(url: string): string;
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export declare function gleifInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike;
+/**
+ * Whether the archive on disk is the publish the API names and the length the receipt records.
+ *
+ * Exported so a test can exercise the decision without a transfer.
+ */
+export declare function isGLEIFArchiveCurrent(recorded: GLEIFGoldenCopyReceipt | null, published: GLEIFPublishedFile, path: PathBuilderLike, verifyDigest: boolean): Promise<boolean>;
+export interface DownloadGLEIFOptions {
+    /**
+     * Where the archive and the receipt are written.
+     * The directory itself is the adapter's `inputPath`.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Re-read the archive's sha256 on a re-run instead of comparing its byte count.
+     */
+    verifyDigest?: boolean;
+    /**
+     * Download even where the receipt already records the current publish.
+     */
+    force?: boolean;
+    retryDelayMs?: number;
+    report?: (line: string) => void;
+}
+export type FetchGLEIFOptions = BaseFetchOptions & Omit<DownloadGLEIFOptions, "outputDir" | "report">;
+/**
+ * Download the current Level 1 golden copy into `options.outputDir` and write its receipt.
+ *
+ * Re-runnable: a receipt naming the current publish, with the archive still on disk
+ * at the recorded length, makes no request for the body.
+ * Archives of earlier publishes are removed once the new one is verified,
+ * so the directory holds one publish.
+ */
+export declare function downloadGLEIF(client: Pick<APIClient, "fetch">, options: DownloadGLEIFOptions): Promise<FetchSummary>;
+/**
+ * Download the current golden copy into `<outRoot>/gleif/`.
+ *
+ * The registry entry point.
+ * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself.
+ */
+export declare function fetchGLEIF(options: FetchGLEIFOptions, report?: (line: string) => void): Promise<FetchSummary>;
+//# sourceMappingURL=gleif.d.ts.map

package/out/tools/fetch/gleif.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gleif.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/gleif.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAM/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAU3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAG3E;;GAEG;AACH,eAAO,MAAM,wBAAwB,uEAAuE,CAAA;AAO5G;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB,EAAE,MAAM,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;CACnB;AAgBD;;;;;GAKG;AACH,wBAAsB,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA4B1G;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAQxD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAEpF;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,CAC1C,QAAQ,EAAE,sBAAsB,GAAG,IAAI,EACvC,SAAS,EAAE,kBAAkB,EAC7B,IAAI,EAAE,eAAe,EACrB,YAAY,EAAE,OAAO,GACnB,OAAO,CAAC,OAAO,CAAC,CAYlB;AAED,MAAM,WAAW,oBAAoB;IACpC;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;OAEG;IACH,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,GAAG,QAAQ,CAAC,CAAA;AAErG;;;;;;;GAOG;AACH,wBAAsB,aAAa,CAClC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,oBAAoB,GAC3B,OAAO,CAAC,YAAY,CAAC,CA6GvB;AAED;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,CAWnH"}

package/out/tools/fetch/gleif.js

@@ -0,0 +1,220 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the GLEIF Level 1 golden copy for the `gleif-lei` adapter.
+ *
+ * GLEIF publishes the whole LEI-CDF record set three times a day as one zipped CSV. The publish API at
+ * {@linkcode GLEIF_LATEST_PUBLISH_URL} names the current file, its byte length, its record count and the
+ * CDF version it follows, so this module reads that answer first and downloads the file it names.
+ *
+ * Three measured properties of the host decide what this module does. Measured on 2026-10-03 against the
+ * `2026-10-03 08:00:00` publish:
+ *
+ * 1. **The API's `size` is the archive's length.** It stated 507,006,221 and the host answered a
+ *    `HEAD` with `content-length` 507,006,221 and a one-byte range with `Content-Range: bytes
+ *    0-9/507006221`. The transfer is checked against the API's figure, so a body cut short or a
+ *    publish replaced mid-transfer is refused rather than recorded.
+ * 2. **The host honors `Range`.** An interrupted transfer resumes through
+ *    {@linkcode resumableDownload}. The request names `accept-encoding: identity`, because the host
+ *    compresses small responses on the fly and a range must count the archive's own bytes.
+ * 3. **Each publish has its own file name**, date-prefixed as
+ *    `20261003-0800-gleif-goldencopy-lei2-golden-copy.csv.zip`. The archive is kept under that
+ *    name, so a partial transfer of one publish never resumes into another, and the adapter reads the
+ *    newest name in the directory.
+ *
+ * The archive is kept zipped. Its one CSV member decompresses to several gigabytes, and the adapter
+ * streams the member out of the archive rather than reading an extracted copy.
+ *
+ * The receipt written beside the archive records the URL, the publish date, the byte count and the
+ * sha256 of the archive, which identify the bytes a corpus row was read from.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { listZipEntries } from "@mailwoman/core/fs/zip";
+import { sha256File } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { GLEIF_ADAPTER_ID, GLEIF_ARCHIVE_PATTERN, GLEIF_ATTRIBUTION, GLEIF_LICENSE, GLEIF_LICENSE_URL, } from "#adapters/gleif/adapter";
+import { readManifest, resumableDownload, writeManifest } from "#tools/fetch/download";
+/**
+ * The publish API's answer for the newest golden copy and its delta files.
+ */
+export const GLEIF_LATEST_PUBLISH_URL = "https://goldencopy.gleif.org/api/v2/golden-copies/publishes/latest";
+/**
+ * The directory the archive and its receipt are written under, which is the adapter's `inputPath`.
+ */
+const SLUG = "gleif";
+/**
+ * Read which file the publish API names as the current Level 1 golden copy in CSV.
+ *
+ * @throws When the answer omits any of the five fields, naming the one it lacks,
+ * rather than downloading a file whose length or date cannot be checked.
+ */
+export async function readGLEIFLatestPublish(client) {
+    const response = await client.fetch({ method: "GET", url: GLEIF_LATEST_PUBLISH_URL });
+    const lei2 = response.data?.data?.lei2;
+    const csv = lei2?.full_file?.csv;
+    const fields = {
+        "data.lei2.publish_date": lei2?.publish_date,
+        "data.lei2.full_file.csv.url": csv?.url,
+        "data.lei2.full_file.csv.size": csv?.size,
+        "data.lei2.full_file.csv.record_count": csv?.record_count,
+        "data.lei2.full_file.csv.cdf_version": csv?.cdf_version,
+    };
+    const missing = Object.entries(fields)
+        .filter(([, value]) => value === undefined || value === null || value === "")
+        .map(([name]) => name);
+    if (missing.length) {
+        throw new Error(`${GLEIF_LATEST_PUBLISH_URL} answered without ${missing.join(", ")}`);
+    }
+    return {
+        publishDate: lei2.publish_date,
+        url: csv.url,
+        bytes: Number(csv.size),
+        recordCount: Number(csv.record_count),
+        cdfVersion: csv.cdf_version,
+    };
+}
+/**
+ * The archive's file name, which is the last path segment of the URL the API names.
+ */
+export function gleifArchiveFilename(url) {
+    const name = new URL(url).pathname.split("/").at(-1) ?? "";
+    if (!GLEIF_ARCHIVE_PATTERN.test(name)) {
+        throw new Error(`${url} does not name a golden-copy archive matching ${GLEIF_ARCHIVE_PATTERN}`);
+    }
+    return name;
+}
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export function gleifInputPath(outRoot) {
+    return outRoot(SLUG);
+}
+/**
+ * Whether the archive on disk is the publish the API names and the length the receipt records.
+ *
+ * Exported so a test can exercise the decision without a transfer.
+ */
+export async function isGLEIFArchiveCurrent(recorded, published, path, verifyDigest) {
+    if (!recorded)
+        return false;
+    if (recorded.source_url !== published.url || recorded.publish_date !== published.publishDate)
+        return false;
+    const stat = await tryStat(path);
+    if (!stat || stat.size !== recorded.bytes)
+        return false;
+    if (!verifyDigest)
+        return true;
+    return (await sha256File(path)) === recorded.sha256;
+}
+/**
+ * Download the current Level 1 golden copy into `options.outputDir` and write its receipt.
+ *
+ * Re-runnable: a receipt naming the current publish, with the archive still on disk
+ * at the recorded length, makes no request for the body.
+ * Archives of earlier publishes are removed once the new one is verified,
+ * so the directory holds one publish.
+ */
+export async function downloadGLEIF(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const summary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] };
+    await makeDirectories(destDir);
+    report?.(`=== ${SLUG}`);
+    const published = await readGLEIFLatestPublish(client);
+    const filename = gleifArchiveFilename(published.url);
+    const archivePath = destDir(filename);
+    const receiptPath = destDir("MANIFEST.json");
+    report?.(`  publish ${published.publishDate}: ${published.recordCount.toLocaleString()} records, ` +
+        `${ByteFormatter.formatIEC(published.bytes)}, ${published.cdfVersion}`);
+    const recorded = await readManifest(receiptPath);
+    if (!options.force &&
+        (await isGLEIFArchiveCurrent(recorded, published, archivePath, options.verifyDigest ?? false))) {
+        report?.(`  ✓ ${filename} already current — no download.`);
+        summary.skipped++;
+        return summary;
+    }
+    let bytes;
+    try {
+        bytes = await resumableDownload({
+            url: published.url,
+            dest: archivePath,
+            headers: { "accept-encoding": "identity", accept: "application/zip" },
+            retryDelayMs: options.retryDelayMs,
+            report,
+        });
+    }
+    catch (error) {
+        report?.(`  ✗ download failed: ${error instanceof Error ? error.message : String(error)}`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    if (bytes !== published.bytes) {
+        await removePathIfPresent(archivePath);
+        report?.(`  ✗ the API stated ${published.bytes} bytes and ${bytes} arrived, so the archive is not recorded`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    const members = (await listZipEntries(archivePath)).filter((entry) => entry.name.toLowerCase().endsWith(".csv"));
+    if (members.length !== 1) {
+        report?.(`  ✗ ${filename} holds ${members.length} CSV members, where the golden copy publishes one`);
+        summary.failed++;
+        summary.failedCodes.push(filename);
+        return summary;
+    }
+    const receipt = {
+        source: GLEIF_ADAPTER_ID,
+        source_url: published.url,
+        api_url: GLEIF_LATEST_PUBLISH_URL,
+        publish_date: published.publishDate,
+        cdf_version: published.cdfVersion,
+        record_count: published.recordCount,
+        downloaded_at: new Date().toISOString(),
+        filename,
+        bytes,
+        sha256: await sha256File(archivePath),
+        csv_member: members[0].name,
+        csv_member_bytes: members[0].uncompressedSize,
+        license: GLEIF_LICENSE,
+        license_url: GLEIF_LICENSE_URL,
+        attribution: GLEIF_ATTRIBUTION,
+    };
+    await writeManifest(receiptPath, receipt);
+    // The adapter reads the newest archive in the directory, so an earlier publish
+    // is removed only after this one is verified and recorded.
+    const earlier = await Globerator.from("*.csv.zip", { cwd: destDir.toString(), absolute: false }).toArray();
+    for (const name of earlier) {
+        if (name !== filename && GLEIF_ARCHIVE_PATTERN.test(name)) {
+            await removePathIfPresent(destDir(name));
+        }
+    }
+    report?.(`  ✓ ${filename}: ${ByteFormatter.formatIEC(bytes)}, sha256 ${receipt.sha256}`);
+    report?.(`  MANIFEST written to ${receiptPath.toString()}`);
+    summary.fetched++;
+    return summary;
+}
+/**
+ * Download the current golden copy into `<outRoot>/gleif/`.
+ *
+ * The registry entry point.
+ * The adapter is pointed at {@linkcode gleifInputPath}, the directory itself.
+ */
+export async function fetchGLEIF(options, report) {
+    await using client = new APIClient({ displayName: SLUG, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits.
+    return await downloadGLEIF(client, {
+        outputDir: options.outRoot(SLUG),
+        verifyDigest: options.verifyDigest,
+        force: options.force,
+        retryDelayMs: options.retryDelayMs,
+        report,
+    });
+}
+//# sourceMappingURL=gleif.js.map

package/out/tools/fetch/gleif.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gleif.js","sourceRoot":"","sources":["../../../lib/tools/fetch/gleif.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AACvD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EACN,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,EACb,iBAAiB,GACjB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAEtF;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,oEAAoE,CAAA;AAE5G;;GAEG;AACH,MAAM,IAAI,GAAG,OAAO,CAAA;AA4DpB;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,MAAgC;IAC5E,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,wBAAwB,EAAE,CAAC,CAAA;IACtG,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAA;IACtC,MAAM,GAAG,GAAG,IAAI,EAAE,SAAS,EAAE,GAAG,CAAA;IAEhC,MAAM,MAAM,GAAG;QACd,wBAAwB,EAAE,IAAI,EAAE,YAAY;QAC5C,6BAA6B,EAAE,GAAG,EAAE,GAAG;QACvC,8BAA8B,EAAE,GAAG,EAAE,IAAI;QACzC,sCAAsC,EAAE,GAAG,EAAE,YAAY;QACzD,qCAAqC,EAAE,GAAG,EAAE,WAAW;KACvD,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;SAC5E,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;IAEvB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,GAAG,wBAAwB,qBAAqB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,OAAO;QACN,WAAW,EAAE,IAAK,CAAC,YAAa;QAChC,GAAG,EAAE,GAAI,CAAC,GAAI;QACd,KAAK,EAAE,MAAM,CAAC,GAAI,CAAC,IAAI,CAAC;QACxB,WAAW,EAAE,MAAM,CAAC,GAAI,CAAC,YAAY,CAAC;QACtC,UAAU,EAAE,GAAI,CAAC,WAAY;KAC7B,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAW;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAE1D,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,GAAG,iDAAiD,qBAAqB,EAAE,CAAC,CAAA;IAChG,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAoC;IAClE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,QAAuC,EACvC,SAA6B,EAC7B,IAAqB,EACrB,YAAqB;IAErB,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAE3B,IAAI,QAAQ,CAAC,UAAU,KAAK,SAAS,CAAC,GAAG,IAAI,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,WAAW;QAAE,OAAO,KAAK,CAAA;IAE1G,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IAEhC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK;QAAE,OAAO,KAAK,CAAA;IAEvD,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAE9B,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAA;AACpD,CAAC;AAsBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC,EAChC,OAA6B;IAE7B,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,OAAO,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAEpF,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACpD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAE5C,MAAM,EAAE,CACP,aAAa,SAAS,CAAC,WAAW,KAAK,SAAS,CAAC,WAAW,CAAC,cAAc,EAAE,YAAY;QACxF,GAAG,aAAa,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,UAAU,EAAE,CACvE,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAyB,WAAW,CAAC,CAAA;IAExE,IACC,CAAC,OAAO,CAAC,KAAK;QACd,CAAC,MAAM,qBAAqB,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,CAAC,EAC7F,CAAC;QACF,MAAM,EAAE,CAAC,OAAO,QAAQ,iCAAiC,CAAC,CAAA;QAE1D,OAAO,CAAC,OAAO,EAAE,CAAA;QAEjB,OAAO,OAAO,CAAA;IACf,CAAC;IAED,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,iBAAiB,CAAC;YAC/B,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,EAAE,iBAAiB,EAAE;YACrE,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAE1F,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAEtC,MAAM,EAAE,CAAC,sBAAsB,SAAS,CAAC,KAAK,cAAc,KAAK,0CAA0C,CAAC,CAAA;QAE5G,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,cAAc,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;IAEhH,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,EAAE,CAAC,OAAO,QAAQ,UAAU,OAAO,CAAC,MAAM,mDAAmD,CAAC,CAAA;QAEpG,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAElC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,OAAO,GAA2B;QACvC,MAAM,EAAE,gBAAgB;QACxB,UAAU,EAAE,SAAS,CAAC,GAAG;QACzB,OAAO,EAAE,wBAAwB;QACjC,YAAY,EAAE,SAAS,CAAC,WAAW;QACnC,WAAW,EAAE,SAAS,CAAC,UAAU;QACjC,YAAY,EAAE,SAAS,CAAC,WAAW;QACnC,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ;QACR,KAAK;QACL,MAAM,EAAE,MAAM,UAAU,CAAC,WAAW,CAAC;QACrC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI;QAC5B,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,gBAAgB;QAC9C,OAAO,EAAE,aAAa;QACtB,WAAW,EAAE,iBAAiB;QAC9B,WAAW,EAAE,iBAAiB;KAC9B,CAAA;IAED,MAAM,aAAa,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAEzC,+EAA+E;IAC/E,2DAA2D;IAC3D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IAE1G,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,mBAAmB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;QACzC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,OAAO,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACxF,MAAM,EAAE,CAAC,yBAAyB,WAAW,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE3D,OAAO,CAAC,OAAO,EAAE,CAAA;IAEjB,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAA0B,EAAE,MAA+B;IAC3F,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtE,yFAAyF;IACzF,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE;QAClC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM;KACN,CAAC,CAAA;AACH,CAAC"}

package/out/tools/fetch/header.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * indexes.
+ *
+ * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * because a renamed column reaches the adapter as an empty string on every row rather than as an
+ * error. The check belongs here rather than in each publisher's module: every caller needs the same
+ * two steps, and each one that wrote them again would also be a second answer to how the file is
+ * parsed.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The header line's column names.
+ *
+ * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
+ * a split of a fetch module's own: which columns a publisher quotes is a property of
+ * the edition, and a second reader would be a second answer to that question.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file,
+ * which the default `"auto"` does even for one row.
+ *
+ * @param path The written file.
+ * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export declare function readDelimitedHeader(path: PathBuilderLike, columnDelimiter?: string): Promise<readonly string[]>;
+/**
+ * Refuse a header that does not name every column a caller reads.
+ *
+ * @param columns The header {@linkcode readDelimitedHeader} returned.
+ * @param required The columns the adapter indexes by name.
+ * @param context What is being checked, which opens the message.
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export declare function assertHeaderColumns(columns: readonly string[], required: readonly string[], context: string): void;
+//# sourceMappingURL=header.d.ts.map

package/out/tools/fetch/header.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"header.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9C;;;;;;;;;;;;;GAaG;AACH,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAgBrH;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CASlH"}

package/out/tools/fetch/header.js

@@ -0,0 +1,59 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Reading a delimited file's header line, and refusing one that stops naming a column an adapter
+ * indexes.
+ *
+ * A fetch module checks the file it just wrote against the columns its adapter reads by name,
+ * because a renamed column reaches the adapter as an empty string on every row rather than as an
+ * error. The check belongs here rather than in each publisher's module: every caller needs the same
+ * two steps, and each one that wrote them again would also be a second answer to how the file is
+ * parsed.
+ */
+import { CSVSpliterator } from "spliterator";
+/**
+ * The header line's column names.
+ *
+ * Read through `CSVSpliterator`, which is the reader the adapters use, rather than through
+ * a split of a fetch module's own: which columns a publisher quotes is a property of
+ * the edition, and a second reader would be a second answer to that question.
+ * `columnScan: "rows"` keeps the first row from decoding the whole file,
+ * which the default `"auto"` does even for one row.
+ *
+ * @param path The written file.
+ * @param columnDelimiter The publisher's delimiter, which defaults to a comma.
+ * @throws When the file holds no row at all, so an empty or truncated file reports itself
+ * rather than reading as a file with no columns.
+ */
+export async function readDelimitedHeader(path, columnDelimiter) {
+    const [header] = await Array.fromAsync(CSVSpliterator.fromAsync(path, {
+        header: false,
+        mode: "array",
+        columnScan: "rows",
+        take: 1,
+        ...(columnDelimiter === undefined ? {} : { columnDelimiter }),
+    }));
+    if (!header) {
+        throw new Error(`${path.toString()}: the file holds no row, so its header could not be read`);
+    }
+    return header;
+}
+/**
+ * Refuse a header that does not name every column a caller reads.
+ *
+ * @param columns The header {@linkcode readDelimitedHeader} returned.
+ * @param required The columns the adapter indexes by name.
+ * @param context What is being checked, which opens the message.
+ * @throws Naming the columns that are absent, so a renamed column is a reported failure
+ * rather than an empty string on every row the adapter emits.
+ */
+export function assertHeaderColumns(columns, required, context) {
+    const present = new Set(columns);
+    const absent = required.filter((column) => !present.has(column));
+    if (absent.length) {
+        throw new Error(`${context}: the header names ${columns.length} columns and not ${absent.join(", ")}, which the adapter reads by name`);
+    }
+}
+//# sourceMappingURL=header.js.map

package/out/tools/fetch/header.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"header.js","sourceRoot":"","sources":["../../../lib/tools/fetch/header.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAqB,EAAE,eAAwB;IACxF,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,SAAS,CACrC,cAAc,CAAC,SAAS,CAAW,IAAI,EAAE;QACxC,MAAM,EAAE,KAAK;QACb,IAAI,EAAE,OAAO;QACb,UAAU,EAAE,MAAM;QAClB,IAAI,EAAE,CAAC;QACP,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC;KAC7D,CAAC,CACF,CAAA;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,0DAA0D,CAAC,CAAA;IAC9F,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAA0B,EAAE,QAA2B,EAAE,OAAe;IAC3G,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;IAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;IAEhE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,sBAAsB,OAAO,CAAC,MAAM,oBAAoB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,mCAAmC,CACtH,CAAA;IACF,CAAC;AACF,CAAC"}