@mailwoman/corpus 10.0.0 → 10.1.0

package/out/fr/tools/fetch/finess.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"finess.d.ts","sourceRoot":"","sources":["../../../../lib/fr/tools/fetch/finess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAM/C,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG3D,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAA4B,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAGrH;;GAEG;AACH,eAAO,MAAM,mBAAmB,oDAAoD,CAAA;AAEpF;;GAEG;AACH,eAAO,MAAM,sBAAsB,6FAAoE,CAAA;AAEvG;;GAEG;AACH,eAAO,MAAM,uBAAuB,sFAA6D,CAAA;AAYjG;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,GAAG,EAAE,MAAM,CAAA;IACX,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,QAAQ,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;CACjD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,SAAS,gBAAgB,EAAE,CAAA;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,cAAc;IAC5D,WAAW,EAAE,MAAM,CAAA;IACnB,sBAAsB,EAAE,MAAM,CAAA;IAC9B;;OAEG;IACH,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B;;OAEG;IACH,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;CACzB;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAMzG;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,eAAe,GAAG,gBAAgB,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAerG;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC,CAQlG;AAWD,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;OAEG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC/B;AAED,MAAM,MAAM,kBAAkB,GAAG,gBAAgB,GAAG,IAAI,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAA;AAExF;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,gBAAgB,CAAC,SAAS,CAAC,GAAG,eAAe,CAErF;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAC1C,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAChC,OAAO,EAAE,qBAAqB,GAC5B,OAAO,CAAC,YAAY,CAAC,CA4HvB;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAUvB"}

package/out/fr/tools/fetch/finess.js

@@ -0,0 +1,228 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Download the FINESS establishment extract for the `fr-finess-overseas` adapter.
+ *
+ * The extract is a resource of the data.gouv.fr dataset
+ * `finess-extraction-du-fichier-des-etablissements`, whose API record at
+ * `https://www.data.gouv.fr/api/1/datasets/finess-extraction-du-fichier-des-etablissements/` lists
+ * every resource with its URL, `last_modified`, `filesize` and a SHA-1 `checksum`. The establishment
+ * extract is the resource whose file is named `etalab-cs1100502-stock-<date>-<time>.csv`; the
+ * geolocated extract `etalab-cs1100507-…` and the history archive sit beside it and are not read.
+ * {@linkcode selectFinessExtract} picks the newest establishment extract the record lists, so the
+ * fetcher follows a republication without a code change.
+ *
+ * The publisher froze the extract at data dated 2026-05-04 and states on the dataset page that it
+ * will not be updated again, so a re-run normally finds the recorded file current and makes no
+ * transfer.
+ *
+ * The transfer is checked against the record: the byte count against `filesize` and the SHA-1
+ * against `checksum`. The record's SHA-1 can describe an earlier upload of the same object. Measured
+ * on 2026-10-03, the record states `d3c2c0e68a413974dffbe0aabf093e3f40414967` and `last_modified`
+ * 2026-05-12, while the object `static.data.gouv.fr` serves has `last-modified` 2026-07-02, SHA-1
+ * `8be4bdc362f3f382e3a7de2a26da6c3a5e6074f3` on two separate transfers, the record's 35,786,629
+ * bytes, and an ETag equal to its MD5. Where the SHA-1 disagrees, the storage host's ETag MD5
+ * decides, and the manifest records all three digests. A file that matches neither is an
+ * interrupted transfer or an error page and is removed rather than recorded. The manifest records the SHA-256 the corpus tooling compares,
+ * the resource's `last_modified`, and the extract's own first line (`finess;etalab;111;2026-05-12`),
+ * which carries the extraction date Licence Ouverte's attribution names.
+ */
+import { APIClient } from "@mailwoman/core/api";
+import { ByteFormatter } from "@mailwoman/core/fs/formatters";
+import { readFileHead, readLocalBuffer, tryStat } from "@mailwoman/core/fs/readers";
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers";
+import { createHash, md5File, sha256File } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { PathBuilder } from "path-ts";
+import { FR_FINESS_ADAPTER_ID, FR_FINESS_ATTRIBUTION, FR_FINESS_LICENSE } from "#fr/adapters/finess/adapter";
+import { downloadToFile, loadCollectionFiles, writeManifest } from "#tools/fetch/download";
+/**
+ * The dataset's slug on data.gouv.fr.
+ */
+export const FINESS_DATASET_SLUG = "finess-extraction-du-fichier-des-etablissements";
+/**
+ * The dataset's API record, which lists its resources.
+ */
+export const FINESS_DATASET_API_URL = `https://www.data.gouv.fr/api/1/datasets/${FINESS_DATASET_SLUG}/`;
+/**
+ * The dataset's page, which states the license.
+ */
+export const FINESS_DATASET_PAGE_URL = `https://www.data.gouv.fr/datasets/${FINESS_DATASET_SLUG}`;
+/**
+ * The file name of an establishment extract, as the resource URL ends.
+ */
+const ESTABLISHMENT_EXTRACT = /\/(etalab-cs1100502-stock-\d{8}-\d{4}\.csv)$/u;
+/**
+ * The opening of the extract's comment line, which an HTML error page does not have.
+ */
+const EXTRACT_HEADER = "finess;";
+/**
+ * The MD5 a storage host's ETag states for an object, or `null` when it sends no ETag of that shape.
+ *
+ * `static.data.gouv.fr` answers `etag: W/"<md5 hex>"`.
+ */
+export async function readServedMD5(client, url) {
+    const response = await client.fetch({ method: "head", url });
+    const headers = response.headers;
+    const match = /^(?:W\/)?"([\da-f]{32})"$/iu.exec(headers?.["etag"] ?? "");
+    return match ? match[1].toLowerCase() : null;
+}
+/**
+ * The newest establishment extract a dataset record lists.
+ *
+ * @throws When the record lists none, so a renamed resource reports itself rather than reading as no data.
+ */
+export function selectFinessExtract(dataset) {
+    const extracts = dataset.resources
+        .map((resource) => ({ resource, match: ESTABLISHMENT_EXTRACT.exec(resource.url) }))
+        .filter((entry) => entry.match !== null)
+        .toSorted((left, right) => left.resource.last_modified.localeCompare(right.resource.last_modified));
+    const newest = extracts.at(-1);
+    if (!newest) {
+        throw new Error(`${FR_FINESS_ADAPTER_ID}: ${FINESS_DATASET_API_URL} lists no resource named etalab-cs1100502-stock-<date>.csv`);
+    }
+    return { ...newest.resource, filename: newest.match[1] };
+}
+/**
+ * Read the dataset's API record.
+ */
+export async function readFinessDataset(client) {
+    const { data } = await client.fetch({ url: FINESS_DATASET_API_URL });
+    if (typeof data !== "object" || data === null || !Array.isArray(data.resources)) {
+        throw new Error(`${FR_FINESS_ADAPTER_ID}: ${FINESS_DATASET_API_URL} did not answer a dataset record`);
+    }
+    return data;
+}
+/**
+ * The SHA-1 of a file, which is the digest data.gouv.fr publishes for a resource.
+ */
+async function sha1File(path) {
+    return createHash("sha1")
+        .update(await readLocalBuffer(path))
+        .digest("hex");
+}
+/**
+ * The directory `#fr/adapters/finess/adapter` reads under a fetch root.
+ */
+export function finessInputPath(outRoot) {
+    return outRoot(FR_FINESS_ADAPTER_ID);
+}
+/**
+ * Download the newest establishment extract into `options.outputDir` and record it in the manifest.
+ *
+ * Re-runnable: the transfer is skipped when the manifest records the same resource URL
+ * and `last_modified` and the file is on disk at the recorded length.
+ */
+export async function downloadFinessExtract(client, options) {
+    const { report } = options;
+    const destDir = PathBuilder.from(options.outputDir);
+    const summary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] };
+    await makeDirectories(destDir);
+    report?.(`=== ${FR_FINESS_ADAPTER_ID}`);
+    const dataset = await readFinessDataset(client);
+    const resource = selectFinessExtract(dataset);
+    const manifestPath = destDir("MANIFEST.json");
+    const recorded = (await loadCollectionFiles(manifestPath));
+    const previous = recorded.get(resource.filename);
+    const dest = destDir(resource.filename);
+    report?.(`  ${resource.filename}: last_modified ${resource.last_modified}, filesize ${resource.filesize ?? "absent"}`);
+    if (!options.force &&
+        previous?.source_url === resource.url &&
+        previous.resource_last_modified === resource.last_modified &&
+        (await tryStat(dest))?.size === previous.bytes) {
+        report?.(`  ✓ ${resource.filename} already current — no download.`);
+        summary.skipped++;
+        return summary;
+    }
+    const fail = async (reason) => {
+        report?.(`  ✗ ${resource.filename}: ${reason}`);
+        await removePathIfPresent(dest);
+        summary.failed++;
+        summary.failedCodes.push(resource.filename);
+        return summary;
+    };
+    let bytes;
+    try {
+        const written = await downloadToFile({
+            url: resource.url,
+            dest,
+            retries: 3,
+            retryDelayMs: options.retryDelayMs,
+            report,
+        });
+        bytes = written.bytes;
+    }
+    catch (error) {
+        return await fail(`download failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (typeof resource.filesize === "number" && resource.filesize !== bytes) {
+        return await fail(`the record states ${resource.filesize} bytes and ${bytes} arrived`);
+    }
+    const sha1 = await sha1File(dest);
+    let servedMD5 = null;
+    if (resource.checksum?.type === "sha1" && sha1 !== resource.checksum.value) {
+        // The record's digest can describe an earlier upload of the same object.
+        // The storage host's ETag is the MD5 of the bytes it serves now, so it decides.
+        servedMD5 = await readServedMD5(client, resource.url);
+        const md5 = await md5File(dest);
+        if (servedMD5 !== md5) {
+            return await fail(`SHA-1 ${sha1} differs from the record's ${resource.checksum.value}, ` +
+                `and MD5 ${md5} differs from the served ETag ${servedMD5 ?? "(absent)"}`);
+        }
+        report?.(`  ! ${resource.filename}: the record's SHA-1 ${resource.checksum.value} describes other bytes; ` +
+            `the transfer matches the served ETag MD5 ${servedMD5} and the record's filesize`);
+    }
+    const head = await readFileHead(dest, 256);
+    const header = head.split(/\r?\n/u)[0] ?? "";
+    if (!header.startsWith(EXTRACT_HEADER)) {
+        return await fail(`the file opens with ${stringifyJSON(header.slice(0, 40))}, not the extract's comment line`);
+    }
+    const entry = {
+        source_url: resource.url,
+        downloaded_at: new Date().toISOString(),
+        filename: resource.filename,
+        sha256: await sha256File(dest),
+        bytes,
+        resource_id: resource.id,
+        resource_last_modified: resource.last_modified,
+        dataset_license: dataset.license ?? null,
+        extract_header: header,
+        sha1,
+        record_sha1: resource.checksum?.type === "sha1" ? resource.checksum.value : null,
+        served_md5: servedMD5,
+    };
+    recorded.set(resource.filename, entry);
+    const manifest = {
+        source: FR_FINESS_ADAPTER_ID,
+        source_url: FINESS_DATASET_PAGE_URL,
+        license: FR_FINESS_LICENSE,
+        attribution: FR_FINESS_ATTRIBUTION,
+        downloaded_at: entry.downloaded_at,
+        files: [...recorded.values()].toSorted((left, right) => left.filename.localeCompare(right.filename)),
+    };
+    await writeManifest(manifestPath, manifest);
+    report?.(`  ✓ ${resource.filename}: ${ByteFormatter.formatIEC(bytes)}, sha256 ${entry.sha256}`);
+    report?.(`  MANIFEST written to ${manifestPath.toString()}`);
+    summary.fetched++;
+    return summary;
+}
+/**
+ * Download the FINESS establishment extract into `<outRoot>/fr-finess-overseas/`.
+ *
+ * The registry entry point.
+ * `#fr/adapters/finess/adapter` is pointed at {@linkcode finessInputPath},
+ * and reads the newest extract in it.
+ */
+export async function fetchFinessOverseas(options, report) {
+    await using client = new APIClient({ displayName: FR_FINESS_ADAPTER_ID, retry: true });
+    // Awaited rather than returned: `await using` disposes the client when this scope exits.
+    return await downloadFinessExtract(client, {
+        outputDir: finessInputPath(options.outRoot),
+        force: options.force,
+        retryDelayMs: options.retryDelayMs,
+        report,
+    });
+}
+//# sourceMappingURL=finess.js.map

package/out/fr/tools/fetch/finess.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"finess.js","sourceRoot":"","sources":["../../../../lib/fr/tools/fetch/finess.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACnF,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;AAE5G,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAE1F;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,iDAAiD,CAAA;AAEpF;;GAEG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,2CAA2C,mBAAmB,GAAG,CAAA;AAEvG;;GAEG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,qCAAqC,mBAAmB,EAAE,CAAA;AAEjG;;GAEG;AACH,MAAM,qBAAqB,GAAG,+CAA+C,CAAA;AAE7E;;GAEG;AACH,MAAM,cAAc,GAAG,SAAS,CAAA;AAkDhC;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAgC,EAAE,GAAW;IAChF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAA;IACrE,MAAM,OAAO,GAAG,QAAQ,CAAC,OAA6C,CAAA;IACtE,MAAM,KAAK,GAAG,6BAA6B,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAEzE,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAwB;IAC3D,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS;SAChC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAClF,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC;SACvC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAA;IAEpG,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAE9B,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CACd,GAAG,oBAAoB,KAAK,sBAAsB,4DAA4D,CAC9G,CAAA;IACF,CAAC;IAED,OAAO,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAM,CAAC,CAAC,CAAE,EAAE,CAAA;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAgC;IACvE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,KAAK,CAA2B,EAAE,GAAG,EAAE,sBAAsB,EAAE,CAAC,CAAA;IAE9F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,GAAG,oBAAoB,KAAK,sBAAsB,kCAAkC,CAAC,CAAA;IACtG,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,QAAQ,CAAC,IAAqB;IAC5C,OAAO,UAAU,CAAC,MAAM,CAAC;SACvB,MAAM,CAAC,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;SACnC,MAAM,CAAC,KAAK,CAAC,CAAA;AAChB,CAAC;AAiBD;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAAoC;IACnE,OAAO,OAAO,CAAC,oBAAoB,CAAC,CAAA;AACrC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAC1C,MAAgC,EAChC,OAA8B;IAE9B,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,OAAO,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAEpF,MAAM,eAAe,CAAC,OAAO,CAAC,CAAA;IAE9B,MAAM,EAAE,CAAC,OAAO,oBAAoB,EAAE,CAAC,CAAA;IAEvC,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;IAC7C,MAAM,QAAQ,GAAG,CAAC,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAuC,CAAA;IAChG,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAEvC,MAAM,EAAE,CAAC,KAAK,QAAQ,CAAC,QAAQ,mBAAmB,QAAQ,CAAC,aAAa,cAAc,QAAQ,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC,CAAA;IAEtH,IACC,CAAC,OAAO,CAAC,KAAK;QACd,QAAQ,EAAE,UAAU,KAAK,QAAQ,CAAC,GAAG;QACrC,QAAQ,CAAC,sBAAsB,KAAK,QAAQ,CAAC,aAAa;QAC1D,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,KAAK,QAAQ,CAAC,KAAK,EAC7C,CAAC;QACF,MAAM,EAAE,CAAC,OAAO,QAAQ,CAAC,QAAQ,iCAAiC,CAAC,CAAA;QAEnE,OAAO,CAAC,OAAO,EAAE,CAAA;QAEjB,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,EAAE,MAAc,EAAyB,EAAE;QAC5D,MAAM,EAAE,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAA;QAC/C,MAAM,mBAAmB,CAAC,IAAI,CAAC,CAAA;QAE/B,OAAO,CAAC,MAAM,EAAE,CAAA;QAChB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAE3C,OAAO,OAAO,CAAA;IACf,CAAC,CAAA;IAED,IAAI,KAAa,CAAA;IAEjB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACpC,GAAG,EAAE,QAAQ,CAAC,GAAG;YACjB,IAAI;YACJ,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM;SACN,CAAC,CAAA;QAEF,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,OAAO,MAAM,IAAI,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAChG,CAAC;IAED,IAAI,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;QAC1E,OAAO,MAAM,IAAI,CAAC,qBAAqB,QAAQ,CAAC,QAAQ,cAAc,KAAK,UAAU,CAAC,CAAA;IACvF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;IACjC,IAAI,SAAS,GAAkB,IAAI,CAAA;IAEnC,IAAI,QAAQ,CAAC,QAAQ,EAAE,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC5E,yEAAyE;QACzE,gFAAgF;QAChF,SAAS,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAA;QACrD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;QAE/B,IAAI,SAAS,KAAK,GAAG,EAAE,CAAC;YACvB,OAAO,MAAM,IAAI,CAChB,SAAS,IAAI,8BAA8B,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI;gBACrE,WAAW,GAAG,iCAAiC,SAAS,IAAI,UAAU,EAAE,CACzE,CAAA;QACF,CAAC;QAED,MAAM,EAAE,CACP,OAAO,QAAQ,CAAC,QAAQ,wBAAwB,QAAQ,CAAC,QAAQ,CAAC,KAAK,0BAA0B;YAChG,4CAA4C,SAAS,4BAA4B,CAClF,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAE5C,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QACxC,OAAO,MAAM,IAAI,CAAC,uBAAuB,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,kCAAkC,CAAC,CAAA;IAC/G,CAAC;IAED,MAAM,KAAK,GAA0B;QACpC,UAAU,EAAE,QAAQ,CAAC,GAAG;QACxB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC;QAC9B,KAAK;QACL,WAAW,EAAE,QAAQ,CAAC,EAAE;QACxB,sBAAsB,EAAE,QAAQ,CAAC,aAAa;QAC9C,eAAe,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;QACxC,cAAc,EAAE,MAAM;QACtB,IAAI;QACJ,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAChF,UAAU,EAAE,SAAS;KACrB,CAAA;IAED,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAA6B;QAC1C,MAAM,EAAE,oBAAoB;QAC5B,UAAU,EAAE,uBAAuB;QACnC,OAAO,EAAE,iBAAiB;QAC1B,WAAW,EAAE,qBAAqB;QAClC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,KAAK,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;KACpG,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC/F,MAAM,EAAE,CAAC,yBAAyB,YAAY,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE5D,OAAO,CAAC,OAAO,EAAE,CAAA;IAEjB,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAA2B,EAC3B,MAA+B;IAE/B,YAAY,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,WAAW,EAAE,oBAAoB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtF,yFAAyF;IACzF,OAAO,MAAM,qBAAqB,CAAC,MAAM,EAAE;QAC1C,SAAS,EAAE,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC;QAC3C,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM;KACN,CAAC,CAAA;AACH,CAAC"}

package/out/gauntlet-inputs.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Every surface that appears as an input on a gauntlet board, so a recipe can refuse to train on one.
+ *
+ *   A recipe's own reserve list holds the strings its author knew about. The boards are a separate instrument with a
+ *   separate history — `cz/bare-postcode.jsonl` was authored months before the recipe that trains on Czech postcodes —
+ *   so neither register knows the other exists. A board row that reaches the corpus measures recall of one string
+ *   instead of a capability.
+ *
+ *   read from disk rather than imported. `mailwoman` depends on `@mailwoman/corpus`, so this package cannot import the
+ *   gauntlet loader without a cycle. A recipe is a build tool reading the repository it is built in.
+ *   This is a file read rather than a dependency. The shipped package never calls this.
+ */
+import { type PathBuilderLike } from "path-ts";
+/**
+ * The committed board corpus, as a path rather than a module.
+ */
+export declare const GAUNTLET_CASES_DIR: PathBuilderLike;
+/**
+ * The comparison surface: case-folded, every run of whitespace removed.
+ *
+ * Whitespace-insensitive because a postcode's board spelling and its corpus spelling
+ * differ by exactly that — `100 00` against `10000` — and a check that missed the
+ * pair would report a clean build over a leaked row.
+ */
+export declare function normalizeGauntletSurface(surface: string): string;
+/**
+ * Every board row's `input`, normalized.
+ *
+ * Reads the whole corpus once.
+ * Callers retain the output.
+ *
+ * A row that does not parse is skipped.
+ * The gauntlet loader validates the corpus.
+ *
+ * A recipe that refused to build over a malformed board row would stop a build
+ * for a check that is advisory to it.
+ */
+export declare function readGauntletInputs(dir?: PathBuilderLike): Promise<ReadonlySet<string>>;
+//# sourceMappingURL=gauntlet-inputs.d.ts.map

package/out/gauntlet-inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gauntlet-inputs.d.ts","sourceRoot":"","sources":["../lib/gauntlet-inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAI3D;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,eAOhC,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEhE;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,kBAAkB,CAAC,GAAG,GAAE,eAAoC,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAmChH"}

package/out/gauntlet-inputs.js

@@ -0,0 +1,78 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Every surface that appears as an input on a gauntlet board, so a recipe can refuse to train on one.
+ *
+ *   A recipe's own reserve list holds the strings its author knew about. The boards are a separate instrument with a
+ *   separate history — `cz/bare-postcode.jsonl` was authored months before the recipe that trains on Czech postcodes —
+ *   so neither register knows the other exists. A board row that reaches the corpus measures recall of one string
+ *   instead of a capability.
+ *
+ *   read from disk rather than imported. `mailwoman` depends on `@mailwoman/corpus`, so this package cannot import the
+ *   gauntlet loader without a cycle. A recipe is a build tool reading the repository it is built in.
+ *   This is a file read rather than a dependency. The shipped package never calls this.
+ */
+import { repoRootPath } from "@mailwoman/core/paths";
+import { PathBuilder } from "path-ts";
+import { JSONSpliterator } from "spliterator";
+import { Globerator } from "spliterator/node/fs";
+/**
+ * The committed board corpus, as a path rather than a module.
+ */
+export const GAUNTLET_CASES_DIR = repoRootPath("packages", "mailwoman", "tools", "eval-harness", "gauntlet", "cases");
+/**
+ * The comparison surface: case-folded, every run of whitespace removed.
+ *
+ * Whitespace-insensitive because a postcode's board spelling and its corpus spelling
+ * differ by exactly that — `100 00` against `10000` — and a check that missed the
+ * pair would report a clean build over a leaked row.
+ */
+export function normalizeGauntletSurface(surface) {
+    return surface.trim().toUpperCase().replaceAll(/\s+/gu, "");
+}
+/**
+ * Every board row's `input`, normalized.
+ *
+ * Reads the whole corpus once.
+ * Callers retain the output.
+ *
+ * A row that does not parse is skipped.
+ * The gauntlet loader validates the corpus.
+ *
+ * A recipe that refused to build over a malformed board row would stop a build
+ * for a check that is advisory to it.
+ */
+export async function readGauntletInputs(dir = GAUNTLET_CASES_DIR) {
+    const inputs = new Set();
+    // Each immediate subdirectory is read independently so `generalization/` parked
+    // passes remain board inputs a recipe must not train on.
+    const entries = await Globerator.from("*", { cwd: dir, withFileTypes: true, onlyFiles: false }).toArray();
+    const directories = entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+    const root = PathBuilder.from(dir);
+    for (const directory of [".", ...directories]) {
+        const here = directory === "." ? root : root(directory);
+        for (const file of await Globerator.files("jsonl", {
+            cwd: here,
+            absolute: false,
+            recursive: false,
+        }).toArray()) {
+            try {
+                for await (const row of JSONSpliterator.fromAsync(here(file))) {
+                    if (typeof row.input === "string" && row.input.trim()) {
+                        inputs.add(normalizeGauntletSurface(row.input));
+                    }
+                }
+            }
+            catch {
+                // A file this cannot read or parse is skipped.
+                // The gauntlet loader is what validates the board corpus and reports the file and line.
+                // A recipe stopping its build over one malformed row would turn an
+                // advisory check into a blocked build.
+                continue;
+            }
+        }
+    }
+    return inputs;
+}
+//# sourceMappingURL=gauntlet-inputs.js.map

package/out/gauntlet-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gauntlet-inputs.js","sourceRoot":"","sources":["../lib/gauntlet-inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAoB,YAAY,CAC9D,UAAU,EACV,WAAW,EACX,OAAO,EACP,cAAc,EACd,UAAU,EACV,OAAO,CACP,CAAA;AAED;;;;;;GAMG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAe;IACvD,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAA;AAC5D,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,GAAG,GAAoB,kBAAkB;IACjF,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAEhC,gFAAgF;IAChF,yDAAyD;IACzD,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IACzG,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAE7F,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAElC,KAAK,MAAM,SAAS,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAEvD,KAAK,MAAM,IAAI,IAAI,MAAM,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE;YAClD,GAAG,EAAE,IAAI;YACT,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,KAAK;SAChB,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YACd,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAsB,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;oBACpF,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;wBACvD,MAAM,CAAC,GAAG,CAAC,wBAAwB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAA;oBAChD,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,MAAM,CAAC;gBACR,+CAA+C;gBAC/C,wFAAwF;gBACxF,mEAAmE;gBACnE,uCAAuC;gBACvC,SAAQ;YACT,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC"}

package/out/index.d.ts

@@ -3,11 +3,17 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  */
-export * from "#adapters/utils/index";
+export * from "#adapters/dedup-key";
+export * from "#adapters/errors";
+export * from "#adapters/libpostal-dictionary";
+export * from "#adapters/registry";
+export * from "#adapters/sha256-stream";
+export * from "#adapters/source-id";
+export * from "#adapters/street-line";
 export * from "#au/adapters/gnaf/assemble";
-export * from "#adapters/index";
+export * from "#adapters";
 export * from "#build";
 export * from "#runner";
-export * from "#recipes/index";
+export * from "#recipes";
 export * from "#types";
 //# sourceMappingURL=index.d.ts.map

package/out/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAA;AACrC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,QAAQ,CAAA;AACtB,cAAc,SAAS,CAAA;AACvB,cAAc,gBAAgB,CAAA;AAC9B,cAAc,QAAQ,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAA;AACnC,cAAc,kBAAkB,CAAA;AAChC,cAAc,gCAAgC,CAAA;AAC9C,cAAc,oBAAoB,CAAA;AAClC,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,uBAAuB,CAAA;AACrC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,WAAW,CAAA;AACzB,cAAc,QAAQ,CAAA;AACtB,cAAc,SAAS,CAAA;AACvB,cAAc,UAAU,CAAA;AACxB,cAAc,QAAQ,CAAA"}

package/out/index.js

@@ -3,11 +3,17 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  */
-export * from "#adapters/utils/index";
+export * from "#adapters/dedup-key";
+export * from "#adapters/errors";
+export * from "#adapters/libpostal-dictionary";
+export * from "#adapters/registry";
+export * from "#adapters/sha256-stream";
+export * from "#adapters/source-id";
+export * from "#adapters/street-line";
 export * from "#au/adapters/gnaf/assemble";
-export * from "#adapters/index";
+export * from "#adapters";
 export * from "#build";
 export * from "#runner";
-export * from "#recipes/index";
+export * from "#recipes";
 export * from "#types";
 //# sourceMappingURL=index.js.map

package/out/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,uBAAuB,CAAA;AACrC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,iBAAiB,CAAA;AAC/B,cAAc,QAAQ,CAAA;AACtB,cAAc,SAAS,CAAA;AACvB,cAAc,gBAAgB,CAAA;AAC9B,cAAc,QAAQ,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,qBAAqB,CAAA;AACnC,cAAc,kBAAkB,CAAA;AAChC,cAAc,gCAAgC,CAAA;AAC9C,cAAc,oBAAoB,CAAA;AAClC,cAAc,yBAAyB,CAAA;AACvC,cAAc,qBAAqB,CAAA;AACnC,cAAc,uBAAuB,CAAA;AACrC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,WAAW,CAAA;AACzB,cAAc,QAAQ,CAAA;AACtB,cAAc,SAAS,CAAA;AACvB,cAAc,UAAU,CAAA;AACxB,cAAc,QAAQ,CAAA"}