@mailwoman/corpus 9.4.0 → 10.1.0

package/out/sk/tools/fetch/inspire.js

@@ -0,0 +1,280 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Harvests Slovakia's INSPIRE Addresses (AD) theme into the GML documents that
+ * `#sk/adapters/inspire/adapter` reads.
+ *
+ * The Ministerstvo vnútra Slovenskej republiky publishes the national address register through
+ * GeoServer at {@linkcode SK_INSPIRE_WFS}. Its register record,
+ * `https://rpi.gov.sk/api/collection_record/86dea70e-a55b-4241-bd63-4024b3f76b72`, names a WMS and
+ * this WFS as the theme's only distributions and states one condition of use, `Neuplatňujú sa žiadne
+ * podmienky (CC0 Voľné dielo)`, which the address-source register elected. There is no INSPIRE ATOM
+ * download service for this theme: the feeds under `inspirews.skgeodesy.sk` belong to the
+ * Geodetický a kartografický ústav, which is a different publisher and a different record.
+ *
+ * `#tools/fetch/wfs-harvest` owns the page read, the appending, the manifest and the resumption.
+ * What is particular to Slovakia is that an address is assembled from five feature types rather
+ * than read from one, which is what shapes the layout this writes.
+ *
+ * ## Five types, five documents, one directory
+ *
+ * `ad:Address` carries a locator and references its street, its settlement part, its administrative
+ * units and its postcode through `ad:component`. The adapter resolves those references locally
+ * against an index it builds in a first pass over every document in `inputPath`, so every type has
+ * to be on disk before the first address is composed. Measured 2026-10-03, `resultType=hits` past
+ * index 0 reports 1,704,196 `ad:Address`, 29,315 `ad:ThoroughfareName`, 3,015 `ad:AdminUnitName`,
+ * 1,769 `ad:AddressAreaName` and 1,415 `ad:PostalDescriptor`.
+ *
+ * Each type is harvested into its own directory under the source root, because a harvest's manifest
+ * sits at a fixed name beside its data file and five types sharing one directory would overwrite
+ * each other's manifest. A run would then re-harvest every type, and the 341 address pages are the
+ * expensive part of this acquisition:
+ *
+ *     sk-inspire/Address/Address.gml            + MANIFEST.json
+ *     sk-inspire/ThoroughfareName/…             + MANIFEST.json
+ *     …
+ *
+ * The adapter reads `inputPath` recursively, so that root is what {@linkcode skInspireInputPath}
+ * states. The four component types are harvested before the addresses, so an interrupted run leaves
+ * the small side of the join complete rather than leaving addresses whose component references the
+ * harvest cannot answer.
+ *
+ * ## What the service states about itself
+ *
+ * Measured 2026-10-03: `ImplementsResultPaging` is `TRUE`, `ImplementsSorting` is `TRUE`,
+ * `CountDefault` is 5000, and the advertised `GetFeature` formats include
+ * `application/gml+xml; version=3.2`, which is the one the adapter's markup reader takes. A
+ * 5,000-feature page of `ad:Address` answered in 17 seconds carrying 13,879,456 bytes, so a full
+ * harvest of that type is 341 requests and roughly 4.7 GB.
+ *
+ * `CountDefault` is 5000 and no type reports 5000 features, so `statedFeatureCount` accepts each
+ * type's `numberMatched` as a count rather than refusing it as a page cap. The register records one
+ * response in nine that carried no `numberMatched` attribute at all;
+ * `readCheckedWFSFeatureCount` reads such a response as no usable count, and the harvest then ends
+ * on the first page that returns no features instead of recording a wrong total.
+ */
+import { readCheckedWFSFeatureCount } from "@mailwoman/core/api";
+import { SK_INSPIRE_ADAPTER_ID, SK_INSPIRE_DEFAULT_LICENSE } from "#sk/adapters/inspire/adapter";
+import { readWFSCapabilities } from "#tools/fetch/inspire-addresses";
+import { collectionOpeningTag, harvestPagedWFS, readWFSMarkupPage, rootContent, runWFSHarvest, statedFeatureCount, } from "#tools/fetch/wfs-harvest";
+/**
+ * The service this module harvests.
+ */
+export const SK_INSPIRE_WFS = "https://rageo.minv.sk/geoserver/ad/wfs";
+/**
+ * The output format the harvest asks for, which is the GML the adapter's markup reader takes.
+ */
+export const SK_INSPIRE_OUTPUT_FORMAT = "application/gml+xml; version=3.2";
+/**
+ * The property the pages are ordered by.
+ *
+ * Every AD feature type carries `ad:inspireId`, so one sort serves all five.
+ * Without a sort, a resumed harvest would rest on the service returning the same
+ * features in the same order as the earlier run, which no WFS guarantees.
+ */
+export const SK_INSPIRE_SORT_BY = "ad:inspireId";
+/**
+ * Features per page.
+ *
+ * The service's own `CountDefault`, which is the largest page it will serve.
+ */
+export const SK_INSPIRE_PAGE_SIZE = 5000;
+/**
+ * The feature types harvested, in the order they are taken.
+ *
+ * The four component types come first, each in the thousands, so an interrupted run
+ * leaves the index side of the adapter's join complete.
+ * `Address` is last and is the 1.7-million-feature side.
+ */
+export const SK_INSPIRE_TYPES = [
+    "ThoroughfareName",
+    "AddressAreaName",
+    "AdminUnitName",
+    "PostalDescriptor",
+    "Address",
+];
+/**
+ * The XML declaration each assembled document opens with.
+ *
+ * The service's own pages declare UTF-8, and the harvest is written as UTF-8.
+ */
+const XML_DECLARATION = '<?xml version="1.0" encoding="UTF-8"?>\n';
+/**
+ * The adapter slug this module fetches for, and the directory it writes under.
+ */
+const SLUG = SK_INSPIRE_ADAPTER_ID;
+/**
+ * Names the service in every refusal this module raises.
+ */
+const CONTEXT = "sk-inspire wfs";
+/**
+ * The spacing between requests.
+ *
+ * A 5,000-feature page answered in 17 seconds, so the service's own latency already
+ * spaces a serial harvest further apart than this.
+ * The interval is the ceiling a run is held to rather than the rate it achieves.
+ */
+const MIN_REQUEST_INTERVAL_MS = 1000;
+/**
+ * The data file one type is harvested into, inside that type's own directory.
+ */
+export function skInspireHarvestFile(type) {
+    return `${type}.gml`;
+}
+/**
+ * What the service states about itself, with the claims this harvest depends on checked.
+ *
+ * The qualified type names come from the capabilities document rather than from a
+ * constant here, because a service's prefix is its own: this one publishes `ad:Address`
+ * and Estonia's publishes `AD_Address:AD.Address`.
+ *
+ * @throws When the service stops publishing one of the five types, does not advertise
+ * the GML format the adapter reads, or does not advertise result paging.
+ */
+export async function readSKInspireCapabilities(client) {
+    const capabilities = await readWFSCapabilities(client, { wfsURL: SK_INSPIRE_WFS, context: CONTEXT });
+    const absent = SK_INSPIRE_TYPES.filter((type) => !capabilities.typeNames[type]);
+    if (absent.length) {
+        throw new Error(`${CONTEXT}: the capabilities document publishes no ${absent.join(", ")} feature type, and an address ` +
+            `references every one of the five, so the adapter's join could not be built from this service`);
+    }
+    if (!capabilities.outputFormats.includes(SK_INSPIRE_OUTPUT_FORMAT)) {
+        throw new Error(`${CONTEXT}: the service does not advertise ${SK_INSPIRE_OUTPUT_FORMAT}, which is the format the adapter reads — it advertises ${capabilities.outputFormats.join(", ")}`);
+    }
+    if (!capabilities.supportsPaging) {
+        throw new Error(`${CONTEXT}: the service does not advertise ImplementsResultPaging, and such a service answers every page with the first, so a paged harvest would store one page repeated`);
+    }
+    return capabilities;
+}
+/**
+ * The count the service states for one type, with how it was obtained.
+ *
+ * `statedFeatureCount` carries no count forward where the reported number equals
+ * the advertised page cap, because such a number describes the largest page the
+ * service will serve rather than what the type holds.
+ * This service's cap is 5000 and no type reports 5000.
+ */
+export async function readSKInspireFeatureCount(client, options) {
+    return statedFeatureCount(await readCheckedWFSFeatureCount(client, {
+        wfsURL: SK_INSPIRE_WFS,
+        typeNames: options.typeName,
+        context: CONTEXT,
+    }), options.pageCap);
+}
+/**
+ * Harvests one feature type into its own directory, recording each page in a manifest beside it.
+ *
+ * @returns The manifest as written.
+ */
+export async function harvestSKInspireType(client, options) {
+    const { typeName, type } = options;
+    const pageSize = Math.min(options.pageSize ?? SK_INSPIRE_PAGE_SIZE, SK_INSPIRE_PAGE_SIZE);
+    const featureCount = await readSKInspireFeatureCount(client, { typeName, pageCap: options.pageCap });
+    options.report?.(`  ${type}: ${featureCount.count ?? "no stated count"} — ${featureCount.because}`);
+    return await harvestPagedWFS({
+        context: `${CONTEXT} ${typeName}`,
+        source: SLUG,
+        outputDir: options.outputDir,
+        filename: skInspireHarvestFile(type),
+        request: {
+            wfsURL: SK_INSPIRE_WFS,
+            typeName,
+            outputFormat: SK_INSPIRE_OUTPUT_FORMAT,
+            sortBy: SK_INSPIRE_SORT_BY,
+        },
+        license: SK_INSPIRE_DEFAULT_LICENSE,
+        // The elected statement names no attribution clause.
+        // Crediting the Ministerstvo vnútra SR stays good practice.
+        attribution: "",
+        featureCount,
+        pageSize,
+        maxPages: options.maxPages,
+        signal: options.signal,
+        report: options.report,
+        readPage: async ({ startIndex, count }) => {
+            const page = await readWFSMarkupPage(client, {
+                wfsURL: SK_INSPIRE_WFS,
+                context: `${CONTEXT} ${typeName}`,
+                params: {
+                    typeNames: typeName,
+                    outputFormat: SK_INSPIRE_OUTPUT_FORMAT,
+                    sortBy: SK_INSPIRE_SORT_BY,
+                    count: String(count),
+                    startIndex: String(startIndex),
+                },
+            });
+            return {
+                payload: rootContent(page.body, page.root, `${CONTEXT} ${typeName}`),
+                header: `${XML_DECLARATION}${collectionOpeningTag(page.root)}`,
+                footer: `</${page.root.name}>\n`,
+                numberReturned: page.numberReturned,
+                numberMatched: page.numberMatched,
+                retrievedAt: page.timeStamp,
+            };
+        },
+    });
+}
+/**
+ * The directory `#sk/adapters/inspire/adapter` reads, given the root a fetch wrote under.
+ *
+ * A directory rather than a file, because the five feature types are five documents
+ * and the adapter indexes four of them before it composes an address out of the fifth.
+ */
+export function skInspireInputPath(outRoot) {
+    return outRoot(SLUG);
+}
+/**
+ * Harvests Slovakia's five AD feature types under `<outRoot>/sk-inspire/<type>/`,
+ * for `mailwoman corpus fetch sk-inspire`.
+ *
+ * Each type is reported separately and a type that fails does not stop the rest, because the
+ * documents are read together by the adapter and a partial directory is still worth resuming.
+ * Re-runnable per type: a complete harvest whose file still hashes to its manifest's digest
+ * makes no request, and an interrupted one resumes at the page after the last that reached disk.
+ */
+export async function fetchSKInspire(options, report) {
+    const pageSize = Math.min(options.pageSize ?? SK_INSPIRE_PAGE_SIZE, SK_INSPIRE_PAGE_SIZE);
+    report?.(`=== ${SLUG}: ${SK_INSPIRE_TYPES.length} feature types from ${SK_INSPIRE_WFS}`);
+    const summary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] };
+    for (const type of SK_INSPIRE_TYPES) {
+        if (options.signal?.aborted)
+            break;
+        const typeSummary = await runWFSHarvest({
+            slug: type,
+            // The harvest directory is the type's own, inside this source's root,
+            // so each type keeps its own manifest instead of overwriting the previous type's.
+            outRoot: (segment) => options.outRoot(SLUG, segment),
+            filename: skInspireHarvestFile(type),
+            expect: { wfsURL: SK_INSPIRE_WFS, sortBy: SK_INSPIRE_SORT_BY, pageSize },
+            displayName: `${SLUG} ${type}`,
+            minRequestIntervalMs: MIN_REQUEST_INTERVAL_MS,
+            client: options.client,
+            maxPages: options.maxPages,
+            harvest: async (client) => {
+                const capabilities = await readSKInspireCapabilities(client);
+                const typeName = capabilities.typeNames[type];
+                if (!typeName) {
+                    throw new Error(`${CONTEXT}: the capabilities document names no qualified type for ${type}, so it could not be requested`);
+                }
+                return await harvestSKInspireType(client, {
+                    outputDir: options.outRoot(SLUG, type),
+                    typeName,
+                    type,
+                    pageCap: capabilities.countDefault,
+                    pageSize,
+                    maxPages: options.maxPages,
+                    signal: options.signal,
+                    report,
+                });
+            },
+        }, report);
+        summary.fetched += typeSummary.fetched;
+        summary.skipped += typeSummary.skipped;
+        summary.failed += typeSummary.failed;
+        summary.failedCodes.push(...typeSummary.failedCodes.map((code) => `${SLUG}/${code}`));
+    }
+    return summary;
+}
+//# sourceMappingURL=inspire.js.map

package/out/sk/tools/fetch/inspire.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inspire.js","sourceRoot":"","sources":["../../../../lib/sk/tools/fetch/inspire.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAEH,OAAO,EAAE,0BAA0B,EAAkB,MAAM,qBAAqB,CAAA;AAGhF,OAAO,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAA;AAEhG,OAAO,EAAE,mBAAmB,EAAwB,MAAM,gCAAgC,CAAA;AAC1F,OAAO,EACN,oBAAoB,EACpB,eAAe,EACf,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,kBAAkB,GAIlB,MAAM,0BAA0B,CAAA;AAEjC;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,wCAAwC,CAAA;AAEtE;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,kCAAkC,CAAA;AAE1E;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAA;AAEhD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CAAA;AAExC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC/B,kBAAkB;IAClB,iBAAiB;IACjB,eAAe;IACf,kBAAkB;IAClB,SAAS;CACA,CAAA;AAOV;;;;GAIG;AACH,MAAM,eAAe,GAAG,0CAA0C,CAAA;AAElE;;GAEG;AACH,MAAM,IAAI,GAAG,qBAAqB,CAAA;AAElC;;GAEG;AACH,MAAM,OAAO,GAAG,gBAAgB,CAAA;AAEhC;;;;;;GAMG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AAEpC;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAmB;IACvD,OAAO,GAAG,IAAI,MAAM,CAAA;AACrB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,MAAgC;IAC/E,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;IAEpG,MAAM,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;IAE/E,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,4CAA4C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,gCAAgC;YACtG,8FAA8F,CAC/F,CAAA;IACF,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,oCAAoC,wBAAwB,2DAA2D,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxK,CAAA;IACF,CAAC;IAED,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,iKAAiK,CAC3K,CAAA;IACF,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,MAAgC,EAChC,OAAqD;IAErD,OAAO,kBAAkB,CACxB,MAAM,0BAA0B,CAAC,MAAM,EAAE;QACxC,MAAM,EAAE,cAAc;QACtB,SAAS,EAAE,OAAO,CAAC,QAAQ;QAC3B,OAAO,EAAE,OAAO;KAChB,CAAC,EACF,OAAO,CAAC,OAAO,CACf,CAAA;AACF,CAAC;AAiCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,MAAgC,EAChC,OAAoC;IAEpC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,OAAO,CAAA;IAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,oBAAoB,EAAE,oBAAoB,CAAC,CAAA;IAEzF,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;IAEpG,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,IAAI,KAAK,YAAY,CAAC,KAAK,IAAI,iBAAiB,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC,CAAA;IAEnG,OAAO,MAAM,eAAe,CAAC;QAC5B,OAAO,EAAE,GAAG,OAAO,IAAI,QAAQ,EAAE;QACjC,MAAM,EAAE,IAAI;QACZ,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,oBAAoB,CAAC,IAAI,CAAC;QACpC,OAAO,EAAE;YACR,MAAM,EAAE,cAAc;YACtB,QAAQ;YACR,YAAY,EAAE,wBAAwB;YACtC,MAAM,EAAE,kBAAkB;SAC1B;QACD,OAAO,EAAE,0BAA0B;QACnC,qDAAqD;QACrD,4DAA4D;QAC5D,WAAW,EAAE,EAAE;QACf,YAAY;QACZ,QAAQ;QACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,EAA6B,EAAE;YACpE,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,MAAM,EAAE;gBAC5C,MAAM,EAAE,cAAc;gBACtB,OAAO,EAAE,GAAG,OAAO,IAAI,QAAQ,EAAE;gBACjC,MAAM,EAAE;oBACP,SAAS,EAAE,QAAQ;oBACnB,YAAY,EAAE,wBAAwB;oBACtC,MAAM,EAAE,kBAAkB;oBAC1B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;oBACpB,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC;iBAC9B;aACD,CAAC,CAAA;YAEF,OAAO;gBACN,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACpE,MAAM,EAAE,GAAG,eAAe,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC9D,MAAM,EAAE,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK;gBAChC,cAAc,EAAE,IAAI,CAAC,cAAc;gBACnC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,WAAW,EAAE,IAAI,CAAC,SAAS;aAC3B,CAAA;QACF,CAAC;KACD,CAAC,CAAA;AACH,CAAC;AAqBD;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAoC;IACtE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,oBAAoB,EAAE,oBAAoB,CAAC,CAAA;IAEzF,MAAM,EAAE,CAAC,OAAO,IAAI,KAAK,gBAAgB,CAAC,MAAM,uBAAuB,cAAc,EAAE,CAAC,CAAA;IAExF,MAAM,OAAO,GAAiB,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAEpF,KAAK,MAAM,IAAI,IAAI,gBAAgB,EAAE,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAK;QAElC,MAAM,WAAW,GAAG,MAAM,aAAa,CACtC;YACC,IAAI,EAAE,IAAI;YACV,sEAAsE;YACtE,kFAAkF;YAClF,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;YACpD,QAAQ,EAAE,oBAAoB,CAAC,IAAI,CAAC;YACpC,MAAM,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,kBAAkB,EAAE,QAAQ,EAAE;YACxE,WAAW,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE;YAC9B,oBAAoB,EAAE,uBAAuB;YAC7C,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;gBACzB,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,MAAM,CAAC,CAAA;gBAC5D,MAAM,QAAQ,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAE7C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,2DAA2D,IAAI,gCAAgC,CACzG,CAAA;gBACF,CAAC;gBAED,OAAO,MAAM,oBAAoB,CAAC,MAAM,EAAE;oBACzC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC;oBACtC,QAAQ;oBACR,IAAI;oBACJ,OAAO,EAAE,YAAY,CAAC,YAAY;oBAClC,QAAQ;oBACR,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,MAAM;iBACN,CAAC,CAAA;YACH,CAAC;SACD,EACD,MAAM,CACN,CAAA;QAED,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC,OAAO,CAAA;QACtC,OAAO,CAAC,OAAO,IAAI,WAAW,CAAC,OAAO,CAAA;QACtC,OAAO,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAA;QACpC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,CAAA;IACtF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC"}

package/out/source-register/effective-manifest.d.ts

@@ -0,0 +1,355 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file What reached the trainer, as distinct from what the corpus holds.
+ *
+ *   `freezeTrainingManifest` records the sources that contributed rows to a corpus. A run reads that
+ *   corpus through a config. Four stages between them remove rows: `country_weights` rejects a row
+ *   whose country it does not list. `source_weights` draws only the sources it lists. A source set to `0.0`
+ *   is drawn zero times. `augment_exclude_sources` withholds a source from augmentation. A model card
+ *   quoting the corpus manifest therefore attributes sources the checkpoint never saw.
+ *
+ *   Measured over `v0.6.0-register-surface` with `v6.0.0-register-surface-60k.yaml` on 2026-09-28: the corpus's
+ *   frozen manifest names 11 sources. The audited epoch emitted 9 of those sources and 48 sources
+ *   in total. The config admits 136 countries. Forty countries drew rows, so 96 admitted countries drew zero of
+ *   the 1,000,000 rows the epoch emitted.
+ *
+ *   **What this establishes and what it does not.** The counts come from one audited epoch at one seed.
+ *   A source drawn zero times here is one the sampler did not reach in that epoch. The measurement does not show
+ *   that no epoch can reach it. The audit reports its seed and draw count. This record stores both. A source the corpus holds
+ *   and the config omits is excluded by construction. The record reports that exclusion instead of a zero draw.
+ */
+import type { PathBuilder } from "path-ts";
+import type { TrainingManifest } from "#source-register/training-manifest";
+/**
+ * Where an `audit_epoch_mixture --json` output for one config is looked for.
+ *
+ * Under the data root rather than committed because the file is a measurement of one
+ * corpus at one seed and takes about ten minutes to produce.
+ * The config determines the filename so two training arms' audits stay apart.
+ *
+ * This avoids the mismatch {@linkcode deriveEffectiveTrainingManifest} refuses.
+ */
+export declare function epochMixtureAuditPath(configPath: string): PathBuilder;
+/**
+ * Why a source the corpus holds contributed no row to the audited epoch.
+ */
+export declare const ExclusionReason: {
+    /**
+     * The config's `source_weights` map does not name the source, so the sampler never considers it.
+     */
+    readonly UnweightedSource: "unweighted-source";
+    /**
+     * `source_weights` names the source at `0.0`.
+     */
+    readonly ZeroWeight: "zero-weight";
+    /**
+     * The source is weighted and the audited epoch drew zero of its rows.
+     *
+     * This is a statement about one seed and one epoch length rather than about the source.
+     */
+    readonly DrewZeroRows: "drew-zero-rows";
+};
+export type ExclusionReason = (typeof ExclusionReason)[keyof typeof ExclusionReason];
+/**
+ * One source as the audited epoch saw it.
+ */
+export interface EffectiveSourceRecord {
+    source: string;
+    /**
+     * Rows the corpus holds for this source, from the frozen manifest.
+     */
+    corpusRows: number;
+    license: string;
+    /**
+     * The weight `source_weights` gives it, or `null` where the map omits it.
+     */
+    weight: number | null;
+    /**
+     * Rows the sampler drew, before augmentation.
+     */
+    drawnRows: number;
+    /**
+     * Rows that survived augmentation and reached the trainer.
+     */
+    emittedRows: number;
+    /**
+     * Whether `augment_exclude_sources` withholds this source from augmentation.
+     */
+    augmentExcluded: boolean;
+    /**
+     * Why it contributed no row, or `null` where it contributed at least one.
+     */
+    excludedBecause: ExclusionReason | null;
+}
+/**
+ * Which stage of the draw chain a manifest's row counts come from.
+ *
+ * The chain runs eligible rows, requested mixture, expected draws, realized draws, checkpoint.
+ * A manifest records the last stage it could read, and a reader treats an earlier
+ * stage as an estimate of the later ones rather than as the fact.
+ */
+export declare const DrawEvidence: {
+    /**
+     * The epoch-mixture audit replays epoch 1 of the sampler with the trainer's seed.
+     *
+     * It reconstructs the draw.
+     * The trainer's own count is {@link DrawEvidence.TrainerLogged}.
+     */
+    readonly ReplayedEpochAudit: "replayed-epoch-audit";
+    /**
+     * The trainer's `exposure-realized-from-<step>.json`, which counts the rows of every batch it trained on.
+     */
+    readonly TrainerLogged: "trainer-logged";
+};
+export type DrawEvidence = (typeof DrawEvidence)[keyof typeof DrawEvidence];
+/**
+ * The head a checkpoint trained: its label set and where each head tag sits in the shared
+ * semantic tag registry (`corpus-python/src/mailwoman_train/semantic_tags.py`).
+ */
+export interface LabelSetContract {
+    labelSetID: string;
+    semanticTagRegistryVersion: number;
+    /**
+     * The global semantic id of each head tag, in head order.
+     */
+    headMapping: number[];
+}
+/**
+ * An attribution a model card owes because a source under that license reached the trainer.
+ */
+export interface RequiredAttribution {
+    license: string;
+    /**
+     * The training sources carrying the license.
+     */
+    sources: string[];
+    /**
+     * The statement to print, or `null` where no statement is recorded for the license
+     * and the attribution has to be written from each source's register entry.
+     */
+    statement: string | null;
+}
+/**
+ * The attributions owed by the sources that reached the trainer, grouped by license.
+ *
+ * Only `trainingSources` are read, so a source the corpus holds and the run never drew owes no attribution.
+ * A license is included when `summarizeLicense` records an attribution obligation for it, or
+ * when it is unrecognized, since an unrecognized license's obligations are unknown rather than absent.
+ */
+export declare function requiredAttributions(sources: readonly EffectiveSourceRecord[]): RequiredAttribution[];
+/**
+ * What one config drew from one corpus in one audited epoch.
+ */
+export interface EffectiveTrainingManifest {
+    manifestID: "corpus-effective-training-manifest";
+    schemaVersion: 2;
+    /**
+     * The stage of the draw chain the row counts below come from.
+     */
+    drawEvidence: DrawEvidence;
+    /**
+     * The head the run trained, or `null` where the derivation was given none.
+     */
+    labelSet: LabelSetContract | null;
+    /**
+     * The attributions the sources in {@linkcode trainingSources} owe.
+     */
+    requiredAttributions: RequiredAttribution[];
+    /**
+     * The corpus the rows came from and the digest of its frozen manifest.
+     * A reader can identify which record this was derived from.
+     */
+    corpusVersion: string;
+    corpusManifestDigest: string;
+    /**
+     * The config file the audit ran under, as the audit recorded it.
+     */
+    config: string;
+    /**
+     * The seed and draw count of the audited epoch.
+     * Both decide the counts below.
+     */
+    seed: number;
+    drawsRequested: number;
+    drawsRealized: number;
+    /**
+     * Number of countries `country_weights` admits and number that drew a row.
+     */
+    admittedCountries: number;
+    countriesDrawingRows: number;
+    /**
+     * Countries admitted at a weight that drew zero rows in this epoch.
+     */
+    admittedCountriesDrawingZero: string[];
+    /**
+     * Every source the corpus holds, in descending emitted order.
+     */
+    sources: EffectiveSourceRecord[];
+    /**
+     * Sources that reached the trainer.
+     * A model card may attribute this set as training data.
+     */
+    trainingSources: string[];
+    /**
+     * Sources the corpus holds that reached no row, each with its reason.
+     */
+    excludedSources: Record<string, ExclusionReason>;
+    /**
+     * Sources the epoch emitted that the corpus's frozen manifest does not name, with their emitted rows.
+     *
+     * `freezeTrainingManifest` runs inside `buildCorpus` and records the base build's sources.
+     * An overlay is merged into the corpus afterwards by `overlay-manifest`,
+     * so its rows are in the corpus and outside that manifest.
+     *
+     * Measured over `v0.6.0-register-surface` on 2026-09-28: the frozen manifest names 11 sources
+     * and the audited epoch emitted 48, of which 39 have no entry in it.
+     * Those 39 account for 890,666 of the 1,000,000 rows emitted, 89.1%, so the frozen
+     * manifest's license set covers 10.9% of what trained the model.
+     *
+     * The set keeps this record from reading as a complete one.
+     * A release assertion over it has to treat a non-empty value as an unread input
+     * rather than as an absence of rows.
+     */
+    emittedButUnrecorded: Record<string, number>;
+    /**
+     * The three stages a source passes through, for each source in {@linkcode emittedButUnrecorded}.
+     *
+     * The entries above give only the emitted rows.
+     * These give the rows the corpus holds, the rows the epoch drew and the rows
+     * it emitted after augmentation.
+     *
+     * A reader can distinguish a source drawn heavily from one drawn barely.
+     *
+     * `corpusRows` comes from `draw_level.per_source`.
+     * That field counts the corpus files.
+     *
+     * A source outside the frozen manifest has a corpus row count nowhere else.
+     * `-1` records that the audit reported none rather than that the corpus holds no such row.
+     */
+    unrecordedSourceStages: Record<string, {
+        corpusRows: number;
+        drawnRows: number;
+        emittedRows: number;
+    }>;
+    totalEmittedRows: number;
+    /**
+     * Sha256 over this manifest with `contentDigest` emptied.
+     */
+    contentDigest: string;
+}
+/**
+ * The fields this reads from an `audit_epoch_mixture --json` output.
+ */
+export interface EpochMixtureAudit {
+    draw_level?: {
+        totals?: Record<string, number>;
+        by_country?: Record<string, number>;
+        admitted_countries_drawn?: Record<string, number>;
+        admitted_countries_drawing_nothing?: string[];
+        /**
+         * Per-source detail the audit computes from the corpus files rather than from the corpus manifest.
+         *
+         * `rows` is how many rows of that source the corpus holds.
+         * It is the only place a source outside the frozen manifest has a corpus row count, so it is what
+         * lets a reader read a source's rows in the corpus beside the rows one epoch drew and emitted.
+         */
+        per_source?: Record<string, {
+            rows?: number;
+            draws?: number;
+        }>;
+    };
+    emitted_level?: {
+        totals?: Record<string, number>;
+    };
+    meta?: {
+        seed?: number;
+        draws_requested?: number;
+        draws_realized?: number;
+        config?: string;
+    };
+}
+/**
+ * The config fields that decide which of a corpus's sources a run reaches.
+ */
+export interface EffectiveConfigView {
+    sourceWeights: Readonly<Record<string, number>>;
+    augmentExcludeSources: readonly string[];
+}
+/**
+ * Read `data.source_weights` and `data.augment_exclude_sources` from a training config.
+ *
+ * A line scanner instead of a YAML parse.
+ * `corpus audit` already reads these blocks this way.
+ *
+ * The `wire-identifiers` check uses the same approach.
+ * The configs use a small hand-written subset: two-space indentation, one entry per line and `#` comments.
+ * They contain no flow syntax.
+ *
+ * YAML 1.1 coerces the bare key `NO` to boolean false.
+ * Norway is a `country_weights` key, so a parser would drop it.
+ *
+ * @throws When the text contains no `source_weights` block.
+ * That map decides which sources the sampler considers.
+ * A config without one is a truncated file rather than a config drawing from every source.
+ */
+export declare function readConfigView(text: string): EffectiveConfigView;
+/**
+ * The digest a manifest should include, computed over the manifest with its digest field emptied.
+ */
+export declare function effectiveManifestDigest(manifest: EffectiveTrainingManifest): string;
+/**
+ * Derive what reached the trainer from the corpus's frozen manifest, one audited epoch and the config.
+ *
+ * @throws When the audit contains no `emitted_level.totals`.
+ * That field answers the question.
+ * An audit without it is either a different report or a truncated one.
+ * A derivation from only `draw_level` would attribute rows augmentation removed.
+ * @throws When the audit ran under a different config file than the one supplied, because the two are
+ * two training arms and one record over both would report a source one of them never weighted.
+ */
+export declare function deriveEffectiveTrainingManifest(input: {
+    corpusManifest: TrainingManifest;
+    audit: EpochMixtureAudit;
+    config: EffectiveConfigView;
+    configPath: string;
+    labelSet?: LabelSetContract | null;
+}): EffectiveTrainingManifest;
+/**
+ * Sources a record declares as training data that the audited epoch never reached,
+ * plus the reverse mismatch.
+ *
+ * A release reads both directions.
+ * A declared source the checkpoint never saw overstates what the model learned from.
+ *
+ * An emitted source the record omits is an attribution a consumer never receives.
+ */
+export declare function provenanceDisagreement(manifest: EffectiveTrainingManifest, declared: readonly string[]): {
+    declaredButNotTrained: string[];
+    trainedButNotDeclared: string[];
+};
+/**
+ * Why a release may not assert that a model's declared provenance equals what trained it.
+ *
+ * An empty array is the only value a caller may read as agreement.
+ * It is reachable only when the effective manifest covers every emitted source.
+ *
+ * A `null` manifest means no release read one at all.
+ * The function reports it rather than passing it.
+ *
+ * `declared` is a list of corpus source ids.
+ * A caller holding a model card's attribution entries passes `null`.
+ *
+ * Those entries are prose naming publishers.
+ * Every field of the card contains prose rather than a source id, so comparing the two
+ * lists as strings reports every entry as a source the epoch never drew.
+ *
+ * The refusal list then records the coverage refusal and one line recording the comparison as unmeasured.
+ */
+export declare function provenanceRefusals(input: {
+    manifest: EffectiveTrainingManifest | null;
+    declared: readonly string[] | null;
+    packageName: string;
+}): string[];
+//# sourceMappingURL=effective-manifest.d.ts.map

package/out/source-register/effective-manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"effective-manifest.d.ts","sourceRoot":"","sources":["../../lib/source-register/effective-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAG1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AAE1E;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAQrE;AAED;;GAEG;AACH,eAAO,MAAM,eAAe;IAC3B;;OAEG;aACH,gBAAgB,EAAE,mBAAmB;IAErC;;OAEG;aACH,UAAU,EAAE,aAAa;IAEzB;;;;OAIG;aACH,YAAY,EAAE,gBAAgB;CACrB,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAA;AAEpF;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAErB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,eAAe,EAAE,OAAO,CAAA;IAExB;;OAEG;IACH,eAAe,EAAE,eAAe,GAAG,IAAI,CAAA;CACvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY;IACxB;;;;;OAKG;aACH,kBAAkB,EAAE,sBAAsB;IAC1C;;OAEG;aACH,aAAa,EAAE,gBAAgB;CACtB,CAAA;AAEV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAE3E;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,0BAA0B,EAAE,MAAM,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAcD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,mBAAmB,EAAE,CAoBrG;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC,UAAU,EAAE,oCAAoC,CAAA;IAChD,aAAa,EAAE,CAAC,CAAA;IAEhB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAA;IAE1B;;OAEG;IACH,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAA;IAEjC;;OAEG;IACH,oBAAoB,EAAE,mBAAmB,EAAE,CAAA;IAE3C;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAA;IACzB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,4BAA4B,EAAE,MAAM,EAAE,CAAA;IAEtC;;OAEG;IACH,OAAO,EAAE,qBAAqB,EAAE,CAAA;IAEhC;;;OAGG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAEhD;;;;;;;;;;;;;;;OAeG;IACH,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAE5C;;;;;;;;;;;;;;OAcG;IACH,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAEtG,gBAAgB,EAAE,MAAM,CAAA;IAExB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,UAAU,CAAC,EAAE;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,wBAAwB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACjD,kCAAkC,CAAC,EAAE,MAAM,EAAE,CAAA;QAC7C;;;;;;WAMG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAC9D,CAAA;IACD,aAAa,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAA;IACnD,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAC5F;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC/C,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CA4DhE;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,yBAAyB,GAAG,MAAM,CAEnF;AAED;;;;;;;;;GASG;AACH,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACtD,cAAc,EAAE,gBAAgB,CAAA;IAChC,KAAK,EAAE,iBAAiB,CAAA;IACxB,MAAM,EAAE,mBAAmB,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAA;CAClC,GAAG,yBAAyB,CA6G5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,yBAAyB,EACnC,QAAQ,EAAE,SAAS,MAAM,EAAE,GACzB;IAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAAC,qBAAqB,EAAE,MAAM,EAAE,CAAA;CAAE,CAQtE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACzC,QAAQ,EAAE,yBAAyB,GAAG,IAAI,CAAA;IAC1C,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IAClC,WAAW,EAAE,MAAM,CAAA;CACnB,GAAG,MAAM,EAAE,CAoDX"}