@mailwoman/corpus 9.4.0 → 10.1.0

package/out/source-register/effective-manifest.js

@@ -0,0 +1,355 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file What reached the trainer, as distinct from what the corpus holds.
+ *
+ *   `freezeTrainingManifest` records the sources that contributed rows to a corpus. A run reads that
+ *   corpus through a config. Four stages between them remove rows: `country_weights` rejects a row
+ *   whose country it does not list. `source_weights` draws only the sources it lists. A source set to `0.0`
+ *   is drawn zero times. `augment_exclude_sources` withholds a source from augmentation. A model card
+ *   quoting the corpus manifest therefore attributes sources the checkpoint never saw.
+ *
+ *   Measured over `v0.6.0-register-surface` with `v6.0.0-register-surface-60k.yaml` on 2026-09-28: the corpus's
+ *   frozen manifest names 11 sources. The audited epoch emitted 9 of those sources and 48 sources
+ *   in total. The config admits 136 countries. Forty countries drew rows, so 96 admitted countries drew zero of
+ *   the 1,000,000 rows the epoch emitted.
+ *
+ *   **What this establishes and what it does not.** The counts come from one audited epoch at one seed.
+ *   A source drawn zero times here is one the sampler did not reach in that epoch. The measurement does not show
+ *   that no epoch can reach it. The audit reports its seed and draw count. This record stores both. A source the corpus holds
+ *   and the config omits is excluded by construction. The record reports that exclusion instead of a zero draw.
+ */
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { sha256Hex } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { LicenseObligation, summarizeLicense } from "@mailwoman/core/license/obligations";
+import { OSM_ATTRIBUTION, OSM_LICENSE } from "#adapters/osm/adapter";
+/**
+ * Where an `audit_epoch_mixture --json` output for one config is looked for.
+ *
+ * Under the data root rather than committed because the file is a measurement of one
+ * corpus at one seed and takes about ten minutes to produce.
+ * The config determines the filename so two training arms' audits stay apart.
+ *
+ * This avoids the mismatch {@linkcode deriveEffectiveTrainingManifest} refuses.
+ */
+export function epochMixtureAuditPath(configPath) {
+    const name = configPath
+        .split("/")
+        .at(-1)
+        ?.replace(/\.ya?ml$/u, "") ?? configPath;
+    return dataRootPath("corpus", "epoch-mixture", `${name}.json`);
+}
+/**
+ * Why a source the corpus holds contributed no row to the audited epoch.
+ */
+export const ExclusionReason = {
+    /**
+     * The config's `source_weights` map does not name the source, so the sampler never considers it.
+     */
+    UnweightedSource: "unweighted-source",
+    /**
+     * `source_weights` names the source at `0.0`.
+     */
+    ZeroWeight: "zero-weight",
+    /**
+     * The source is weighted and the audited epoch drew zero of its rows.
+     *
+     * This is a statement about one seed and one epoch length rather than about the source.
+     */
+    DrewZeroRows: "drew-zero-rows",
+};
+/**
+ * Which stage of the draw chain a manifest's row counts come from.
+ *
+ * The chain runs eligible rows, requested mixture, expected draws, realized draws, checkpoint.
+ * A manifest records the last stage it could read, and a reader treats an earlier
+ * stage as an estimate of the later ones rather than as the fact.
+ */
+export const DrawEvidence = {
+    /**
+     * The epoch-mixture audit replays epoch 1 of the sampler with the trainer's seed.
+     *
+     * It reconstructs the draw.
+     * The trainer's own count is {@link DrawEvidence.TrainerLogged}.
+     */
+    ReplayedEpochAudit: "replayed-epoch-audit",
+    /**
+     * The trainer's `exposure-realized-from-<step>.json`, which counts the rows of every batch it trained on.
+     */
+    TrainerLogged: "trainer-logged",
+};
+/**
+ * Recorded attribution statements, keyed by license.
+ *
+ * A license appears here only when one statement covers every source carrying it.
+ * OSM's statement covers every OSM-derived row.
+ *
+ * A CC BY source credits its own publisher, so its statement comes from the register.
+ */
+const ATTRIBUTION_STATEMENTS = {
+    [OSM_LICENSE]: OSM_ATTRIBUTION,
+};
+/**
+ * The attributions owed by the sources that reached the trainer, grouped by license.
+ *
+ * Only `trainingSources` are read, so a source the corpus holds and the run never drew owes no attribution.
+ * A license is included when `summarizeLicense` records an attribution obligation for it, or
+ * when it is unrecognized, since an unrecognized license's obligations are unknown rather than absent.
+ */
+export function requiredAttributions(sources) {
+    const byLicense = new Map();
+    for (const record of sources) {
+        if (record.emittedRows <= 0)
+            continue;
+        const summary = summarizeLicense(record.license);
+        if (summary.recognized && !summary.obligations.includes(LicenseObligation.Attribution))
+            continue;
+        byLicense.set(record.license, [...(byLicense.get(record.license) ?? []), record.source]);
+    }
+    return [...byLicense]
+        .toSorted(([a], [b]) => a.localeCompare(b))
+        .map(([license, licensed]) => ({
+        license,
+        sources: licensed.toSorted(),
+        statement: ATTRIBUTION_STATEMENTS[license] ?? null,
+    }));
+}
+/**
+ * Read `data.source_weights` and `data.augment_exclude_sources` from a training config.
+ *
+ * A line scanner instead of a YAML parse.
+ * `corpus audit` already reads these blocks this way.
+ *
+ * The `wire-identifiers` check uses the same approach.
+ * The configs use a small hand-written subset: two-space indentation, one entry per line and `#` comments.
+ * They contain no flow syntax.
+ *
+ * YAML 1.1 coerces the bare key `NO` to boolean false.
+ * Norway is a `country_weights` key, so a parser would drop it.
+ *
+ * @throws When the text contains no `source_weights` block.
+ * That map decides which sources the sampler considers.
+ * A config without one is a truncated file rather than a config drawing from every source.
+ */
+export function readConfigView(text) {
+    const sourceWeights = {};
+    const augmentExcludeSources = [];
+    let field = null;
+    let fieldIndent = -1;
+    let sawWeights = false;
+    // oxlint-disable-next-line mailwoman/prefer-spliterator -- one training config, a few hundred lines read whole
+    for (const raw of text.split("\n")) {
+        if (/^[\t ]*(#|$)/.test(raw))
+            continue;
+        const indent = /^[\t ]*/.exec(raw)[0].length;
+        if (field !== null && indent <= fieldIndent) {
+            field = null;
+        }
+        if (field === null) {
+            const opener = /^([\t ]*)(source_weights|augment_exclude_sources):\s*(#.*)?$/.exec(raw);
+            if (opener) {
+                field = opener[2];
+                fieldIndent = opener[1].length;
+                if (field === "source_weights") {
+                    sawWeights = true;
+                }
+            }
+            continue;
+        }
+        if (field === "source_weights") {
+            const entry = /^[\t ]+([\w-]+):\s*([\d.]+)/.exec(raw);
+            if (entry) {
+                sourceWeights[entry[1]] = Number.parseFloat(entry[2]);
+            }
+            continue;
+        }
+        const item = /^[\t ]+-\s*([\w-]+)\s*(#.*)?$/.exec(raw);
+        if (item) {
+            augmentExcludeSources.push(item[1]);
+        }
+    }
+    if (!sawWeights) {
+        throw new Error("the config carries no `data.source_weights` block. That map decides which sources the sampler " +
+            "considers, so a config without one is a truncated file rather than a config drawing from every source.");
+    }
+    return { sourceWeights, augmentExcludeSources };
+}
+/**
+ * The digest a manifest should include, computed over the manifest with its digest field emptied.
+ */
+export function effectiveManifestDigest(manifest) {
+    return sha256Hex(stringifyJSON({ ...manifest, contentDigest: "" }));
+}
+/**
+ * Derive what reached the trainer from the corpus's frozen manifest, one audited epoch and the config.
+ *
+ * @throws When the audit contains no `emitted_level.totals`.
+ * That field answers the question.
+ * An audit without it is either a different report or a truncated one.
+ * A derivation from only `draw_level` would attribute rows augmentation removed.
+ * @throws When the audit ran under a different config file than the one supplied, because the two are
+ * two training arms and one record over both would report a source one of them never weighted.
+ */
+export function deriveEffectiveTrainingManifest(input) {
+    const emitted = input.audit.emitted_level?.totals;
+    if (!emitted) {
+        throw new Error("the audit carries no `emitted_level.totals`, which is the field that says what reached the trainer. " +
+            "Deriving from `draw_level` alone would attribute the rows augmentation removed.");
+    }
+    const audited = input.audit.meta?.config?.split("/").at(-1);
+    const wanted = input.configPath.split("/").at(-1);
+    if (audited && wanted && audited !== wanted) {
+        throw new Error(`the audit ran under ${audited} and the config supplied is ${wanted}. Those are two training arms, and ` +
+            `one record over both would report a source one of them never weighted.`);
+    }
+    const drawn = input.audit.draw_level?.totals ?? {};
+    const admittedDrawn = input.audit.draw_level?.admitted_countries_drawn ?? {};
+    const drawingZero = input.audit.draw_level?.admitted_countries_drawing_nothing ?? [];
+    const augmentExcluded = new Set(input.config.augmentExcludeSources);
+    const sources = input.corpusManifest.sources.map((record) => {
+        const weight = Object.hasOwn(input.config.sourceWeights, record.source)
+            ? input.config.sourceWeights[record.source]
+            : null;
+        const drawnRows = drawn[record.source] ?? 0;
+        const emittedRows = emitted[record.source] ?? 0;
+        const excludedBecause = emittedRows > 0
+            ? null
+            : weight === null
+                ? ExclusionReason.UnweightedSource
+                : weight === 0
+                    ? ExclusionReason.ZeroWeight
+                    : ExclusionReason.DrewZeroRows;
+        return {
+            source: record.source,
+            corpusRows: record.rows,
+            license: record.license,
+            weight,
+            drawnRows,
+            emittedRows,
+            augmentExcluded: augmentExcluded.has(record.source),
+            excludedBecause,
+        };
+    });
+    sources.sort((a, b) => b.emittedRows - a.emittedRows);
+    const excludedSources = {};
+    for (const record of sources) {
+        if (record.excludedBecause) {
+            excludedSources[record.source] = record.excludedBecause;
+        }
+    }
+    const recorded = new Set(input.corpusManifest.sources.map((record) => record.source));
+    const perSource = input.audit.draw_level?.per_source ?? {};
+    const emittedButUnrecorded = {};
+    const unrecordedSourceStages = {};
+    for (const [source, rows] of Object.entries(emitted)) {
+        if (rows > 0 && !recorded.has(source)) {
+            emittedButUnrecorded[source] = rows;
+            unrecordedSourceStages[source] = {
+                // `-1` records that the audit reported no corpus row count for this source.
+                // A 0 would say the corpus holds no row for this source.
+                // That contradicts its emitted rows.
+                corpusRows: perSource[source]?.rows ?? -1,
+                drawnRows: drawn[source] ?? 0,
+                emittedRows: rows,
+            };
+        }
+    }
+    const manifest = {
+        manifestID: "corpus-effective-training-manifest",
+        schemaVersion: 2,
+        drawEvidence: DrawEvidence.ReplayedEpochAudit,
+        labelSet: input.labelSet ?? null,
+        requiredAttributions: requiredAttributions(sources),
+        corpusVersion: input.corpusManifest.corpusVersion,
+        corpusManifestDigest: input.corpusManifest.contentDigest,
+        config: input.configPath,
+        seed: input.audit.meta?.seed ?? -1,
+        drawsRequested: input.audit.meta?.draws_requested ?? 0,
+        drawsRealized: input.audit.meta?.draws_realized ?? 0,
+        admittedCountries: Object.keys(admittedDrawn).length,
+        countriesDrawingRows: Object.values(admittedDrawn).filter((count) => count > 0).length,
+        admittedCountriesDrawingZero: [...drawingZero].toSorted(),
+        sources,
+        trainingSources: sources.filter((record) => record.emittedRows > 0).map((record) => record.source),
+        excludedSources,
+        emittedButUnrecorded,
+        unrecordedSourceStages,
+        totalEmittedRows: Object.values(emitted).reduce((sum, count) => sum + count, 0),
+        contentDigest: "",
+    };
+    return { ...manifest, contentDigest: effectiveManifestDigest(manifest) };
+}
+/**
+ * Sources a record declares as training data that the audited epoch never reached,
+ * plus the reverse mismatch.
+ *
+ * A release reads both directions.
+ * A declared source the checkpoint never saw overstates what the model learned from.
+ *
+ * An emitted source the record omits is an attribution a consumer never receives.
+ */
+export function provenanceDisagreement(manifest, declared) {
+    const trained = new Set(manifest.trainingSources);
+    const named = new Set(declared);
+    return {
+        declaredButNotTrained: [...named].filter((source) => !trained.has(source)).toSorted(),
+        trainedButNotDeclared: [...trained].filter((source) => !named.has(source)).toSorted(),
+    };
+}
+/**
+ * Why a release may not assert that a model's declared provenance equals what trained it.
+ *
+ * An empty array is the only value a caller may read as agreement.
+ * It is reachable only when the effective manifest covers every emitted source.
+ *
+ * A `null` manifest means no release read one at all.
+ * The function reports it rather than passing it.
+ *
+ * `declared` is a list of corpus source ids.
+ * A caller holding a model card's attribution entries passes `null`.
+ *
+ * Those entries are prose naming publishers.
+ * Every field of the card contains prose rather than a source id, so comparing the two
+ * lists as strings reports every entry as a source the epoch never drew.
+ *
+ * The refusal list then records the coverage refusal and one line recording the comparison as unmeasured.
+ */
+export function provenanceRefusals(input) {
+    if (!input.manifest) {
+        return [
+            `${input.packageName}: no effective training manifest was read, so whether its declared provenance ` +
+                `equals what trained it is unmeasured. \`effective-manifest.run.ts\` writes one from the corpus's ` +
+                `frozen manifest and an audit_epoch_mixture output.`,
+        ];
+    }
+    const refusals = [];
+    const unrecorded = Object.entries(input.manifest.emittedButUnrecorded);
+    if (unrecorded.length) {
+        const total = unrecorded.reduce((sum, [, rows]) => sum + rows, 0);
+        refusals.push(`${input.packageName}: the audited epoch emitted ${unrecorded.length} source(s) the corpus's frozen ` +
+            `manifest does not name, ${total.toLocaleString()} rows in total, so the effective manifest covers ` +
+            `part of what trained the model. Those sources reached the corpus through an overlay merged after ` +
+            `the frozen manifest was written: ${unrecorded.map(([source]) => source).join(", ")}.`);
+    }
+    if (input.declared === null) {
+        refusals.push(`${input.packageName}: whether its declared provenance equals the ${input.manifest.trainingSources.length} ` +
+            `source(s) the audited epoch drew from is unmeasured. The model card states its attribution as prose ` +
+            `naming publishers, and an effective manifest states corpus source ids, so the two carry no common key.`);
+        return refusals;
+    }
+    const { declaredButNotTrained, trainedButNotDeclared } = provenanceDisagreement(input.manifest, input.declared);
+    if (declaredButNotTrained.length) {
+        refusals.push(`${input.packageName} declares ${declaredButNotTrained.length} source(s) the audited epoch drew zero ` +
+            `rows from: ${declaredButNotTrained.join(", ")}. Attributing them says the model learned from rows ` +
+            `it never saw.`);
+    }
+    if (trainedButNotDeclared.length) {
+        refusals.push(`${input.packageName} omits ${trainedButNotDeclared.length} source(s) the audited epoch emitted: ` +
+            `${trainedButNotDeclared.join(", ")}. A consumer installing this package receives those rows' ` +
+            `contribution without their attribution.`);
+    }
+    return refusals;
+}
+//# sourceMappingURL=effective-manifest.js.map

package/out/source-register/effective-manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"effective-manifest.js","sourceRoot":"","sources":["../../lib/source-register/effective-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAA;AAGzF,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAA;AAGpE;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,UAAkB;IACvD,MAAM,IAAI,GACT,UAAU;SACR,KAAK,CAAC,GAAG,CAAC;SACV,EAAE,CAAC,CAAC,CAAC,CAAC;QACP,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,IAAI,UAAU,CAAA;IAE1C,OAAO,YAAY,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,IAAI,OAAO,CAAC,CAAA;AAC/D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B;;OAEG;IACH,gBAAgB,EAAE,mBAAmB;IAErC;;OAEG;IACH,UAAU,EAAE,aAAa;IAEzB;;;;OAIG;IACH,YAAY,EAAE,gBAAgB;CACrB,CAAA;AA2CV;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG;IAC3B;;;;;OAKG;IACH,kBAAkB,EAAE,sBAAsB;IAC1C;;OAEG;IACH,aAAa,EAAE,gBAAgB;CACtB,CAAA;AAiCV;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAqC;IAChE,CAAC,WAAW,CAAC,EAAE,eAAe;CAC9B,CAAA;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAyC;IAC7E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE7C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,WAAW,IAAI,CAAC;YAAE,SAAQ;QAErC,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAEhD,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,CAAC;YAAE,SAAQ;QAEhG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IACzF,CAAC;IAED,OAAO,CAAC,GAAG,SAAS,CAAC;SACnB,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9B,OAAO;QACP,OAAO,EAAE,QAAQ,CAAC,QAAQ,EAAE;QAC5B,SAAS,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,IAAI;KAClD,CAAC,CAAC,CAAA;AACL,CAAC;AAgJD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,MAAM,aAAa,GAA2B,EAAE,CAAA;IAChD,MAAM,qBAAqB,GAAa,EAAE,CAAA;IAI1C,IAAI,KAAK,GAAuB,IAAI,CAAA;IACpC,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IACpB,IAAI,UAAU,GAAG,KAAK,CAAA;IAEtB,+GAA+G;IAC/G,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QAEtC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE7C,IAAI,KAAK,KAAK,IAAI,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC7C,KAAK,GAAG,IAAI,CAAA;QACb,CAAC;QAED,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,8DAA8D,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEvF,IAAI,MAAM,EAAE,CAAC;gBACZ,KAAK,GAAG,MAAM,CAAC,CAAC,CAAgB,CAAA;gBAChC,WAAW,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;gBAE/B,IAAI,KAAK,KAAK,gBAAgB,EAAE,CAAC;oBAChC,UAAU,GAAG,IAAI,CAAA;gBAClB,CAAC;YACF,CAAC;YAED,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,KAAK,gBAAgB,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,6BAA6B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAErD,IAAI,KAAK,EAAE,CAAC;gBACX,aAAa,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;YACxD,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,IAAI,GAAG,+BAA+B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEtD,IAAI,IAAI,EAAE,CAAC;YACV,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CACd,gGAAgG;YAC/F,wGAAwG,CACzG,CAAA;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,QAAmC;IAC1E,OAAO,SAAS,CAAC,aAAa,CAAC,EAAE,GAAG,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;AACpE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,+BAA+B,CAAC,KAM/C;IACA,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,aAAa,EAAE,MAAM,CAAA;IAEjD,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,sGAAsG;YACrG,iFAAiF,CAClF,CAAA;IACF,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEjD,IAAI,OAAO,IAAI,MAAM,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACd,uBAAuB,OAAO,+BAA+B,MAAM,qCAAqC;YACvG,wEAAwE,CACzE,CAAA;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,wBAAwB,IAAI,EAAE,CAAA;IAC5E,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,kCAAkC,IAAI,EAAE,CAAA;IACpF,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAA;IAEnE,MAAM,OAAO,GAA4B,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACpF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,MAAM,CAAC;YACtE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAE;YAC5C,CAAC,CAAC,IAAI,CAAA;QAEP,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAE/C,MAAM,eAAe,GACpB,WAAW,GAAG,CAAC;YACd,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,MAAM,KAAK,IAAI;gBAChB,CAAC,CAAC,eAAe,CAAC,gBAAgB;gBAClC,CAAC,CAAC,MAAM,KAAK,CAAC;oBACb,CAAC,CAAC,eAAe,CAAC,UAAU;oBAC5B,CAAC,CAAC,eAAe,CAAC,YAAY,CAAA;QAElC,OAAO;YACN,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM;YACN,SAAS;YACT,WAAW;YACX,eAAe,EAAE,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;YACnD,eAAe;SACf,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;IAErD,MAAM,eAAe,GAAoC,EAAE,CAAA;IAE3D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YAC5B,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,eAAe,CAAA;QACxD,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IACrF,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,IAAI,EAAE,CAAA;IAC1D,MAAM,oBAAoB,GAA2B,EAAE,CAAA;IACvD,MAAM,sBAAsB,GAAwD,EAAE,CAAA;IAEtF,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACvC,oBAAoB,CAAC,MAAM,CAAC,GAAG,IAAI,CAAA;YAEnC,sBAAsB,CAAC,MAAM,CAAC,GAAG;gBAChC,4EAA4E;gBAC5E,yDAAyD;gBACzD,qCAAqC;gBACrC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;gBACzC,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;gBAC7B,WAAW,EAAE,IAAI;aACjB,CAAA;QACF,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAA8B;QAC3C,UAAU,EAAE,oCAAoC;QAChD,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,YAAY,CAAC,kBAAkB;QAC7C,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;QAChC,oBAAoB,EAAE,oBAAoB,CAAC,OAAO,CAAC;QACnD,aAAa,EAAE,KAAK,CAAC,cAAc,CAAC,aAAa;QACjD,oBAAoB,EAAE,KAAK,CAAC,cAAc,CAAC,aAAa;QACxD,MAAM,EAAE,KAAK,CAAC,UAAU;QACxB,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;QAClC,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,eAAe,IAAI,CAAC;QACtD,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,cAAc,IAAI,CAAC;QACpD,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM;QACpD,oBAAoB,EAAE,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM;QACtF,4BAA4B,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,QAAQ,EAAE;QACzD,OAAO;QACP,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAClG,eAAe;QACf,oBAAoB;QACpB,sBAAsB;QACtB,gBAAgB,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;QAC/E,aAAa,EAAE,EAAE;KACjB,CAAA;IAED,OAAO,EAAE,GAAG,QAAQ,EAAE,aAAa,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAA;AACzE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CACrC,QAAmC,EACnC,QAA2B;IAE3B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAA;IACjD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAA;IAE/B,OAAO;QACN,qBAAqB,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;QACrF,qBAAqB,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;KACrF,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAIlC;IACA,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACrB,OAAO;YACN,GAAG,KAAK,CAAC,WAAW,gFAAgF;gBACnG,mGAAmG;gBACnG,oDAAoD;SACrD,CAAA;IACF,CAAC;IAED,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAA;IAEtE,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAA;QAEjE,QAAQ,CAAC,IAAI,CACZ,GAAG,KAAK,CAAC,WAAW,+BAA+B,UAAU,CAAC,MAAM,iCAAiC;YACpG,2BAA2B,KAAK,CAAC,cAAc,EAAE,mDAAmD;YACpG,mGAAmG;YACnG,oCAAoC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvF,CAAA;IACF,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CACZ,GAAG,KAAK,CAAC,WAAW,gDAAgD,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG;YAC3G,sGAAsG;YACtG,wGAAwG,CACzG,CAAA;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED,MAAM,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE/G,IAAI,qBAAqB,CAAC,MAAM,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CACZ,GAAG,KAAK,CAAC,WAAW,aAAa,qBAAqB,CAAC,MAAM,yCAAyC;YACrG,cAAc,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,sDAAsD;YACpG,eAAe,CAChB,CAAA;IACF,CAAC;IAED,IAAI,qBAAqB,CAAC,MAAM,EAAE,CAAC;QAClC,QAAQ,CAAC,IAAI,CACZ,GAAG,KAAK,CAAC,WAAW,UAAU,qBAAqB,CAAC,MAAM,wCAAwC;YACjG,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,4DAA4D;YAC/F,yCAAyC,CAC1C,CAAA;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/source-register/register.d.ts

@@ -0,0 +1,79 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads and audits the address-source register.
+ */
+import type { PathBuilderLike } from "path-ts";
+import { SourceOperation, type AddressSourceRecord, type AddressSourceRegister, type LicenseDecision, type OperationDecision } from "#source-register/types";
+/**
+ * Resolves the path of the committed register through the package's `./data/*` export.
+ *
+ * Resolution happens on call so that importing this module works before the build has written the register.
+ */
+export declare function addressSourceRegisterPath(): string;
+/**
+ * Returns the license label that the exclude filter in `@mailwoman/corpus/utils/license` matches by prefix.
+ *
+ * The function returns `undefined` until someone has elected terms for the decision.
+ */
+export declare function electedLicenseLabel(decision: LicenseDecision): string | undefined;
+/**
+ * Replaces generated license decisions with recorded ones that share a license id.
+ *
+ * The build generates an `unchecked` decision for each license and rewrites the register
+ * whole, so reviewed decisions live in a separate input and merge here.
+ *
+ * @throws When a recorded decision refers to a license that the generated set lacks.
+ * Such a decision is a typo or refers to a removed source.
+ */
+export declare function applyLicenseDecisions(generated: readonly LicenseDecision[], recorded: ReadonlyMap<string, LicenseDecision>): LicenseDecision[];
+/**
+ * The operations that ingesting a source into the corpus performs.
+ *
+ * Redistribution and commercial sublicensing happen at release, so this list omits them.
+ */
+export declare const INGEST_OPERATIONS: readonly SourceOperation[];
+/**
+ * The operations that releasing a model trained on a source adds to {@link INGEST_OPERATIONS}.
+ */
+export declare const MODEL_RELEASE_OPERATIONS: readonly SourceOperation[];
+/**
+ * Returns the recorded permission for one operation under a license decision.
+ *
+ * The result is `unreviewed` when the decision is not elected or does not mention the operation.
+ * The selected terms establish which terms apply.
+ * Each operation still needs its own review.
+ */
+export declare function permissionFor(decision: LicenseDecision, operation: SourceOperation): OperationDecision;
+/**
+ * Returns every reason that a source may not enter a training corpus, or an empty array when it may.
+ *
+ * The function returns reasons so that callers can tell the different blockers apart.
+ */
+export declare function ingestEligibilityProblems(source: AddressSourceRecord, register: AddressSourceRegister, operations?: readonly SourceOperation[]): readonly string[];
+/**
+ * Returns one message for each structural problem in a register.
+ *
+ * The content digest is checked separately on read, because the build audits registers
+ * before it writes a digest.
+ */
+export declare function auditAddressSourceRegister(register: AddressSourceRegister): string[];
+/**
+ * Computes the SHA-256 digest of the register without its `contentDigest` field.
+ *
+ * The build and the read audit share this function so that they serialize identically.
+ * The digest covers the parsed object, so reformatting the file keeps it valid.
+ *
+ * Key order is part of the serialization, so reordering keys by hand changes the digest.
+ */
+export declare function registerContentDigest(register: AddressSourceRegister): string;
+/**
+ * Reads the committed register and checks its digest and structure.
+ *
+ * @throws When the file does not parse or the audit reports a problem.
+ * The message lists every problem.
+ */
+export declare function readAddressSourceRegister(path?: PathBuilderLike): Promise<AddressSourceRegister>;
+//# sourceMappingURL=register.d.ts.map

package/out/source-register/register.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"register.d.ts","sourceRoot":"","sources":["../../lib/source-register/register.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,OAAO,EAQN,eAAe,EAGf,KAAK,mBAAmB,EACxB,KAAK,qBAAqB,EAE1B,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,MAAM,wBAAwB,CAAA;AAG/B;;;;GAIG;AACH,wBAAgB,yBAAyB,IAAI,MAAM,CAElD;AAWD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,GAAG,SAAS,CAIjF;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,SAAS,eAAe,EAAE,EACrC,QAAQ,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,GAC5C,eAAe,EAAE,CAanB;AAED;;;;GAIG;AACH,eAAO,MAAM,iBAAiB,EAAE,SAAS,eAAe,EAKvD,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,SAAS,eAAe,EAG9D,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,eAAe,EAAE,SAAS,EAAE,eAAe,GAAG,iBAAiB,CActG;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACxC,MAAM,EAAE,mBAAmB,EAC3B,QAAQ,EAAE,qBAAqB,EAC/B,UAAU,GAAE,SAAS,eAAe,EAAsB,GACxD,SAAS,MAAM,EAAE,CA2CnB;AAyBD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,EAAE,CAiBpF;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,qBAAqB,GAAG,MAAM,CAI7E;AAmRD;;;;;GAKG;AACH,wBAAsB,yBAAyB,CAAC,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAUtG"}