@mailwoman/corpus 9.4.0 → 10.1.0

package/out/source-register/register.js

@@ -0,0 +1,400 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Reads and audits the address-source register.
+ */
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { sha256Hex } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { resolveModulePath } from "@mailwoman/core/module/resolvers";
+import { AssertedProposition } from "@mailwoman/evidence/status";
+import { BackboneState, JurisdictionResearchState, LicenseReviewState, OperationPermission, PersonalDataReading, REGISTER_SECTORS, SourceGeometry, SourceOperation, SourceStatus, UNRESOLVED_FIELDS, } from "#source-register/types";
+import { AddressRole } from "#types";
+/**
+ * Resolves the path of the committed register through the package's `./data/*` export.
+ *
+ * Resolution happens on call so that importing this module works before the build has written the register.
+ */
+export function addressSourceRegisterPath() {
+    return resolveModulePath("@mailwoman/corpus/data/address-source-register.json");
+}
+const PROPOSITIONS = new Set(Object.values(AssertedProposition));
+const SECTORS = new Set(REGISTER_SECTORS);
+const BACKBONE_STATES = new Set(Object.values(BackboneState));
+const RESEARCH_STATES = new Set(Object.values(JurisdictionResearchState));
+const SOURCE_STATUSES = new Set(Object.values(SourceStatus));
+const GEOMETRIES = new Set(Object.values(SourceGeometry));
+const PERSONAL_DATA_READINGS = new Set(Object.values(PersonalDataReading));
+const ADDRESS_ROLES = new Set(Object.values(AddressRole));
+/**
+ * Returns the license label that the exclude filter in `@mailwoman/corpus/utils/license` matches by prefix.
+ *
+ * The function returns `undefined` until someone has elected terms for the decision.
+ */
+export function electedLicenseLabel(decision) {
+    if (decision.state !== LicenseReviewState.Elected)
+        return undefined;
+    return decision.spdx ?? decision.electedTerms;
+}
+/**
+ * Replaces generated license decisions with recorded ones that share a license id.
+ *
+ * The build generates an `unchecked` decision for each license and rewrites the register
+ * whole, so reviewed decisions live in a separate input and merge here.
+ *
+ * @throws When a recorded decision refers to a license that the generated set lacks.
+ * Such a decision is a typo or refers to a removed source.
+ */
+export function applyLicenseDecisions(generated, recorded) {
+    const known = new Set(generated.map((decision) => decision.licenseID));
+    for (const licenseID of recorded.keys()) {
+        if (!known.has(licenseID)) {
+            throw new Error(`a recorded license decision names ${stringifyJSON(licenseID)}, which the register does not carry. ` +
+                "A decision that licenses nothing is a typo or a source that has been removed.");
+        }
+    }
+    return generated.map((decision) => recorded.get(decision.licenseID) ?? decision);
+}
+/**
+ * The operations that ingesting a source into the corpus performs.
+ *
+ * Redistribution and commercial sublicensing happen at release, so this list omits them.
+ */
+export const INGEST_OPERATIONS = [
+    SourceOperation.Fetch,
+    SourceOperation.Extract,
+    SourceOperation.Transform,
+    SourceOperation.Train,
+];
+/**
+ * The operations that releasing a model trained on a source adds to {@link INGEST_OPERATIONS}.
+ */
+export const MODEL_RELEASE_OPERATIONS = [
+    SourceOperation.RedistributeModel,
+    SourceOperation.CommercialSublicense,
+];
+/**
+ * Returns the recorded permission for one operation under a license decision.
+ *
+ * The result is `unreviewed` when the decision is not elected or does not mention the operation.
+ * The selected terms establish which terms apply.
+ * Each operation still needs its own review.
+ */
+export function permissionFor(decision, operation) {
+    if (decision.state !== LicenseReviewState.Elected) {
+        return {
+            permission: OperationPermission.Unreviewed,
+            because: `the decision reads ${decision.state}, so no operation has been read against terms`,
+        };
+    }
+    return (decision.operations?.[operation] ?? {
+        permission: OperationPermission.Unreviewed,
+        because: `the elected terms do not state whether ${operation} is permitted`,
+    });
+}
+/**
+ * Returns every reason that a source may not enter a training corpus, or an empty array when it may.
+ *
+ * The function returns reasons so that callers can tell the different blockers apart.
+ */
+export function ingestEligibilityProblems(source, register, operations = INGEST_OPERATIONS) {
+    const problems = [];
+    const decision = register.licenses.find((entry) => entry.licenseID === source.license);
+    if (!decision) {
+        problems.push(`license ${stringifyJSON(source.license)} is not declared in the register`);
+    }
+    else if (decision.state !== LicenseReviewState.Elected) {
+        problems.push(`license ${stringifyJSON(source.license)} is ${decision.state}, and only elected terms admit a source`);
+    }
+    else {
+        for (const operation of operations) {
+            const reading = permissionFor(decision, operation);
+            if (reading.permission === OperationPermission.Permitted)
+                continue;
+            problems.push(`the elected terms read ${reading.permission} for ${operation}: ${reading.because}`);
+        }
+    }
+    if (source.status === SourceStatus.RetainedOriginal) {
+        problems.push("the source was carried from the earlier memo and has not been re-resolved");
+    }
+    if (source.status === SourceStatus.VerifiedCorpusStale) {
+        problems.push("the bulk copy that was examined has stopped being updated");
+    }
+    if (source.status === SourceStatus.VerifiedAuthority) {
+        problems.push("the register was confirmed but its address fields and bulk access were not inspected");
+    }
+    if (!Object.keys(source.addressRoles ?? {}).length) {
+        problems.push("no address column has a resolved role, so the grammar the rows carry is unknown");
+    }
+    if (!source.coverage) {
+        problems.push("no coverage has been measured");
+    }
+    problems.push(...personalDataProblems(source));
+    return problems;
+}
+/**
+ * Returns the personal-data problems that block ingest of a source.
+ *
+ * A missing review blocks ingest, because an unexamined publication has not been found clear.
+ */
+function personalDataProblems(source) {
+    const review = source.personalDataReview;
+    if (!review) {
+        return ["no personal-data review is recorded, and an unexamined publication is not one found clear"];
+    }
+    if (review.reading === PersonalDataReading.Present) {
+        return [`the publication carries records about identifiable people and no analysis is complete: ${review.because}`];
+    }
+    if (review.reading === PersonalDataReading.Assessed && !review.record) {
+        return ["the personal-data review reads assessed and names no record, so the analysis cannot be read"];
+    }
+    return [];
+}
+/**
+ * Returns one message for each structural problem in a register.
+ *
+ * The content digest is checked separately on read, because the build audits registers
+ * before it writes a digest.
+ */
+export function auditAddressSourceRegister(register) {
+    const problems = [];
+    if (!register.version) {
+        problems.push("the register carries no version");
+    }
+    if (!register.jurisdictions.length) {
+        problems.push("the jurisdiction table is empty");
+    }
+    problems.push(...auditLicenses(register));
+    problems.push(...auditJurisdictions(register));
+    problems.push(...auditSources(register));
+    problems.push(...auditUnresolvedClaim(register));
+    return problems;
+}
+/**
+ * Computes the SHA-256 digest of the register without its `contentDigest` field.
+ *
+ * The build and the read audit share this function so that they serialize identically.
+ * The digest covers the parsed object, so reformatting the file keeps it valid.
+ *
+ * Key order is part of the serialization, so reordering keys by hand changes the digest.
+ */
+export function registerContentDigest(register) {
+    const { contentDigest: _omitted, ...rest } = register;
+    return sha256Hex(stringifyJSON(rest));
+}
+/**
+ * Checks that a register read from disk still matches the digest that its build wrote.
+ *
+ * Only the read path runs this check, because the build and test fixtures
+ * audit registers that have no digest.
+ */
+function auditContentDigest(register) {
+    if (!register.contentDigest) {
+        return [
+            "the register carries no `contentDigest`. It is written by `mailwoman corpus source-register`, and a register " +
+                "without one cannot be told apart from a hand-edited copy.",
+        ];
+    }
+    const expected = registerContentDigest(register);
+    if (expected === register.contentDigest)
+        return [];
+    return [
+        `the register's \`contentDigest\` reads ${register.contentDigest} and its content hashes to ${expected}. ` +
+            "Something edited the file after the build wrote it. Regenerate it with the command in " +
+            "`packages/corpus/data/PROVENANCE.md` rather than correcting the digest by hand.",
+    ];
+}
+function auditLicenses(register) {
+    const problems = [];
+    const seen = new Set();
+    const referenced = new Set(register.sources.map((source) => source.license));
+    for (const decision of register.licenses) {
+        const named = stringifyJSON(decision.licenseID);
+        if (seen.has(decision.licenseID)) {
+            problems.push(`license ${named} is declared twice`);
+        }
+        seen.add(decision.licenseID);
+        if (!referenced.has(decision.licenseID)) {
+            problems.push(`license ${named} is declared and no source points at it`);
+        }
+        switch (decision.state) {
+            case LicenseReviewState.Unchecked: {
+                if (!decision.publisherStatement) {
+                    problems.push(`license ${named} is unchecked and records no publisher statement`);
+                }
+                break;
+            }
+            case LicenseReviewState.Elected: {
+                problems.push(...auditElected(decision, named));
+                break;
+            }
+            case LicenseReviewState.Refused: {
+                if (!decision.refusedBecause) {
+                    problems.push(`license ${named} is refused and gives no reason`);
+                }
+                break;
+            }
+            default: {
+                problems.push(`license ${named} carries an unknown review state`);
+            }
+        }
+    }
+    return problems;
+}
+function auditElected(decision, named) {
+    const problems = [];
+    if (!decision.electedTerms) {
+        problems.push(`license ${named} is elected and names no terms`);
+    }
+    if (!decision.retrievedCopy) {
+        problems.push(`license ${named} is elected and names no retrieved copy`);
+    }
+    if (!decision.electedBecause) {
+        problems.push(`license ${named} is elected and gives no reason`);
+    }
+    for (const [operation, reading] of Object.entries(decision.operations ?? {})) {
+        if (!reading.because) {
+            problems.push(`license ${named} reads ${reading.permission} for ${operation} and gives no reason`);
+        }
+        // A permission must cite its basis in the terms so that a reviewer can check it.
+        // A refusal needs no basis.
+        if (reading.permission === OperationPermission.Permitted && !reading.basis) {
+            problems.push(`license ${named} permits ${operation} and names no basis for the permission`);
+        }
+        if (!Object.values(OperationPermission).includes(reading.permission)) {
+            problems.push(`license ${named} reads an unknown permission ${stringifyJSON(reading.permission)} for ${operation}`);
+        }
+        if (!Object.values(SourceOperation).includes(operation)) {
+            problems.push(`license ${named} names ${stringifyJSON(operation)}, which is not an operation this repository performs`);
+        }
+    }
+    return problems;
+}
+function auditJurisdictions(register) {
+    const problems = [];
+    const seen = new Set();
+    const sourceCounts = new Map();
+    for (const source of register.sources) {
+        sourceCounts.set(source.iso2, (sourceCounts.get(source.iso2) ?? 0) + 1);
+    }
+    for (const jurisdiction of register.jurisdictions) {
+        const named = stringifyJSON(jurisdiction.iso2);
+        if (seen.has(jurisdiction.iso2)) {
+            problems.push(`jurisdiction ${named} appears twice`);
+        }
+        seen.add(jurisdiction.iso2);
+        if (!BACKBONE_STATES.has(jurisdiction.backboneState)) {
+            problems.push(`jurisdiction ${named} carries an unknown backbone state`);
+        }
+        if (!RESEARCH_STATES.has(jurisdiction.researchState)) {
+            problems.push(`jurisdiction ${named} carries an unknown research state`);
+        }
+        const count = sourceCounts.get(jurisdiction.iso2) ?? 0;
+        const seeded = jurisdiction.researchState === JurisdictionResearchState.Seeded;
+        if (seeded && count === 0) {
+            problems.push(`jurisdiction ${named} is seeded and has no sources`);
+        }
+        if (!seeded && count > 0) {
+            problems.push(`jurisdiction ${named} is ${jurisdiction.researchState} and has ${count} sources`);
+        }
+        if (!seeded && !jurisdiction.stateReason) {
+            problems.push(`jurisdiction ${named} has no sources and does not say why`);
+        }
+    }
+    return problems;
+}
+function auditSources(register) {
+    const problems = [];
+    const known = new Set(register.jurisdictions.map((jurisdiction) => jurisdiction.iso2));
+    const declared = new Set(register.licenses.map((decision) => decision.licenseID));
+    const seen = new Set();
+    for (const source of register.sources) {
+        const named = stringifyJSON(source.sourceID);
+        if (seen.has(source.sourceID)) {
+            problems.push(`source ${named} appears twice`);
+        }
+        seen.add(source.sourceID);
+        if (!known.has(source.iso2)) {
+            problems.push(`source ${named} names jurisdiction ${stringifyJSON(source.iso2)}, which the jurisdiction table does not carry`);
+        }
+        if (!SECTORS.has(source.sector)) {
+            problems.push(`source ${named} names an unlisted sector`);
+        }
+        if (!SOURCE_STATUSES.has(source.status)) {
+            problems.push(`source ${named} carries an unknown status`);
+        }
+        if (!GEOMETRIES.has(source.geometry)) {
+            problems.push(`source ${named} carries an unknown geometry state`);
+        }
+        if (!declared.has(source.license)) {
+            problems.push(`source ${named} points at an undeclared license decision`);
+        }
+        if (source.addressRoles && !Object.keys(source.addressRoles).length) {
+            problems.push(`source ${named} carries an empty \`addressRoles\`, which reads as resolved and states no role. ` +
+                "Leave the field out until a column has one.");
+        }
+        for (const [column, role] of Object.entries(source.addressRoles ?? {})) {
+            if (!ADDRESS_ROLES.has(role)) {
+                problems.push(`source ${named} gives column ${stringifyJSON(column)} the role ${stringifyJSON(role)}, ` +
+                    "which is not an `AddressRole`");
+            }
+        }
+        const review = source.personalDataReview;
+        if (review && !PERSONAL_DATA_READINGS.has(review.reading)) {
+            problems.push(`source ${named} carries an unknown personal-data reading`);
+        }
+        if (review && !review.because) {
+            problems.push(`source ${named} carries a personal-data review stating no reason, so nothing records what it rests on`);
+        }
+        if (review?.reading === PersonalDataReading.Assessed && !review.record) {
+            problems.push(`source ${named} is assessed for personal data and names no record of the analysis`);
+        }
+        if (!source.asserts.length) {
+            problems.push(`source ${named} asserts nothing`);
+        }
+        for (const proposition of source.asserts) {
+            if (!PROPOSITIONS.has(proposition)) {
+                problems.push(`source ${named} asserts ${stringifyJSON(proposition)}, which is not a proposition`);
+            }
+        }
+    }
+    return problems;
+}
+function auditUnresolvedClaim(register) {
+    const problems = [];
+    const listed = new Set(register.unresolved);
+    for (const field of register.unresolved) {
+        if (!UNRESOLVED_FIELDS.includes(field)) {
+            problems.push(`${stringifyJSON(field)} is named unresolved and is not a field of a source`);
+        }
+    }
+    for (const field of UNRESOLVED_FIELDS) {
+        const resolved = register.sources.filter((source) => source[field] !== undefined);
+        if (listed.has(field) && resolved.length) {
+            problems.push(`${stringifyJSON(field)} is declared unresolved and ${resolved.length} sources carry it — ` +
+                `drop it from \`unresolved\``);
+        }
+        if (!listed.has(field) && !resolved.length) {
+            problems.push(`${stringifyJSON(field)} is not declared unresolved and no source carries it`);
+        }
+    }
+    return problems;
+}
+/**
+ * Reads the committed register and checks its digest and structure.
+ *
+ * @throws When the file does not parse or the audit reports a problem.
+ * The message lists every problem.
+ */
+export async function readAddressSourceRegister(path) {
+    const resolved = path ?? addressSourceRegisterPath();
+    const register = await readLocalJSONFile(resolved);
+    const problems = [...auditContentDigest(register), ...auditAddressSourceRegister(register)];
+    if (problems.length) {
+        throw new Error(`${resolved} failed the register audit:\n  ${problems.join("\n  ")}`);
+    }
+    return register;
+}
+//# sourceMappingURL=register.js.map

package/out/source-register/register.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"register.js","sourceRoot":"","sources":["../../lib/source-register/register.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAGhE,OAAO,EACN,aAAa,EACb,yBAAyB,EACzB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,YAAY,EACZ,iBAAiB,GAMjB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAEpC;;;;GAIG;AACH,MAAM,UAAU,yBAAyB;IACxC,OAAO,iBAAiB,CAAC,qDAAqD,CAAC,CAAA;AAChF,CAAC;AAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAA;AACxE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,gBAAgB,CAAC,CAAA;AACjD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA;AACrE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAC,CAAA;AACjF,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAA;AACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAA;AACjE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC,CAAA;AAClF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAA;AAEjE;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAyB;IAC5D,IAAI,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO;QAAE,OAAO,SAAS,CAAA;IAEnE,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,YAAY,CAAA;AAC9C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAqC,EACrC,QAA8C;IAE9C,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACd,qCAAqC,aAAa,CAAC,SAAS,CAAC,uCAAuC;gBACnG,+EAA+E,CAChF,CAAA;QACF,CAAC;IACF,CAAC;IAED,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,CAAA;AACjF,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA+B;IAC5D,eAAe,CAAC,KAAK;IACrB,eAAe,CAAC,OAAO;IACvB,eAAe,CAAC,SAAS;IACzB,eAAe,CAAC,KAAK;CACrB,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA+B;IACnE,eAAe,CAAC,iBAAiB;IACjC,eAAe,CAAC,oBAAoB;CACpC,CAAA;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,QAAyB,EAAE,SAA0B;IAClF,IAAI,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO,EAAE,CAAC;QACnD,OAAO;YACN,UAAU,EAAE,mBAAmB,CAAC,UAAU;YAC1C,OAAO,EAAE,sBAAsB,QAAQ,CAAC,KAAK,+CAA+C;SAC5F,CAAA;IACF,CAAC;IAED,OAAO,CACN,QAAQ,CAAC,UAAU,EAAE,CAAC,SAAS,CAAC,IAAI;QACnC,UAAU,EAAE,mBAAmB,CAAC,UAAU;QAC1C,OAAO,EAAE,0CAA0C,SAAS,eAAe;KAC3E,CACD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,yBAAyB,CACxC,MAA2B,EAC3B,QAA+B,EAC/B,UAAU,GAA+B,iBAAiB;IAE1D,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,OAAO,CAAC,CAAA;IAEtF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,QAAQ,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAA;IAC1F,CAAC;SAAM,IAAI,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO,EAAE,CAAC;QAC1D,QAAQ,CAAC,IAAI,CACZ,WAAW,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,QAAQ,CAAC,KAAK,yCAAyC,CACtG,CAAA;IACF,CAAC;SAAM,CAAC;QACP,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YAElD,IAAI,OAAO,CAAC,UAAU,KAAK,mBAAmB,CAAC,SAAS;gBAAE,SAAQ;YAElE,QAAQ,CAAC,IAAI,CAAC,0BAA0B,OAAO,CAAC,UAAU,QAAQ,SAAS,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;QACnG,CAAC;IACF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,gBAAgB,EAAE,CAAC;QACrD,QAAQ,CAAC,IAAI,CAAC,2EAA2E,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,mBAAmB,EAAE,CAAC;QACxD,QAAQ,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAA;IAC3E,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,CAAC,iBAAiB,EAAE,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,sFAAsF,CAAC,CAAA;IACtG,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACpD,QAAQ,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAA;IACjG,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAA;IAC/C,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAA;IAE9C,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,MAA2B;IACxD,MAAM,MAAM,GAAG,MAAM,CAAC,kBAAkB,CAAA;IAExC,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,OAAO,CAAC,2FAA2F,CAAC,CAAA;IACrG,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,mBAAmB,CAAC,OAAO,EAAE,CAAC;QACpD,OAAO,CAAC,0FAA0F,MAAM,CAAC,OAAO,EAAE,CAAC,CAAA;IACpH,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,KAAK,mBAAmB,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACvE,OAAO,CAAC,6FAA6F,CAAC,CAAA;IACvG,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,QAA+B;IACzE,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;IACjD,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAA;IACjD,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAA;IACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAA;IAC9C,QAAQ,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAA;IACxC,QAAQ,CAAC,IAAI,CAAC,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAA;IAEhD,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAA+B;IACpE,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAA;IAErD,OAAO,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAA;AACtC,CAAC;AAED;;;;;GAKG;AACH,SAAS,kBAAkB,CAAC,QAA+B;IAC1D,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC7B,OAAO;YACN,+GAA+G;gBAC9G,2DAA2D;SAC5D,CAAA;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAA;IAEhD,IAAI,QAAQ,KAAK,QAAQ,CAAC,aAAa;QAAE,OAAO,EAAE,CAAA;IAElD,OAAO;QACN,0CAA0C,QAAQ,CAAC,aAAa,8BAA8B,QAAQ,IAAI;YACzG,wFAAwF;YACxF,iFAAiF;KAClF,CAAA;AACF,CAAC;AAED,SAAS,aAAa,CAAC,QAA+B;IACrD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;IAE5E,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;QAE/C,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,oBAAoB,CAAC,CAAA;QACpD,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAA;QAE5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,yCAAyC,CAAC,CAAA;QACzE,CAAC;QAED,QAAQ,QAAQ,CAAC,KAAK,EAAE,CAAC;YACxB,KAAK,kBAAkB,CAAC,SAAS,EAAE,CAAC;gBACnC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC;oBAClC,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,kDAAkD,CAAC,CAAA;gBAClF,CAAC;gBAED,MAAK;YACN,CAAC;YAED,KAAK,kBAAkB,CAAC,OAAO,EAAE,CAAC;gBACjC,QAAQ,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAE/C,MAAK;YACN,CAAC;YAED,KAAK,kBAAkB,CAAC,OAAO,EAAE,CAAC;gBACjC,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;oBAC9B,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,iCAAiC,CAAC,CAAA;gBACjE,CAAC;gBAED,MAAK;YACN,CAAC;YAED,SAAS,CAAC;gBACT,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,kCAAkC,CAAC,CAAA;YAClE,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,QAAwB,EAAE,KAAa;IAC5D,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,gCAAgC,CAAC,CAAA;IAChE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,yCAAyC,CAAC,CAAA;IACzE,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC9B,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,iCAAiC,CAAC,CAAA;IACjE,CAAC;IAED,KAAK,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;QAC9E,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,UAAU,OAAO,CAAC,UAAU,QAAQ,SAAS,sBAAsB,CAAC,CAAA;QACnG,CAAC;QAED,iFAAiF;QACjF,4BAA4B;QAC5B,IAAI,OAAO,CAAC,UAAU,KAAK,mBAAmB,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC5E,QAAQ,CAAC,IAAI,CAAC,WAAW,KAAK,YAAY,SAAS,wCAAwC,CAAC,CAAA;QAC7F,CAAC;QAED,IAAI,CAAE,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAc,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YACpF,QAAQ,CAAC,IAAI,CACZ,WAAW,KAAK,gCAAgC,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,SAAS,EAAE,CACpG,CAAA;QACF,CAAC;QAED,IAAI,CAAE,MAAM,CAAC,MAAM,CAAC,eAAe,CAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvE,QAAQ,CAAC,IAAI,CACZ,WAAW,KAAK,UAAU,aAAa,CAAC,SAAS,CAAC,sDAAsD,CACxG,CAAA;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,QAA+B;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE9C,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACxE,CAAC;IAED,KAAK,MAAM,YAAY,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,aAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QAE9C,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,gBAAgB,CAAC,CAAA;QACrD,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QAE3B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,oCAAoC,CAAC,CAAA;QACzE,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,YAAY,CAAC,aAAa,CAAC,EAAE,CAAC;YACtD,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,oCAAoC,CAAC,CAAA;QACzE,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtD,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,KAAK,yBAAyB,CAAC,MAAM,CAAA;QAE9E,IAAI,MAAM,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,+BAA+B,CAAC,CAAA;QACpE,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,OAAO,YAAY,CAAC,aAAa,YAAY,KAAK,UAAU,CAAC,CAAA;QACjG,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CAAC,gBAAgB,KAAK,sCAAsC,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,QAA+B;IACpD,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAA;IACtF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAA;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAE5C,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,gBAAgB,CAAC,CAAA;QAC/C,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAEzB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CACZ,UAAU,KAAK,uBAAuB,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAC/G,CAAA;QACF,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,2BAA2B,CAAC,CAAA;QAC1D,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,4BAA4B,CAAC,CAAA;QAC3D,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtC,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,oCAAoC,CAAC,CAAA;QACnE,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACnC,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,2CAA2C,CAAC,CAAA;QAC1E,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,CAAC;YACrE,QAAQ,CAAC,IAAI,CACZ,UAAU,KAAK,kFAAkF;gBAChG,6CAA6C,CAC9C,CAAA;QACF,CAAC;QAED,KAAK,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CAAC;YACxE,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9B,QAAQ,CAAC,IAAI,CACZ,UAAU,KAAK,iBAAiB,aAAa,CAAC,MAAM,CAAC,aAAa,aAAa,CAAC,IAAI,CAAC,IAAI;oBACxF,+BAA+B,CAChC,CAAA;YACF,CAAC;QACF,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,kBAAkB,CAAA;QAExC,IAAI,MAAM,IAAI,CAAC,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3D,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,2CAA2C,CAAC,CAAA;QAC1E,CAAC;QAED,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CACZ,UAAU,KAAK,wFAAwF,CACvG,CAAA;QACF,CAAC;QAED,IAAI,MAAM,EAAE,OAAO,KAAK,mBAAmB,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxE,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,oEAAoE,CAAC,CAAA;QACnG,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,kBAAkB,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAC1C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpC,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,YAAY,aAAa,CAAC,WAAW,CAAC,8BAA8B,CAAC,CAAA;YACnG,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA+B;IAC5D,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,QAAQ,CAAC,UAAU,CAAC,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAA;QAC5F,CAAC;IACF,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC,CAAA;QAEjF,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC1C,QAAQ,CAAC,IAAI,CACZ,GAAG,aAAa,CAAC,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,sBAAsB;gBAC1F,6BAA6B,CAC9B,CAAA;QACF,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;QAC7F,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAsB;IACrE,MAAM,QAAQ,GAAG,IAAI,IAAI,yBAAyB,EAAE,CAAA;IACpD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAwB,QAAQ,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,CAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,EAAE,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE3F,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,kCAAkC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/source-register/training-manifest.d.ts

@@ -0,0 +1,111 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The frozen record of sources that contributed rows to one corpus build.
+ *   It also records the applicable terms.
+ *
+ *   Its digest is over the manifest with `contentDigest` emptied, so two builds over identical inputs
+ *   produce the same digest. A change to a source, its elected grant, row count or recipe changes the digest.
+ */
+import { LicenseReviewState, type LicenseDecision, type SourceOperation } from "#source-register/types";
+/**
+ * What one source contributed to a build.
+ *
+ * Also records what the build had established about its terms.
+ */
+export interface TrainingSourceRecord {
+    /**
+     * The adapter id stamped into every row's `source`, and the join key to the register's `sourceID`.
+     */
+    source: string;
+    /**
+     * Rows this source contributed after license exclusion and eligibility but before augmentation.
+     * A synthetic row counts under the source it was fanned from.
+     */
+    rows: number;
+    license: string;
+    /**
+     * The register's decision for that license when the build ran, or `null`
+     * when the register contains no decision for it.
+     */
+    decision: {
+        licenseID: string;
+        state: LicenseReviewState;
+        electedTerms: string | null;
+        /**
+         * The operations the elected grant permitted at build time.
+         *
+         * An operation absent here was `unreviewed` or `refused`, which the decision records separately.
+         */
+        permitted: SourceOperation[];
+    } | null;
+}
+/**
+ * One corpus build's frozen source record.
+ */
+export interface TrainingManifest {
+    manifestID: "corpus-training-manifest";
+    schemaVersion: 1;
+    corpusVersion: string;
+    builtAt: string;
+    /**
+     * The build profile, so a reader can tell a corpus whose sources were checked
+     * from one whose sources were not.
+     */
+    profile: string;
+    /**
+     * Sources in descending row order.
+     */
+    sources: TrainingSourceRecord[];
+    /**
+     * Sources the build refused, with the reasons, so the record shows what was
+     * left out as well as what went in.
+     *
+     * Each key is `<adapter id>:<ISO 3166-1 alpha-2>`, the pair that addresses one register source,
+     * because one adapter can emit several jurisdictions and each carries its own license decision.
+     */
+    refused: Record<string, readonly string[]>;
+    totalRows: number;
+    /**
+     * Sha256 over this manifest with `contentDigest` emptied.
+     */
+    contentDigest: string;
+}
+/**
+ * The digest a manifest should include, computed over the manifest with its digest field emptied.
+ */
+export declare function trainingManifestDigest(manifest: TrainingManifest): string;
+/**
+ * Freeze one build's source observations into a manifest.
+ *
+ * A license absent from the decision table records `decision: null` rather than being omitted,
+ * so a build that read an unregistered adapter stays distinguishable from one that read no source.
+ */
+export declare function freezeTrainingManifest(input: {
+    corpusVersion: string;
+    builtAt: string;
+    profile: string;
+    rowsBySource: ReadonlyMap<string, {
+        rows: number;
+        license: string;
+    }>;
+    decisionsByLicense: ReadonlyMap<string, LicenseDecision>;
+    refused: ReadonlyMap<string, readonly string[]>;
+}): TrainingManifest;
+/**
+ * Everything wrong with a training manifest, one message per problem.
+ *
+ * The digest check is what makes the record frozen rather than merely written.
+ */
+export declare function auditTrainingManifest(manifest: TrainingManifest): string[];
+/**
+ * The sources in a manifest whose terms did not permit an operation
+ * when the build ran, with the reason for each.
+ */
+export declare function sourcesNotPermitting(manifest: TrainingManifest, operation: SourceOperation): Array<{
+    source: string;
+    because: string;
+}>;
+//# sourceMappingURL=training-manifest.d.ts.map

package/out/source-register/training-manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-manifest.d.ts","sourceRoot":"","sources":["../../lib/source-register/training-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,OAAO,EAAE,kBAAkB,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAEvG;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,QAAQ,EAAE;QACT,SAAS,EAAE,MAAM,CAAA;QACjB,KAAK,EAAE,kBAAkB,CAAA;QACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;QAC3B;;;;WAIG;QACH,SAAS,EAAE,eAAe,EAAE,CAAA;KAC5B,GAAG,IAAI,CAAA;CACR;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,UAAU,EAAE,0BAA0B,CAAA;IACtC,aAAa,EAAE,CAAC,CAAA;IAChB,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC/B;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAC1C,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,CAEzE;AAED;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE;IAC7C,aAAa,EAAE,MAAM,CAAA;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,WAAW,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACpE,kBAAkB,EAAE,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IACxD,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC/C,GAAG,gBAAgB,CAwCnB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,EAAE,CA4B1E;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CACnC,QAAQ,EAAE,gBAAgB,EAC1B,SAAS,EAAE,eAAe,GACxB,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CA+B5C"}

package/out/source-register/training-manifest.js

@@ -0,0 +1,120 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The frozen record of sources that contributed rows to one corpus build.
+ *   It also records the applicable terms.
+ *
+ *   Its digest is over the manifest with `contentDigest` emptied, so two builds over identical inputs
+ *   produce the same digest. A change to a source, its elected grant, row count or recipe changes the digest.
+ */
+import { sha256Hex } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { LicenseReviewState } from "#source-register/types";
+/**
+ * The digest a manifest should include, computed over the manifest with its digest field emptied.
+ */
+export function trainingManifestDigest(manifest) {
+    return sha256Hex(stringifyJSON({ ...manifest, contentDigest: "" }));
+}
+/**
+ * Freeze one build's source observations into a manifest.
+ *
+ * A license absent from the decision table records `decision: null` rather than being omitted,
+ * so a build that read an unregistered adapter stays distinguishable from one that read no source.
+ */
+export function freezeTrainingManifest(input) {
+    const sources = [...input.rowsBySource.entries()]
+        .map(([source, { rows, license }]) => {
+        const decision = input.decisionsByLicense.get(license);
+        return {
+            source,
+            rows,
+            license,
+            decision: decision
+                ? {
+                    licenseID: decision.licenseID,
+                    state: decision.state,
+                    electedTerms: decision.state === LicenseReviewState.Elected ? decision.electedTerms : null,
+                    permitted: decision.state === LicenseReviewState.Elected
+                        ? Object.entries(decision.operations ?? {})
+                            .filter(([, reading]) => reading.permission === "permitted")
+                            .map(([operation]) => operation)
+                            .toSorted()
+                        : [],
+                }
+                : null,
+        };
+    })
+        .toSorted((left, right) => right.rows - left.rows || left.source.localeCompare(right.source));
+    const manifest = {
+        manifestID: "corpus-training-manifest",
+        schemaVersion: 1,
+        corpusVersion: input.corpusVersion,
+        builtAt: input.builtAt,
+        profile: input.profile,
+        sources,
+        refused: Object.fromEntries([...input.refused.entries()].toSorted(([a], [b]) => a.localeCompare(b))),
+        totalRows: sources.reduce((total, record) => total + record.rows, 0),
+        contentDigest: "",
+    };
+    return { ...manifest, contentDigest: trainingManifestDigest(manifest) };
+}
+/**
+ * Everything wrong with a training manifest, one message per problem.
+ *
+ * The digest check is what makes the record frozen rather than merely written.
+ */
+export function auditTrainingManifest(manifest) {
+    const problems = [];
+    if (manifest.contentDigest !== trainingManifestDigest(manifest)) {
+        problems.push("the manifest's content digest does not match its contents, so something edited it after its build");
+    }
+    const counted = manifest.sources.reduce((total, record) => total + record.rows, 0);
+    if (counted !== manifest.totalRows) {
+        problems.push(`the manifest totals ${manifest.totalRows} rows and its sources carry ${counted}`);
+    }
+    const seen = new Set();
+    for (const record of manifest.sources) {
+        if (seen.has(record.source)) {
+            problems.push(`source ${stringifyJSON(record.source)} appears more than once`);
+        }
+        seen.add(record.source);
+        if (record.rows <= 0) {
+            problems.push(`source ${stringifyJSON(record.source)} contributed ${record.rows} rows and should not be listed`);
+        }
+    }
+    return problems;
+}
+/**
+ * The sources in a manifest whose terms did not permit an operation
+ * when the build ran, with the reason for each.
+ */
+export function sourcesNotPermitting(manifest, operation) {
+    const blocking = [];
+    for (const record of manifest.sources) {
+        if (!record.decision) {
+            blocking.push({
+                source: record.source,
+                because: `the register carried no decision for license ${stringifyJSON(record.license)} when the corpus was built`,
+            });
+            continue;
+        }
+        if (record.decision.state !== LicenseReviewState.Elected) {
+            blocking.push({
+                source: record.source,
+                because: `its license decision read ${record.decision.state} when the corpus was built`,
+            });
+            continue;
+        }
+        if (!record.decision.permitted.includes(operation)) {
+            blocking.push({
+                source: record.source,
+                because: `the elected terms ${stringifyJSON(record.decision.electedTerms ?? "")} did not permit ${operation}`,
+            });
+        }
+    }
+    return blocking;
+}
+//# sourceMappingURL=training-manifest.js.map

package/out/source-register/training-manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-manifest.js","sourceRoot":"","sources":["../../lib/source-register/training-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAEpD,OAAO,EAAE,kBAAkB,EAA8C,MAAM,wBAAwB,CAAA;AAmEvG;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAA0B;IAChE,OAAO,SAAS,CAAC,aAAa,CAAC,EAAE,GAAG,QAAQ,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;AACpE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAOtC;IACA,MAAM,OAAO,GAA2B,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;SACvE,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAwB,EAAE;QAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAEtD,OAAO;YACN,MAAM;YACN,IAAI;YACJ,OAAO;YACP,QAAQ,EAAE,QAAQ;gBACjB,CAAC,CAAC;oBACA,SAAS,EAAE,QAAQ,CAAC,SAAS;oBAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;oBACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;oBAC1F,SAAS,EACR,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO;wBAC5C,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC;6BACxC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,WAAW,CAAC;6BAC3D,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,SAA4B,CAAC;6BAClD,QAAQ,EAAE;wBACb,CAAC,CAAC,EAAE;iBACN;gBACF,CAAC,CAAC,IAAI;SACP,CAAA;IACF,CAAC,CAAC;SACD,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IAE9F,MAAM,QAAQ,GAAqB;QAClC,UAAU,EAAE,0BAA0B;QACtC,aAAa,EAAE,CAAC;QAChB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO;QACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;QACpG,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACpE,aAAa,EAAE,EAAE;KACjB,CAAA;IAED,OAAO,EAAE,GAAG,QAAQ,EAAE,aAAa,EAAE,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAA;AACxE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAA0B;IAC/D,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,IAAI,QAAQ,CAAC,aAAa,KAAK,sBAAsB,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,mGAAmG,CAAC,CAAA;IACnH,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAElF,IAAI,OAAO,KAAK,QAAQ,CAAC,SAAS,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,QAAQ,CAAC,SAAS,+BAA+B,OAAO,EAAE,CAAC,CAAA;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7B,QAAQ,CAAC,IAAI,CAAC,UAAU,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,yBAAyB,CAAC,CAAA;QAC/E,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAEvB,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,UAAU,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,gBAAgB,MAAM,CAAC,IAAI,gCAAgC,CAAC,CAAA;QACjH,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CACnC,QAA0B,EAC1B,SAA0B;IAE1B,MAAM,QAAQ,GAA+C,EAAE,CAAA;IAE/D,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,gDAAgD,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,4BAA4B;aAClH,CAAC,CAAA;YAEF,SAAQ;QACT,CAAC;QAED,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO,EAAE,CAAC;YAC1D,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,6BAA6B,MAAM,CAAC,QAAQ,CAAC,KAAK,4BAA4B;aACvF,CAAC,CAAA;YAEF,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACpD,QAAQ,CAAC,IAAI,CAAC;gBACb,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,OAAO,EAAE,qBAAqB,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC,mBAAmB,SAAS,EAAE;aAC7G,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}