@mailwoman/corpus 10.6.0 → 10.7.0

package/out/source-register/corpus-inputs.d.ts

@@ -0,0 +1,240 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Joins a corpus source id to the register sources whose data its rows are built from.
+ *
+ *   A corpus source id is the `source` value stored on every corpus row, keyed by a training config's
+ *   `source_weights` and listed in a training manifest. The register scopes a source to one publisher in one
+ *   jurisdiction and records its license decision. This table is the one record joining the two. Before it, a source whose
+ *   register row was refused or never read could feed a corpus build and a training run: HM Land Registry Price Paid
+ *   Data reached `rendered-gb` and `spliced-sub-venue` while its row read `unchecked` (#2635).
+ *
+ *   The table is `packages/corpus/data/corpus-source-inputs.json`. A derived or synthetic source lists every input
+ *   whose data reaches one of its rows. An input with no register row gets one in `source-additions.json`, under an
+ *   `unchecked` decision, so the gap reads as a gap rather than as an absence.
+ */
+import type { PathBuilderLike } from "path-ts";
+import { SourceOperation, type AddressSourceRegister, type LicenseDecision } from "#source-register/types";
+/**
+ * The operations a register decision must permit before a source's rows may
+ * be built into a corpus or trained on.
+ *
+ * Both need adaptation: normalizing an address into labeled components is a derived work,
+ * and fitting a model on it is another.
+ */
+export declare const TRAINING_OPERATIONS: readonly SourceOperation[];
+/**
+ * One register source whose data reaches a corpus source's rows.
+ */
+export interface CorpusSourceInput {
+    /**
+     * The register's `sourceID`.
+     */
+    registerSource: string;
+    /**
+     * The `licenseID` that register source points at, repeated so a reviewer reads the decision key here.
+     *
+     * {@linkcode auditCorpusSourceInputs} refuses a value that disagrees with the register.
+     */
+    license: string;
+    /**
+     * What the corpus source reads from it: the file, the fields and the component they fill.
+     */
+    reads: string;
+}
+/**
+ * Everything one corpus source's rows are built from.
+ */
+export interface CorpusSourceMapping {
+    /**
+     * Other spellings rows of this source are stored under, such as the retired `synth-*` spelling.
+     */
+    aliases?: readonly string[];
+    /**
+     * Every register source whose data reaches a row.
+     */
+    inputs: readonly CorpusSourceInput[];
+    /**
+     * Inputs written in this repository, which need no register row: a hand-written word list, a template.
+     */
+    authored?: readonly string[];
+    note?: string;
+    /**
+     * What the source read in corpora built before its producer changed inputs under the same id.
+     *
+     * `inputs` describes what a build reads today.
+     * The build and training checks read it.
+     *
+     * A corpus already built keeps the rows it was built from, so the release audit reads the
+     * entry naming that corpus: `spliced-sub-venue` kept its name when its GB skeletons moved from
+     * Price Paid Data to OS OpenData, and the published model trained on the Price Paid Data rows.
+     */
+    history?: readonly CorpusSourceHistory[];
+}
+/**
+ * The inputs one source read in the corpora named.
+ */
+export interface CorpusSourceHistory {
+    corpora: readonly string[];
+    inputs: readonly CorpusSourceInput[];
+    because: string;
+}
+/**
+ * The committed table.
+ */
+export interface CorpusSourceInputsFile {
+    sources: Readonly<Record<string, CorpusSourceMapping>>;
+}
+/**
+ * The table indexed for lookup under every spelling it records.
+ */
+export interface CorpusSourceInputs {
+    readonly file: CorpusSourceInputsFile;
+    /**
+     * The mapping a spelling resolves to, with the id it is filed under.
+     */
+    resolve(source: string): {
+        id: string;
+        mapping: CorpusSourceMapping;
+    } | null;
+}
+/**
+ * Resolves the committed table through the package's `./data/*` export.
+ */
+export declare function corpusSourceInputsPath(): string;
+/**
+ * Indexes a table by id and alias.
+ *
+ * @throws When two entries claim one spelling, because a lookup would then answer for whichever came last.
+ */
+export declare function indexCorpusSourceInputs(file: CorpusSourceInputsFile): CorpusSourceInputs;
+/**
+ * One message per structural problem in the table, read against the register.
+ */
+export declare function auditCorpusSourceInputs(file: CorpusSourceInputsFile, register: AddressSourceRegister): string[];
+/**
+ * Reads the committed table and checks it against the register.
+ *
+ * @throws When the file does not parse or the audit reports a problem.
+ */
+export declare function readCorpusSourceInputs(register: AddressSourceRegister, path?: PathBuilderLike): Promise<CorpusSourceInputs>;
+/**
+ * Why one corpus source lacks the permission {@linkcode TRAINING_OPERATIONS} need.
+ */
+export interface SourceRightsGap {
+    /**
+     * The corpus source id as the caller named it.
+     */
+    source: string;
+    /**
+     * The register source lacking the permission, or `null` when the table has no entry for the corpus source.
+     */
+    registerSource: string | null;
+    licenseID: string | null;
+    /**
+     * The decision's review state, or `null` when there is no decision to read.
+     */
+    state: LicenseDecision["state"] | null;
+    /**
+     * The operations the decision does not permit.
+     */
+    missing: SourceOperation[];
+    because: string;
+}
+/**
+ * Every reason a corpus source's rows may not be built into a corpus or trained on.
+ *
+ * An empty array is the only answer a caller may read as permission.
+ * A source the table does not map answers one gap whose `registerSource` is `null`,
+ * because an unmapped source is one nobody has traced to its publishers.
+ *
+ * `corpusVersion` names a corpus already built, so a source whose inputs changed under
+ * the same id is read against what that corpus's rows were built from.
+ * A build or training check omits it and reads today's inputs.
+ */
+export declare function sourceRightsGaps(source: string, inputs: CorpusSourceInputs, register: AddressSourceRegister, corpusVersion?: string): SourceRightsGap[];
+/**
+ * One line naming the corpus source, the register row and the missing permission.
+ */
+export declare function describeSourceRightsGap(gap: SourceRightsGap): string;
+/**
+ * Raised when a build or training run would admit a source lacking the permission.
+ */
+export declare class SourceRightsError extends Error {
+    readonly gaps: readonly SourceRightsGap[];
+    constructor(context: string, gaps: readonly SourceRightsGap[]);
+}
+/**
+ * What a checked output records about the sources it admitted.
+ *
+ * Its field names are snake_case because it is written into corpus manifests,
+ * which a Python reader also reads.
+ */
+export interface SourceRightsRecord {
+    /**
+     * False when an override admitted a source lacking the permission.
+     */
+    shippable: boolean;
+    unshippable_reason: string | null;
+    /**
+     * Every corpus source the check read, sorted.
+     */
+    sources_checked: string[];
+    gaps: SourceRightsGap[];
+    /**
+     * Inputs whose corpus sources could not be read, so the check could not judge them,
+     * such as a base corpus whose files name no source and whose manifest carries no record.
+     */
+    unverified: string[];
+}
+/**
+ * Checks corpus sources one at a time as an output admits them.
+ */
+export interface SourceRightsAdmission {
+    /**
+     * Admits a source or throws {@linkcode SourceRightsError}.
+     *
+     * Under an override it logs each gap once and returns them instead.
+     */
+    admit(source: string): readonly SourceRightsGap[];
+    /**
+     * Admits several sources at once, so a refusal names every one of them.
+     */
+    admitAll(sources: Iterable<string>): readonly SourceRightsGap[];
+    /**
+     * Refuses an input whose sources cannot be read, or records it under an override.
+     *
+     * An input the check cannot read stays unverified rather than clear.
+     */
+    admitUnverified(description: string): void;
+    record(): SourceRightsRecord;
+}
+/**
+ * Options for {@linkcode createSourceRightsAdmission}.
+ */
+export interface SourceRightsAdmissionOptions {
+    /**
+     * Names the output in a refusal, as `corpus build v0.7.7` or `recipe sub-venue`.
+     */
+    context: string;
+    /**
+     * Why a local experiment admits sources lacking the permission.
+     *
+     * Setting it marks the output unshippable rather than relaxing the rule.
+     */
+    unshippableReason?: string | null;
+    log?: (line: string) => void;
+}
+/**
+ * Builds the admission check over an already-read register and table.
+ *
+ * @throws When `unshippableReason` is set and blank, because an override must say why.
+ */
+export declare function createSourceRightsAdmission(register: AddressSourceRegister, inputs: CorpusSourceInputs, options: SourceRightsAdmissionOptions): SourceRightsAdmission;
+/**
+ * Reads the committed register and table and builds the admission check over them.
+ */
+export declare function openSourceRightsAdmission(options: SourceRightsAdmissionOptions): Promise<SourceRightsAdmission>;
+//# sourceMappingURL=corpus-inputs.d.ts.map

package/out/source-register/corpus-inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-inputs.d.ts","sourceRoot":"","sources":["../../lib/source-register/corpus-inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9C,OAAO,EAGN,eAAe,EACf,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EACpB,MAAM,wBAAwB,CAAA;AAE/B;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,EAAE,SAAS,eAAe,EAAuD,CAAA;AAEjH;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;CACb;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;OAEG;IACH,MAAM,EAAE,SAAS,iBAAiB,EAAE,CAAA;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;;;;;;OASG;IACH,OAAO,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAA;CACxC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE,SAAS,iBAAiB,EAAE,CAAA;IACpC,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAA;CACtD;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAA;IACrC;;OAEG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,mBAAmB,CAAA;KAAE,GAAG,IAAI,CAAA;CAC5E;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,kBAAkB,CAuBxF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,qBAAqB,GAAG,MAAM,EAAE,CA2C/G;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,QAAQ,EAAE,qBAAqB,EAC/B,IAAI,CAAC,EAAE,eAAe,GACpB,OAAO,CAAC,kBAAkB,CAAC,CAU7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB;;OAEG;IACH,KAAK,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;IACtC;;OAEG;IACH,OAAO,EAAE,eAAe,EAAE,CAAA;IAC1B,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAC/B,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,kBAAkB,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,aAAa,CAAC,EAAE,MAAM,GACpB,eAAe,EAAE,CAoEnB;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,eAAe,GAAG,MAAM,CAWpE;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;IAC3C,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,CAAA;IAEzC,YAAY,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,eAAe,EAAE,EAY5D;CACD;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,OAAO,CAAA;IAClB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC;;OAEG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,IAAI,EAAE,eAAe,EAAE,CAAA;IACvB;;;OAGG;IACH,UAAU,EAAE,MAAM,EAAE,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;;;OAIG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,eAAe,EAAE,CAAA;IACjD;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,SAAS,eAAe,EAAE,CAAA;IAC/D;;;;OAIG;IACH,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1C,MAAM,IAAI,kBAAkB,CAAA;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACjC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAC5B;AAED;;;;GAIG;AACH,wBAAgB,2BAA2B,CAC1C,QAAQ,EAAE,qBAAqB,EAC/B,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,4BAA4B,GACnC,qBAAqB,CA0EvB;AAED;;GAEG;AACH,wBAAsB,yBAAyB,CAAC,OAAO,EAAE,4BAA4B,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAKrH"}

package/out/source-register/corpus-inputs.js

@@ -0,0 +1,275 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Joins a corpus source id to the register sources whose data its rows are built from.
+ *
+ *   A corpus source id is the `source` value stored on every corpus row, keyed by a training config's
+ *   `source_weights` and listed in a training manifest. The register scopes a source to one publisher in one
+ *   jurisdiction and records its license decision. This table is the one record joining the two. Before it, a source whose
+ *   register row was refused or never read could feed a corpus build and a training run: HM Land Registry Price Paid
+ *   Data reached `rendered-gb` and `spliced-sub-venue` while its row read `unchecked` (#2635).
+ *
+ *   The table is `packages/corpus/data/corpus-source-inputs.json`. A derived or synthetic source lists every input
+ *   whose data reaches one of its rows. An input with no register row gets one in `source-additions.json`, under an
+ *   `unchecked` decision, so the gap reads as a gap rather than as an absence.
+ */
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { resolveModulePath } from "@mailwoman/core/module/resolvers";
+import { permissionFor, readAddressSourceRegister } from "#source-register/register";
+import { LicenseReviewState, OperationPermission, SourceOperation, } from "#source-register/types";
+/**
+ * The operations a register decision must permit before a source's rows may
+ * be built into a corpus or trained on.
+ *
+ * Both need adaptation: normalizing an address into labeled components is a derived work,
+ * and fitting a model on it is another.
+ */
+export const TRAINING_OPERATIONS = [SourceOperation.Transform, SourceOperation.Train];
+/**
+ * Resolves the committed table through the package's `./data/*` export.
+ */
+export function corpusSourceInputsPath() {
+    return resolveModulePath("@mailwoman/corpus/data/corpus-source-inputs.json");
+}
+/**
+ * Indexes a table by id and alias.
+ *
+ * @throws When two entries claim one spelling, because a lookup would then answer for whichever came last.
+ */
+export function indexCorpusSourceInputs(file) {
+    const bySpelling = new Map();
+    for (const [id, mapping] of Object.entries(file.sources)) {
+        for (const spelling of [id, ...(mapping.aliases ?? [])]) {
+            const claimed = bySpelling.get(spelling);
+            if (claimed && claimed !== id) {
+                throw new Error(`corpus source spelling ${stringifyJSON(spelling)} is claimed by both ${claimed} and ${id}`);
+            }
+            bySpelling.set(spelling, id);
+        }
+    }
+    return {
+        file,
+        resolve(source) {
+            const id = bySpelling.get(source);
+            return id ? { id, mapping: file.sources[id] } : null;
+        },
+    };
+}
+/**
+ * One message per structural problem in the table, read against the register.
+ */
+export function auditCorpusSourceInputs(file, register) {
+    const problems = [];
+    const sources = new Map(register.sources.map((source) => [source.sourceID, source]));
+    for (const [id, mapping] of Object.entries(file.sources)) {
+        const named = stringifyJSON(id);
+        if (!mapping.inputs.length && !mapping.authored?.length) {
+            problems.push(`corpus source ${named} lists no input, so nothing records what its rows are built from`);
+        }
+        for (const entry of mapping.history ?? []) {
+            if (!entry.corpora.length || !entry.because) {
+                problems.push(`corpus source ${named} has a history entry naming no corpus or giving no reason`);
+            }
+        }
+        for (const input of [...mapping.inputs, ...(mapping.history ?? []).flatMap((entry) => entry.inputs)]) {
+            const row = sources.get(input.registerSource);
+            if (!row) {
+                problems.push(`corpus source ${named} reads register source ${stringifyJSON(input.registerSource)}, which the register ` +
+                    "does not carry. Add the row to `source-additions.json` and regenerate the register.");
+                continue;
+            }
+            if (row.license !== input.license) {
+                problems.push(`corpus source ${named} records license ${stringifyJSON(input.license)} for ${input.registerSource}, ` +
+                    `and the register points that source at ${stringifyJSON(row.license)}`);
+            }
+            if (!input.reads) {
+                problems.push(`corpus source ${named} reads ${input.registerSource} and does not say what it reads`);
+            }
+        }
+    }
+    return problems;
+}
+/**
+ * Reads the committed table and checks it against the register.
+ *
+ * @throws When the file does not parse or the audit reports a problem.
+ */
+export async function readCorpusSourceInputs(register, path) {
+    const resolved = path ?? corpusSourceInputsPath();
+    const file = await readLocalJSONFile(resolved);
+    const problems = auditCorpusSourceInputs(file, register);
+    if (problems.length) {
+        throw new Error(`${resolved} failed its audit against the register:\n  ${problems.join("\n  ")}`);
+    }
+    return indexCorpusSourceInputs(file);
+}
+/**
+ * Every reason a corpus source's rows may not be built into a corpus or trained on.
+ *
+ * An empty array is the only answer a caller may read as permission.
+ * A source the table does not map answers one gap whose `registerSource` is `null`,
+ * because an unmapped source is one nobody has traced to its publishers.
+ *
+ * `corpusVersion` names a corpus already built, so a source whose inputs changed under
+ * the same id is read against what that corpus's rows were built from.
+ * A build or training check omits it and reads today's inputs.
+ */
+export function sourceRightsGaps(source, inputs, register, corpusVersion) {
+    const resolved = inputs.resolve(source);
+    if (!resolved) {
+        return [
+            {
+                source,
+                registerSource: null,
+                licenseID: null,
+                state: null,
+                missing: [...TRAINING_OPERATIONS],
+                because: "no entry in packages/corpus/data/corpus-source-inputs.json says which register sources its rows are " +
+                    "built from",
+            },
+        ];
+    }
+    const decisions = new Map(register.licenses.map((decision) => [decision.licenseID, decision]));
+    const rows = new Map(register.sources.map((row) => [row.sourceID, row]));
+    const gaps = [];
+    const former = corpusVersion
+        ? resolved.mapping.history?.find((entry) => entry.corpora.includes(corpusVersion))
+        : undefined;
+    for (const input of former?.inputs ?? resolved.mapping.inputs) {
+        const row = rows.get(input.registerSource);
+        const decision = row ? decisions.get(row.license) : undefined;
+        if (!row || !decision) {
+            gaps.push({
+                source,
+                registerSource: input.registerSource,
+                licenseID: row?.license ?? null,
+                state: null,
+                missing: [...TRAINING_OPERATIONS],
+                because: row
+                    ? `the register declares no decision ${stringifyJSON(row.license)}`
+                    : "the register has no such row",
+            });
+            continue;
+        }
+        const readings = TRAINING_OPERATIONS.map((operation) => ({
+            operation,
+            reading: permissionFor(decision, operation),
+        }));
+        const missing = readings.filter(({ reading }) => reading.permission !== OperationPermission.Permitted);
+        if (!missing.length)
+            continue;
+        gaps.push({
+            source,
+            registerSource: row.sourceID,
+            licenseID: decision.licenseID,
+            state: decision.state,
+            missing: missing.map(({ operation }) => operation),
+            because: decision.state === LicenseReviewState.Refused
+                ? `the decision is refused: ${decision.refusedBecause}`
+                : missing[0].reading.because,
+        });
+    }
+    return gaps;
+}
+/**
+ * One line naming the corpus source, the register row and the missing permission.
+ */
+export function describeSourceRightsGap(gap) {
+    const missing = gap.missing.join(" and ");
+    if (!gap.registerSource) {
+        return `corpus source ${gap.source} is unmapped and so lacks a permission for ${missing}: ${gap.because}`;
+    }
+    return (`corpus source ${gap.source} reads register source ${gap.registerSource} ` +
+        `(license ${gap.licenseID ?? "none"}, ${gap.state ?? "no decision"}), which does not permit ${missing}: ${gap.because}`);
+}
+/**
+ * Raised when a build or training run would admit a source lacking the permission.
+ */
+export class SourceRightsError extends Error {
+    gaps;
+    constructor(context, gaps) {
+        super(`${context} refuses ${new Set(gaps.map((gap) => gap.source)).size} corpus source(s) whose register rows ` +
+            `lack an elected decision permitting ${TRAINING_OPERATIONS.join(" and ")}:\n  ` +
+            gaps.map(describeSourceRightsGap).join("\n  ") +
+            "\nRecord the decision in packages/corpus/data/license-decisions.json, map the source in " +
+            "packages/corpus/data/corpus-source-inputs.json, or pass --unshippable-reason <why> for a local " +
+            "experiment, which marks the output as not shippable.");
+        this.name = "SourceRightsError";
+        this.gaps = gaps;
+    }
+}
+/**
+ * Builds the admission check over an already-read register and table.
+ *
+ * @throws When `unshippableReason` is set and blank, because an override must say why.
+ */
+export function createSourceRightsAdmission(register, inputs, options) {
+    const reason = options.unshippableReason ?? null;
+    if (reason !== null && !reason.trim()) {
+        throw new Error(`${options.context}: --unshippable-reason must say why the output is not shippable`);
+    }
+    const log = options.log ?? ((line) => console.error(line));
+    const checked = new Map();
+    const admitAll = (sources) => {
+        const fresh = [];
+        const all = [];
+        for (const source of sources) {
+            const known = checked.get(source);
+            if (known) {
+                all.push(...known);
+                continue;
+            }
+            const gaps = sourceRightsGaps(source, inputs, register);
+            fresh.push(...gaps);
+            all.push(...gaps);
+            if (!gaps.length || reason !== null) {
+                checked.set(source, gaps);
+            }
+        }
+        if (!fresh.length)
+            return all;
+        if (reason === null)
+            throw new SourceRightsError(options.context, fresh);
+        for (const gap of fresh) {
+            log(`UNSHIPPABLE ${options.context}: ${describeSourceRightsGap(gap)} (override: ${reason})`);
+        }
+        return all;
+    };
+    const unverified = [];
+    return {
+        admit: (source) => admitAll([source]),
+        admitAll,
+        admitUnverified: (description) => {
+            if (reason === null) {
+                throw new Error(`${options.context} cannot check the corpus sources of ${description}, and an input it cannot read is ` +
+                    "not one it found clear. Pass --unshippable-reason <why> for a local experiment, which marks the " +
+                    "output as not shippable.");
+            }
+            log(`UNSHIPPABLE ${options.context}: the corpus sources of ${description} were not checked (override: ${reason})`);
+            unverified.push(description);
+        },
+        record: () => {
+            const gaps = [...checked.values()].flat();
+            const shippable = gaps.length === 0 && unverified.length === 0;
+            return {
+                shippable,
+                unshippable_reason: shippable ? null : reason,
+                sources_checked: [...checked.keys()].toSorted(),
+                gaps,
+                unverified: [...unverified],
+            };
+        },
+    };
+}
+/**
+ * Reads the committed register and table and builds the admission check over them.
+ */
+export async function openSourceRightsAdmission(options) {
+    const register = await readAddressSourceRegister();
+    const inputs = await readCorpusSourceInputs(register);
+    return createSourceRightsAdmission(register, inputs, options);
+}
+//# sourceMappingURL=corpus-inputs.js.map

package/out/source-register/corpus-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-inputs.js","sourceRoot":"","sources":["../../lib/source-register/corpus-inputs.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAA;AAGpE,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAA;AACpF,OAAO,EACN,kBAAkB,EAClB,mBAAmB,EACnB,eAAe,GAGf,MAAM,wBAAwB,CAAA;AAE/B;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAA+B,CAAC,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,KAAK,CAAC,CAAA;AA+EjH;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACrC,OAAO,iBAAiB,CAAC,kDAAkD,CAAC,CAAA;AAC7E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAA4B;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE5C,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,KAAK,MAAM,QAAQ,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YAExC,IAAI,OAAO,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,0BAA0B,aAAa,CAAC,QAAQ,CAAC,uBAAuB,OAAO,QAAQ,EAAE,EAAE,CAAC,CAAA;YAC7G,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;QAC7B,CAAC;IACF,CAAC;IAED,OAAO;QACN,IAAI;QACJ,OAAO,CAAC,MAAM;YACb,MAAM,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEjC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;QACtD,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAA4B,EAAE,QAA+B;IACpG,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAA;IAEpF,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,aAAa,CAAC,EAAE,CAAC,CAAA;QAE/B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;YACzD,QAAQ,CAAC,IAAI,CAAC,iBAAiB,KAAK,kEAAkE,CAAC,CAAA;QACxG,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC,iBAAiB,KAAK,2DAA2D,CAAC,CAAA;YACjG,CAAC;QACF,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YACtG,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;YAE7C,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,QAAQ,CAAC,IAAI,CACZ,iBAAiB,KAAK,0BAA0B,aAAa,CAAC,KAAK,CAAC,cAAc,CAAC,uBAAuB;oBACzG,qFAAqF,CACtF,CAAA;gBAED,SAAQ;YACT,CAAC;YAED,IAAI,GAAG,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CACZ,iBAAiB,KAAK,oBAAoB,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,cAAc,IAAI;oBACrG,0CAA0C,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACvE,CAAA;YACF,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBAClB,QAAQ,CAAC,IAAI,CAAC,iBAAiB,KAAK,UAAU,KAAK,CAAC,cAAc,iCAAiC,CAAC,CAAA;YACrG,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC3C,QAA+B,EAC/B,IAAsB;IAEtB,MAAM,QAAQ,GAAG,IAAI,IAAI,sBAAsB,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAyB,QAAQ,CAAC,CAAA;IACtE,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAExD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,GAAG,QAAQ,8CAA8C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAClG,CAAC;IAED,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAA;AACrC,CAAC;AA0BD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAC/B,MAAc,EACd,MAA0B,EAC1B,QAA+B,EAC/B,aAAsB;IAEtB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAEvC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACf,OAAO;YACN;gBACC,MAAM;gBACN,cAAc,EAAE,IAAI;gBACpB,SAAS,EAAE,IAAI;gBACf,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,CAAC,GAAG,mBAAmB,CAAC;gBACjC,OAAO,EACN,sGAAsG;oBACtG,YAAY;aACb;SACD,CAAA;IACF,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAA;IAC9F,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;IACxE,MAAM,IAAI,GAAsB,EAAE,CAAA;IAElC,MAAM,MAAM,GAAG,aAAa;QAC3B,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAClF,CAAC,CAAC,SAAS,CAAA;IAEZ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,MAAM,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QAE7D,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC;gBACT,MAAM;gBACN,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,SAAS,EAAE,GAAG,EAAE,OAAO,IAAI,IAAI;gBAC/B,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,CAAC,GAAG,mBAAmB,CAAC;gBACjC,OAAO,EAAE,GAAG;oBACX,CAAC,CAAC,qCAAqC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;oBACnE,CAAC,CAAC,8BAA8B;aACjC,CAAC,CAAA;YAEF,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACxD,SAAS;YACT,OAAO,EAAE,aAAa,CAAC,QAAQ,EAAE,SAAS,CAAC;SAC3C,CAAC,CAAC,CAAA;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,KAAK,mBAAmB,CAAC,SAAS,CAAC,CAAA;QAEtG,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,IAAI,CAAC,IAAI,CAAC;YACT,MAAM;YACN,cAAc,EAAE,GAAG,CAAC,QAAQ;YAC5B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC;YAClD,OAAO,EACN,QAAQ,CAAC,KAAK,KAAK,kBAAkB,CAAC,OAAO;gBAC5C,CAAC,CAAC,4BAA4B,QAAQ,CAAC,cAAc,EAAE;gBACvD,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,OAAO;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,GAAoB;IAC3D,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QACzB,OAAO,iBAAiB,GAAG,CAAC,MAAM,8CAA8C,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CAAA;IAC1G,CAAC;IAED,OAAO,CACN,iBAAiB,GAAG,CAAC,MAAM,0BAA0B,GAAG,CAAC,cAAc,GAAG;QAC1E,YAAY,GAAG,CAAC,SAAS,IAAI,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,aAAa,4BAA4B,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CACvH,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAClC,IAAI,CAA4B;IAEzC,YAAY,OAAe,EAAE,IAAgC;QAC5D,KAAK,CACJ,GAAG,OAAO,YAAY,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,wCAAwC;YACxG,uCAAuC,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO;YAC/E,IAAI,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,0FAA0F;YAC1F,iGAAiG;YACjG,sDAAsD,CACvD,CAAA;QAED,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAA;QAC/B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IACjB,CAAC;CACD;AAkED;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CAC1C,QAA+B,EAC/B,MAA0B,EAC1B,OAAqC;IAErC,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,IAAI,IAAI,CAAA;IAEhD,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,iEAAiE,CAAC,CAAA;IACrG,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IAClE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAsC,CAAA;IAE7D,MAAM,QAAQ,GAAG,CAAC,OAAyB,EAA8B,EAAE;QAC1E,MAAM,KAAK,GAAsB,EAAE,CAAA;QACnC,MAAM,GAAG,GAAsB,EAAE,CAAA;QAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEjC,IAAI,KAAK,EAAE,CAAC;gBACX,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAA;gBAElB,SAAQ;YACT,CAAC;YAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;YAEvD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;YAEjB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAC1B,CAAC;QACF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,OAAO,GAAG,CAAA;QAE7B,IAAI,MAAM,KAAK,IAAI;YAAE,MAAM,IAAI,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;QAExE,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACzB,GAAG,CAAC,eAAe,OAAO,CAAC,OAAO,KAAK,uBAAuB,CAAC,GAAG,CAAC,eAAe,MAAM,GAAG,CAAC,CAAA;QAC7F,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC,CAAA;IAED,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,OAAO;QACN,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC;QACrC,QAAQ;QACR,eAAe,EAAE,CAAC,WAAW,EAAE,EAAE;YAChC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CACd,GAAG,OAAO,CAAC,OAAO,uCAAuC,WAAW,mCAAmC;oBACtG,kGAAkG;oBAClG,0BAA0B,CAC3B,CAAA;YACF,CAAC;YAED,GAAG,CAAC,eAAe,OAAO,CAAC,OAAO,2BAA2B,WAAW,gCAAgC,MAAM,GAAG,CAAC,CAAA;YAClH,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;QAC7B,CAAC;QACD,MAAM,EAAE,GAAG,EAAE;YACZ,MAAM,IAAI,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,CAAA;YAE9D,OAAO;gBACN,SAAS;gBACT,kBAAkB,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gBAC7C,eAAe,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE;gBAC/C,IAAI;gBACJ,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC;aAC3B,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,OAAqC;IACpF,MAAM,QAAQ,GAAG,MAAM,yBAAyB,EAAE,CAAA;IAClD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAA;IAErD,OAAO,2BAA2B,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AAC9D,CAAC"}

package/out/source-register/effective-manifest.d.ts

@@ -139,6 +139,8 @@ export interface RequiredAttribution {
  * Only `trainingSources` are read, so a source the corpus holds and the run never drew owes no attribution.
  * A license is included when `summarizeLicense` records an attribution obligation for it, or
  * when it is unrecognized, since an unrecognized license's obligations are unknown rather than absent.
+ *
+ * A source joins the group of every license in {@linkcode CONTENT_LICENSES} beside its own.
  */
 export declare function requiredAttributions(sources: readonly EffectiveSourceRecord[]): RequiredAttribution[];
 /**

package/out/source-register/effective-manifest.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"effective-manifest.d.ts","sourceRoot":"","sources":["../../lib/source-register/effective-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAG1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AAE1E;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAQrE;AAED;;GAEG;AACH,eAAO,MAAM,eAAe;IAC3B;;;OAGG;aACH,gBAAgB,EAAE,mBAAmB;IAErC;;OAEG;aACH,UAAU,EAAE,aAAa;IAEzB;;;;OAIG;aACH,YAAY,EAAE,gBAAgB;CACrB,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAA;AAEpF;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAErB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,eAAe,EAAE,OAAO,CAAA;IAExB;;OAEG;IACH,eAAe,EAAE,eAAe,GAAG,IAAI,CAAA;CACvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY;IACxB;;;;;OAKG;aACH,kBAAkB,EAAE,sBAAsB;IAC1C;;OAEG;aACH,aAAa,EAAE,gBAAgB;CACtB,CAAA;AAEV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAE3E;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,0BAA0B,EAAE,MAAM,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAcD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,mBAAmB,EAAE,CAoBrG;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC,UAAU,EAAE,oCAAoC,CAAA;IAChD,aAAa,EAAE,CAAC,CAAA;IAEhB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAA;IAE1B;;OAEG;IACH,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAA;IAEjC;;OAEG;IACH,oBAAoB,EAAE,mBAAmB,EAAE,CAAA;IAE3C;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAA;IACzB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,4BAA4B,EAAE,MAAM,EAAE,CAAA;IAEtC;;OAEG;IACH,OAAO,EAAE,qBAAqB,EAAE,CAAA;IAEhC;;;OAGG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAEhD;;;;;;;;;;;;;;;OAeG;IACH,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAE5C;;;;;;;;;;;;;;OAcG;IACH,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAEtG,gBAAgB,EAAE,MAAM,CAAA;IAExB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,UAAU,CAAC,EAAE;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,wBAAwB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACjD,kCAAkC,CAAC,EAAE,MAAM,EAAE,CAAA;QAC7C;;;;;;WAMG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAC9D,CAAA;IACD,aAAa,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAA;IACnD,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAC5F;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC/C,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CA4DhE;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,yBAAyB,GAAG,MAAM,CAEnF;AAED;;;;;;;;;GASG;AACH,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACtD,cAAc,EAAE,gBAAgB,CAAA;IAChC,KAAK,EAAE,iBAAiB,CAAA;IACxB,MAAM,EAAE,mBAAmB,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAA;CAClC,GAAG,yBAAyB,CA6G5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,yBAAyB,EACnC,QAAQ,EAAE,SAAS,MAAM,EAAE,GACzB;IAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAAC,qBAAqB,EAAE,MAAM,EAAE,CAAA;CAAE,CAQtE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACzC,QAAQ,EAAE,yBAAyB,GAAG,IAAI,CAAA;IAC1C,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IAClC,WAAW,EAAE,MAAM,CAAA;CACnB,GAAG,MAAM,EAAE,CAoDX"}
+{"version":3,"file":"effective-manifest.d.ts","sourceRoot":"","sources":["../../lib/source-register/effective-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAI1C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AAE1E;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,WAAW,CAQrE;AAED;;GAEG;AACH,eAAO,MAAM,eAAe;IAC3B;;;OAGG;aACH,gBAAgB,EAAE,mBAAmB;IAErC;;OAEG;aACH,UAAU,EAAE,aAAa;IAEzB;;;;OAIG;aACH,YAAY,EAAE,gBAAgB;CACrB,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAA;AAEpF;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,MAAM,EAAE,MAAM,CAAA;IAEd;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAElB,OAAO,EAAE,MAAM,CAAA;IAEf;;;OAGG;IACH,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IAErB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IAEnB;;OAEG;IACH,eAAe,EAAE,OAAO,CAAA;IAExB;;OAEG;IACH,eAAe,EAAE,eAAe,GAAG,IAAI,CAAA;CACvC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY;IACxB;;;;;OAKG;aACH,kBAAkB,EAAE,sBAAsB;IAC1C;;OAEG;aACH,aAAa,EAAE,gBAAgB;CACtB,CAAA;AAEV,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,OAAO,YAAY,CAAC,CAAA;AAE3E;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,0BAA0B,EAAE,MAAM,CAAA;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB;;;OAGG;IACH,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AA6CD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,SAAS,qBAAqB,EAAE,GAAG,mBAAmB,EAAE,CAkBrG;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACzC,UAAU,EAAE,oCAAoC,CAAA;IAChD,aAAa,EAAE,CAAC,CAAA;IAEhB;;OAEG;IACH,YAAY,EAAE,YAAY,CAAA;IAE1B;;OAEG;IACH,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAA;IAEjC;;OAEG;IACH,oBAAoB,EAAE,mBAAmB,EAAE,CAAA;IAE3C;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,iBAAiB,EAAE,MAAM,CAAA;IACzB,oBAAoB,EAAE,MAAM,CAAA;IAE5B;;OAEG;IACH,4BAA4B,EAAE,MAAM,EAAE,CAAA;IAEtC;;OAEG;IACH,OAAO,EAAE,qBAAqB,EAAE,CAAA;IAEhC;;;OAGG;IACH,eAAe,EAAE,MAAM,EAAE,CAAA;IAEzB;;OAEG;IACH,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAEhD;;;;;;;;;;;;;;;OAeG;IACH,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAE5C;;;;;;;;;;;;;;OAcG;IACH,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAEtG,gBAAgB,EAAE,MAAM,CAAA;IAExB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,UAAU,CAAC,EAAE;QACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,wBAAwB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACjD,kCAAkC,CAAC,EAAE,MAAM,EAAE,CAAA;QAC7C;;;;;;WAMG;QACH,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAC9D,CAAA;IACD,aAAa,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAA;IACnD,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,eAAe,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAC5F;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC/C,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAA;CACxC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CA4DhE;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,yBAAyB,GAAG,MAAM,CAEnF;AAED;;;;;;;;;GASG;AACH,wBAAgB,+BAA+B,CAAC,KAAK,EAAE;IACtD,cAAc,EAAE,gBAAgB,CAAA;IAChC,KAAK,EAAE,iBAAiB,CAAA;IACxB,MAAM,EAAE,mBAAmB,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAAA;CAClC,GAAG,yBAAyB,CA6G5B;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,yBAAyB,EACnC,QAAQ,EAAE,SAAS,MAAM,EAAE,GACzB;IAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAAC,qBAAqB,EAAE,MAAM,EAAE,CAAA;CAAE,CAQtE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE;IACzC,QAAQ,EAAE,yBAAyB,GAAG,IAAI,CAAA;IAC1C,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;IAClC,WAAW,EAAE,MAAM,CAAA;CACnB,GAAG,MAAM,EAAE,CAoDX"}

package/out/source-register/effective-manifest.js

@@ -25,6 +25,7 @@ import { sha256Hex } from "@mailwoman/core/hash";
 import { stringifyJSON } from "@mailwoman/core/json";
 import { LicenseObligation, summarizeLicense } from "@mailwoman/core/license/obligations";
 import { OSM_ATTRIBUTION, OSM_LICENSE } from "#adapters/osm/adapter";
+import { currentSourceName } from "#recipes/sources";
 /**
  * Where an `audit_epoch_mixture --json` output for one config is looked for.
  *
@@ -92,22 +93,50 @@ export const DrawEvidence = {
 const ATTRIBUTION_STATEMENTS = {
     [OSM_LICENSE]: OSM_ATTRIBUTION,
 };
+/**
+ * Recognized licenses carried by contents of a source whose rows state one free-text
+ * license, keyed by the source's current spelling.
+ *
+ * The derivation groups sources by the license their rows state.
+ * The `sub-venue` recipe states one license over three publications.
+ *
+ * That license is free text, and the OpenStreetMap venue names it carries are under ODbL 1.0.
+ * Grouping by that text alone leaves the source out of the ODbL group,
+ * the group that carries the recorded OpenStreetMap statement.
+ *
+ * The source keeps its own group as well, because that text also credits the
+ * publishers of the address skeletons.
+ */
+const CONTENT_LICENSES = {
+    "spliced-sub-venue": [OSM_LICENSE],
+};
+/**
+ * The licenses a source owes attribution under: the one its rows state and those its contents carry.
+ */
+function attributedLicenses(record) {
+    const current = currentSourceName(record.source) ?? record.source;
+    return [record.license, ...(CONTENT_LICENSES[current] ?? [])].filter((license) => {
+        const summary = summarizeLicense(license);
+        return !summary.recognized || summary.obligations.includes(LicenseObligation.Attribution);
+    });
+}
 /**
  * The attributions owed by the sources that reached the trainer, grouped by license.
  *
  * Only `trainingSources` are read, so a source the corpus holds and the run never drew owes no attribution.
  * A license is included when `summarizeLicense` records an attribution obligation for it, or
  * when it is unrecognized, since an unrecognized license's obligations are unknown rather than absent.
+ *
+ * A source joins the group of every license in {@linkcode CONTENT_LICENSES} beside its own.
  */
 export function requiredAttributions(sources) {
     const byLicense = new Map();
     for (const record of sources) {
         if (record.emittedRows <= 0)
             continue;
-        const summary = summarizeLicense(record.license);
-        if (summary.recognized && !summary.obligations.includes(LicenseObligation.Attribution))
-            continue;
-        byLicense.set(record.license, [...(byLicense.get(record.license) ?? []), record.source]);
+        for (const license of attributedLicenses(record)) {
+            byLicense.set(license, [...(byLicense.get(license) ?? []), record.source]);
+        }
     }
     return [...byLicense]
         .toSorted(([a], [b]) => a.localeCompare(b))