@mailwoman/corpus 9.4.0 → 10.1.0

package/out/build/adapters.js

@@ -0,0 +1,68 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The adapter phase: drive each configured adapter to its own `canonical.jsonl`, or reuse the run a
+ *   previous build finished.
+ */
+import { pathExists, readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers";
+import { $public } from "#env";
+import { runAdapter } from "#runner";
+/**
+ * Reuse a finished adapter run, or return `null` when this build has to drive the adapter.
+ *
+ * @throws When the cached manifest and the file on disk disagree on the byte count.
+ * The writer flushes the canonical file before writing the manifest.
+ * Manifest presence establishes that the writer reached the end of its run.
+ * The manifest describes the file as of that moment.
+ * A truncated copy, an interrupted move or a partial transfer leaves the
+ * manifest intact beside a shorter file.
+ * The recorded byte count distinguishes those cases.
+ * A resumed build that reuses a short file can produce a corpus whose row count later readers accept.
+ */
+async function reusableRun(adapterID, adapterDir) {
+    if ($public.MAILWOMAN_RESUME !== "1")
+        return null;
+    const cachedManifest = adapterDir("MANIFEST.json");
+    const canonicalPath = adapterDir("canonical.jsonl");
+    if (!(await pathExists(cachedManifest)) || !(await pathExists(canonicalPath)))
+        return null;
+    const cached = await readLocalJSONFile(cachedManifest);
+    const onDisk = (await statPath(canonicalPath)).size;
+    if (onDisk !== cached.bytes) {
+        throw new Error(`resume refused for ${adapterID}: ${canonicalPath} is ${onDisk.toLocaleString()} bytes and its ` +
+            `manifest records ${cached.bytes.toLocaleString()}. Delete ${cachedManifest} to ` +
+            `re-run the adapter, or restore the file this manifest describes.`);
+    }
+    return cached;
+}
+/**
+ * Drive every configured adapter, reusing whatever a previous build already finished.
+ */
+export async function runAdapterPhase(opts) {
+    const runs = [];
+    const skipped = [];
+    for (const adapter of opts.adapters) {
+        const adapterOptions = opts.adapterInputs[adapter.id];
+        if (!adapterOptions) {
+            skipped.push(adapter.id);
+            opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`);
+            continue;
+        }
+        const cached = await reusableRun(adapter.id, opts.intermediateDir(adapter.id));
+        if (cached) {
+            opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`);
+            runs.push(cached);
+            continue;
+        }
+        opts.onProgress?.("adapter-run", `running ${adapter.id}`);
+        runs.push(await runAdapter({
+            adapter,
+            adapterOptions,
+            outputDir: opts.intermediateDir,
+            corpusVersion: opts.corpusVersion,
+        }));
+    }
+    return { runs, skipped };
+}
+//# sourceMappingURL=adapters.js.map

package/out/build/adapters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../../lib/build/adapters.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AAIpF,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,UAAU,EAA2B,MAAM,SAAS,CAAA;AAuB7D;;;;;;;;;;;GAWG;AACH,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,UAAuB;IACpE,IAAI,OAAO,CAAC,gBAAgB,KAAK,GAAG;QAAE,OAAO,IAAI,CAAA;IAEjD,MAAM,cAAc,GAAG,UAAU,CAAC,eAAe,CAAC,CAAA;IAClD,MAAM,aAAa,GAAG,UAAU,CAAC,iBAAiB,CAAC,CAAA;IAEnD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,aAAa,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1F,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAqB,cAAc,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAA;IAEnD,IAAI,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACd,sBAAsB,SAAS,KAAK,aAAa,OAAO,MAAM,CAAC,cAAc,EAAE,iBAAiB;YAC/F,oBAAoB,MAAM,CAAC,KAAK,CAAC,cAAc,EAAE,YAAY,cAAc,MAAM;YACjF,kEAAkE,CACnE,CAAA;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAyB;IAC9D,MAAM,IAAI,GAAyB,EAAE,CAAA;IACrC,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QACrC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAE/E,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;QAE9E,IAAI,MAAM,EAAE,CAAC;YACZ,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,CACR,MAAM,UAAU,CAAC;YAChB,OAAO;YACP,cAAc;YACd,SAAS,EAAE,IAAI,CAAC,eAAe;YAC/B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CACF,CAAA;IACF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AACzB,CAAC"}

package/out/build/align.d.ts

@@ -0,0 +1,73 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The align phase reads each adapter's canonical rows and applies license and register refusals.
+ *   It expands synthesis and aligns each row. It then routes the result to a split or quarantine.
+ *
+ *   The phase appends to four files. It writes a checkpoint at each adapter boundary, allowing an interrupted
+ *   build to resume at the next adapter without aligning earlier adapters again.
+ */
+import type { PathBuilder } from "path-ts";
+import type { BuildProfile, BuildStage } from "#build/types";
+import { type AdapterRunManifest } from "#runner";
+import { type LicensePolicy, type LicenseRefusalKind, type LicenseVerdictCache } from "#utils/license";
+import { type CountryHoldout, type SplitName } from "#utils/split";
+export interface AlignPhaseOptions {
+    /**
+     * The adapter runs to align, in the order their rows enter the output files.
+     */
+    adapterRuns: readonly AdapterRunManifest[];
+    corpusVersion: string;
+    synthesize: boolean;
+    profile: BuildProfile;
+    licensePolicy: LicensePolicy;
+    excludeLicenses: readonly RegExp[];
+    holdouts: Record<string, CountryHoldout>;
+    /**
+     * Each register source's ingest-eligibility reasons, or `null` under the exploratory profile.
+     */
+    eligibility: ReadonlyMap<string, readonly string[]> | null;
+    labeledPaths: Record<SplitName, PathBuilder>;
+    quarantinePath: PathBuilder;
+    checkpointPath: PathBuilder;
+    onProgress?: (stage: BuildStage, message: string) => void;
+}
+/**
+ * The counts produced by the align phase.
+ *
+ * The build uses them in its manifest and frozen source record.
+ */
+export interface AlignPhaseResult {
+    aligned: number;
+    quarantined: number;
+    counts: Record<SplitName, number>;
+    licenseCounts: Map<string, number>;
+    refusedByKind: Map<LicenseRefusalKind, number>;
+    /**
+     * Rows the license policy refused, counted per `source`.
+     *
+     * A source here contributed zero rows to the corpus whatever its adapter reported writing, which is
+     * the distinction between a source present in corpus provenance and one present in model provenance.
+     */
+    refusedBySource: Map<string, number>;
+    excludedByLicense: number;
+    admittedUnresolved: number;
+    excludedByEligibility: number;
+    rowsBySource: Map<string, {
+        rows: number;
+        license: string;
+    }>;
+    ineligibleSources: Map<string, readonly string[]>;
+    /**
+     * The license cache used by this phase.
+     *
+     * Its `refusedValues()` method returns every value it refused.
+     */
+    licenses: LicenseVerdictCache;
+}
+/**
+ * Align every adapter run's rows into the four output files, writing a checkpoint at each adapter boundary.
+ */
+export declare function runAlignPhase(opts: AlignPhaseOptions): Promise<AlignPhaseResult>;
+//# sourceMappingURL=align.d.ts.map

package/out/build/align.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"align.d.ts","sourceRoot":"","sources":["../../lib/build/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAY1C,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAE5D,OAAO,EAAqB,KAAK,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAIpE,OAAO,EAEN,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,EACxB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAe,KAAK,cAAc,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAA;AAE/E,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,WAAW,EAAE,SAAS,kBAAkB,EAAE,CAAA;IAE1C,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,OAAO,CAAA;IACnB,OAAO,EAAE,YAAY,CAAA;IACrB,aAAa,EAAE,aAAa,CAAA;IAC5B,eAAe,EAAE,SAAS,MAAM,EAAE,CAAA;IAClC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAExC;;OAEG;IACH,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,GAAG,IAAI,CAAA;IAE1D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAC5C,cAAc,EAAE,WAAW,CAAA;IAC3B,cAAc,EAAE,WAAW,CAAA;IAE3B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACzD;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,aAAa,EAAE,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IAC9C;;;;;OAKG;IACH,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,iBAAiB,EAAE,MAAM,CAAA;IACzB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,qBAAqB,EAAE,MAAM,CAAA;IAC7B,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC5D,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAEjD;;;;OAIG;IACH,QAAQ,EAAE,mBAAmB,CAAA;CAC7B;AA2DD;;GAEG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA6MtF"}

package/out/build/align.js

@@ -0,0 +1,226 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The align phase reads each adapter's canonical rows and applies license and register refusals.
+ *   It expands synthesis and aligns each row. It then routes the result to a split or quarantine.
+ *
+ *   The phase appends to four files. It writes a checkpoint at each adapter boundary, allowing an interrupted
+ *   build to resume at the next adapter without aligning earlier adapters again.
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { truncateFile } from "@mailwoman/core/fs/writers";
+import { sha256Hex } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { ALIGN_OUTPUTS, readAlignCheckpoint, writeAlignCheckpoint, } from "#build/checkpoint";
+import { createIneligibilityReader } from "#build/eligibility";
+import { streamJSONL } from "#build/rows";
+import { $public } from "#env";
+import { flushStream, once } from "#runner";
+import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils";
+import { alignRow } from "#utils/align";
+import { createLicenseVerdictCache, } from "#utils/license";
+import { splitForRow } from "#utils/split";
+/**
+ * Return each output to the length the checkpoint recorded.
+ *
+ * Whatever the interrupted adapter wrote past the boundary is discarded, because the checkpoint's
+ * accumulators do not count those rows and the split manifests are computed from the accumulators.
+ */
+async function restoreOutputs(checkpoint, paths) {
+    for (const name of ALIGN_OUTPUTS) {
+        await truncateFile(paths[name], checkpoint.offsets[name]);
+    }
+}
+/**
+ * Restore the accumulators a checkpoint recorded into this run's containers.
+ */
+function restoreTally(checkpoint, result) {
+    for (const name of ["train", "val", "test"]) {
+        result.counts[name] = checkpoint.counts[name];
+    }
+    for (const [license, count] of Object.entries(checkpoint.license_counts)) {
+        result.licenseCounts.set(license, count);
+    }
+    for (const [kind, count] of Object.entries(checkpoint.refused_by_kind)) {
+        result.refusedByKind.set(kind, count);
+    }
+    for (const [source, count] of Object.entries(checkpoint.refused_by_source)) {
+        result.refusedBySource.set(source, count);
+    }
+    for (const [source, contributed] of Object.entries(checkpoint.rows_by_source)) {
+        result.rowsBySource.set(source, contributed);
+    }
+    return {
+        aligned: checkpoint.aligned,
+        quarantined: checkpoint.quarantined,
+        excludedByLicense: checkpoint.excluded_by_license,
+        admittedUnresolved: checkpoint.admitted_unresolved_license_rows,
+        excludedByEligibility: checkpoint.excluded_by_eligibility,
+    };
+}
+/**
+ * Align every adapter run's rows into the four output files, writing a checkpoint at each adapter boundary.
+ */
+export async function runAlignPhase(opts) {
+    const ineligibility = createIneligibilityReader(opts.eligibility);
+    const licenses = createLicenseVerdictCache(opts.licensePolicy, opts.excludeLicenses);
+    const result = {
+        aligned: 0,
+        quarantined: 0,
+        counts: { train: 0, val: 0, test: 0 },
+        licenseCounts: new Map(),
+        refusedByKind: new Map(),
+        excludedByLicense: 0,
+        admittedUnresolved: 0,
+        excludedByEligibility: 0,
+        refusedBySource: new Map(),
+        rowsBySource: new Map(),
+        ineligibleSources: ineligibility.refused,
+        licenses,
+    };
+    const settings = {
+        corpus_version: opts.corpusVersion,
+        synthesize: opts.synthesize,
+        license_policy: opts.licensePolicy,
+        exclude_licenses: opts.excludeLicenses.map((pattern) => pattern.source).toSorted(),
+        profile: opts.profile,
+        holdout_digest: sha256Hex(stringifyJSON(opts.holdouts)),
+    };
+    const paths = { ...opts.labeledPaths, quarantine: opts.quarantinePath };
+    const checkpoint = $public.MAILWOMAN_RESUME === "1"
+        ? await readAlignCheckpoint(opts.checkpointPath, {
+            settings,
+            adapterOrder: opts.adapterRuns.map((run) => run.adapter_id),
+        })
+        : null;
+    let tally = {
+        aligned: 0,
+        quarantined: 0,
+        excludedByLicense: 0,
+        admittedUnresolved: 0,
+        excludedByEligibility: 0,
+    };
+    if (checkpoint) {
+        await restoreOutputs(checkpoint, paths);
+        tally = restoreTally(checkpoint, result);
+        opts.onProgress?.("align", `resumed align after ${checkpoint.completed_adapters.join(", ")} ` +
+            `(${tally.aligned.toLocaleString()} aligned rows)`);
+    }
+    // `a` rather than the default `w` on a resume, because the files hold the rows the checkpoint counted.
+    const flags = checkpoint ? "a" : "w";
+    const streams = {
+        train: openWriteStream(paths.train, { encoding: "utf8", flags }),
+        val: openWriteStream(paths.val, { encoding: "utf8", flags }),
+        test: openWriteStream(paths.test, { encoding: "utf8", flags }),
+        quarantine: openWriteStream(paths.quarantine, { encoding: "utf8", flags }),
+    };
+    const offsets = checkpoint
+        ? { ...checkpoint.offsets }
+        : { train: 0, val: 0, test: 0, quarantine: 0 };
+    const writeLine = (name, line) => {
+        offsets[name] += Buffer.byteLength(line, "utf8");
+        streams[name].write(line);
+    };
+    const quarantine = (row, reason) => {
+        writeLine("quarantine", `${stringifyJSON({ row, reason })}\n`);
+    };
+    const completed = checkpoint ? [...checkpoint.completed_adapters] : [];
+    for (const adapterRun of opts.adapterRuns) {
+        if (completed.includes(adapterRun.adapter_id))
+            continue;
+        opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
+        for await (const row of streamJSONL(adapterRun.jsonl_path)) {
+            result.licenseCounts.set(row.license, (result.licenseCounts.get(row.license) ?? 0) + 1);
+            // The license count is incremented before the drop, so the manifest's license set
+            // reflects what the corpus contained and `excluded_by_license` what was removed.
+            const verdict = licenses.read(row.license);
+            if (verdict.refusal) {
+                tally.excludedByLicense++;
+                result.refusedByKind.set(verdict.refusal, (result.refusedByKind.get(verdict.refusal) ?? 0) + 1);
+                // Counted per source as well as per refusal kind, because an adapter's own
+                // `written` count records what it emitted rather than what entered the corpus.
+                // A source refused here contributed no row, and model provenance generated from
+                // `adapters[].written` alone would credit it with rows it did not supply.
+                result.refusedBySource.set(row.source, (result.refusedBySource.get(row.source) ?? 0) + 1);
+                continue;
+            }
+            // An admitted row whose license resolves to no expression entered with
+            // unknown obligations rather than with none.
+            // The manifest states the count so a consumer can tell the two apart.
+            if (!verdict.resolved) {
+                tally.admittedUnresolved++;
+            }
+            // Eligibility runs before augmentation: a synthetic row inherits its ancestor's `source`,
+            // so refusing the ancestor here refuses everything fanned from it.
+            const ineligible = ineligibility.read(row);
+            if (ineligible) {
+                tally.excludedByEligibility++;
+                quarantine(row, `source-ineligible:${ineligible.join("; ")}`);
+                continue;
+            }
+            // Counted after both refusals, so the frozen manifest records what a source
+            // contributed rather than what it offered.
+            const contributed = result.rowsBySource.get(row.source);
+            result.rowsBySource.set(row.source, { rows: (contributed?.rows ?? 0) + 1, license: row.license });
+            const fanned = [row];
+            if (opts.synthesize) {
+                for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
+                    fanned.push(aug);
+                }
+            }
+            for (const r of fanned) {
+                let aligned;
+                try {
+                    aligned = alignRow(r);
+                }
+                catch (error) {
+                    // No single row may crash a multi-hour build.
+                    // An unknown throw is quarantined as `align-threw`, and a rise in
+                    // that reason's count is a finding.
+                    quarantine(r, `align-threw:${error.message.slice(0, 160)}`);
+                    tally.quarantined++;
+                    continue;
+                }
+                if (aligned.kind === "labeled") {
+                    const split = splitForRow(aligned.row, opts.holdouts);
+                    writeLine(split, `${stringifyJSON(aligned.row)}\n`);
+                    result.counts[split]++;
+                    tally.aligned++;
+                }
+                else {
+                    quarantine(r, aligned.row.reason);
+                    tally.quarantined++;
+                }
+            }
+        }
+        // The checkpoint is written at the adapter boundary, after the bytes it counts have reached disk.
+        // Mid-adapter recovery is unavailable.
+        // The largest adapter takes about twenty minutes to align.
+        // A row offset inside one adapter's file would still require replaying that adapter's synthesis.
+        await Promise.all(ALIGN_OUTPUTS.map((name) => flushStream(streams[name])));
+        completed.push(adapterRun.adapter_id);
+        await writeAlignCheckpoint(opts.checkpointPath, {
+            settings,
+            completed_adapters: [...completed],
+            offsets: { ...offsets },
+            counts: { ...result.counts },
+            aligned: tally.aligned,
+            quarantined: tally.quarantined,
+            license_counts: Object.fromEntries(result.licenseCounts),
+            refused_by_kind: Object.fromEntries(result.refusedByKind),
+            refused_by_source: Object.fromEntries(result.refusedBySource),
+            excluded_by_license: tally.excludedByLicense,
+            admitted_unresolved_license_rows: tally.admittedUnresolved,
+            excluded_by_eligibility: tally.excludedByEligibility,
+            rows_by_source: Object.fromEntries(result.rowsBySource),
+            written_at: new Date().toISOString(),
+        });
+    }
+    for (const stream of Object.values(streams)) {
+        stream.end();
+    }
+    await Promise.all(Object.values(streams).map((stream) => once(stream, "close")));
+    return { ...result, ...tally };
+}
+//# sourceMappingURL=align.js.map

package/out/build/align.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"align.js","sourceRoot":"","sources":["../../lib/build/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpD,OAAO,EACN,aAAa,EACb,mBAAmB,EACnB,oBAAoB,GAIpB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,WAAW,EAAE,IAAI,EAA2B,MAAM,SAAS,CAAA;AACpE,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAEnF,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvC,OAAO,EACN,yBAAyB,GAIzB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,WAAW,EAAuC,MAAM,cAAc,CAAA;AAuE/E;;;;;GAKG;AACH,KAAK,UAAU,cAAc,CAAC,UAA2B,EAAE,KAA2C;IACrG,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IAC1D,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,UAA2B,EAAE,MAAwB;IAC1E,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QAC1E,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IACzC,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC,EAAE,CAAC;QACxE,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAA0B,EAAE,KAAK,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC5E,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC1C,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/E,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO;QACN,OAAO,EAAE,UAAU,CAAC,OAAO;QAC3B,WAAW,EAAE,UAAU,CAAC,WAAW;QACnC,iBAAiB,EAAE,UAAU,CAAC,mBAAmB;QACjD,kBAAkB,EAAE,UAAU,CAAC,gCAAgC;QAC/D,qBAAqB,EAAE,UAAU,CAAC,uBAAuB;KACzD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAuB;IAC1D,MAAM,aAAa,GAAG,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACjE,MAAM,QAAQ,GAAG,yBAAyB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,eAAe,CAAC,CAAA;IAEpF,MAAM,MAAM,GAAqB;QAChC,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;QACrC,aAAa,EAAE,IAAI,GAAG,EAAE;QACxB,aAAa,EAAE,IAAI,GAAG,EAAE;QACxB,iBAAiB,EAAE,CAAC;QACpB,kBAAkB,EAAE,CAAC;QACrB,qBAAqB,EAAE,CAAC;QACxB,eAAe,EAAE,IAAI,GAAG,EAAE;QAC1B,YAAY,EAAE,IAAI,GAAG,EAAE;QACvB,iBAAiB,EAAE,aAAa,CAAC,OAAO;QACxC,QAAQ;KACR,CAAA;IAED,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE;QAClF,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,cAAc,EAAE,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KACvD,CAAA;IAED,MAAM,KAAK,GAAyC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,cAAc,EAAE,CAAA;IAE7G,MAAM,UAAU,GACf,OAAO,CAAC,gBAAgB,KAAK,GAAG;QAC/B,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE;YAC/C,QAAQ;YACR,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;SAC3D,CAAC;QACH,CAAC,CAAC,IAAI,CAAA;IAER,IAAI,KAAK,GAAe;QACvB,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,CAAC;QACpB,kBAAkB,EAAE,CAAC;QACrB,qBAAqB,EAAE,CAAC;KACxB,CAAA;IAED,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;QACvC,KAAK,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAExC,IAAI,CAAC,UAAU,EAAE,CAChB,OAAO,EACP,uBAAuB,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACjE,IAAI,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,gBAAgB,CACnD,CAAA;IACF,CAAC;IAED,uGAAuG;IACvG,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAA;IAEpC,MAAM,OAAO,GAAyC;QACrD,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAChE,GAAG,EAAE,eAAe,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC5D,IAAI,EAAE,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC9D,UAAU,EAAE,eAAe,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;KAC1E,CAAA;IAED,MAAM,OAAO,GAAoC,UAAU;QAC1D,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,EAAE;QAC3B,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAE/C,MAAM,SAAS,GAAG,CAAC,IAAqB,EAAE,IAAY,EAAQ,EAAE;QAC/D,OAAO,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC1B,CAAC,CAAA;IAED,MAAM,UAAU,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QAC9D,SAAS,CAAC,YAAY,EAAE,GAAG,aAAa,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAEtE,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,SAAQ;QAEvD,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEvF,kFAAkF;YAClF,iFAAiF;YACjF,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YAE1C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,KAAK,CAAC,iBAAiB,EAAE,CAAA;gBACzB,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAE/F,2EAA2E;gBAC3E,+EAA+E;gBAC/E,gFAAgF;gBAChF,0EAA0E;gBAC1E,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAEzF,SAAQ;YACT,CAAC;YAED,uEAAuE;YACvE,6CAA6C;YAC7C,sEAAsE;YACtE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvB,KAAK,CAAC,kBAAkB,EAAE,CAAA;YAC3B,CAAC;YAED,0FAA0F;YAC1F,mEAAmE;YACnE,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAE1C,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,CAAC,qBAAqB,EAAE,CAAA;gBAE7B,UAAU,CAAC,GAAG,EAAE,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBAE7D,SAAQ;YACT,CAAC;YAED,4EAA4E;YAC5E,2CAA2C;YAC3C,MAAM,WAAW,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAEvD,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;YAEjG,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,OAAoC,CAAA;gBAExC,IAAI,CAAC;oBACJ,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACtB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,8CAA8C;oBAC9C,kEAAkE;oBAClE,oCAAoC;oBACpC,UAAU,CAAC,CAAC,EAAE,eAAgB,KAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBAEtE,KAAK,CAAC,WAAW,EAAE,CAAA;oBAEnB,SAAQ;gBACT,CAAC;gBAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;oBAErD,SAAS,CAAC,KAAK,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAEnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBAEtB,KAAK,CAAC,OAAO,EAAE,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAEjC,KAAK,CAAC,WAAW,EAAE,CAAA;gBACpB,CAAC;YACF,CAAC;QACF,CAAC;QAED,kGAAkG;QAClG,uCAAuC;QACvC,2DAA2D;QAC3D,iGAAiG;QACjG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;QAE1E,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;QAErC,MAAM,oBAAoB,CAAC,IAAI,CAAC,cAAc,EAAE;YAC/C,QAAQ;YACR,kBAAkB,EAAE,CAAC,GAAG,SAAS,CAAC;YAClC,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE;YACvB,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC;YACxD,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC;YACzD,iBAAiB,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,CAAC;YAC7D,mBAAmB,EAAE,KAAK,CAAC,iBAAiB;YAC5C,gCAAgC,EAAE,KAAK,CAAC,kBAAkB;YAC1D,uBAAuB,EAAE,KAAK,CAAC,qBAAqB;YACpD,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC;YACvD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;IAEhF,OAAO,EAAE,GAAG,MAAM,EAAE,GAAG,KAAK,EAAE,CAAA;AAC/B,CAAC"}

package/out/build/checkpoint.d.ts

@@ -0,0 +1,94 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The align phase's resume point. It records aligned adapters and each output's length at that moment.
+ *   It also records every accumulator used to compute the manifest.
+ *
+ *   The adapter phase recovers from its per-adapter `MANIFEST.json`. Align, split and parquet ran 6 h 35 min
+ *   of the 8 h 27 min `v0.7.0-de-holdout` build with no recovery point, because the four labeled streams are
+ *   truncated at open and every accumulator is a local. This file is what a resumed align phase reads
+ *   instead of starting over.
+ *
+ *   The phase is replayable only when the same rows produce the same output, so the checkpoint records the
+ *   settings that change alignment and refuses a resume under different ones. `align-determinism` in
+ *   `build.test.ts` compares four output digests across two builds over one intermediate directory.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * The four append-only outputs of the align phase, by the name in each file.
+ */
+export declare const ALIGN_OUTPUTS: readonly ["train", "val", "test", "quarantine"];
+export type AlignOutputName = (typeof ALIGN_OUTPUTS)[number];
+/**
+ * The settings that change what the align phase writes for a given input row.
+ *
+ * A resume under different settings would mix two policies inside one output file,
+ * so the checkpoint records these and a mismatch refuses the resume rather than continuing.
+ */
+export interface AlignSettings {
+    corpus_version: string;
+    synthesize: boolean;
+    license_policy: string;
+    exclude_licenses: readonly string[];
+    profile: string;
+    holdout_digest: string;
+}
+/**
+ * The align phase's state at an adapter boundary.
+ */
+export interface AlignCheckpoint {
+    settings: AlignSettings;
+    /**
+     * Adapter ids whose rows are already in the four outputs, in the order they were aligned.
+     *
+     * A resume requires this to be a prefix of the current run's adapter order.
+     * Alignment appends, so a different order would place a later adapter's rows
+     * before an earlier one's and change every split file's contents.
+     */
+    completed_adapters: string[];
+    /**
+     * Byte length of each output at the checkpoint, after its writes were flushed.
+     *
+     * A resumed build shortens each file to this length.
+     * The shortening discards whatever the interrupted adapter wrote past the boundary.
+     */
+    offsets: Record<AlignOutputName, number>;
+    counts: Record<"train" | "val" | "test", number>;
+    aligned: number;
+    quarantined: number;
+    license_counts: Record<string, number>;
+    refused_by_kind: Record<string, number>;
+    /**
+     * Rows the license policy refused, counted per `source`, so a resumed build carries the tally
+     * rather than restarting it at zero and understating the refusal in its manifest.
+     */
+    refused_by_source: Record<string, number>;
+    excluded_by_license: number;
+    admitted_unresolved_license_rows: number;
+    excluded_by_eligibility: number;
+    rows_by_source: Record<string, {
+        rows: number;
+        license: string;
+    }>;
+    written_at: string;
+}
+/**
+ * Read the align checkpoint, or return `null` when the build has not reached one.
+ *
+ * @throws When the checkpoint describes different settings, or an adapter order
+ * the current run does not begin with.
+ * Both cases would append this run's rows to a file written under different rules.
+ * The resulting corpus could report a row count that later readers accept.
+ */
+export declare function readAlignCheckpoint(path: PathBuilderLike, expected: {
+    settings: AlignSettings;
+    adapterOrder: readonly string[];
+}): Promise<AlignCheckpoint | null>;
+/**
+ * Write the align checkpoint.
+ *
+ * The caller flushes the four outputs before calling this, so every byte the offsets count is on disk.
+ */
+export declare function writeAlignCheckpoint(path: PathBuilderLike, checkpoint: AlignCheckpoint): Promise<void>;
+//# sourceMappingURL=checkpoint.d.ts.map

package/out/build/checkpoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint.d.ts","sourceRoot":"","sources":["../../lib/build/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;GAEG;AACH,eAAO,MAAM,aAAa,YAAI,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAU,CAAA;AAE5E,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAA;AAE5D;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,UAAU,EAAE,OAAO,CAAA;IACnB,cAAc,EAAE,MAAM,CAAA;IACtB,gBAAgB,EAAE,SAAS,MAAM,EAAE,CAAA;IACnC,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,aAAa,CAAA;IAEvB;;;;;;OAMG;IACH,kBAAkB,EAAE,MAAM,EAAE,CAAA;IAE5B;;;;;OAKG;IACH,OAAO,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;IAExC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,KAAK,GAAG,MAAM,EAAE,MAAM,CAAC,CAAA;IAChD,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACtC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACvC;;;OAGG;IACH,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,mBAAmB,EAAE,MAAM,CAAA;IAC3B,gCAAgC,EAAE,MAAM,CAAA;IACxC,uBAAuB,EAAE,MAAM,CAAA;IAC/B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACjE,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;GAOG;AACH,wBAAsB,mBAAmB,CACxC,IAAI,EAAE,eAAe,EACrB,QAAQ,EAAE;IAAE,QAAQ,EAAE,aAAa,CAAC;IAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACpE,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CA0BjC;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtG"}

package/out/build/checkpoint.js

@@ -0,0 +1,59 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The align phase's resume point. It records aligned adapters and each output's length at that moment.
+ *   It also records every accumulator used to compute the manifest.
+ *
+ *   The adapter phase recovers from its per-adapter `MANIFEST.json`. Align, split and parquet ran 6 h 35 min
+ *   of the 8 h 27 min `v0.7.0-de-holdout` build with no recovery point, because the four labeled streams are
+ *   truncated at open and every accumulator is a local. This file is what a resumed align phase reads
+ *   instead of starting over.
+ *
+ *   The phase is replayable only when the same rows produce the same output, so the checkpoint records the
+ *   settings that change alignment and refuses a resume under different ones. `align-determinism` in
+ *   `build.test.ts` compares four output digests across two builds over one intermediate directory.
+ */
+import { readLocalJSONFile, pathExists } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { stringifyJSON } from "@mailwoman/core/json";
+/**
+ * The four append-only outputs of the align phase, by the name in each file.
+ */
+export const ALIGN_OUTPUTS = ["train", "val", "test", "quarantine"];
+/**
+ * Read the align checkpoint, or return `null` when the build has not reached one.
+ *
+ * @throws When the checkpoint describes different settings, or an adapter order
+ * the current run does not begin with.
+ * Both cases would append this run's rows to a file written under different rules.
+ * The resulting corpus could report a row count that later readers accept.
+ */
+export async function readAlignCheckpoint(path, expected) {
+    if (!(await pathExists(path)))
+        return null;
+    const checkpoint = await readLocalJSONFile(path);
+    const found = stringifyJSON(checkpoint.settings);
+    const wanted = stringifyJSON(expected.settings);
+    if (found !== wanted) {
+        throw new Error(`resume refused: ${path.toString()} was written under ${found} and this build runs under ${wanted}. ` +
+            `Delete the intermediate directory's labeled and quarantine files together with this checkpoint to ` +
+            `align from the start.`);
+    }
+    const prefix = expected.adapterOrder.slice(0, checkpoint.completed_adapters.length);
+    if (stringifyJSON(prefix) !== stringifyJSON(checkpoint.completed_adapters)) {
+        throw new Error(`resume refused: ${path.toString()} recorded ${stringifyJSON(checkpoint.completed_adapters)} as aligned ` +
+            `and this build's adapter order begins ${stringifyJSON(prefix)}. Alignment appends, so the rows would ` +
+            `land in a different order than the recorded offsets describe.`);
+    }
+    return checkpoint;
+}
+/**
+ * Write the align checkpoint.
+ *
+ * The caller flushes the four outputs before calling this, so every byte the offsets count is on disk.
+ */
+export function writeAlignCheckpoint(path, checkpoint) {
+    return writeLocalJSONFile(checkpoint, path).then(() => undefined);
+}
+//# sourceMappingURL=checkpoint.js.map

package/out/build/checkpoint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint.js","sourceRoot":"","sources":["../../lib/build/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAGpD;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAU,CAAA;AA2D5E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,IAAqB,EACrB,QAAsE;IAEtE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAkB,IAAI,CAAC,CAAA;IACjE,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAE/C,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACd,mBAAmB,IAAI,CAAC,QAAQ,EAAE,sBAAsB,KAAK,8BAA8B,MAAM,IAAI;YACpG,oGAAoG;YACpG,uBAAuB,CACxB,CAAA;IACF,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEnF,IAAI,aAAa,CAAC,MAAM,CAAC,KAAK,aAAa,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CACd,mBAAmB,IAAI,CAAC,QAAQ,EAAE,aAAa,aAAa,CAAC,UAAU,CAAC,kBAAkB,CAAC,cAAc;YACxG,yCAAyC,aAAa,CAAC,MAAM,CAAC,yCAAyC;YACvG,+DAA+D,CAChE,CAAA;IACF,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAqB,EAAE,UAA2B;IACtF,OAAO,kBAAkB,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;AAClE,CAAC"}

package/out/build/eligibility.d.ts

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file What the address-source register records about a row's source, read once per build.
+ */
+import { type LicenseDecision } from "#source-register";
+import type { CanonicalRow } from "#types";
+/**
+ * The key a row is looked up under: the adapter that emitted it and the jurisdiction it describes.
+ *
+ * The register scopes a source to one publisher in one jurisdiction,
+ * so an adapter id alone cannot address a record.
+ * One adapter serves several: `ban` emits eleven jurisdictions from one schema
+ * and each is its own source under its own license decision.
+ */
+export declare function sourceEligibilityKey(adapterID: string, iso2: string): string;
+/**
+ * Every register source's ingest-eligibility reasons, keyed by the adapter
+ * and jurisdiction of the rows that carry it.
+ *
+ * A source declaring no `adapterID` is absent from the map, because no adapter emits it
+ * and no row can arrive under it.
+ * An empty array is the only value a caller may read as permission.
+ * An absent key is refused rather than admitted.
+ */
+export declare function readSourceEligibility(): Promise<ReadonlyMap<string, readonly string[]>>;
+/**
+ * The register's license decisions keyed by the label in a row.
+ *
+ * An unmatched adapter label records a `null` decision rather than inventing one.
+ */
+export declare function readRegisterDecisions(): Promise<ReadonlyMap<string, LicenseDecision>>;
+/**
+ * Reads why a row's source may be refused from a release-eligible corpus.
+ * Records each refused source.
+ *
+ * `refused` accumulates as rows arrive.
+ * The build manifest reports the collected values.
+ *
+ * A build whose `eligibility` is `null` runs under the exploratory profile and every row reads eligible.
+ */
+export declare function createIneligibilityReader(eligibility: ReadonlyMap<string, readonly string[]> | null): {
+    refused: Map<string, readonly string[]>;
+    read: (row: CanonicalRow) => readonly string[] | null;
+};
+//# sourceMappingURL=eligibility.d.ts.map

package/out/build/eligibility.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eligibility.d.ts","sourceRoot":"","sources":["../../lib/build/eligibility.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAwD,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAC7G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAE1C;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAE5E;AAED;;;;;;;;GAQG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAC,CAW7F;AAED;;;;GAIG;AACH,wBAAsB,qBAAqB,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAI3F;AAED;;;;;;;;GAQG;AACH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG;IACtG,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IACvC,IAAI,EAAE,CAAC,GAAG,EAAE,YAAY,KAAK,SAAS,MAAM,EAAE,GAAG,IAAI,CAAA;CACrD,CA0BA"}