@mailwoman/corpus 10.0.0 → 10.1.0

package/out/build.d.ts

@@ -2,131 +2,16 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
+ * @file The corpus build, as the four phases it runs in order: adapters, align, split, parquet.
  *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` slices.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet slices** — `writeSlices` streams labeled rows into 1M-row `.parquet` slices per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-slice checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-slice
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final slices for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
- */
-import { type AdapterRunManifest } from "#runner";
-import type { AdapterOptions, CorpusAdapter } from "#types";
-import { type SliceManifest } from "#utils/parquet";
-import { type SplitManifest } from "#utils/split";
-/**
- * Stage tags surfaced to `onProgress`.
- */
-export type BuildStage = "adapter-run" | "align" | "split" | "slice" | "manifest";
-/**
- * Per-invocation options for `buildCorpus`.
+ *   Each phase is its own module. This module defines their sequence and shared paths.
+ *   It assembles the manifest from the phase reports.
  */
-export interface BuildCorpusOptions {
-    /**
-     * Root output directory. All build artifacts land beneath it.
-     */
-    outputDir: string;
-    /**
-     * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
-     */
-    corpusVersion: string;
-    /**
-     * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
-     * `[wofAdminAdapter]` for a smoke run).
-     */
-    adapters?: readonly CorpusAdapter[];
-    /**
-     * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
-     * noted in the manifest).
-     */
-    adapterInputs: Record<string, AdapterOptions>;
-    /**
-     * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
-     */
-    synthesize?: boolean;
-    /**
-     * Forwarded to `writeSlices`. Default 1_000_000.
-     */
-    rowsPerSlice?: number;
-    /**
-     * Progress hook. Errors thrown abort the build.
-     */
-    onProgress?: (stage: BuildStage, message: string) => void;
-    /**
-     * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
-     * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
-     * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
-     * share-alike set (`--exclude-share-alike`).
-     */
-    excludeLicenses?: readonly RegExp[];
-}
-/**
- * Top-level manifest tying every stage together.
- */
-export interface BuildCorpusManifest {
-    corpus_version: string;
-    built_at: string;
-    adapters: AdapterRunManifest[];
-    skipped_adapters: string[];
-    splits: {
-        counts: SplitManifest["counts"];
-        holdouts: SplitManifest["holdouts"];
-    };
-    slices: {
-        counts: SliceManifest["counts"];
-        total_rows: number;
-    };
-    quarantine_count: number;
-    total_aligned_rows: number;
-    /**
-     * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
-     * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
-     */
-    licenses: Record<string, number>;
-    excluded_by_license: number;
-}
+import { type BuildCorpusManifest, type BuildCorpusOptions } from "#build/types";
+export * from "#build/heap";
+export * from "#build/types";
 /**
  * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → slice hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
  */
 export declare function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest>;
 //# sourceMappingURL=build.d.ts.map

package/out/build.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../lib/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAUH,OAAO,EAAoB,KAAK,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAEnE,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,QAAQ,CAAA;AAGrF,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,cAAc,CAAA;AAErB;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyNxF"}
+{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../lib/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAYH,OAAO,EAAwC,KAAK,mBAAmB,EAAE,KAAK,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAUtH,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAE5B;;GAEG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyJxF"}

package/out/build.js

@@ -2,244 +2,149 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
+ * @file The corpus build, as the four phases it runs in order: adapters, align, split, parquet.
  *
- *   End-to-end corpus build (Phase 1 task #10 in the plan).
- *
- *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
- *
- *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
- *        `<intermediate>/<adapter.id>/canonical.jsonl` slices.
- *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
- *        country-default policy emits an augmented row alongside the original.
- *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
- *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
- *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
- *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
- *        `test.txt`.
- *   5. **Parquet slices** — `writeSlices` streams labeled rows into 1M-row `.parquet` slices per split
- *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
- *        row groups), with per-slice checksums + per-stage manifest in
- *        `corpus-v<version>/MANIFEST.json`.
- *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
- *        a top-level corpus_version, built_at, and aggregate counts.
- *
- *   Output layout:
- *
- *   ```
- *   <outputDir>/
- *   MANIFEST.json
- *   intermediate/
- *     <adapter.id>/canonical.jsonl   # one per adapter
- *     labeled.jsonl                  # post-alignment, pre-slice
- *     quarantine.jsonl               # rows that failed alignment
- *   splits/
- *     SPLIT_MANIFEST.json
- *     train.txt / val.txt / test.txt
- *   corpus-v<version>/
- *     MANIFEST.json
- *     train/part-NNNN.parquet
- *     val/part-NNNN.parquet
- *     test/part-NNNN.parquet
- * ```
- *
- *   The intermediate files live alongside the final slices for reproducibility + debugging. Operators
- *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
- *   self-contained.
+ *   Each phase is its own module. This module defines their sequence and shared paths.
+ *   It assembles the manifest from the phase reports.
  */
-import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers";
-import { openWriteStream } from "@mailwoman/core/fs/streams";
-import { writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers";
-import { join } from "path-ts";
-import { JSONSpliterator } from "spliterator";
-import { defaultAdapterRegistry } from "#adapters/utils";
+import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils";
+import { heapLimitBytes } from "@mailwoman/core/utils/system";
+import { PathBuilder } from "path-ts";
+import { defaultAdapterRegistry } from "#adapters/registry";
+import { runAdapterPhase } from "#build/adapters";
+import { runAlignPhase } from "#build/align";
+import { readRegisterDecisions, readSourceEligibility } from "#build/eligibility";
+import { streamJSONL } from "#build/rows";
+import { BuildProfile, TRAINING_MANIFEST_FILE } from "#build/types";
 import { $public } from "#env";
-import { once, runAdapter } from "#runner";
-import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils";
-import { alignRow } from "#utils/align";
-import { licenseExcluded } from "#utils/license";
-import { writeSlices } from "#utils/parquet";
-import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "#utils/split";
+import { ROWS_PER_FILE } from "#parquet/schema";
+import { DEFAULT_SHUFFLE_SEED, DEFAULT_SHUFFLE_WINDOW, shuffleWithinWindow } from "#parquet/shuffle";
+import { writeParquetSplits } from "#parquet/writers";
+import { freezeTrainingManifest } from "#source-register/training-manifest";
+import { LicensePolicy } from "#utils/license";
+import { defaultHoldouts, writeSplitManifestsFromLabeledFiles } from "#utils/split";
+export * from "#build/heap";
+export * from "#build/types";
 /**
  * Drive the full corpus build to completion.
- *
- * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → slice hand-off.
- * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
- * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
  */
 export async function buildCorpus(opts) {
     const adapters = opts.adapters ?? defaultAdapterRegistry.list();
     const synthesize = opts.synthesize ?? true;
-    const rowsPerSlice = opts.rowsPerSlice ?? 1_000_000;
+    const rowsPerFile = opts.rowsPerFile ?? ROWS_PER_FILE;
+    const shuffleWindow = opts.shuffleWindow ?? DEFAULT_SHUFFLE_WINDOW;
+    const shuffleSeed = opts.shuffleSeed ?? DEFAULT_SHUFFLE_SEED;
+    const profile = opts.profile ?? BuildProfile.Exploratory;
+    const licensePolicy = opts.licensePolicy ?? LicensePolicy.All;
     const built_at = new Date().toISOString();
-    await makeDirectories(opts.outputDir);
-    const intermediateDir = join(opts.outputDir, "intermediate");
-    await makeDirectories(intermediateDir);
-    // 1. Adapter runs.
-    const adapterRuns = [];
-    const skipped = [];
-    for (const adapter of adapters) {
-        const adapterOptions = opts.adapterInputs[adapter.id];
-        if (!adapterOptions) {
-            skipped.push(adapter.id);
-            opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`);
-            continue;
-        }
-        // Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
-        // already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
-        // canonical is fully flushed, so its presence guarantees completeness; row order is identical,
-        // so downstream holdout-split determinism is preserved. Recovers an align-phase crash without
-        // redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
-        const adapterDir = join(intermediateDir, adapter.id);
-        const cachedManifest = join(adapterDir, "MANIFEST.json");
-        if ($public.MAILWOMAN_RESUME === "1" &&
-            (await pathExists(cachedManifest)) &&
-            (await pathExists(join(adapterDir, "canonical.jsonl")))) {
-            const cached = await readLocalJSONFile(cachedManifest);
-            opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`);
-            adapterRuns.push(cached);
-            continue;
-        }
-        opts.onProgress?.("adapter-run", `running ${adapter.id}`);
-        const m = await runAdapter({
-            adapter,
-            adapterOptions,
-            outputDir: intermediateDir,
-            corpusVersion: opts.corpusVersion,
-        });
-        adapterRuns.push(m);
-    }
-    // 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
-    // and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
-    // jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIDMap`
-    // + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
-    // `splitForRow` (a pure function of source_id + region + holdout policy).
+    // The build reports heap state when it aborts with SIGABRT.
+    // These aborts arrive twelve minutes into the adapter phase.
+    const heapLimit = heapLimitBytes();
+    const configured = adapters.filter((adapter) => opts.adapterInputs[adapter.id]).map((adapter) => adapter.id);
+    opts.onProgress?.("adapter-run", `heap limit ${(heapLimit / 1024 / 1024 / 1024).toFixed(1)} GiB, ${configured.length} adapters configured`);
+    const outputDir = PathBuilder.from(opts.outputDir);
+    const intermediateDir = outputDir("intermediate");
+    await makeDirectories(outputDir, intermediateDir);
+    const { runs: adapterRuns, skipped } = await runAdapterPhase({
+        adapters,
+        adapterInputs: opts.adapterInputs,
+        corpusVersion: opts.corpusVersion,
+        intermediateDir,
+        onProgress: opts.onProgress,
+    });
     const labeledPaths = {
-        train: join(intermediateDir, "labeled-train.jsonl"),
-        val: join(intermediateDir, "labeled-val.jsonl"),
-        test: join(intermediateDir, "labeled-test.jsonl"),
-    };
-    const labeledStreams = {
-        train: openWriteStream(labeledPaths.train, { encoding: "utf8" }),
-        val: openWriteStream(labeledPaths.val, { encoding: "utf8" }),
-        test: openWriteStream(labeledPaths.test, { encoding: "utf8" }),
+        train: intermediateDir("labeled-train.jsonl"),
+        val: intermediateDir("labeled-val.jsonl"),
+        test: intermediateDir("labeled-test.jsonl"),
     };
-    const quarantinePath = join(intermediateDir, "quarantine.jsonl");
-    const quarantineStream = openWriteStream(quarantinePath, { encoding: "utf8" });
-    let aligned = 0;
-    let quarantined = 0;
-    const counts = { train: 0, val: 0, test: 0 };
     const holdouts = defaultHoldouts();
-    // License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
-    // license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
-    // everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
-    const excludeLicenses = opts.excludeLicenses ?? [];
-    const licenseCounts = new Map();
-    let excludedByLicense = 0;
-    const writeQuarantine = (row, reason) => {
-        quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
-    };
-    for (const adapterRun of adapterRuns) {
-        opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
-        for await (const row of streamJSONL(adapterRun.jsonl_path)) {
-            licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
-            // Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
-            // kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
-            // purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
-            // set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was removed.
-            if (licenseExcluded(row.license, excludeLicenses)) {
-                excludedByLicense++;
-                continue;
-            }
-            const fanned = [row];
-            if (synthesize) {
-                for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
-                    fanned.push(aug);
-                }
-            }
-            for (const r of fanned) {
-                let result;
-                try {
-                    result = alignRow(r);
-                }
-                catch (error) {
-                    // Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
-                    // alignRow's targeted paths normalize/quarantine known issues with specific reasons;
-                    // this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
-                    // shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
-                    writeQuarantine(r, `align-threw:${error.message.slice(0, 160)}`);
-                    quarantined++;
-                    continue;
-                }
-                if (result.kind === "labeled") {
-                    const split = splitForRow(result.row, holdouts);
-                    labeledStreams[split].write(`${JSON.stringify(result.row)}\n`);
-                    counts[split]++;
-                    aligned++;
-                }
-                else {
-                    writeQuarantine(r, result.row.reason);
-                    quarantined++;
-                }
-            }
-        }
-    }
-    for (const s of Object.values(labeledStreams)) {
-        s.end();
-    }
-    quarantineStream.end();
-    await Promise.all([...Object.values(labeledStreams).map((s) => once(s, "close")), once(quarantineStream, "close")]);
-    // 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
-    // source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
-    // manifests with disk spill for splits that exceed in-memory thresholds.
-    opts.onProgress?.("split", `splitting ${aligned} aligned rows`);
-    const splitsDir = join(opts.outputDir, "splits");
+    const registerDecisions = await readRegisterDecisions();
+    const align = await runAlignPhase({
+        adapterRuns,
+        corpusVersion: opts.corpusVersion,
+        synthesize,
+        profile,
+        licensePolicy,
+        excludeLicenses: opts.excludeLicenses ?? [],
+        holdouts,
+        eligibility: profile === BuildProfile.ReleaseEligible ? await readSourceEligibility() : null,
+        labeledPaths,
+        quarantinePath: intermediateDir("quarantine.jsonl"),
+        checkpointPath: intermediateDir("align-checkpoint.json"),
+        onProgress: opts.onProgress,
+    });
+    opts.onProgress?.("split", `splitting ${align.aligned} aligned rows`);
     const splitCounts = await writeSplitManifestsFromLabeledFiles({
         labeledPaths,
-        outputDir: splitsDir,
+        outputDir: outputDir("splits"),
         corpusVersion: opts.corpusVersion,
-        counts,
+        counts: align.counts,
         holdouts,
     });
-    // 5. Parquet slices — per-split labeled JSONL streams in, sliced `.parquet` out. The prior
-    // `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
-    opts.onProgress?.("slice", "writing parquet slices");
-    const sliceManifest = await writeSlices({
-        train: streamJSONL(labeledPaths.train),
-        val: streamJSONL(labeledPaths.val),
-        test: streamJSONL(labeledPaths.test),
+    opts.onProgress?.("parquet", "writing parquet files");
+    // Each split gets its own generator seeded from one base, so a split's row order does
+    // not depend on how many rows the earlier splits happened to contain.
+    const shuffled = (path, salt) => shuffleWithinWindow(streamJSONL(path), makeMulberry32(shuffleSeed + salt), shuffleWindow);
+    opts.onProgress?.("parquet", shuffleWindow > 1
+        ? `shuffling each split within a ${shuffleWindow.toLocaleString("en-US")}-row window, seed ${shuffleSeed}`
+        : "writing rows in arrival order (shuffle window disabled)");
+    const parquetManifest = await writeParquetSplits({
+        train: shuffled(labeledPaths.train, 0),
+        val: shuffled(labeledPaths.val, 1),
+        test: shuffled(labeledPaths.test, 2),
     }, {
-        outputDir: opts.outputDir,
+        outputDir,
         corpusVersion: opts.corpusVersion,
-        rowsPerSlice,
+        rowsPerFile,
+        resume: $public.MAILWOMAN_RESUME === "1",
     });
-    // License-set visibility (#26): loudly report the resolved license set so a build is an obvious
-    // deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
-    const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1]);
+    const licenseSummary = [...align.licenseCounts.entries()].toSorted((a, b) => b[1] - a[1]);
+    const refusedValues = align.licenses.refusedValues();
+    const refusalTally = [...align.refusedByKind.entries()].map(([kind, rows]) => `${kind}=${rows}`).join(", ");
     opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
-        (excludedByLicense > 0
-            ? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
-            : " | NO license exclusion applied (all rows kept)"));
-    // 6. Top-level manifest.
+        ` | policy ${licensePolicy}` +
+        (align.excludedByLicense > 0
+            ? ` | refused ${align.excludedByLicense} rows over ${refusedValues.size} values (${refusalTally})`
+            : ` | refused 0 rows`) +
+        ` | admitted ${align.admittedUnresolved} rows whose license resolves to no expression`);
     opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
+    // Frozen from what this build observed rather than re-derived later, because the register moves
+    // as terms are reviewed and a re-derived record would describe a build that never happened.
+    const trainingManifest = freezeTrainingManifest({
+        corpusVersion: opts.corpusVersion,
+        builtAt: built_at,
+        profile,
+        rowsBySource: align.rowsBySource,
+        decisionsByLicense: registerDecisions,
+        refused: align.ineligibleSources,
+    });
+    await writeLocalJSONFile(trainingManifest, outputDir(TRAINING_MANIFEST_FILE));
     const manifest = {
         corpus_version: opts.corpusVersion,
         built_at,
         adapters: adapterRuns,
         skipped_adapters: skipped,
         splits: { counts: splitCounts, holdouts },
-        slices: { counts: sliceManifest.counts, total_rows: sliceManifest.total_rows },
-        quarantine_count: quarantined,
-        total_aligned_rows: aligned,
+        slices: { counts: parquetManifest.counts, total_rows: parquetManifest.total_rows },
+        quarantine_count: align.quarantined,
+        total_aligned_rows: align.aligned,
         licenses: Object.fromEntries(licenseSummary),
-        excluded_by_license: excludedByLicense,
+        licenses_cover: "the rows this build aligned from its adapters. An overlay parquet assembled afterwards carries its " +
+            "own licenses, and the overlay corpus's own MANIFEST.json states no license set at all, so a reader " +
+            "attributing an overlay corpus reads this set plus each overlay's own record rather than this set alone.",
+        excluded_by_license: align.excludedByLicense,
+        license_policy: licensePolicy,
+        refused_by_license_kind: Object.fromEntries(align.refusedByKind),
+        refused_license_values: Object.fromEntries(refusedValues),
+        refused_rows_by_source: Object.fromEntries(align.refusedBySource),
+        admitted_unresolved_license_rows: align.admittedUnresolved,
+        profile,
+        excluded_by_eligibility: align.excludedByEligibility,
+        ineligible_sources: Object.fromEntries(align.ineligibleSources),
+        training_manifest_digest: trainingManifest.contentDigest,
     };
-    await writeLocalJSONFile(manifest, opts.outputDir, "MANIFEST.json");
+    await writeLocalJSONFile(manifest, outputDir("MANIFEST.json"));
     return manifest;
 }
-async function* streamJSONL(path) {
-    // JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
-    // lines are dropped at the row level) and throws SyntaxError on a malformed row —
-    // same fail-loud behavior as the prior readline + bare `JSON.parse`.
-    yield* JSONSpliterator.fromAsync(path);
-}
 //# sourceMappingURL=build.js.map

package/out/build.js.map

@@ -1 +1 @@
-{"version":3,"file":"build.js","sourceRoot":"","sources":["../lib/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAA;AACxD,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,IAAI,EAAE,UAAU,EAA2B,MAAM,SAAS,CAAA;AACnE,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAEnF,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAE,WAAW,EAAsB,MAAM,gBAAgB,CAAA;AAChE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,cAAc,CAAA;AA6ErB;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,eAAe,CAAC,eAAe,CAAC,CAAA;IAEtC,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAE/E,SAAQ;QACT,CAAC;QAED,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAExD,IACC,OAAO,CAAC,gBAAgB,KAAK,GAAG;YAChC,CAAC,MAAM,UAAU,CAAC,cAAc,CAAC,CAAC;YAClC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,CAAC,EACtD,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAqB,cAAc,CAAC,CAAA;YAC1E,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAExB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QAEzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QAEF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IAED,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,eAAe,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAChE,GAAG,EAAE,eAAe,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC5D,IAAI,EAAE,eAAe,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAC9D,CAAA;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,eAAe,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAE9E,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,+FAA+F;YAC/F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBAEnB,SAAQ;YACT,CAAC;YAED,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBAEvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,KAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBAE3E,WAAW,EAAE,CAAA;oBAEb,SAAQ;gBACT,CAAC;gBAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAE9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBAEf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAErC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,CAAC;IAED,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;IAEnH,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAEhD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,2FAA2F;IAC3F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IAEpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAEnE,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,mFAAmF;IACnF,kFAAkF;IAClF,qEAAqE;IACrE,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAI,IAAI,CAAC,CAAA;AAC1C,CAAC"}
+{"version":3,"file":"build.js","sourceRoot":"","sources":["../lib/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,UAAU,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAA;AAC7D,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAA;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,sBAAsB,EAAqD,MAAM,cAAc,CAAA;AACtH,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACpG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAA;AAE3E,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAC9C,OAAO,EAAE,eAAe,EAAE,mCAAmC,EAAkB,MAAM,cAAc,CAAA;AAEnG,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAE5B;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,aAAa,CAAA;IACrD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,sBAAsB,CAAA;IAClE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,oBAAoB,CAAA;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,WAAW,CAAA;IACxD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,GAAG,CAAA;IAC7D,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,4DAA4D;IAC5D,6DAA6D;IAC7D,MAAM,SAAS,GAAG,cAAc,EAAE,CAAA;IAClC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAE5G,IAAI,CAAC,UAAU,EAAE,CAChB,aAAa,EACb,cAAc,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,MAAM,sBAAsB,CACzG,CAAA;IAED,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAClD,MAAM,eAAe,GAAG,SAAS,CAAC,cAAc,CAAC,CAAA;IAEjD,MAAM,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAEjD,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,eAAe,CAAC;QAC5D,QAAQ;QACR,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,eAAe;QACf,UAAU,EAAE,IAAI,CAAC,UAAU;KAC3B,CAAC,CAAA;IAEF,MAAM,YAAY,GAAmC;QACpD,KAAK,EAAE,eAAe,CAAC,qBAAqB,CAAC;QAC7C,GAAG,EAAE,eAAe,CAAC,mBAAmB,CAAC;QACzC,IAAI,EAAE,eAAe,CAAC,oBAAoB,CAAC;KAC3C,CAAA;IAED,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,MAAM,iBAAiB,GAAG,MAAM,qBAAqB,EAAE,CAAA;IAEvD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC;QACjC,WAAW;QACX,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,UAAU;QACV,OAAO;QACP,aAAa;QACb,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,EAAE;QAC3C,QAAQ;QACR,WAAW,EAAE,OAAO,KAAK,YAAY,CAAC,eAAe,CAAC,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,CAAC,CAAC,IAAI;QAC5F,YAAY;QACZ,cAAc,EAAE,eAAe,CAAC,kBAAkB,CAAC;QACnD,cAAc,EAAE,eAAe,CAAC,uBAAuB,CAAC;QACxD,UAAU,EAAE,IAAI,CAAC,UAAU;KAC3B,CAAC,CAAA;IAEF,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,KAAK,CAAC,OAAO,eAAe,CAAC,CAAA;IAErE,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC;QAC9B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ;KACR,CAAC,CAAA;IAEF,IAAI,CAAC,UAAU,EAAE,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAA;IAErD,sFAAsF;IACtF,sEAAsE;IACtE,MAAM,QAAQ,GAAG,CAAC,IAAqB,EAAE,IAAY,EAAE,EAAE,CACxD,mBAAmB,CAAC,WAAW,CAAa,IAAI,CAAC,EAAE,cAAc,CAAC,WAAW,GAAG,IAAI,CAAC,EAAE,aAAa,CAAC,CAAA;IAEtG,IAAI,CAAC,UAAU,EAAE,CAChB,SAAS,EACT,aAAa,GAAG,CAAC;QAChB,CAAC,CAAC,iCAAiC,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,qBAAqB,WAAW,EAAE;QAC1G,CAAC,CAAC,yDAAyD,CAC5D,CAAA;IAED,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAC/C;QACC,KAAK,EAAE,QAAQ,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC;QACtC,GAAG,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,CAAC;QAClC,IAAI,EAAE,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;KACpC,EACD;QACC,SAAS;QACT,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,WAAW;QACX,MAAM,EAAE,OAAO,CAAC,gBAAgB,KAAK,GAAG;KACxC,CACD,CAAA;IAED,MAAM,cAAc,GAAG,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACzF,MAAM,aAAa,GAAG,KAAK,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAA;IACpD,MAAM,YAAY,GAAG,CAAC,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE3G,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,aAAa,aAAa,EAAE;QAC5B,CAAC,KAAK,CAAC,iBAAiB,GAAG,CAAC;YAC3B,CAAC,CAAC,cAAc,KAAK,CAAC,iBAAiB,cAAc,aAAa,CAAC,IAAI,YAAY,YAAY,GAAG;YAClG,CAAC,CAAC,mBAAmB,CAAC;QACvB,eAAe,KAAK,CAAC,kBAAkB,+CAA+C,CACvF,CAAA;IAED,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAEhE,gGAAgG;IAChG,4FAA4F;IAC5F,MAAM,gBAAgB,GAAG,sBAAsB,CAAC;QAC/C,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,OAAO,EAAE,QAAQ;QACjB,OAAO;QACP,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,kBAAkB,EAAE,iBAAiB;QACrC,OAAO,EAAE,KAAK,CAAC,iBAAiB;KAChC,CAAC,CAAA;IAEF,MAAM,kBAAkB,CAAC,gBAAgB,EAAE,SAAS,CAAC,sBAAsB,CAAC,CAAC,CAAA;IAE7E,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,eAAe,CAAC,UAAU,EAAE;QAClF,gBAAgB,EAAE,KAAK,CAAC,WAAW;QACnC,kBAAkB,EAAE,KAAK,CAAC,OAAO;QACjC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,cAAc,EACb,qGAAqG;YACrG,qGAAqG;YACrG,yGAAyG;QAC1G,mBAAmB,EAAE,KAAK,CAAC,iBAAiB;QAC5C,cAAc,EAAE,aAAa;QAC7B,uBAAuB,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC;QAChE,sBAAsB,EAAE,MAAM,CAAC,WAAW,CAAC,aAAa,CAAC;QACzD,sBAAsB,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,eAAe,CAAC;QACjE,gCAAgC,EAAE,KAAK,CAAC,kBAAkB;QAC1D,OAAO;QACP,uBAAuB,EAAE,KAAK,CAAC,qBAAqB;QACpD,kBAAkB,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC/D,wBAAwB,EAAE,gBAAgB,CAAC,aAAa;KACxD,CAAA;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,SAAS,CAAC,eAAe,CAAC,CAAC,CAAA;IAE9D,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/cn/recipes/organizational-units.d.ts

@@ -2,35 +2,18 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
- *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
- *   character by character for the CJK sibling model.
+ * @file Labels Chinese rows whose settlement ends in an organizational unit chain.
  *
- *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
- *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
- *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
- *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
- *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
- *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
- *
- *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
- *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
- *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
- *   licence and the inference is stated in the `license` field rather than hidden behind it.
- *
- *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
- *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
- *   on this recipe's rows; its label set has no `locality_unit`.
+ *   The unit grammar tags the whole chain as `locality_unit` and its head unit as `dependent_locality`. Input is JSONL
+ *   with `{ raw, country }`. Rows are aligned per character for the CJK model.
  */
 import { type CorpusRecipe } from "#recipes/scaffold";
 /**
- * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
- * no organizational chain and so teaches nothing this recipe exists for.
+ * Labels the components of a row, or returns `null` when the row has no unit chain.
  */
 export declare function labelCNOrganizationalRow(raw: string): Record<string, string> | null;
 /**
- * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
- * reading of a generic rather than a guess.
+ * The recipe registered with the corpus builder.
  */
 export declare const cnOrganizationalUnitsRecipe: CorpusRecipe;
 //# sourceMappingURL=organizational-units.d.ts.map

package/out/cn/recipes/organizational-units.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"organizational-units.d.ts","sourceRoot":"","sources":["../../../lib/cn/recipes/organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,KAAK,YAAY,EAA6B,MAAM,mBAAmB,CAAA;AAmBhF;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAkDnF;AAmBD;;;GAGG;AACH,eAAO,MAAM,2BAA2B,EAAE,YA2EzC,CAAA"}
+{"version":3,"file":"organizational-units.d.ts","sourceRoot":"","sources":["../../../lib/cn/recipes/organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,EAAE,KAAK,YAAY,EAA8B,MAAM,mBAAmB,CAAA;AAkBjF;;GAEG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CA8CnF;AAmBD;;GAEG;AACH,eAAO,MAAM,2BAA2B,EAAE,YA0EzC,CAAA"}