@mailwoman/corpus 9.2.0 → 9.3.0

package/out/utils/slice-stats.js

@@ -0,0 +1,58 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
+ *   and the slice linter (`tools/lint-slice.ts`). The two sides MUST agree on the bigram key format:
+ *   the linter looks its slice's bigrams up in the stats file's `bigrams` table, and a key built with
+ *   a different separator never matches — which silently blanks the bigram-collision check.
+ */
+import { streamParquetRows } from "#utils/parquet";
+/**
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`): U+001F UNIT SEPARATOR, a
+ * character no address token contains. Render a key for humans with `key.split(SLICE_STATS_SEP).join(" ")`.
+ */
+export const SLICE_STATS_SEP = "";
+/**
+ * An empty pair of tables for {@link accumulateCooccurrences}.
+ */
+export function createCooccurrenceStats() {
+    return { tokens: new Map(), bigrams: new Map() };
+}
+function bump(table, key, sub) {
+    let counter = table.get(key);
+    if (!counter) {
+        counter = new Map();
+        table.set(key, counter);
+    }
+    counter.set(sub, (counter.get(sub) ?? 0) + 1);
+}
+/**
+ * Fold one row's parallel `tokens`/`labels` into the tables. The caller has already checked the arrays are the same
+ * length.
+ */
+export function accumulateCooccurrences(stats, tokens, labels) {
+    for (let i = 0; i < tokens.length; i++) {
+        const tk = tokens[i];
+        const lb = labels[i];
+        bump(stats.tokens, tk, lb);
+        if (i + 1 < tokens.length) {
+            bump(stats.bigrams, tk + SLICE_STATS_SEP + tokens[i + 1], lb + SLICE_STATS_SEP + labels[i + 1]);
+        }
+    }
+}
+/**
+ * Stream a slice's `tokens`/`labels` columns.
+ *
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched. `limit` stops the
+ * iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
+ */
+export async function* streamTokenLabelRows(slicePath, limit) {
+    let emitted = 0;
+    for await (const row of streamParquetRows(slicePath, ["tokens", "labels"], { limit })) {
+        yield row;
+        emitted++;
+    }
+}
+//# sourceMappingURL=slice-stats.js.map

package/out/utils/slice-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"slice-stats.js","sourceRoot":"","sources":["../../lib/utils/slice-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAA;AAelC;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACtC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;AACjD,CAAC;AAED,SAAS,IAAI,CAAC,KAAwB,EAAE,GAAW,EAAE,GAAW;IAC/D,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACtC,KAAwB,EACxB,MAAyB,EACzB,MAAyB;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QAErB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;QAE1B,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,eAAe,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,EAAE,EAAE,GAAG,eAAe,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAA;QAClG,CAAC;IACF,CAAC;AACF,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CAAC,SAAiB,EAAE,KAAc;IAC5E,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,iBAAiB,CAAgB,SAAS,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtG,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}

package/out/utils/split.d.ts

@@ -0,0 +1,128 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { CanonicalRow, LabeledRow } from "#types";
+export type SplitName = "train" | "val" | "test";
+export interface SplitOptions {
+    /**
+     * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
+     * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
+     * `defaultHoldouts()`.
+     */
+    holdouts?: Record<string, readonly string[]>;
+}
+/**
+ * Output manifest: source_id lists per split.
+ */
+export interface SplitManifest {
+    train: string[];
+    val: string[];
+    test: string[];
+    /**
+     * Echoes the holdouts used, so the manifest is self-describing.
+     */
+    holdouts: Record<string, readonly string[]>;
+    /**
+     * Corpus version stamped onto the manifest. Read from the first row.
+     */
+    corpus_version: string;
+    /**
+     * Counts for quick sanity checks.
+     */
+    counts: {
+        train: number;
+        val: number;
+        test: number;
+        total: number;
+    };
+}
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
+ *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
+ *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
+ *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ */
+export declare function defaultHoldouts(): Record<string, readonly string[]>;
+type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">;
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
+ * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
+ * array-based path so the decision is stable regardless of caller.
+ */
+export declare function splitForRow(row: Pick<SplitInputRow, "source_id" | "country" | "components">, holdouts?: Record<string, readonly string[]>): SplitName;
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
+ * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
+ * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
+ * split membership in heap.
+ */
+export declare function splitRows(rows: Iterable<SplitInputRow>, opts?: SplitOptions): SplitManifest;
+/**
+ * Lightweight deterministic 0..(n-1) bucket from a string id.
+ */
+/**
+ * Deterministic bucket for a stable id.
+ *
+ * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
+ * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
+ * line it saves.
+ */
+export declare function hashBucket(id: string, n: number): number;
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
+ * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
+ * including holdouts + counts + corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export declare function writeSplitManifests(manifest: SplitManifest, outputDir: PathBuilderLike): Promise<void>;
+/**
+ * Type re-export for callers that want to ingest LabeledRow specifically.
+ */
+export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">;
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
+ * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
+ * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
+ * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ */
+export declare function writeSplitManifestsFromLabeledFiles(opts: {
+    labeledPaths: Record<SplitName, string>;
+    outputDir: PathBuilderLike;
+    corpusVersion: string;
+    counts: Record<SplitName, number>;
+    holdouts?: Record<string, readonly string[]>;
+}): Promise<SplitManifest["counts"]>;
+export {};
+//# sourceMappingURL=split.d.ts.map

package/out/utils/split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAGpD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAEtD,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;AAEhD,MAAM,WAAW,YAAY;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,EAAE,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAC3C;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACnE;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAMnE;AAED,KAAK,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAElG;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC,EAChE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAqB,GAC7D,SAAS,CASX;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,GAAE,YAAiB,GAAG,aAAa,CAiC/F;AAED;;GAEG;AACH;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAMxD;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB5G;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAE9G;;;;;;;GAOG;AACH,wBAAsB,mCAAmC,CAAC,IAAI,EAAE;IAC/D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACvC,SAAS,EAAE,eAAe,CAAA;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAqBnC"}

package/out/utils/split.js

@@ -0,0 +1,204 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories, removePath } from "@mailwoman/core/fs/writers";
+import { createHash } from "@mailwoman/core/hash";
+import { spawnProcess } from "@mailwoman/core/process";
+import { childEnv } from "@mailwoman/core/scripting/utils";
+import { join } from "path-ts";
+import { JSONSpliterator } from "spliterator";
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
+ *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
+ *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
+ *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ */
+export function defaultHoldouts() {
+    return {
+        US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
+        FR: ["Corse", "Lozère", "Lozere", "Creuse"],
+        DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+    };
+}
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
+ * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
+ * array-based path so the decision is stable regardless of caller.
+ */
+export function splitForRow(row, holdouts = defaultHoldouts()) {
+    const region = row.components.region;
+    const countryHoldouts = holdouts[row.country] ?? [];
+    const isHeldOut = region !== undefined && countryHoldouts.includes(region);
+    if (!isHeldOut)
+        return "train";
+    // 50/50 deterministic by source_id hash. Same input always lands in the same split.
+    return hashBucket(row.source_id, 2) === 0 ? "val" : "test";
+}
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
+ * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
+ * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
+ * split membership in heap.
+ */
+export function splitRows(rows, opts = {}) {
+    const holdouts = opts.holdouts ?? defaultHoldouts();
+    const train = [];
+    const val = [];
+    const test = [];
+    let corpus_version = "";
+    for (const row of rows) {
+        if (!corpus_version && row.corpus_version) {
+            corpus_version = row.corpus_version;
+        }
+        const split = splitForRow(row, holdouts);
+        if (split === "train") {
+            train.push(row.source_id);
+        }
+        else if (split === "val") {
+            val.push(row.source_id);
+        }
+        else {
+            test.push(row.source_id);
+        }
+    }
+    const total = train.length + val.length + test.length;
+    return {
+        train,
+        val,
+        test,
+        holdouts,
+        corpus_version,
+        counts: { train: train.length, val: val.length, test: test.length, total },
+    };
+}
+/**
+ * Lightweight deterministic 0..(n-1) bucket from a string id.
+ */
+/**
+ * Deterministic bucket for a stable id.
+ *
+ * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
+ * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
+ * line it saves.
+ */
+export function hashBucket(id, n) {
+    const digest = createHash("sha256").update(id).digest();
+    // Read 4 bytes as uint32 to avoid bigint overhead.
+    const u = digest[0] * 0x01_00_00_00 + digest[1] * 0x01_00_00 + digest[2] * 0x01_00 + digest[3];
+    return u % n;
+}
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
+ * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
+ * including holdouts + counts + corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export async function writeSplitManifests(manifest, outputDir) {
+    await makeDirectories(outputDir);
+    for (const name of ["train", "val", "test"]) {
+        const sorted = [...manifest[name]].toSorted();
+        await writeLocalTextFile(sorted.join("\n") + (sorted.length ? "\n" : ""), outputDir, `${name}.txt`);
+    }
+    const summary = {
+        corpus_version: manifest.corpus_version,
+        holdouts: manifest.holdouts,
+        counts: manifest.counts,
+    };
+    await writeLocalJSONFile(summary, outputDir, "SPLIT_MANIFEST.json");
+}
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
+ * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
+ * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
+ * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ */
+export async function writeSplitManifestsFromLabeledFiles(opts) {
+    await makeDirectories(opts.outputDir);
+    const holdouts = opts.holdouts ?? defaultHoldouts();
+    for (const split of ["train", "val", "test"]) {
+        const labeledPath = opts.labeledPaths[split];
+        const outPath = join(opts.outputDir, `${split}.txt`);
+        await streamSortedSourceIDs(labeledPath, outPath);
+    }
+    const total = opts.counts.train + opts.counts.val + opts.counts.test;
+    const summary = {
+        corpus_version: opts.corpusVersion,
+        holdouts,
+        counts: { ...opts.counts, total },
+    };
+    await writeLocalJSONFile(summary, opts.outputDir, "SPLIT_MANIFEST.json");
+    return summary.counts;
+}
+/**
+ * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
+ * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
+ */
+async function streamSortedSourceIDs(labeledJsonlPath, outPath) {
+    const unsortedPath = `${outPath}.unsorted`;
+    const out = openWriteStream(unsortedPath, { encoding: "utf8" });
+    const outClosed = new Promise((resolve, reject) => {
+        out.on("close", () => resolve());
+        out.on("error", reject);
+    });
+    // JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
+    // SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
+    // always ends the write stream so `sort` reads a complete file even if the read throws.
+    try {
+        for await (const obj of JSONSpliterator.fromAsync(labeledJsonlPath)) {
+            if (typeof obj.source_id === "string") {
+                out.write(`${obj.source_id}\n`);
+            }
+        }
+    }
+    finally {
+        out.end();
+    }
+    await outClosed;
+    await new Promise((resolve, reject) => {
+        // LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+        const proc = spawnProcess("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) });
+        proc.on("error", reject);
+        proc.on("exit", (code) => {
+            if (code === 0) {
+                resolve();
+            }
+            else {
+                reject(new Error(`sort exited with code ${code}`));
+            }
+        });
+    });
+    await removePath(unsortedPath).catch(() => { });
+}
+//# sourceMappingURL=split.js.map

package/out/utils/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAChH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAoC7C;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;QAC3C,EAAE,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,CAAC;KACtD,CAAA;AACF,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,WAA8C,eAAe,EAAE;IAE/D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAE1E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAE9B,oFAAoF;IACpF,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,OAAqB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAC3C,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAExC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAErD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,mDAAmD;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAElG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAA0B;IAC5F,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE7C,MAAM,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,CAAA;IACpG,CAAC;IAED,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAA;AACpE,CAAC;AAOD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QACpD,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IAEpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;IAExE,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAwB,EAAE,OAAe;IAC7E,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,eAAe,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAE/D,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,wFAAwF;IACxF,8FAA8F;IAC9F,wFAAwF;IACxF,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAyB,gBAAgB,CAAC,EAAE,CAAC;YAC7F,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,GAAG,CAAC,GAAG,EAAE,CAAA;IACV,CAAC;IAED,MAAM,SAAS,CAAA;IAEf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,wEAAwE;QACxE,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QACpG,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAExB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;YACnD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC/C,CAAC"}

package/out/utils/tokenize.d.ts

@@ -0,0 +1,64 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+/**
+ * A token with its character span in the source string.
+ */
+export interface TokenSpan {
+    /**
+     * The token text, possibly normalized (case unchanged here; tokenizers may differ).
+     */
+    text: string;
+    /**
+     * Inclusive start offset (UTF-16 code-unit index) in the source string.
+     */
+    start: number;
+    /**
+     * Exclusive end offset in the source string. `text === source.slice(start, end)`.
+     */
+    end: number;
+}
+/**
+ * A tokenizer that maps a string to a sequence of `TokenSpan`s.
+ */
+export interface Tokenizer {
+    tokenize(text: string): readonly TokenSpan[];
+}
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
+ * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
+ * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
+ * tokenizer will preserve all bytes via byte-fallback.
+ */
+export declare function whitespaceTokenizer(): Tokenizer;
+/**
+ * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ *
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
+ * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
+ * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ */
+export declare function cjkAwareTokenizer(): Tokenizer;
+//# sourceMappingURL=tokenize.d.ts.map

package/out/utils/tokenize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;CAC5C;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAkB/C;AAQD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,IAAI,SAAS,CA2B7C"}

package/out/utils/tokenize.js

@@ -0,0 +1,82 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
+ * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
+ * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
+ * tokenizer will preserve all bytes via byte-fallback.
+ */
+export function whitespaceTokenizer() {
+    // Maximal runs of letters/digits/marks plus the joiners common to addresses
+    // (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
+    const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu;
+    return {
+        tokenize(text) {
+            const out = [];
+            tokenRe.lastIndex = 0;
+            let m;
+            while ((m = tokenRe.exec(text))) {
+                out.push({ text: m[0], start: m.index, end: m.index + m[0].length });
+            }
+            return out;
+        },
+    };
+}
+/**
+ * Han characters, the script CJK addresses are written in and the one the whitespace tokenizer cannot split: a run like
+ * `三分场八队` is one "word" to `\p{L}+`, so the aligner could give it one label and never two.
+ */
+const HAN = /\p{Script=Han}/u;
+/**
+ * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ *
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
+ * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
+ * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ */
+export function cjkAwareTokenizer() {
+    const base = whitespaceTokenizer();
+    return {
+        tokenize(text) {
+            const out = [];
+            for (const token of base.tokenize(text)) {
+                if (!HAN.test(token.text)) {
+                    out.push(token);
+                    continue;
+                }
+                // A Han run may carry Latin letters or digits inside it (`3分场2队`); those stay glued to their neighbours
+                // only if they are Han too, so every code point of a Han-bearing token becomes its own token.
+                let offset = token.start;
+                for (const codePoint of token.text) {
+                    out.push({ text: codePoint, start: offset, end: offset + codePoint.length });
+                    offset += codePoint.length;
+                }
+            }
+            return out;
+        },
+    };
+}
+//# sourceMappingURL=tokenize.js.map

package/out/utils/tokenize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA6BH;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB;IAClC,4EAA4E;IAC5E,gFAAgF;IAChF,MAAM,OAAO,GAAG,yBAAyB,CAAA;IAEzC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAC3B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;YACrB,IAAI,CAAyB,CAAA;YAE7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YACrE,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,GAAG,GAAG,iBAAiB,CAAA;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB;IAChC,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAA;IAElC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAE3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEf,SAAQ;gBACT,CAAC;gBAED,wGAAwG;gBACxG,8FAA8F;gBAC9F,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAA;gBAExB,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACpC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC5E,MAAM,IAAI,SAAS,CAAC,MAAM,CAAA;gBAC3B,CAAC;YACF,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC"}