@mailwoman/corpus 9.2.0 → 9.4.0

package/out/src/utils/split.js

@@ -1,204 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Train / val / test split with **locality holdout** per the Phase 1 plan.
- *
- *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
- *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
- *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
- *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
- *   memorization, not by learning the underlying schema.
- *
- *   Phase 1 holdouts (chosen for low data density + administrative isolation):
- *
- *   - **US**: Vermont, Wyoming, North Dakota
- *   - **FR**: Corse, Lozère, Creuse
- *
- *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
- *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
- *   the locality boundary, not the exact split percentages.
- *
- *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
- *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
- */
-import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
-import { createWriteStream } from "node:fs";
-import { mkdir, unlink, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { childEnv } from "@mailwoman/core/scripting/utils";
-import { JSONSpliterator } from "spliterator";
-/**
- * Phase 1 default holdouts (per plan).
- *
- * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
- * - FR: Corse, Lozère, Creuse (small departments / regions).
- * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
- *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
- *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
- *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
- */
-export function defaultHoldouts() {
-    return {
-        US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
-        FR: ["Corse", "Lozère", "Lozere", "Creuse"],
-        DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
-    };
-}
-/**
- * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
- * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
- * array-based path so the decision is stable regardless of caller.
- */
-export function splitForRow(row, holdouts = defaultHoldouts()) {
-    const region = row.components.region;
-    const countryHoldouts = holdouts[row.country] ?? [];
-    const isHeldOut = region !== undefined && countryHoldouts.includes(region);
-    if (!isHeldOut)
-        return "train";
-    // 50/50 deterministic by source_id hash. Same input always lands in the same split.
-    return hashBucket(row.source_id, 2) === 0 ? "val" : "test";
-}
-/**
- * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
- * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
- *
- * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
- * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
- * split membership in heap.
- */
-export function splitRows(rows, opts = {}) {
-    const holdouts = opts.holdouts ?? defaultHoldouts();
-    const train = [];
-    const val = [];
-    const test = [];
-    let corpus_version = "";
-    for (const row of rows) {
-        if (!corpus_version && row.corpus_version) {
-            corpus_version = row.corpus_version;
-        }
-        const split = splitForRow(row, holdouts);
-        if (split === "train") {
-            train.push(row.source_id);
-        }
-        else if (split === "val") {
-            val.push(row.source_id);
-        }
-        else {
-            test.push(row.source_id);
-        }
-    }
-    const total = train.length + val.length + test.length;
-    return {
-        train,
-        val,
-        test,
-        holdouts,
-        corpus_version,
-        counts: { train: train.length, val: val.length, test: test.length, total },
-    };
-}
-/**
- * Lightweight deterministic 0..(n-1) bucket from a string id.
- */
-/**
- * Deterministic bucket for a stable id.
- *
- * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
- * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
- * line it saves.
- */
-export function hashBucket(id, n) {
-    const digest = createHash("sha256").update(id).digest();
-    // Read 4 bytes as uint32 to avoid bigint overhead.
-    const u = digest[0] * 0x01_00_00_00 + digest[1] * 0x01_00_00 + digest[2] * 0x01_00 + digest[3];
-    return u % n;
-}
-/**
- * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
- * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
- * including holdouts + counts + corpus version.
- *
- * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
- */
-export async function writeSplitManifests(manifest, outputDir) {
-    await mkdir(outputDir, { recursive: true });
-    for (const name of ["train", "val", "test"]) {
-        const sorted = [...manifest[name]].toSorted();
-        await writeFile(join(outputDir, `${name}.txt`), sorted.join("\n") + (sorted.length ? "\n" : ""), "utf8");
-    }
-    const summary = {
-        corpus_version: manifest.corpus_version,
-        holdouts: manifest.holdouts,
-        counts: manifest.counts,
-    };
-    await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
-}
-/**
- * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
- * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
- * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
- *
- * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
- * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
- */
-export async function writeSplitManifestsFromLabeledFiles(opts) {
-    await mkdir(opts.outputDir, { recursive: true });
-    const holdouts = opts.holdouts ?? defaultHoldouts();
-    for (const split of ["train", "val", "test"]) {
-        const labeledPath = opts.labeledPaths[split];
-        const outPath = join(opts.outputDir, `${split}.txt`);
-        await streamSortedSourceIDs(labeledPath, outPath);
-    }
-    const total = opts.counts.train + opts.counts.val + opts.counts.test;
-    const summary = {
-        corpus_version: opts.corpusVersion,
-        holdouts,
-        counts: { ...opts.counts, total },
-    };
-    await writeFile(join(opts.outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
-    return summary.counts;
-}
-/**
- * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
- * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
- */
-async function streamSortedSourceIDs(labeledJsonlPath, outPath) {
-    const unsortedPath = `${outPath}.unsorted`;
-    const out = createWriteStream(unsortedPath, { encoding: "utf8" });
-    const outClosed = new Promise((resolve, reject) => {
-        out.on("close", () => resolve());
-        out.on("error", reject);
-    });
-    // JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
-    // SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
-    // always ends the write stream so `sort` reads a complete file even if the read throws.
-    try {
-        for await (const obj of JSONSpliterator.fromAsync(labeledJsonlPath)) {
-            if (typeof obj.source_id === "string") {
-                out.write(`${obj.source_id}\n`);
-            }
-        }
-    }
-    finally {
-        out.end();
-    }
-    await outClosed;
-    await new Promise((resolve, reject) => {
-        // LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
-        const proc = spawn("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) });
-        proc.on("error", reject);
-        proc.on("exit", (code) => {
-            if (code === 0) {
-                resolve();
-            }
-            else {
-                reject(new Error(`sort exited with code ${code}`));
-            }
-        });
-    });
-    await unlink(unsortedPath).catch(() => { });
-}
-//# sourceMappingURL=split.js.map

package/out/src/utils/split.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"split.js","sourceRoot":"","sources":["../../../src/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAoC7C;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;QAC3C,EAAE,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,CAAC;KACtD,CAAA;AACF,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,WAA8C,eAAe,EAAE;IAE/D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAE1E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAE9B,oFAAoF;IACpF,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,OAAqB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAC3C,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAExC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAErD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,mDAAmD;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAElG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAAiB;IACnF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC7C,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IACzG,CAAC;IAED,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AACzG,CAAC;AAOD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QACpD,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IAEpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE7G,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAwB,EAAE,OAAe;IAC7E,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,iBAAiB,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEjE,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,wFAAwF;IACxF,8FAA8F;IAC9F,wFAAwF;IACxF,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAyB,gBAAgB,CAAC,EAAE,CAAC;YAC7F,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,GAAG,CAAC,GAAG,EAAE,CAAA;IACV,CAAC;IAED,MAAM,SAAS,CAAA;IAEf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,wEAAwE;QACxE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7F,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAExB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;YACnD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC3C,CAAC"}

package/out/src/utils/tokenize.d.ts

@@ -1,56 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Tokenizer interface for alignment.
- *
- *   Two implementations live in the corpus package:
- *
- *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
- *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
- *        default for in-container alignment tests and as a fallback when no SentencePiece model is
- *        available.
- *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
- *        trained on the corpus. Same interface, different splits. Locked against the corpus version
- *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
- *
- *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
- *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
- *   spans, independent of how the tokenizer chose its splits.
- */
-/**
- * A token with its character span in the source string.
- */
-export interface TokenSpan {
-    /**
-     * The token text, possibly normalized (case unchanged here; tokenizers may differ).
-     */
-    text: string;
-    /**
-     * Inclusive start offset (UTF-16 code-unit index) in the source string.
-     */
-    start: number;
-    /**
-     * Exclusive end offset in the source string. `text === source.slice(start, end)`.
-     */
-    end: number;
-}
-/**
- * A tokenizer that maps a string to a sequence of `TokenSpan`s.
- */
-export interface Tokenizer {
-    tokenize(text: string): readonly TokenSpan[];
-}
-/**
- * Whitespace + punctuation tokenizer (pure JS).
- *
- * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
- * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
- * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
- *
- * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
- * tokenizer will preserve all bytes via byte-fallback.
- */
-export declare function whitespaceTokenizer(): Tokenizer;
-//# sourceMappingURL=tokenize.d.ts.map

package/out/src/utils/tokenize.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../../src/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;CAC5C;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAkB/C"}

package/out/src/utils/tokenize.js

@@ -1,48 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Tokenizer interface for alignment.
- *
- *   Two implementations live in the corpus package:
- *
- *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
- *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
- *        default for in-container alignment tests and as a fallback when no SentencePiece model is
- *        available.
- *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
- *        trained on the corpus. Same interface, different splits. Locked against the corpus version
- *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
- *
- *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
- *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
- *   spans, independent of how the tokenizer chose its splits.
- */
-/**
- * Whitespace + punctuation tokenizer (pure JS).
- *
- * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
- * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
- * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
- *
- * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
- * tokenizer will preserve all bytes via byte-fallback.
- */
-export function whitespaceTokenizer() {
-    // Maximal runs of letters/digits/marks plus the joiners common to addresses
-    // (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
-    const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu;
-    return {
-        tokenize(text) {
-            const out = [];
-            tokenRe.lastIndex = 0;
-            let m;
-            while ((m = tokenRe.exec(text))) {
-                out.push({ text: m[0], start: m.index, end: m.index + m[0].length });
-            }
-            return out;
-        },
-    };
-}
-//# sourceMappingURL=tokenize.js.map

package/out/src/utils/tokenize.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../../src/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA6BH;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB;IAClC,4EAA4E;IAC5E,gFAAgF;IAChF,MAAM,OAAO,GAAG,yBAAyB,CAAA;IAEzC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAC3B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;YACrB,IAAI,CAAyB,CAAA;YAE7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YACrE,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC"}

package/out/src/utils/wof-json.d.ts

@@ -1,109 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
- *
- *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
- *   `names` table) and onto the per-record GeoJSON bundles published as
- *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
- *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
- *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
- *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
- *   about.
- *
- *   This module provides:
- *
- *   - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
- *   - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
- *       deprecated records).
- *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
- *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
- *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
- *       in `source_id` suffixes.
- *
- *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
- *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
- *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
- *   silently emitted zero rows from the real corpus.
- */
-/**
- * A WOF GeoJSON feature, as published by the per-record bundles.
- */
-export interface WOFFeature {
-    type?: string;
-    id?: number | string;
-    properties?: Record<string, unknown> | null;
-}
-/**
- * Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped — it's 95% of the file
- * weight and the adapters never consult it.
- */
-export interface WOFRecord {
-    id: number;
-    parent_id: number | null;
-    /**
-     * Canonical `wof:name` of the record.
-     */
-    name: string;
-    placetype: string;
-    /**
-     * ISO 3166-1 alpha-2 from `wof:country`.
-     */
-    country: string;
-    /**
-     * Localized name variants from `name:*` properties.
-     *
-     * Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the underlying array (WOF
-     * stores variants as arrays even when only one form is present). The canonical `wof:name` is NOT included here —
-     * adapters add a synthetic `"default"` slot for it.
-     */
-    nameVariants: Map<string, string>;
-}
-/**
- * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
- *
- * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
- * alongside `1`. Tightening the predicate to `= 1` is the trap: the postalcode distros then contribute ZERO rows to the
- * corpus, silently, with nothing raised to notice it by.
- */
-export declare function isCurrentFeature(props: Record<string, unknown>): boolean;
-/**
- * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
- * Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
- * not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
- */
-export declare function extractNameVariants(props: Record<string, unknown>): Map<string, string>;
-/**
- * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
- *
- * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
- * existing source_id separator vocabulary and downstream consumers split on `-`.
- */
-export declare function normalizeNameKey(rawKey: string): string;
-/**
- * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
- *
- * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
- * (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
- * `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
- * records.
- *
- * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
- * walk. Adapters can add stricter validation downstream if they need it.
- */
-export declare function walkFeatures(repoDir: string, opts?: {
-    signal?: AbortSignal;
-}): AsyncIterable<WOFRecord>;
-/**
- * Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id` upward and stopping
- * at the first missing link. A cycle guard halts at any re-visit (defensive — WOF data is acyclic by construction but
- * corrupt fixtures shouldn't infinite-loop the adapter).
- *
- * Records whose ancestors aren't in `byID` (e.g. an FR locality whose region wasn't included in the cloned repo set)
- * get a shorter chain; the variant emission gracefully degrades.
- */
-export type AncestryIndex = Map<number, WOFRecord[]>;
-export declare function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex;
-//# sourceMappingURL=wof-json.d.ts.map

package/out/src/utils/wof-json.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../../src/utils/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAOH;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IACf;;;;;;OAMG;IACH,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAKxE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAkBvF;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD;AA+CD;;;;;;;;;;GAUG;AACH,wBAAuB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAgClH;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;AAEpD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,aAAa,CAuB9E"}

package/out/src/utils/wof-json.js

@@ -1,173 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
- *
- *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
- *   `names` table) and onto the per-record GeoJSON bundles published as
- *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
- *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
- *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
- *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
- *   about.
- *
- *   This module provides:
- *
- *   - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
- *   - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
- *       deprecated records).
- *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
- *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
- *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
- *       in `source_id` suffixes.
- *
- *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
- *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
- *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
- *   silently emitted zero rows from the real corpus.
- */
-import { readFile } from "node:fs/promises";
-import { tryParsingJSON } from "@mailwoman/core/objects";
-import FastGlob from "fast-glob";
-/**
- * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
- *
- * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
- * alongside `1`. Tightening the predicate to `= 1` is the trap: the postalcode distros then contribute ZERO rows to the
- * corpus, silently, with nothing raised to notice it by.
- */
-export function isCurrentFeature(props) {
-    const raw = props["mz:is_current"];
-    const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : 1;
-    return n !== 0;
-}
-/**
- * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
- * Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
- * not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
- */
-export function extractNameVariants(props) {
-    const out = new Map();
-    for (const [key, value] of Object.entries(props)) {
-        if (!key.startsWith("name:"))
-            continue;
-        const candidate = Array.isArray(value)
-            ? value.find((v) => typeof v === "string" && v.trim().length > 0)
-            : typeof value === "string" && value.trim().length
-                ? value
-                : undefined;
-        if (candidate) {
-            out.set(key, candidate.trim());
-        }
-    }
-    return out;
-}
-/**
- * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
- *
- * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
- * existing source_id separator vocabulary and downstream consumers split on `-`.
- */
-export function normalizeNameKey(rawKey) {
-    return rawKey.replaceAll(/[:_]/g, "-");
-}
-/**
- * Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason).
- */
-function recordFromFeature(feature) {
-    if (!feature || feature.type !== "Feature" || !feature.properties)
-        return null;
-    const props = feature.properties;
-    const rawID = typeof feature.id === "number" ? feature.id : props["wof:id"];
-    const id = typeof rawID === "number" ? rawID : typeof rawID === "string" ? Number(rawID) : Number.NaN;
-    if (!Number.isFinite(id))
-        return null;
-    const name = props["wof:name"];
-    if (typeof name !== "string" || !name.trim())
-        return null;
-    const placetype = props["wof:placetype"];
-    if (typeof placetype !== "string" || !placetype)
-        return null;
-    const country = props["wof:country"];
-    if (typeof country !== "string" || !country)
-        return null;
-    if (!isCurrentFeature(props))
-        return null;
-    const parentRaw = props["wof:parent_id"];
-    const parent_id = typeof parentRaw === "number"
-        ? parentRaw
-        : typeof parentRaw === "string" && parentRaw.trim()
-            ? Number(parentRaw)
-            : null;
-    return {
-        id,
-        parent_id: Number.isFinite(parent_id) ? parent_id : null,
-        name: name.trim(),
-        placetype,
-        country,
-        nameVariants: extractNameVariants(props),
-    };
-}
-/**
- * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
- *
- * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
- * (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
- * `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
- * records.
- *
- * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
- * walk. Adapters can add stricter validation downstream if they need it.
- */
-export async function* walkFeatures(repoDir, opts = {}) {
-    const stream = FastGlob.stream(["**/*.geojson"], {
-        cwd: repoDir,
-        absolute: true,
-        onlyFiles: true,
-        suppressErrors: true,
-    });
-    for await (const entry of stream) {
-        if (opts.signal?.aborted)
-            return;
-        const filePath = String(entry);
-        if (filePath.includes("-alt-"))
-            continue;
-        let text;
-        try {
-            text = await readFile(filePath, "utf8");
-        }
-        catch {
-            continue;
-        }
-        const parsed = tryParsingJSON(text);
-        if (parsed === null)
-            continue;
-        const rec = recordFromFeature(parsed);
-        if (rec) {
-            yield rec;
-        }
-    }
-}
-export function buildAncestryIndex(byID) {
-    const index = new Map();
-    for (const [id, rec] of byID) {
-        const chain = [];
-        const guard = new Set([id]);
-        let cur = rec.parent_id;
-        while (cur !== null && cur > 0) {
-            const parent = byID.get(cur);
-            if (!parent)
-                break;
-            if (guard.has(parent.id))
-                break;
-            chain.push(parent);
-            guard.add(parent.id);
-            cur = parent.parent_id;
-        }
-        index.set(id, chain);
-    }
-    return index;
-}
-//# sourceMappingURL=wof-json.js.map

package/out/src/utils/wof-json.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../../src/utils/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,QAAQ,MAAM,WAAW,CAAA;AAqChC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA8B;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;IAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAQ;QAEtC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC9E,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM;gBACjD,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,SAAS,CAAA;QAEb,IAAI,SAAS,EAAE,CAAC;YACf,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC9C,OAAO,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;AACvC,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,OAAmB;IAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,UAAU;QAAE,OAAO,IAAI,CAAA;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAA;IAEhC,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC3E,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAA;IAErG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;IAE9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAA;IAEzD,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAExC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAA;IAE5D,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;IAEpC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAExD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAExC,MAAM,SAAS,GACd,OAAO,SAAS,KAAK,QAAQ;QAC5B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;YACnB,CAAC,CAAC,IAAI,CAAA;IAET,OAAO;QACN,EAAE;QACF,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAmB,CAAC,CAAC,CAAC,CAAE,SAAoB,CAAC,CAAC,CAAC,IAAI;QAC9E,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACjB,SAAS;QACT,OAAO;QACP,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC;KACxC,CAAA;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,OAAe,EAAE,OAAiC,EAAE;IACvF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE;QAChD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAQ;QAExC,IAAI,IAAY,CAAA;QAEhB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,cAAc,CAAa,IAAI,CAAC,CAAA;QAE/C,IAAI,MAAM,KAAK,IAAI;YAAE,SAAQ;QAE7B,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QAErC,IAAI,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;AACF,CAAC;AAYD,MAAM,UAAU,kBAAkB,CAAC,IAA4B;IAC9D,MAAM,KAAK,GAAkB,IAAI,GAAG,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,CAAC,EAAE,CAAC,CAAC,CAAA;QACnC,IAAI,GAAG,GAAkB,GAAG,CAAC,SAAS,CAAA;QAEtC,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,MAAM;gBAAE,MAAK;YAElB,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAE,MAAK;YAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAClB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACpB,GAAG,GAAG,MAAM,CAAC,SAAS,CAAA;QACvB,CAAC;QAED,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}