@mailwoman/corpus 9.4.0 → 10.1.0

package/out/utils/split.js

@@ -3,71 +3,172 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ * Splits corpus rows into train, val and test by holding out whole places: a random row split leaks by
+ * neighborhood, because the model can memorize a street it saw in train. Rows inside a held-out place
+ * go to val or test 50/50 by a hash of `source_id`; every other row goes to train.
  *
- *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
- *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
- *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
- *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
- *   memorization, not by learning the underlying schema.
- *
- *   Phase 1 holdouts (chosen for low data density + administrative isolation):
- *
- *   - **US**: Vermont, Wyoming, North Dakota
- *   - **FR**: Corse, Lozère, Creuse
- *
- *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
- *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
- *   the locality boundary, not the exact split percentages.
- *
- *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
- *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ * A holdout change takes effect at the next corpus rebuild. Each versioned corpus keeps the
+ * `SPLIT_MANIFEST.json` it was built with, because adding a holdout to a built corpus would leak rows
+ * the model already trained on.
  */
+import { delimitedSource, preferCompressed } from "@mailwoman/core/fs/delimited";
 import { openWriteStream } from "@mailwoman/core/fs/streams";
 import { writeLocalJSONFile, writeLocalTextFile, makeDirectories, removePath } from "@mailwoman/core/fs/writers";
 import { createHash } from "@mailwoman/core/hash";
 import { spawnProcess } from "@mailwoman/core/process";
 import { childEnv } from "@mailwoman/core/scripting/utils";
-import { join } from "path-ts";
+import { resolvePathBuilder } from "path-ts";
 import { JSONSpliterator } from "spliterator";
 /**
- * Phase 1 default holdouts (per plan).
- *
- * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
- * - FR: Corse, Lozère, Creuse (small departments / regions).
- * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
- *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
- *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
- *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ * Returns the default holdouts — small, peripheral places in each country.
+ * A change here takes effect at the next base corpus rebuild.
  */
 export function defaultHoldouts() {
     return {
         US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
-        FR: ["Corse", "Lozère", "Lozere", "Creuse"],
-        DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+        FR: {
+            regions: ["Corse", "Lozère", "Lozere", "Creuse"],
+            // BAN street rows have no region, so these postcode departments hold out the
+            // same three places: Corse 20, Creuse 23 and Lozère 48.
+            postcodePrefixes: ["20", "23", "48"],
+        },
+        DE: {
+            // Only `wof-admin` has a DE region, so these Länder hold out admin rows only.
+            regions: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+            // DE street rows come from two OpenAddresses members, Berlin and Sachsen,
+            // so the regions above match no street row.
+            // Postcode area `02` (Upper Lusatia in eastern Sachsen) is the street-level holdout.
+            // German postcodes have five digits, so this holdout cannot collide.
+            postcodePrefixes: ["02"],
+        },
+        GB: {
+            // GB street rows have no region, so the holdout matches postcode areas —
+            // Truro, Llandudno and Halifax.
+            // Each prefix has two letters and none is a prefix of another
+            // (`L` for Liverpool would also match `LL`).
+            postcodePrefixes: ["TR", "LL", "HX"],
+            // Cornwall matches the admin rows for the same place `TR` covers.
+            regions: ["Cornwall"],
+        },
+        // The eight countries below enter training through OpenStreetMap rows, most of which carry no
+        // region and many no postcode, so each holdout matches its place three ways: the region an admin
+        // row carries, the postcode prefix, and the city as the OSM rows spell it. The spellings are the
+        // ones the 2026-10-02 extracts write.
+        CN: {
+            // Gansu by its province and capital, and Qinghai `81` and Tibet `85` by postcode.
+            regions: ["甘肃省", "甘肃", "Gansu"],
+            postcodePrefixes: ["81", "85"],
+            localities: ["兰州市", "兰州", "Lanzhou"],
+        },
+        IN: {
+            // Kerala, whose PIN codes run 67 to 69.
+            regions: ["Kerala"],
+            postcodePrefixes: ["67", "68", "69"],
+            localities: ["Kochi", "Kollam", "Thiruvananthapuram", "Kozhikode"],
+        },
+        ID: {
+            // Bali, whose postcodes begin `80`.
+            regions: ["Bali"],
+            postcodePrefixes: ["80"],
+            localities: ["Denpasar", "Bali"],
+        },
+        NG: {
+            // Enugu State.
+            // OSM Nigeria rows carry almost no postcode or region, so the cities carry it.
+            regions: ["Enugu"],
+            localities: ["Enugu", "Nsukka"],
+        },
+        PK: {
+            // Islamabad Capital Territory, in both scripts the extract writes.
+            regions: ["Islamabad Capital Territory"],
+            postcodePrefixes: ["44"],
+            localities: ["Islamabad", "islamabad", "اسلام آباد"],
+        },
+        BD: {
+            // Chattogram, Khulna and Sylhet, whose postcodes begin 4, 9 and 3 respectively.
+            postcodePrefixes: ["40", "41", "42", "43", "90", "91", "92", "31"],
+            localities: ["Chittagong", "Chattogram", "Khulna", "Sylhet"],
+        },
+        VN: {
+            regions: ["Thành phố Cần Thơ", "Cần Thơ", "Thừa Thiên Huế"],
+            localities: ["Cần Thơ", "Thành phố Cần Thơ", "Huế", "Thành phố Huế"],
+        },
+        RU: {
+            // Primorsky Krai, whose postcodes run 690 to 692.
+            regions: ["Приморский край"],
+            postcodePrefixes: ["690", "691", "692"],
+            localities: ["Владивосток", "Уссурийск", "Находка"],
+        },
+        // Each French overseas jurisdiction is one INSEE department with one postcode range,
+        // so the BAN rows hold out by commune, spelled as BAN's `nom_commune` writes it.
+        // Saint-Barthélemy and Saint-Martin are each one commune under one postcode,
+        // so neither has a place to hold out.
+        GP: { localities: ["Le Moule"] },
+        MQ: { localities: ["Le Robert", "Ducos"] },
+        GF: { localities: ["Kourou"] },
+        RE: { localities: ["Saint-Louis", "Saint-Leu"] },
+        PM: { localities: ["Miquelon-Langlade"] },
+        YT: { localities: ["Dzaoudzi", "Pamandzi"] },
+        // Tumaraa holds 1,522 of PF's 6,755 rows with a street, more than a tenth,
+        // because Pirae's 958 rows carry no street name and Papeete holds the remaining 5,233.
+        PF: { localities: ["Tumaraa"] },
+        NC: { localities: ["Koné", "Pouembout", "Kaala-Gomen"] },
+        LV: {
+            // Latgale's south-east: the Krāslava, Ludza, Balvi, Preiļi and Līvāni
+            // municipalities, whose postcodes begin LV-56, LV-57, LV-45 and LV-53,
+            // and the state city Rēzekne, which lies in no municipality.
+            // The address register writes `nov.`; other sources spell the municipality out.
+            regions: [
+                "Krāslavas nov.",
+                "Ludzas nov.",
+                "Balvu nov.",
+                "Preiļu nov.",
+                "Līvānu nov.",
+                "Krāslavas novads",
+                "Ludzas novads",
+                "Balvu novads",
+                "Preiļu novads",
+                "Līvānu novads",
+            ],
+            postcodePrefixes: ["LV-45", "LV-53", "LV-56", "LV-57"],
+            localities: ["Rēzekne"],
+        },
+        RO: {
+            // Alba 51, Hunedoara 33, Mureș 54 and Sibiu 55: a Romanian postcode's first two digits name its county.
+            regions: ["Alba", "Hunedoara", "Mureș", "Sibiu"],
+            postcodePrefixes: ["51", "33", "54", "55"],
+        },
     };
 }
 /**
- * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
- * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
- * array-based path so the decision is stable regardless of caller.
+ * Returns the matchers for one country, reading a bare array as a region list.
+ */
+export function holdoutPolicyFor(holdout) {
+    if (!holdout)
+        return {};
+    return Array.isArray(holdout) ? { regions: holdout } : holdout;
+}
+/**
+ * Returns the split for one row.
+ *
+ * Both `splitRows` and the streaming `buildCorpus` loop call this, so every
+ * caller assigns a row to the same split.
  */
 export function splitForRow(row, holdouts = defaultHoldouts()) {
-    const region = row.components.region;
-    const countryHoldouts = holdouts[row.country] ?? [];
-    const isHeldOut = region !== undefined && countryHoldouts.includes(region);
+    const policy = holdoutPolicyFor(holdouts[row.country]);
+    const { region, postcode, locality } = row.components;
+    const isHeldOut = (region !== undefined && (policy.regions?.includes(region) ?? false)) ||
+        (postcode !== undefined && (policy.postcodePrefixes?.some((prefix) => postcode.startsWith(prefix)) ?? false)) ||
+        (locality !== undefined && (policy.localities?.includes(locality) ?? false));
     if (!isHeldOut)
         return "train";
-    // 50/50 deterministic by source_id hash. Same input always lands in the same split.
     return hashBucket(row.source_id, 2) === 0 ? "val" : "test";
 }
 /**
- * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
- * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ * Builds a `SplitManifest` in memory from labeled or canonical rows.
  *
- * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
- * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
- * split membership in heap.
+ * Tests and small fixtures use this, while `buildCorpus` uses `splitForRow`
+ * and `writeSplitManifestsFromLabeledFiles` so it never holds every row's split in memory.
  */
 export function splitRows(rows, opts = {}) {
     const holdouts = opts.holdouts ?? defaultHoldouts();
@@ -101,33 +202,23 @@ export function splitRows(rows, opts = {}) {
     };
 }
 /**
- * Lightweight deterministic 0..(n-1) bucket from a string id.
- */
-/**
- * Deterministic bucket for a stable id.
- *
- * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
- * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
- * line it saves.
+ * Returns a deterministic bucket in `0..n-1` for an id, from the first four bytes
+ * of the SHA-256 digest read as a big-endian uint32.
  */
 export function hashBucket(id, n) {
     const digest = createHash("sha256").update(id).digest();
-    // Read 4 bytes as uint32 to avoid bigint overhead.
     const u = digest[0] * 0x01_00_00_00 + digest[1] * 0x01_00_00 + digest[2] * 0x01_00 + digest[3];
     return u % n;
 }
 /**
- * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
- * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
- * including holdouts + counts + corpus version.
- *
- * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ * Writes a `SplitManifest` to `<outputDir>/{train,val,test}.txt` (sorted `source_id` values, one per line)
+ * and `SPLIT_MANIFEST.json` (the corpus version, holdouts and counts).
  */
 export async function writeSplitManifests(manifest, outputDir) {
     await makeDirectories(outputDir);
     for (const name of ["train", "val", "test"]) {
         const sorted = [...manifest[name]].toSorted();
-        await writeLocalTextFile(sorted.join("\n") + (sorted.length ? "\n" : ""), outputDir, `${name}.txt`);
+        await writeLocalTextFile(sorted, outputDir, `${name}.txt`);
     }
     const summary = {
         corpus_version: manifest.corpus_version,
@@ -137,19 +228,18 @@ export async function writeSplitManifests(manifest, outputDir) {
     await writeLocalJSONFile(summary, outputDir, "SPLIT_MANIFEST.json");
 }
 /**
- * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
- * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
- * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ * Writes the same files as `writeSplitManifests` by streaming one labeled JSONL file per split.
  *
- * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
- * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ * `buildCorpus` calls this after its align loop partitions rows with `splitForRow`.
+ * The caller passes the counts to avoid rescanning the files.
+ * `sort(1)` spills to disk, so memory stays constant.
  */
 export async function writeSplitManifestsFromLabeledFiles(opts) {
     await makeDirectories(opts.outputDir);
     const holdouts = opts.holdouts ?? defaultHoldouts();
     for (const split of ["train", "val", "test"]) {
         const labeledPath = opts.labeledPaths[split];
-        const outPath = join(opts.outputDir, `${split}.txt`);
+        const outPath = resolvePathBuilder(opts.outputDir, `${split}.txt`);
         await streamSortedSourceIDs(labeledPath, outPath);
     }
     const total = opts.counts.train + opts.counts.val + opts.counts.test;
@@ -162,8 +252,8 @@ export async function writeSplitManifestsFromLabeledFiles(opts) {
     return summary.counts;
 }
 /**
- * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
- * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
+ * Writes the sorted `source_id` values from a labeled JSONL file to `outPath`;
+ * an empty input still produces an empty output file.
  */
 async function streamSortedSourceIDs(labeledJsonlPath, outPath) {
     const unsortedPath = `${outPath}.unsorted`;
@@ -172,11 +262,10 @@ async function streamSortedSourceIDs(labeledJsonlPath, outPath) {
         out.on("close", () => resolve());
         out.on("error", reject);
     });
-    // JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
-    // SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
-    // always ends the write stream so `sort` reads a complete file even if the read throws.
+    // A malformed row throws out of the loop.
+    // The `finally` block still closes the write stream.
     try {
-        for await (const obj of JSONSpliterator.fromAsync(labeledJsonlPath)) {
+        for await (const obj of JSONSpliterator.fromAsync(delimitedSource(await preferCompressed(labeledJsonlPath)))) {
             if (typeof obj.source_id === "string") {
                 out.write(`${obj.source_id}\n`);
             }
@@ -187,7 +276,7 @@ async function streamSortedSourceIDs(labeledJsonlPath, outPath) {
     }
     await outClosed;
     await new Promise((resolve, reject) => {
-        // LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+        // `LC_ALL=C` sorts by byte, so the order is the same on every host.
         const proc = spawnProcess("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) });
         proc.on("error", reject);
         proc.on("exit", (code) => {

package/out/utils/split.js.map

@@ -1 +1 @@
-{"version":3,"file":"split.js","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAChH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAA;AAC1D,OAAO,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAoC7C;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;QAC3C,EAAE,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,CAAC;KACtD,CAAA;AACF,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,WAA8C,eAAe,EAAE;IAE/D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAE1E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAE9B,oFAAoF;IACpF,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,OAAqB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAC3C,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAExC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAErD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED;;GAEG;AACH;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,mDAAmD;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAElG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAA0B;IAC5F,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE7C,MAAM,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,CAAA;IACpG,CAAC;IAED,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAA;AACpE,CAAC;AAOD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QACpD,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IAEpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;IAExE,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAwB,EAAE,OAAe;IAC7E,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,eAAe,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAE/D,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,wFAAwF;IACxF,8FAA8F;IAC9F,wFAAwF;IACxF,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAAyB,gBAAgB,CAAC,EAAE,CAAC;YAC7F,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,GAAG,CAAC,GAAG,EAAE,CAAA;IACV,CAAC;IAED,MAAM,SAAS,CAAA;IAEf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,wEAAwE;QACxE,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QACpG,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAExB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;YACnD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC/C,CAAC"}
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAA;AAChF,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAChH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAA;AAC1D,OAAO,EAAwB,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AA2D7C;;;GAGG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE;YACH,OAAO,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;YAChD,6EAA6E;YAC7E,wDAAwD;YACxD,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;SACpC;QACD,EAAE,EAAE;YACH,8EAA8E;YAC9E,OAAO,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,CAAC;YAC3D,0EAA0E;YAC1E,4CAA4C;YAC5C,qFAAqF;YACrF,qEAAqE;YACrE,gBAAgB,EAAE,CAAC,IAAI,CAAC;SACxB;QACD,EAAE,EAAE;YACH,yEAAyE;YACzE,gCAAgC;YAChC,8DAA8D;YAC9D,6CAA6C;YAC7C,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;YACpC,kEAAkE;YAClE,OAAO,EAAE,CAAC,UAAU,CAAC;SACrB;QACD,8FAA8F;QAC9F,iGAAiG;QACjG,iGAAiG;QACjG,sCAAsC;QACtC,EAAE,EAAE;YACH,kFAAkF;YAClF,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC;YAC/B,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;YAC9B,UAAU,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC;SACpC;QACD,EAAE,EAAE;YACH,wCAAwC;YACxC,OAAO,EAAE,CAAC,QAAQ,CAAC;YACnB,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;YACpC,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,WAAW,CAAC;SAClE;QACD,EAAE,EAAE;YACH,oCAAoC;YACpC,OAAO,EAAE,CAAC,MAAM,CAAC;YACjB,gBAAgB,EAAE,CAAC,IAAI,CAAC;YACxB,UAAU,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;SAChC;QACD,EAAE,EAAE;YACH,eAAe;YACf,+EAA+E;YAC/E,OAAO,EAAE,CAAC,OAAO,CAAC;YAClB,UAAU,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;SAC/B;QACD,EAAE,EAAE;YACH,mEAAmE;YACnE,OAAO,EAAE,CAAC,6BAA6B,CAAC;YACxC,gBAAgB,EAAE,CAAC,IAAI,CAAC;YACxB,UAAU,EAAE,CAAC,WAAW,EAAE,WAAW,EAAE,YAAY,CAAC;SACpD;QACD,EAAE,EAAE;YACH,gFAAgF;YAChF,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;YAClE,UAAU,EAAE,CAAC,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC;SAC5D;QACD,EAAE,EAAE;YACH,OAAO,EAAE,CAAC,mBAAmB,EAAE,SAAS,EAAE,gBAAgB,CAAC;YAC3D,UAAU,EAAE,CAAC,SAAS,EAAE,mBAAmB,EAAE,KAAK,EAAE,eAAe,CAAC;SACpE;QACD,EAAE,EAAE;YACH,kDAAkD;YAClD,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,gBAAgB,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC;YACvC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,EAAE,SAAS,CAAC;SACnD;QACD,qFAAqF;QACrF,iFAAiF;QACjF,6EAA6E;QAC7E,sCAAsC;QACtC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE;QAChC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE;QAC1C,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,QAAQ,CAAC,EAAE;QAC9B,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE;QAChD,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,mBAAmB,CAAC,EAAE;QACzC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,EAAE,UAAU,CAAC,EAAE;QAC5C,2EAA2E;QAC3E,uFAAuF;QACvF,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,SAAS,CAAC,EAAE;QAC/B,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,aAAa,CAAC,EAAE;QACxD,EAAE,EAAE;YACH,sEAAsE;YACtE,uEAAuE;YACvE,6DAA6D;YAC7D,gFAAgF;YAChF,OAAO,EAAE;gBACR,gBAAgB;gBAChB,aAAa;gBACb,YAAY;gBACZ,aAAa;gBACb,aAAa;gBACb,kBAAkB;gBAClB,eAAe;gBACf,cAAc;gBACd,eAAe;gBACf,eAAe;aACf;YACD,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;YACtD,UAAU,EAAE,CAAC,SAAS,CAAC;SACvB;QACD,EAAE,EAAE;YACH,wGAAwG;YACxG,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,CAAC;YAChD,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;SAC1C;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAmC;IACnE,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,CAAA;IAEvB,OAAO,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAE,OAAyB,CAAA;AAClF,CAAC;AAID;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,QAAQ,GAAmC,eAAe,EAAE;IAE5D,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;IACtD,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAC,UAAU,CAAA;IAErD,MAAM,SAAS,GACd,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC;QACrE,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC;QAC7G,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC,CAAA;IAE7E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAE9B,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,IAAI,GAAiB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAC3C,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QACpC,CAAC;QAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAExC,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC1B,CAAC;aAAM,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACzB,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAErD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAElG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAA0B;IAC5F,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE7C,MAAM,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAA;AACpE,CAAC;AAOD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QAClE,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IAEpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IAED,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;IAExE,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAiC,EAAE,OAAwB;IAC/F,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,eAAe,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAE/D,MAAM,SAAS,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACvD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,0CAA0C;IAC1C,qDAAqD;IACrD,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAChD,eAAe,CAAC,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CACzD,EAAE,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YAChC,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,GAAG,CAAC,GAAG,EAAE,CAAA;IACV,CAAC;IAED,MAAM,SAAS,CAAA;IAEf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,oEAAoE;QACpE,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QACpG,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAExB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;YACnD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC/C,CAAC"}

package/out/utils/tokenize.d.ts

@@ -3,36 +3,25 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Tokenizer interface for alignment.
- *
- *   Two implementations live in the corpus package:
- *
- *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
- *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
- *        default for in-container alignment tests and as a fallback when no SentencePiece model is
- *        available.
- *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
- *        trained on the corpus. Same interface, different splits. Locked against the corpus version
- *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
- *
- *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
- *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
- *   spans, independent of how the tokenizer chose its splits.
+ * Tokenizer interface for alignment. Each token comes back with its (start, end) character offsets so
+ * BIO labels can be assigned by span overlap with component spans, independent of how the tokenizer
+ * chose its splits. The interface is deliberately minimal — only what alignment needs.
  */
 /**
  * A token with its character span in the source string.
  */
 export interface TokenSpan {
     /**
-     * The token text, possibly normalized (case unchanged here; tokenizers may differ).
+     * The token text, possibly normalized.
+     * This tokenizer leaves case unchanged.
      */
     text: string;
     /**
-     * Inclusive start offset (UTF-16 code-unit index) in the source string.
+     * Inclusive start offset, a UTF-16 code-unit index into the source string.
      */
     start: number;
     /**
-     * Exclusive end offset in the source string. `text === source.slice(start, end)`.
+     * Exclusive end offset, with `text === source.slice(start, end)`.
      */
     end: number;
 }
@@ -45,20 +34,19 @@ export interface Tokenizer {
 /**
  * Whitespace + punctuation tokenizer (pure JS).
  *
- * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
- * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
- * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
- *
- * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
- * tokenizer will preserve all bytes via byte-fallback.
+ * Tokens are maximal runs of unicode word characters (`\p{L}`, `\p{N}`, `\p{M}`, plus `'`, `-`, `_`);
+ * everything else is a separator and is not emitted, so the spans cover only token regions.
+ * The lossiness at the edges is intentional, because alignment can still label every meaningful span.
  */
 export declare function whitespaceTokenizer(): Tokenizer;
 /**
- * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ * Whitespace tokenizer for Latin runs, one token per character for Han runs.
  *
- * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
- * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
- * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the
+ * unit it labels, while the Latin tail of a mixed row (`赵光三分场二十九队, Heilongjiang, China`)
+ * keeps the word tokens the Latin aligner has always used.
+ * Spans still come back as `[start, end)` offsets over the source string,
+ * so the aligner's span-overlap rule needs no change.
  */
 export declare function cjkAwareTokenizer(): Tokenizer;
 //# sourceMappingURL=tokenize.d.ts.map

package/out/utils/tokenize.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;CAC5C;AAED;;;;;;;;;GASG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAkB/C;AAQD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,IAAI,SAAS,CA2B7C"}
+{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAA;IAEZ;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;CACX;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,SAAS,EAAE,CAAA;CAC5C;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,IAAI,SAAS,CAgB/C;AASD;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,IAAI,SAAS,CA4B7C"}

package/out/utils/tokenize.js

@@ -3,35 +3,18 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Tokenizer interface for alignment.
- *
- *   Two implementations live in the corpus package:
- *
- *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
- *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
- *        default for in-container alignment tests and as a fallback when no SentencePiece model is
- *        available.
- *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
- *        trained on the corpus. Same interface, different splits. Locked against the corpus version
- *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
- *
- *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
- *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
- *   spans, independent of how the tokenizer chose its splits.
+ * Tokenizer interface for alignment. Each token comes back with its (start, end) character offsets so
+ * BIO labels can be assigned by span overlap with component spans, independent of how the tokenizer
+ * chose its splits. The interface is deliberately minimal — only what alignment needs.
  */
 /**
  * Whitespace + punctuation tokenizer (pure JS).
  *
- * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
- * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
- * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
- *
- * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
- * tokenizer will preserve all bytes via byte-fallback.
+ * Tokens are maximal runs of unicode word characters (`\p{L}`, `\p{N}`, `\p{M}`, plus `'`, `-`, `_`);
+ * everything else is a separator and is not emitted, so the spans cover only token regions.
+ * The lossiness at the edges is intentional, because alignment can still label every meaningful span.
  */
 export function whitespaceTokenizer() {
-    // Maximal runs of letters/digits/marks plus the joiners common to addresses
-    // (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
     const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu;
     return {
         tokenize(text) {
@@ -46,16 +29,19 @@ export function whitespaceTokenizer() {
     };
 }
 /**
- * Han characters, the script CJK addresses are written in and the one the whitespace tokenizer cannot split: a run like
- * `三分场八队` is one "word" to `\p{L}+`, so the aligner could give it one label and never two.
+ * Han characters, the script CJK addresses are written in and the one the
+ * whitespace tokenizer cannot split: a run like `三分场八队` is one "word" to `\p{L}+`,
+ * so the aligner could give it one label and never two.
  */
 const HAN = /\p{Script=Han}/u;
 /**
- * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ * Whitespace tokenizer for Latin runs, one token per character for Han runs.
  *
- * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
- * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
- * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the
+ * unit it labels, while the Latin tail of a mixed row (`赵光三分场二十九队, Heilongjiang, China`)
+ * keeps the word tokens the Latin aligner has always used.
+ * Spans still come back as `[start, end)` offsets over the source string,
+ * so the aligner's span-overlap rule needs no change.
  */
 export function cjkAwareTokenizer() {
     const base = whitespaceTokenizer();
@@ -67,8 +53,9 @@ export function cjkAwareTokenizer() {
                     out.push(token);
                     continue;
                 }
-                // A Han run may carry Latin letters or digits inside it (`3分场2队`); those stay glued to their neighbours
-                // only if they are Han too, so every code point of a Han-bearing token becomes its own token.
+                // A Han run may contain Latin letters or digits (`3分场2队`); those stay glued
+                // to their neighbors only if they are Han too, so every code point of a
+                // Han-containing token becomes its own token.
                 let offset = token.start;
                 for (const codePoint of token.text) {
                     out.push({ text: codePoint, start: offset, end: offset + codePoint.length });

package/out/utils/tokenize.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AA6BH;;;;;;;;;GASG;AACH,MAAM,UAAU,mBAAmB;IAClC,4EAA4E;IAC5E,gFAAgF;IAChF,MAAM,OAAO,GAAG,yBAAyB,CAAA;IAEzC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAC3B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;YACrB,IAAI,CAAyB,CAAA;YAE7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YACrE,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,GAAG,GAAG,iBAAiB,CAAA;AAE7B;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB;IAChC,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAA;IAElC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAE3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEf,SAAQ;gBACT,CAAC;gBAED,wGAAwG;gBACxG,8FAA8F;gBAC9F,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAA;gBAExB,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACpC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC5E,MAAM,IAAI,SAAS,CAAC,MAAM,CAAA;gBAC3B,CAAC;YACF,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC"}
+{"version":3,"file":"tokenize.js","sourceRoot":"","sources":["../../lib/utils/tokenize.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AA8BH;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB;IAClC,MAAM,OAAO,GAAG,yBAAyB,CAAA;IAEzC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAC3B,OAAO,CAAC,SAAS,GAAG,CAAC,CAAA;YACrB,IAAI,CAAyB,CAAA;YAE7B,OAAO,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YACrE,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,GAAG,GAAG,iBAAiB,CAAA;AAE7B;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB;IAChC,MAAM,IAAI,GAAG,mBAAmB,EAAE,CAAA;IAElC,OAAO;QACN,QAAQ,CAAC,IAAY;YACpB,MAAM,GAAG,GAAgB,EAAE,CAAA;YAE3B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAEf,SAAQ;gBACT,CAAC;gBAED,4EAA4E;gBAC5E,wEAAwE;gBACxE,8CAA8C;gBAC9C,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAA;gBAExB,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;oBACpC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAA;oBAC5E,MAAM,IAAI,SAAS,CAAC,MAAM,CAAA;gBAC3B,CAAC;YACF,CAAC;YAED,OAAO,GAAG,CAAA;QACX,CAAC;KACD,CAAA;AACF,CAAC"}