@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/utils/split.ts

@@ -3,115 +3,242 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ * Splits corpus rows into train, val and test by holding out whole places: a random row split leaks by
+ * neighborhood, because the model can memorize a street it saw in train. Rows inside a held-out place
+ * go to val or test 50/50 by a hash of `source_id`; every other row goes to train.
  *
- *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
- *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
- *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
- *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
- *   memorization, not by learning the underlying schema.
- *
- *   Phase 1 holdouts (chosen for low data density + administrative isolation):
- *
- *   - **US**: Vermont, Wyoming, North Dakota
- *   - **FR**: Corse, Lozère, Creuse
- *
- *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
- *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
- *   the locality boundary, not the exact split percentages.
- *
- *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
- *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ * A holdout change takes effect at the next corpus rebuild. Each versioned corpus keeps the
+ * `SPLIT_MANIFEST.json` it was built with, because adding a holdout to a built corpus would leak rows
+ * the model already trained on.
  */
 
+import { delimitedSource, preferCompressed } from "@mailwoman/core/fs/delimited"
 import { openWriteStream } from "@mailwoman/core/fs/streams"
 import { writeLocalJSONFile, writeLocalTextFile, makeDirectories, removePath } from "@mailwoman/core/fs/writers"
 import { createHash } from "@mailwoman/core/hash"
 import { spawnProcess } from "@mailwoman/core/process"
 import { childEnv } from "@mailwoman/core/scripting/utils"
-import { join, type PathBuilderLike } from "path-ts"
+import { type PathBuilderLike, resolvePathBuilder } from "path-ts"
 import { JSONSpliterator } from "spliterator"
 
 import type { CanonicalRow, LabeledRow } from "#types"
 
+/**
+ * The name of one corpus split.
+ */
 export type SplitName = "train" | "val" | "test"
 
+/**
+ * The component values that identify the places one country holds out.
+ *
+ * Sources represent a place in different components: `usgov-nad` and `tiger` emit `region`,
+ * but BAN emits none, so a French holdout must also match on postcode.
+ * A row is held out when any declared matcher fires.
+ * A policy with no matchers holds out no entry.
+ */
+export interface HoldoutPolicy {
+	regions?: readonly string[]
+	/**
+	 * Prefixes matched against `row.components.postcode`; use one only where it identifies
+	 * a single place by itself, such as the first two digits of a French postcode.
+	 */
+	postcodePrefixes?: readonly string[]
+	localities?: readonly string[]
+}
+
+/**
+ * One country's holdout as a bare region list or a {@link HoldoutPolicy}.
+ *
+ * Older committed `SPLIT_MANIFEST.json` files use a bare array of region values.
+ */
+export type CountryHoldout = readonly string[] | HoldoutPolicy
+
+/**
+ * Options for {@link splitRows}.
+ */
 export interface SplitOptions {
 	/**
-	 * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
-	 * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
-	 * `defaultHoldouts()`.
+	 * The holdout policy keyed by ISO 3166-1 alpha-2 country code.
 	 */
-	holdouts?: Record<string, readonly string[]>
+	holdouts?: Record<string, CountryHoldout>
 }
 
 /**
- * Output manifest: source_id lists per split.
+ * The `source_id` lists for each split.
  */
 export interface SplitManifest {
 	train: string[]
 	val: string[]
 	test: string[]
+	holdouts: Record<string, CountryHoldout>
 	/**
-	 * Echoes the holdouts used, so the manifest is self-describing.
-	 */
-	holdouts: Record<string, readonly string[]>
-	/**
-	 * Corpus version stamped onto the manifest. Read from the first row.
+	 * The corpus version, read from the first row that has one.
 	 */
 	corpus_version: string
-	/**
-	 * Counts for quick sanity checks.
-	 */
 	counts: { train: number; val: number; test: number; total: number }
 }
 
 /**
- * Phase 1 default holdouts (per plan).
- *
- * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
- * - FR: Corse, Lozère, Creuse (small departments / regions).
- * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
- *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
- *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
- *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ * Returns the default holdouts — small, peripheral places in each country.
+ * A change here takes effect at the next base corpus rebuild.
  */
-export function defaultHoldouts(): Record<string, readonly string[]> {
+export function defaultHoldouts(): Record<string, CountryHoldout> {
 	return {
 		US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
-		FR: ["Corse", "Lozère", "Lozere", "Creuse"],
-		DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+		FR: {
+			regions: ["Corse", "Lozère", "Lozere", "Creuse"],
+			// BAN street rows have no region, so these postcode departments hold out the
+			// same three places: Corse 20, Creuse 23 and Lozère 48.
+			postcodePrefixes: ["20", "23", "48"],
+		},
+		DE: {
+			// Only `wof-admin` has a DE region, so these Länder hold out admin rows only.
+			regions: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+			// DE street rows come from two OpenAddresses members, Berlin and Sachsen,
+			// so the regions above match no street row.
+			// Postcode area `02` (Upper Lusatia in eastern Sachsen) is the street-level holdout.
+			// German postcodes have five digits, so this holdout cannot collide.
+			postcodePrefixes: ["02"],
+		},
+		GB: {
+			// GB street rows have no region, so the holdout matches postcode areas —
+			// Truro, Llandudno and Halifax.
+			// Each prefix has two letters and none is a prefix of another
+			// (`L` for Liverpool would also match `LL`).
+			postcodePrefixes: ["TR", "LL", "HX"],
+			// Cornwall matches the admin rows for the same place `TR` covers.
+			regions: ["Cornwall"],
+		},
+		// The eight countries below enter training through OpenStreetMap rows, most of which carry no
+		// region and many no postcode, so each holdout matches its place three ways: the region an admin
+		// row carries, the postcode prefix, and the city as the OSM rows spell it. The spellings are the
+		// ones the 2026-10-02 extracts write.
+		CN: {
+			// Gansu by its province and capital, and Qinghai `81` and Tibet `85` by postcode.
+			regions: ["甘肃省", "甘肃", "Gansu"],
+			postcodePrefixes: ["81", "85"],
+			localities: ["兰州市", "兰州", "Lanzhou"],
+		},
+		IN: {
+			// Kerala, whose PIN codes run 67 to 69.
+			regions: ["Kerala"],
+			postcodePrefixes: ["67", "68", "69"],
+			localities: ["Kochi", "Kollam", "Thiruvananthapuram", "Kozhikode"],
+		},
+		ID: {
+			// Bali, whose postcodes begin `80`.
+			regions: ["Bali"],
+			postcodePrefixes: ["80"],
+			localities: ["Denpasar", "Bali"],
+		},
+		NG: {
+			// Enugu State.
+			// OSM Nigeria rows carry almost no postcode or region, so the cities carry it.
+			regions: ["Enugu"],
+			localities: ["Enugu", "Nsukka"],
+		},
+		PK: {
+			// Islamabad Capital Territory, in both scripts the extract writes.
+			regions: ["Islamabad Capital Territory"],
+			postcodePrefixes: ["44"],
+			localities: ["Islamabad", "islamabad", "اسلام آباد"],
+		},
+		BD: {
+			// Chattogram, Khulna and Sylhet, whose postcodes begin 4, 9 and 3 respectively.
+			postcodePrefixes: ["40", "41", "42", "43", "90", "91", "92", "31"],
+			localities: ["Chittagong", "Chattogram", "Khulna", "Sylhet"],
+		},
+		VN: {
+			regions: ["Thành phố Cần Thơ", "Cần Thơ", "Thừa Thiên Huế"],
+			localities: ["Cần Thơ", "Thành phố Cần Thơ", "Huế", "Thành phố Huế"],
+		},
+		RU: {
+			// Primorsky Krai, whose postcodes run 690 to 692.
+			regions: ["Приморский край"],
+			postcodePrefixes: ["690", "691", "692"],
+			localities: ["Владивосток", "Уссурийск", "Находка"],
+		},
+		// Each French overseas jurisdiction is one INSEE department with one postcode range,
+		// so the BAN rows hold out by commune, spelled as BAN's `nom_commune` writes it.
+		// Saint-Barthélemy and Saint-Martin are each one commune under one postcode,
+		// so neither has a place to hold out.
+		GP: { localities: ["Le Moule"] },
+		MQ: { localities: ["Le Robert", "Ducos"] },
+		GF: { localities: ["Kourou"] },
+		RE: { localities: ["Saint-Louis", "Saint-Leu"] },
+		PM: { localities: ["Miquelon-Langlade"] },
+		YT: { localities: ["Dzaoudzi", "Pamandzi"] },
+		// Tumaraa holds 1,522 of PF's 6,755 rows with a street, more than a tenth,
+		// because Pirae's 958 rows carry no street name and Papeete holds the remaining 5,233.
+		PF: { localities: ["Tumaraa"] },
+		NC: { localities: ["Koné", "Pouembout", "Kaala-Gomen"] },
+		LV: {
+			// Latgale's south-east: the Krāslava, Ludza, Balvi, Preiļi and Līvāni
+			// municipalities, whose postcodes begin LV-56, LV-57, LV-45 and LV-53,
+			// and the state city Rēzekne, which lies in no municipality.
+			// The address register writes `nov.`; other sources spell the municipality out.
+			regions: [
+				"Krāslavas nov.",
+				"Ludzas nov.",
+				"Balvu nov.",
+				"Preiļu nov.",
+				"Līvānu nov.",
+				"Krāslavas novads",
+				"Ludzas novads",
+				"Balvu novads",
+				"Preiļu novads",
+				"Līvānu novads",
+			],
+			postcodePrefixes: ["LV-45", "LV-53", "LV-56", "LV-57"],
+			localities: ["Rēzekne"],
+		},
+		RO: {
+			// Alba 51, Hunedoara 33, Mureș 54 and Sibiu 55: a Romanian postcode's first two digits name its county.
+			regions: ["Alba", "Hunedoara", "Mureș", "Sibiu"],
+			postcodePrefixes: ["51", "33", "54", "55"],
+		},
 	}
 }
 
+/**
+ * Returns the matchers for one country, reading a bare array as a region list.
+ */
+export function holdoutPolicyFor(holdout: CountryHoldout | undefined): HoldoutPolicy {
+	if (!holdout) return {}
+
+	return Array.isArray(holdout) ? { regions: holdout } : (holdout as HoldoutPolicy)
+}
+
 type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">
 
 /**
- * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
- * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
- * array-based path so the decision is stable regardless of caller.
+ * Returns the split for one row.
+ *
+ * Both `splitRows` and the streaming `buildCorpus` loop call this, so every
+ * caller assigns a row to the same split.
  */
 export function splitForRow(
 	row: Pick<SplitInputRow, "source_id" | "country" | "components">,
-	holdouts: Record<string, readonly string[]> = defaultHoldouts()
+	holdouts: Record<string, CountryHoldout> = defaultHoldouts()
 ): SplitName {
-	const region = row.components.region
-	const countryHoldouts = holdouts[row.country] ?? []
-	const isHeldOut = region !== undefined && countryHoldouts.includes(region)
+	const policy = holdoutPolicyFor(holdouts[row.country])
+	const { region, postcode, locality } = row.components
+
+	const isHeldOut =
+		(region !== undefined && (policy.regions?.includes(region) ?? false)) ||
+		(postcode !== undefined && (policy.postcodePrefixes?.some((prefix) => postcode.startsWith(prefix)) ?? false)) ||
+		(locality !== undefined && (policy.localities?.includes(locality) ?? false))
 
 	if (!isHeldOut) return "train"
 
-	// 50/50 deterministic by source_id hash. Same input always lands in the same split.
 	return hashBucket(row.source_id, 2) === 0 ? "val" : "test"
 }
 
 /**
- * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
- * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ * Builds a `SplitManifest` in memory from labeled or canonical rows.
  *
- * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
- * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
- * split membership in heap.
+ * Tests and small fixtures use this, while `buildCorpus` uses `splitForRow`
+ * and `writeSplitManifestsFromLabeledFiles` so it never holds every row's split in memory.
  */
 export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}): SplitManifest {
 	const holdouts = opts.holdouts ?? defaultHoldouts()
@@ -149,29 +276,19 @@ export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}
 }
 
 /**
- * Lightweight deterministic 0..(n-1) bucket from a string id.
- */
-/**
- * Deterministic bucket for a stable id.
- *
- * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
- * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
- * line it saves.
+ * Returns a deterministic bucket in `0..n-1` for an id, from the first four bytes
+ * of the SHA-256 digest read as a big-endian uint32.
  */
 export function hashBucket(id: string, n: number): number {
 	const digest = createHash("sha256").update(id).digest()
-	// Read 4 bytes as uint32 to avoid bigint overhead.
 	const u = digest[0]! * 0x01_00_00_00 + digest[1]! * 0x01_00_00 + digest[2]! * 0x01_00 + digest[3]!
 
 	return u % n
 }
 
 /**
- * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
- * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
- * including holdouts + counts + corpus version.
- *
- * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ * Writes a `SplitManifest` to `<outputDir>/{train,val,test}.txt` (sorted `source_id` values, one per line)
+ * and `SPLIT_MANIFEST.json` (the corpus version, holdouts and counts).
  */
 export async function writeSplitManifests(manifest: SplitManifest, outputDir: PathBuilderLike): Promise<void> {
 	await makeDirectories(outputDir)
@@ -179,7 +296,7 @@ export async function writeSplitManifests(manifest: SplitManifest, outputDir: Pa
 	for (const name of ["train", "val", "test"] as const) {
 		const sorted = [...manifest[name]].toSorted()
 
-		await writeLocalTextFile(sorted.join("\n") + (sorted.length ? "\n" : ""), outputDir, `${name}.txt`)
+		await writeLocalTextFile(sorted, outputDir, `${name}.txt`)
 	}
 
 	const summary = {
@@ -192,31 +309,30 @@ export async function writeSplitManifests(manifest: SplitManifest, outputDir: Pa
 }
 
 /**
- * Type re-export for callers that want to ingest LabeledRow specifically.
+ * The `LabeledRow` fields that the split functions read.
  */
 export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">
 
 /**
- * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
- * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
- * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ * Writes the same files as `writeSplitManifests` by streaming one labeled JSONL file per split.
  *
- * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
- * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ * `buildCorpus` calls this after its align loop partitions rows with `splitForRow`.
+ * The caller passes the counts to avoid rescanning the files.
+ * `sort(1)` spills to disk, so memory stays constant.
  */
 export async function writeSplitManifestsFromLabeledFiles(opts: {
-	labeledPaths: Record<SplitName, string>
+	labeledPaths: Record<SplitName, PathBuilderLike>
 	outputDir: PathBuilderLike
 	corpusVersion: string
 	counts: Record<SplitName, number>
-	holdouts?: Record<string, readonly string[]>
+	holdouts?: Record<string, CountryHoldout>
 }): Promise<SplitManifest["counts"]> {
 	await makeDirectories(opts.outputDir)
 	const holdouts = opts.holdouts ?? defaultHoldouts()
 
 	for (const split of ["train", "val", "test"] as const) {
 		const labeledPath = opts.labeledPaths[split]
-		const outPath = join(opts.outputDir, `${split}.txt`)
+		const outPath = resolvePathBuilder(opts.outputDir, `${split}.txt`)
 		await streamSortedSourceIDs(labeledPath, outPath)
 	}
 
@@ -234,10 +350,10 @@ export async function writeSplitManifestsFromLabeledFiles(opts: {
 }
 
 /**
- * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
- * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
+ * Writes the sorted `source_id` values from a labeled JSONL file to `outPath`;
+ * an empty input still produces an empty output file.
  */
-async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string): Promise<void> {
+async function streamSortedSourceIDs(labeledJsonlPath: PathBuilderLike, outPath: PathBuilderLike): Promise<void> {
 	const unsortedPath = `${outPath}.unsorted`
 	const out = openWriteStream(unsortedPath, { encoding: "utf8" })
 
@@ -246,11 +362,12 @@ async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string):
 		out.on("error", reject)
 	})
 
-	// JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
-	// SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
-	// always ends the write stream so `sort` reads a complete file even if the read throws.
+	// A malformed row throws out of the loop.
+	// The `finally` block still closes the write stream.
 	try {
-		for await (const obj of JSONSpliterator.fromAsync<{ source_id?: string }>(labeledJsonlPath)) {
+		for await (const obj of JSONSpliterator.fromAsync<{ source_id?: string }>(
+			delimitedSource(await preferCompressed(labeledJsonlPath))
+		)) {
 			if (typeof obj.source_id === "string") {
 				out.write(`${obj.source_id}\n`)
 			}
@@ -262,7 +379,7 @@ async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string):
 	await outClosed
 
 	await new Promise<void>((resolve, reject) => {
-		// LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+		// `LC_ALL=C` sorts by byte, so the order is the same on every host.
 		const proc = spawnProcess("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) })
 		proc.on("error", reject)
 

package/lib/utils/tokenize.ts

@@ -3,21 +3,9 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Tokenizer interface for alignment.
- *
- *   Two implementations live in the corpus package:
- *
- *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
- *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
- *        default for in-container alignment tests and as a fallback when no SentencePiece model is
- *        available.
- *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
- *        trained on the corpus. Same interface, different splits. Locked against the corpus version
- *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
- *
- *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
- *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
- *   spans, independent of how the tokenizer chose its splits.
+ * Tokenizer interface for alignment. Each token comes back with its (start, end) character offsets so
+ * BIO labels can be assigned by span overlap with component spans, independent of how the tokenizer
+ * chose its splits. The interface is deliberately minimal — only what alignment needs.
  */
 
 /**
@@ -25,17 +13,18 @@
  */
 export interface TokenSpan {
 	/**
-	 * The token text, possibly normalized (case unchanged here; tokenizers may differ).
+	 * The token text, possibly normalized.
+	 * This tokenizer leaves case unchanged.
 	 */
 	text: string
 
 	/**
-	 * Inclusive start offset (UTF-16 code-unit index) in the source string.
+	 * Inclusive start offset, a UTF-16 code-unit index into the source string.
 	 */
 	start: number
 
 	/**
-	 * Exclusive end offset in the source string. `text === source.slice(start, end)`.
+	 * Exclusive end offset, with `text === source.slice(start, end)`.
 	 */
 	end: number
 }
@@ -50,16 +39,11 @@ export interface Tokenizer {
 /**
  * Whitespace + punctuation tokenizer (pure JS).
  *
- * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
- * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
- * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
- *
- * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
- * tokenizer will preserve all bytes via byte-fallback.
+ * Tokens are maximal runs of unicode word characters (`\p{L}`, `\p{N}`, `\p{M}`, plus `'`, `-`, `_`);
+ * everything else is a separator and is not emitted, so the spans cover only token regions.
+ * The lossiness at the edges is intentional, because alignment can still label every meaningful span.
  */
 export function whitespaceTokenizer(): Tokenizer {
-	// Maximal runs of letters/digits/marks plus the joiners common to addresses
-	// (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
 	const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu
 
 	return {
@@ -78,17 +62,20 @@ export function whitespaceTokenizer(): Tokenizer {
 }
 
 /**
- * Han characters, the script CJK addresses are written in and the one the whitespace tokenizer cannot split: a run like
- * `三分场八队` is one "word" to `\p{L}+`, so the aligner could give it one label and never two.
+ * Han characters, the script CJK addresses are written in and the one the
+ * whitespace tokenizer cannot split: a run like `三分场八队` is one "word" to `\p{L}+`,
+ * so the aligner could give it one label and never two.
  */
 const HAN = /\p{Script=Han}/u
 
 /**
- * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ * Whitespace tokenizer for Latin runs, one token per character for Han runs.
  *
- * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
- * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
- * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the
+ * unit it labels, while the Latin tail of a mixed row (`赵光三分场二十九队, Heilongjiang, China`)
+ * keeps the word tokens the Latin aligner has always used.
+ * Spans still come back as `[start, end)` offsets over the source string,
+ * so the aligner's span-overlap rule needs no change.
  */
 export function cjkAwareTokenizer(): Tokenizer {
 	const base = whitespaceTokenizer()
@@ -104,8 +91,9 @@ export function cjkAwareTokenizer(): Tokenizer {
 					continue
 				}
 
-				// A Han run may carry Latin letters or digits inside it (`3分场2队`); those stay glued to their neighbours
-				// only if they are Han too, so every code point of a Han-bearing token becomes its own token.
+				// A Han run may contain Latin letters or digits (`3分场2队`); those stay glued
+				// to their neighbors only if they are Han too, so every code point of a
+				// Han-containing token becomes its own token.
 				let offset = token.start
 
 				for (const codePoint of token.text) {