@mailwoman/corpus 9.4.0 → 10.1.0

package/out/utils/license.js

@@ -3,45 +3,175 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Corpus licensing — the single source of truth for the training-data license policy (#26).
+ *   Corpus licensing — the training-data license policy, read from a row's obligations rather than from
+ *   the shape of its license string.
  *
- *   Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
- *   build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
- *   license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
- *   inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
- *   })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
- *   string unless the operator named it. This avoids the trap of silently dropping allowed data
- *   mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
- *   ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
- *   dropped 48M allowed rows).
+ *   Exclusion is a deliberate act rather than a silent default: the build includes every row an adapter
+ *   yields (stamping its `license`), and a build that needs a license set carrying no share-alike
+ *   obligation asks for {@linkcode LicensePolicy.ShareAlikeFree}, so no source is dropped unless the
+ *   operator selected the policy.
  *
- *   Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
- *   — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
- *   proprietary-weights build via `--exclude-share-alike`.
+ *   The refusal reads `readLicenseRecord`, which resolves a value to an SPDX expression and looks up
+ *   that expression's obligations. An anchored prefix match over the raw column cannot do this. Measured
+ *   over `v0.7.0-de-holdout` on 2026-09-26, the column holds 71 distinct values across 703,835,753 rows,
+ *   of which 4 are SPDX identifiers and 66 are prose. A row whose license reads "OpenStreetMap venue +
+ *   sub-venue names (ODbL, © OpenStreetMap contributors) …" is share-alike-derived and starts with
+ *   neither `ODbL` nor `CC-BY-SA`, so `/^ODbL/` admitted it.
+ *
+ *   Three refusal classes exist because they are different claims. A value that resolves to an
+ *   expression carrying share-alike states the obligation. A value whose prose mentions a share-alike
+ *   license while resolving to no expression is share-alike-derived without stating its own grant. A
+ *   value that resolves to no expression has unknown obligations. Unknown obligations differ from an empty
+ *   obligation set. Of these rows, 628,203,119 contain prose this repository writes about its own renderings. The
+ *   third class is therefore reported rather than refused under {@linkcode LicensePolicy.ShareAlikeFree},
+ *   and {@linkcode LicensePolicy.ResolvedOnly} refuses it for a caller that needs every grant stated.
+ */
+import { carriesShareAlike, mentionsShareAlike, readLicenseRecord } from "@mailwoman/core/license/record";
+import { extractDelimited } from "@mailwoman/core/scripting/arguments";
+import { escapeRegExp } from "@mailwoman/core/strings/regexp";
+/**
+ * Which rows a corpus build admits on the evidence of their license values.
  */
+export const LicensePolicy = {
+    /**
+     * Admit every row an adapter yields, whatever its license value states.
+     * The policy a measurement runs under.
+     */
+    All: "all",
+    /**
+     * Refuse a row whose license imposes a share-alike obligation.
+     *
+     * Also refuse a row whose license text mentions a share-alike license while stating no grant of its own.
+     *
+     * A row whose license resolves to no expression and mentions no share-alike license is admitted
+     * and counted, because its obligations are unknown rather than known to include share-alike.
+     */
+    ShareAlikeFree: "share-alike-free",
+    /**
+     * Apply all refusals from {@linkcode LicensePolicy.ShareAlikeFree}.
+     * Also refuse a row whose license resolves to no expression.
+     *
+     * Measured over `v0.7.0-de-holdout`, this refuses 628,203,119 of 703,835,753 rows,
+     * so a caller asking for it is asking for the subset whose grant is stated as an identifier.
+     */
+    ResolvedOnly: "resolved-only",
+};
 /**
- * Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
- * `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use it.
+ * Why a row's license value kept it out of the corpus, counted separately per class
+ * because the classes rest on different evidence.
  */
-export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
+export const LicenseRefusalKind = {
+    /**
+     * The operator supplied this license prefix in `--exclude-licenses`.
+     */
+    OperatorExcluded: "operator-excluded",
+    /**
+     * The value resolves to an expression whose recorded obligations include share-alike.
+     */
+    ShareAlikeCarried: "share-alike-carried",
+    /**
+     * The value's text mentions a share-alike license while resolving to no expression of its own.
+     */
+    ShareAlikeMentioned: "share-alike-mentioned",
+    /**
+     * The value resolves to no expression, so its obligations are unknown.
+     */
+    Unresolved: "unresolved",
+};
 /**
- * Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored, case-insensitive prefix
- * patterns. Each entry matches a license string that STARTS with it, so `CC-BY-SA` catches `CC-BY-SA-3.0`,
- * `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the spec is a literal license prefix, not a user-supplied
- * regex.
+ * Compile an `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
+ * case-insensitive prefix patterns, so `CC-BY-SA` catches `CC-BY-SA-3.0`.
+ *
+ * The spec is a literal license prefix the operator typed.
+ * Regex metacharacters are escaped.
+ *
+ * This is the one place a prefix match is the right reading, because the operator
+ * is naming a spelling rather than asking about an obligation.
  */
 export function compileLicenseExcludes(spec) {
-    return spec
-        .split(",")
-        .map((s) => s.trim())
-        .filter((entry) => entry !== "")
-        .map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"));
+    return extractDelimited(spec).map((s) => new RegExp("^" + escapeRegExp(s), "i"));
 }
 /**
- * True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
+ * True when `license` matches any of the operator's exclude `patterns`; empty patterns never exclude.
  */
 export function licenseExcluded(license, patterns) {
     const l = license ?? "";
     return patterns.some((p) => p.test(l));
 }
+/**
+ * Reads one license value against a policy.
+ *
+ * This resolves an expression and runs the mention patterns on every call.
+ * A corpus build asks about hundreds of millions of rows holding tens of distinct values,
+ * so a row loop calls {@linkcode createLicenseVerdictCache} instead.
+ */
+export function licenseVerdict(license, policy, excluded = []) {
+    const record = readLicenseRecord(license);
+    const resolved = record.expression !== null;
+    const mentions = mentionsShareAlike(record);
+    const verdict = (refusal) => ({
+        refusal,
+        resolved,
+        mentionsShareAlike: mentions || carriesShareAlike(record),
+    });
+    if (licenseExcluded(license, excluded))
+        return verdict(LicenseRefusalKind.OperatorExcluded);
+    if (policy === LicensePolicy.All)
+        return verdict(null);
+    if (carriesShareAlike(record))
+        return verdict(LicenseRefusalKind.ShareAlikeCarried);
+    if (mentions)
+        return verdict(LicenseRefusalKind.ShareAlikeMentioned);
+    if (!resolved && policy === LicensePolicy.ResolvedOnly)
+        return verdict(LicenseRefusalKind.Unresolved);
+    return verdict(null);
+}
+/**
+ * Every license value in a built corpus's license set that imposes or mentions share-alike.
+ *
+ * Takes the `licenses` map recorded by `BuildCorpusManifest`.
+ * The map counts the rows adapters yielded.
+ *
+ * A caller about to move or publish a corpus reads this rather than its own list of source ids,
+ * because the obligation is a property of the license value on the row.
+ *
+ * Ordered by row count, most rows first.
+ */
+export function shareAlikeFindings(licenses) {
+    const findings = [];
+    for (const [license, rows] of Object.entries(licenses)) {
+        const { refusal } = licenseVerdict(license, LicensePolicy.ShareAlikeFree);
+        if (refusal) {
+            findings.push({ license, rows, kind: refusal });
+        }
+    }
+    return findings.toSorted((a, b) => b.rows - a.rows);
+}
+/**
+ * Builds a cache that reads each distinct license value once.
+ *
+ * The corpus holds tens of distinct values over hundreds of millions of rows,
+ * so the memo turns one expression resolution per row into one per value.
+ */
+export function createLicenseVerdictCache(policy, excluded = []) {
+    const verdicts = new Map();
+    const refused = new Map();
+    return {
+        read(license) {
+            const key = license ?? "";
+            const cached = verdicts.get(key);
+            if (cached)
+                return cached;
+            const verdict = licenseVerdict(license, policy, excluded);
+            verdicts.set(key, verdict);
+            if (verdict.refusal) {
+                refused.set(key, verdict.refusal);
+            }
+            return verdict;
+        },
+        refusedValues() {
+            return new Map(refused);
+        },
+    };
+}
 //# sourceMappingURL=license.js.map

package/out/utils/license.js.map

@@ -1 +1 @@
-{"version":3,"file":"license.js","sourceRoot":"","sources":["../../lib/utils/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IAEvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
+{"version":3,"file":"license.js","sourceRoot":"","sources":["../../lib/utils/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AACzG,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAA;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAA;AAE7D;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG;IAC5B;;;OAGG;IACH,GAAG,EAAE,KAAK;IACV;;;;;;;OAOG;IACH,cAAc,EAAE,kBAAkB;IAClC;;;;;;OAMG;IACH,YAAY,EAAE,eAAe;CACpB,CAAA;AAOV;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG;IACjC;;OAEG;IACH,gBAAgB,EAAE,mBAAmB;IACrC;;OAEG;IACH,iBAAiB,EAAE,qBAAqB;IACxC;;OAEG;IACH,mBAAmB,EAAE,uBAAuB;IAC5C;;OAEG;IACH,UAAU,EAAE,YAAY;CACf,CAAA;AA2BV;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,gBAAgB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IAEvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC7B,OAA2B,EAC3B,MAAqB,EACrB,QAAQ,GAAsB,EAAE;IAEhC,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,KAAK,IAAI,CAAA;IAC3C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAE3C,MAAM,OAAO,GAAG,CAAC,OAAkC,EAAkB,EAAE,CAAC,CAAC;QACxE,OAAO;QACP,QAAQ;QACR,kBAAkB,EAAE,QAAQ,IAAI,iBAAiB,CAAC,MAAM,CAAC;KACzD,CAAC,CAAA;IAEF,IAAI,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC;QAAE,OAAO,OAAO,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAA;IAE3F,IAAI,MAAM,KAAK,aAAa,CAAC,GAAG;QAAE,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;IAEtD,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,OAAO,CAAC,kBAAkB,CAAC,iBAAiB,CAAC,CAAA;IAEnF,IAAI,QAAQ;QAAE,OAAO,OAAO,CAAC,kBAAkB,CAAC,mBAAmB,CAAC,CAAA;IAEpE,IAAI,CAAC,QAAQ,IAAI,MAAM,KAAK,aAAa,CAAC,YAAY;QAAE,OAAO,OAAO,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;IAErG,OAAO,OAAO,CAAC,IAAI,CAAC,CAAA;AACrB,CAAC;AAiBD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAA0C;IAC5E,MAAM,QAAQ,GAAwB,EAAE,CAAA;IAExC,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxD,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,CAAA;QAEzE,IAAI,OAAO,EAAE,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;QAChD,CAAC;IACF,CAAC;IAED,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAA;AACpD,CAAC;AAiBD;;;;;GAKG;AACH,MAAM,UAAU,yBAAyB,CACxC,MAAqB,EACrB,QAAQ,GAAsB,EAAE;IAEhC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAA;IAClD,MAAM,OAAO,GAAG,IAAI,GAAG,EAA8B,CAAA;IAErD,OAAO;QACN,IAAI,CAAC,OAAO;YACX,MAAM,GAAG,GAAG,OAAO,IAAI,EAAE,CAAA;YACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEhC,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAA;YAEzB,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAA;YAEzD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;YAE1B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;YAClC,CAAC;YAED,OAAO,OAAO,CAAA;QACf,CAAC;QACD,aAAa;YACZ,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAA;QACxB,CAAC;KACD,CAAA;AACF,CAAC"}

package/out/utils/split.d.ts

@@ -3,55 +3,64 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ * Splits corpus rows into train, val and test by holding out whole places: a random row split leaks by
+ * neighborhood, because the model can memorize a street it saw in train. Rows inside a held-out place
+ * go to val or test 50/50 by a hash of `source_id`; every other row goes to train.
  *
- *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
- *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
- *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
- *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
- *   memorization, not by learning the underlying schema.
- *
- *   Phase 1 holdouts (chosen for low data density + administrative isolation):
- *
- *   - **US**: Vermont, Wyoming, North Dakota
- *   - **FR**: Corse, Lozère, Creuse
- *
- *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
- *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
- *   the locality boundary, not the exact split percentages.
- *
- *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
- *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ * A holdout change takes effect at the next corpus rebuild. Each versioned corpus keeps the
+ * `SPLIT_MANIFEST.json` it was built with, because adding a holdout to a built corpus would leak rows
+ * the model already trained on.
  */
 import { type PathBuilderLike } from "path-ts";
 import type { CanonicalRow, LabeledRow } from "#types";
+/**
+ * The name of one corpus split.
+ */
 export type SplitName = "train" | "val" | "test";
+/**
+ * The component values that identify the places one country holds out.
+ *
+ * Sources represent a place in different components: `usgov-nad` and `tiger` emit `region`,
+ * but BAN emits none, so a French holdout must also match on postcode.
+ * A row is held out when any declared matcher fires.
+ * A policy with no matchers holds out no entry.
+ */
+export interface HoldoutPolicy {
+    regions?: readonly string[];
+    /**
+     * Prefixes matched against `row.components.postcode`; use one only where it identifies
+     * a single place by itself, such as the first two digits of a French postcode.
+     */
+    postcodePrefixes?: readonly string[];
+    localities?: readonly string[];
+}
+/**
+ * One country's holdout as a bare region list or a {@link HoldoutPolicy}.
+ *
+ * Older committed `SPLIT_MANIFEST.json` files use a bare array of region values.
+ */
+export type CountryHoldout = readonly string[] | HoldoutPolicy;
+/**
+ * Options for {@link splitRows}.
+ */
 export interface SplitOptions {
     /**
-     * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
-     * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
-     * `defaultHoldouts()`.
+     * The holdout policy keyed by ISO 3166-1 alpha-2 country code.
      */
-    holdouts?: Record<string, readonly string[]>;
+    holdouts?: Record<string, CountryHoldout>;
 }
 /**
- * Output manifest: source_id lists per split.
+ * The `source_id` lists for each split.
  */
 export interface SplitManifest {
     train: string[];
     val: string[];
     test: string[];
+    holdouts: Record<string, CountryHoldout>;
     /**
-     * Echoes the holdouts used, so the manifest is self-describing.
-     */
-    holdouts: Record<string, readonly string[]>;
-    /**
-     * Corpus version stamped onto the manifest. Read from the first row.
+     * The corpus version, read from the first row that has one.
      */
     corpus_version: string;
-    /**
-     * Counts for quick sanity checks.
-     */
     counts: {
         train: number;
         val: number;
@@ -60,69 +69,56 @@ export interface SplitManifest {
     };
 }
 /**
- * Phase 1 default holdouts (per plan).
- *
- * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
- * - FR: Corse, Lozère, Creuse (small departments / regions).
- * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
- *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
- *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
- *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ * Returns the default holdouts — small, peripheral places in each country.
+ * A change here takes effect at the next base corpus rebuild.
  */
-export declare function defaultHoldouts(): Record<string, readonly string[]>;
-type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">;
+export declare function defaultHoldouts(): Record<string, CountryHoldout>;
 /**
- * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
- * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
- * array-based path so the decision is stable regardless of caller.
+ * Returns the matchers for one country, reading a bare array as a region list.
  */
-export declare function splitForRow(row: Pick<SplitInputRow, "source_id" | "country" | "components">, holdouts?: Record<string, readonly string[]>): SplitName;
+export declare function holdoutPolicyFor(holdout: CountryHoldout | undefined): HoldoutPolicy;
+type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">;
 /**
- * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
- * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ * Returns the split for one row.
  *
- * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
- * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
- * split membership in heap.
+ * Both `splitRows` and the streaming `buildCorpus` loop call this, so every
+ * caller assigns a row to the same split.
  */
-export declare function splitRows(rows: Iterable<SplitInputRow>, opts?: SplitOptions): SplitManifest;
+export declare function splitForRow(row: Pick<SplitInputRow, "source_id" | "country" | "components">, holdouts?: Record<string, CountryHoldout>): SplitName;
 /**
- * Lightweight deterministic 0..(n-1) bucket from a string id.
+ * Builds a `SplitManifest` in memory from labeled or canonical rows.
+ *
+ * Tests and small fixtures use this, while `buildCorpus` uses `splitForRow`
+ * and `writeSplitManifestsFromLabeledFiles` so it never holds every row's split in memory.
  */
+export declare function splitRows(rows: Iterable<SplitInputRow>, opts?: SplitOptions): SplitManifest;
 /**
- * Deterministic bucket for a stable id.
- *
- * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
- * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
- * line it saves.
+ * Returns a deterministic bucket in `0..n-1` for an id, from the first four bytes
+ * of the SHA-256 digest read as a big-endian uint32.
  */
 export declare function hashBucket(id: string, n: number): number;
 /**
- * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
- * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
- * including holdouts + counts + corpus version.
- *
- * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ * Writes a `SplitManifest` to `<outputDir>/{train,val,test}.txt` (sorted `source_id` values, one per line)
+ * and `SPLIT_MANIFEST.json` (the corpus version, holdouts and counts).
  */
 export declare function writeSplitManifests(manifest: SplitManifest, outputDir: PathBuilderLike): Promise<void>;
 /**
- * Type re-export for callers that want to ingest LabeledRow specifically.
+ * The `LabeledRow` fields that the split functions read.
  */
 export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">;
 /**
- * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
- * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
- * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ * Writes the same files as `writeSplitManifests` by streaming one labeled JSONL file per split.
  *
- * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
- * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ * `buildCorpus` calls this after its align loop partitions rows with `splitForRow`.
+ * The caller passes the counts to avoid rescanning the files.
+ * `sort(1)` spills to disk, so memory stays constant.
  */
 export declare function writeSplitManifestsFromLabeledFiles(opts: {
-    labeledPaths: Record<SplitName, string>;
+    labeledPaths: Record<SplitName, PathBuilderLike>;
     outputDir: PathBuilderLike;
     corpusVersion: string;
     counts: Record<SplitName, number>;
-    holdouts?: Record<string, readonly string[]>;
+    holdouts?: Record<string, CountryHoldout>;
 }): Promise<SplitManifest["counts"]>;
 export {};
 //# sourceMappingURL=split.d.ts.map

package/out/utils/split.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAGpD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAEtD,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;AAEhD,MAAM,WAAW,YAAY;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,EAAE,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAC3C;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACnE;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAMnE;AAED,KAAK,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAElG;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC,EAChE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAqB,GAC7D,SAAS,CASX;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,GAAE,YAAiB,GAAG,aAAa,CAiC/F;AAED;;GAEG;AACH;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAMxD;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB5G;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAE9G;;;;;;;GAOG;AACH,wBAAsB,mCAAmC,CAAC,IAAI,EAAE;IAC/D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACvC,SAAS,EAAE,eAAe,CAAA;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAqBnC"}
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../lib/utils/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,SAAS,CAAA;AAGlE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAEtD;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;AAEhD;;;;;;;GAOG;AACH,MAAM,WAAW,aAAa;IAC7B,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACpC,UAAU,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC9B;AAED;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,SAAS,MAAM,EAAE,GAAG,aAAa,CAAA;AAE9D;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IACxC;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACnE;AAED;;;GAGG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAoHhE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,cAAc,GAAG,SAAS,GAAG,aAAa,CAInF;AAED,KAAK,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAElG;;;;;GAKG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC,EAChE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAqB,GAC1D,SAAS,CAYX;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,GAAE,YAAiB,GAAG,aAAa,CAiC/F;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAKxD;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB5G;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAE9G;;;;;;GAMG;AACH,wBAAsB,mCAAmC,CAAC,IAAI,EAAE;IAC/D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAChD,SAAS,EAAE,eAAe,CAAA;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;CACzC,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAqBnC"}