@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/overlay/manifest.js

@@ -0,0 +1,260 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assembles a corpus manifest that keeps every file of a base corpus and adds new parquet files.
+ */
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { readLocalBuffer, readLocalJSONFile, statPath } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { sha256Hex } from "@mailwoman/core/hash";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { basename, dirname, PathBuilder } from "path-ts";
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb";
+/**
+ * The legacy manifest keys for the file list and the rows per file.
+ *
+ * Earlier corpora are immutable and still use these keys, so readers accept both
+ * spellings and writers emit only the current ones.
+ * The keys are built by concatenation because a repository check bans the legacy word.
+ */
+const PRE_RENAME_FILES_KEY = `sh${"ards"}`;
+const PRE_RENAME_ROWS_PER_FILE_KEY = `rows_per_sh${"ard"}`;
+/**
+ * Returns a manifest's file list from `slices` or from the legacy key.
+ *
+ * A non-empty `slices` wins, then a legacy list, then an empty `slices`.
+ * An empty list is a valid answer, but a manifest with neither key throws, because an
+ * empty result would misreport a missing list as a corpus without files.
+ */
+export function baseManifestFiles(manifest) {
+    const slices = manifest.slices;
+    const preRename = manifest[PRE_RENAME_FILES_KEY];
+    if (Array.isArray(slices) && slices.length)
+        return slices;
+    if (Array.isArray(preRename))
+        return preRename;
+    if (Array.isArray(slices))
+        return slices;
+    throw new Error(`manifest names no file list: neither "slices" nor the pre-rename key holds an array. An empty list here ` +
+        `would read as a corpus of no files, which is a measurement this manifest does not support.`);
+}
+/**
+ * Returns the base's rows-per-file value under either key.
+ */
+function baseRowsPerFile(base) {
+    return base.rows_per_slice ?? base[PRE_RENAME_ROWS_PER_FILE_KEY];
+}
+/**
+ * Returns the split encoded in a `<stem>.<split>.parquet` filename, or `null` for any other name.
+ *
+ * Only `splitOverlaySlice` writes names with this suffix.
+ */
+export function splitFromFilename(parquet) {
+    const match = /\.(train|val|test)\.parquet$/.exec(parquet);
+    return match ? match[1] : null;
+}
+/**
+ * Reads one overlay parquet and describes it for the manifest.
+ *
+ * The loader groups rows by `file_source_counts`.
+ * The manifest records the `source` value supplied by the caller.
+ *
+ * A stale argument therefore writes a manifest that disagrees with the rows.
+ * The run trains on the rows.
+ *
+ * The next plan then repeats the stale label from the manifest.
+ *
+ * This reads the column and refuses the mismatch rather than recording the claim.
+ *
+ * `source_id` is materialized rather than aggregated because `first_source_id`
+ * and `last_source_id` mean the first and last row of the file.
+ * DuckDB's `first` and `last` aggregates read whichever row a parallel scan reaches first.
+ *
+ * That result can differ from the first and last rows in the file.
+ */
+async function descriptor(localPath, modalPath, split, source) {
+    // `assembleOverlayManifest` calls this once for each file.
+    using db = await openDuckDB();
+    const result = await db.runAndReadAll(`SELECT source_id, source FROM read_parquet('${escapeSQLString(localPath)}')`);
+    const rows = result.getRowObjects();
+    const sids = rows.map((r) => r.source_id);
+    const stored = new Set(rows.map((r) => r.source));
+    if (stored.size !== 1 || !stored.has(source)) {
+        const found = [...stored].toSorted().join(", ");
+        throw new Error(`${localPath} holds source ${stored.size === 1 ? `"${found}"` : `${stored.size} values (${found})`}, ` +
+            `and the manifest was asked to record "${source}". The loader groups rows by the column, so a ` +
+            `manifest that disagrees with it is a label nothing reads and a plan derived from it carries forward.`);
+    }
+    return {
+        split,
+        path: modalPath,
+        format: "parquet",
+        compression: "SNAPPY",
+        rows: sids.length,
+        bytes: (await statPath(localPath)).size,
+        sha256: sha256Hex(await readLocalBuffer(localPath)),
+        first_source_id: sids[0],
+        last_source_id: sids.at(-1),
+        source,
+    };
+}
+/**
+ * Options for {@link assembleOverlayManifest}.
+ */
+/**
+ * The filename {@linkcode assembleOverlayManifest} writes its plan under, beside `MANIFEST.json`.
+ */
+export const OVERLAY_PLAN_FILE = "OVERLAY_PLAN.json";
+/**
+ * How one overlay file's split was decided.
+ */
+export const SplitRoute = {
+    /**
+     * `corpus split-slice` applied `splitForRow` to every row and wrote the split into the filename.
+     */
+    SplitSlice: "split-slice",
+    /**
+     * The caller supplied the split.
+     * The holdout policy never saw these rows.
+     */
+    Caller: "caller",
+};
+/**
+ * Which route decided one file's split.
+ */
+export function splitRouteFor(parquet) {
+    return splitFromFilename(parquet) ? SplitRoute.SplitSlice : SplitRoute.Caller;
+}
+/**
+ * Rewrites a base manifest's file path to its location in the corpus tree mounted on Modal at `/data`.
+ */
+export function rerootBaseFilePath(path, baseManifestPath) {
+    const versionedIndex = path.indexOf("/corpus/versioned/");
+    if (versionedIndex !== -1)
+        return "/data" + path.slice(versionedIndex);
+    if (/^\/data\/(?:train|val|test)\//u.test(path)) {
+        const localBaseDir = dirname(baseManifestPath);
+        const baseModalRoot = `/data/corpus/versioned/${basename(dirname(localBaseDir))}/${basename(localBaseDir)}`;
+        return baseModalRoot + path.slice("/data".length);
+    }
+    return path;
+}
+/**
+ * Maps a Modal path under `/data/` to the same file under the local data root
+ * and returns any other path unchanged.
+ */
+export function localManifestFilePath(path) {
+    return path.startsWith("/data/") ? dataRootPath(path.slice("/data/".length)).toString() : path;
+}
+/**
+ * Writes a corpus manifest that keeps every file of `args.base` and appends `args.files`.
+ *
+ * A `val` or `test` file must use the `.<split>.parquet` suffix written by `splitOverlaySlice`.
+ * The suffix shows that the holdout policy chose its rows.
+ * A `train` file has no naming requirement.
+ *
+ * @throws When the base lists no files or a held-out file lacks the matching filename suffix.
+ */
+export async function assembleOverlayManifest(args) {
+    if (!args.files.length)
+        throw new Error("an overlay must add at least one parquet file");
+    const base = await readLocalJSONFile(args.base);
+    const baseFiles = baseManifestFiles(base);
+    // An overlay on a base without files would contain only the added files.
+    if (!baseFiles.length) {
+        throw new Error(`${args.base} lists no files, so an overlay on it would carry only the ${args.files.length} file(s) added ` +
+            `here while naming itself an extension of that base.`);
+    }
+    for (const file of args.files) {
+        if (baseFiles.some((s) => s.source === file.source)) {
+            console.log(`WARN: base already contains source '${file.source}' — is this the right base?`);
+        }
+    }
+    // Held-out rows must come from the holdout policy.
+    // The filename records the split.
+    // A caller that picks held-out rows by hand can leak them into train.
+    for (const file of args.files) {
+        if (!file.split || file.split === "train")
+            continue;
+        const named = splitFromFilename(file.parquet);
+        if (named === file.split)
+            continue;
+        throw new Error(named
+            ? `${file.parquet} names split '${named}' and the caller asked for '${file.split}'. The filename is ` +
+                `what splitOverlaySlice wrote, so it decides; pass the split the name carries or route the file again.`
+            : `${file.parquet} was asked for split '${file.split}' and carries no '.${file.split}.parquet' suffix, ` +
+                `so it did not come from 'mailwoman corpus split-slice' and its held-out rows were chosen by the ` +
+                `caller rather than by the holdout policy. Route it through split-slice and add the per-split ` +
+                `outputs it writes. A file appended to 'train' needs none of this.`);
+    }
+    const kept = baseFiles.map((s) => ({ ...s, path: rerootBaseFilePath(s.path, args.base) }));
+    const added = [];
+    const newDir = PathBuilder.from(args.newDir);
+    for (const file of args.files) {
+        const split = file.split ?? "train";
+        added.push(await descriptor(
+        // DuckDB reads the path inside SQL text, so it must be a string.
+        newDir(split, file.parquet).toString(), `${args.modalRoot}/${split}/${file.parquet}`, split, file.source));
+    }
+    const addedRows = { train: 0, val: 0, test: 0 };
+    for (const file of added) {
+        addedRows[file.split] += file.rows;
+    }
+    const sources = args.files.map((file) => file.source).join(", ");
+    const manifest = {
+        corpus_version: args.version,
+        overlay_base: base.corpus_version ?? null,
+        note: args.note || `${base.corpus_version} files (all kept verbatim) + ${sources}. Pure overlay add.`,
+        schema: base.schema,
+        rows_per_slice: baseRowsPerFile(base),
+        row_group_size: base.row_group_size,
+        slices: [...kept, ...added],
+        counts: {
+            train: base.counts.train + addedRows.train,
+            val: base.counts.val + addedRows.val,
+            test: base.counts.test + addedRows.test,
+        },
+        total_rows: base.total_rows + addedRows.train + addedRows.val + addedRows.test,
+        // An overlay corpus's manifest reads no row's `license` column, so it states no license set.
+        // This value in the artifact prevents a reader from deriving an attribution table
+        // from `slices` and treating the base build's set as the whole.
+        // The overlay sources listed here have their own terms, recorded per source in
+        // `packages/corpus/lib/recipes/sources.ts` and per row in the `license` column itself.
+        licenses_cover: `no license set is measured here. The base build's MANIFEST.json covers the rows it aligned, and ` +
+            `these ${added.length} added file(s) carry their own: ${[...new Set(args.files.map((file) => file.source))].toSorted().join(", ")}.`,
+    };
+    const out = newDir("MANIFEST.json");
+    await writeLocalJSONFile(manifest, out);
+    // The plan is written beside the manifest it produced, because the previous overlay
+    // assembly's plan existed only as three scripts in a session scratch directory
+    // and was copied into the next assembly as a hand-written list.
+    // Commit this file under `packages/corpus/data/builds/<version>/`.
+    const plan = {
+        corpusVersion: args.version,
+        overlayBase: base.corpus_version ?? null,
+        baseManifest: args.base,
+        modalRoot: args.modalRoot,
+        note: manifest.note,
+        appliedAt: new Date().toISOString(),
+        files: args.files.map((file) => ({
+            parquet: file.parquet,
+            source: file.source,
+            split: file.split ?? "train",
+            route: splitRouteFor(file.parquet),
+        })),
+    };
+    const planPath = newDir(OVERLAY_PLAN_FILE);
+    await writeLocalJSONFile(plan, planPath);
+    console.log(`wrote ${planPath}`);
+    console.log(`wrote ${out}`);
+    console.log(`  files: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`);
+    console.log(`  counts: ${stringifyJSON(manifest.counts)}  total: ${manifest.total_rows}`);
+    // The route prints beside the split, because a hand-placed split is otherwise visible only in a filename.
+    for (const [index, file] of added.entries()) {
+        const route = plan.files[index]?.route ?? SplitRoute.Caller;
+        console.log(`  ${file.source} ${file.split} via ${route}: ${file.rows} rows (${file.bytes} bytes)`);
+    }
+}
+//# sourceMappingURL=manifest.js.map

package/out/tools/overlay/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../lib/tools/overlay/manifest.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE9E,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAkB7D;;;;;;GAMG;AACH,MAAM,oBAAoB,GAAG,KAAK,MAAM,EAAW,CAAA;AACnD,MAAM,4BAA4B,GAAG,cAAc,KAAK,EAAW,CAAA;AAcnE;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgE;IACjG,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC9B,MAAM,SAAS,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAAA;IAEhD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM;QAAE,OAAO,MAAwB,CAAA;IAE3E,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,OAAO,SAA2B,CAAA;IAEhE,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAwB,CAAA;IAE1D,MAAM,IAAI,KAAK,CACd,0GAA0G;QACzG,4FAA4F,CAC7F,CAAA;AACF,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,IAAkB;IAC1C,OAAO,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,4BAA4B,CAAC,CAAA;AACjE,CAAC;AAkBD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAChD,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAE1D,OAAO,KAAK,CAAC,CAAC,CAAE,KAAK,CAAC,CAAC,CAAe,CAAC,CAAC,CAAC,IAAI,CAAA;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAgB,EAChB,MAAc;IAEd,2DAA2D;IAC3D,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,+CAA+C,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAEpH,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAA;IACnC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAgB,CAAC,CAAC,CAAA;IAE3D,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE/C,MAAM,IAAI,KAAK,CACd,GAAG,SAAS,iBAAiB,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,YAAY,KAAK,GAAG,IAAI;YACrG,yCAAyC,MAAM,gDAAgD;YAC/F,sGAAsG,CACvG,CAAA;IACF,CAAC;IAED,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAED;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,mBAAmB,CAAA;AA6BpD;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG;IACzB;;OAEG;IACH,UAAU,EAAE,aAAa;IAEzB;;;OAGG;IACH,MAAM,EAAE,QAAQ;CACP,CAAA;AAIV;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC5C,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAA;AAC9E,CAAC;AAcD;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,gBAAwB;IACxE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IACjD,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;AAC/F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAExF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAA;IAEzC,yEAAyE;IACzE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACd,GAAG,IAAI,CAAC,IAAI,6DAA6D,IAAI,CAAC,KAAK,CAAC,MAAM,iBAAiB;YAC1G,qDAAqD,CACtD,CAAA;IACF,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,uCAAuC,IAAI,CAAC,MAAM,6BAA6B,CAAC,CAAA;QAC7F,CAAC;IACF,CAAC;IAED,mDAAmD;IACnD,kCAAkC;IAClC,sEAAsE;IACtE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO;YAAE,SAAQ;QAEnD,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAE7C,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK;YAAE,SAAQ;QAElC,MAAM,IAAI,KAAK,CACd,KAAK;YACJ,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,iBAAiB,KAAK,+BAA+B,IAAI,CAAC,KAAK,qBAAqB;gBACnG,uGAAuG;YACzG,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,yBAAyB,IAAI,CAAC,KAAK,sBAAsB,IAAI,CAAC,KAAK,oBAAoB;gBACtG,kGAAkG;gBAClG,+FAA+F;gBAC/F,mEAAmE,CACtE,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAC1F,MAAM,KAAK,GAA4B,EAAE,CAAA;IAEzC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAE5C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAA;QAEnC,KAAK,CAAC,IAAI,CACT,MAAM,UAAU;QACf,iEAAiE;QACjE,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EACtC,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,EAC5C,KAAK,EACL,IAAI,CAAC,MAAM,CACX,CACD,CAAA;IACF,CAAC;IAED,MAAM,SAAS,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IAE1E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,CAAA;IACnC,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,gCAAgC,OAAO,qBAAqB;QACrG,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,eAAe,CAAC,IAAI,CAAC;QACrC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK;YAC1C,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG;YACpC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI;SACvC;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC,KAAK,GAAG,SAAS,CAAC,GAAG,GAAG,SAAS,CAAC,IAAI;QAC9E,6FAA6F;QAC7F,kFAAkF;QAClF,gEAAgE;QAChE,+EAA+E;QAC/E,uFAAuF;QACvF,cAAc,EACb,kGAAkG;YAClG,SAAS,KAAK,CAAC,MAAM,mCAAmC,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;KACrI,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,eAAe,CAAC,CAAA;IACnC,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,oFAAoF;IACpF,+EAA+E;IAC/E,gEAAgE;IAChE,mEAAmE;IACnE,MAAM,IAAI,GAAgB;QACzB,aAAa,EAAE,IAAI,CAAC,OAAO;QAC3B,WAAW,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACxC,YAAY,EAAE,IAAI,CAAC,IAAI;QACvB,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAChC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,OAAO;YAC5B,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC;SAClC,CAAC,CAAC;KACH,CAAA;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,iBAAiB,CAAC,CAAA;IAE1C,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAExC,OAAO,CAAC,GAAG,CAAC,SAAS,QAAQ,EAAE,CAAC,CAAA;IAChC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAA;IACpG,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAEzF,0GAA0G;IAC1G,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,UAAU,CAAC,MAAM,CAAA;QAE3D,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,QAAQ,KAAK,KAAK,IAAI,CAAC,IAAI,UAAU,IAAI,CAAC,KAAK,SAAS,CAAC,CAAA;IACpG,CAAC;AACF,CAAC"}

package/out/tools/overlay/merge-source.d.ts

@@ -0,0 +1,73 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Merges one source's overlay parquet files into shuffled output files.
+ *
+ *   The training sampler can read a whole per-source draw from one row group,
+ *   so a source split into per-country files would reach only one country per draw.
+ *   A shuffle across the inputs mixes each source's countries within each row group.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * Options for {@link mergeSourceFiles}.
+ */
+export interface MergeSourceOptions {
+    /**
+     * The parquet files to merge, read in the order given.
+     */
+    inputs: readonly PathBuilderLike[];
+    /**
+     * The output path.
+     * Each written file receives a five-digit index in its stem.
+     */
+    output: PathBuilderLike;
+    /**
+     * The number of rows held in memory while shuffling.
+     * It defaults to the corpus writer's window.
+     */
+    windowSize?: number;
+    /**
+     * The shuffle seed.
+     *
+     * The default matches the corpus writer's seed.
+     * This makes rebuilds reproducible.
+     */
+    seed?: number;
+    /**
+     * The maximum rows per output file.
+     *
+     * `writeParquetFile` builds one Arrow table per file.
+     * Arrow's list builder overflows on very large tables.
+     */
+    rowsPerFile?: number;
+}
+/**
+ * The files that {@link mergeSourceFiles} wrote and the row counts they hold.
+ */
+export interface MergeSourceResult {
+    inputs: readonly string[];
+    /**
+     * The files written, in order.
+     */
+    outputs: readonly string[];
+    rows: number;
+    /**
+     * Row counts per `country` value.
+     */
+    byCountry: Record<string, number>;
+    /**
+     * Row counts per `source` value.
+     * A successful merge has one entry.
+     */
+    bySource: Record<string, number>;
+}
+/**
+ * Merges the inputs through a windowed shuffle into one or more parquet files and reports their contents.
+ *
+ * @throws When the inputs contain more than one `source`, because a manifest
+ * entry records one source per file.
+ * The check runs after the stream ends, so files flushed before it remain on disk.
+ */
+export declare function mergeSourceFiles(options: MergeSourceOptions): Promise<MergeSourceResult>;
+//# sourceMappingURL=merge-source.d.ts.map

package/out/tools/overlay/merge-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merge-source.d.ts","sourceRoot":"","sources":["../../../lib/tools/overlay/merge-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAO9C;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,MAAM,EAAE,SAAS,eAAe,EAAE,CAAA;IAClC;;;OAGG;IACH,MAAM,EAAE,eAAe,CAAA;IACvB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;;OAKG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB;;OAEG;IACH,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACjC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAChC;AAQD;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkD9F"}

package/out/tools/overlay/merge-source.js

@@ -0,0 +1,63 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Merges one source's overlay parquet files into shuffled output files.
+ *
+ *   The training sampler can read a whole per-source draw from one row group,
+ *   so a source split into per-country files would reach only one country per draw.
+ *   A shuffle across the inputs mixes each source's countries within each row group.
+ */
+import { mulberry32 } from "@mailwoman/core/utils";
+import { ROWS_PER_FILE } from "#parquet/schema";
+import { DEFAULT_SHUFFLE_SEED, DEFAULT_SHUFFLE_WINDOW, shuffleWithinWindow } from "#parquet/shuffle";
+import { openParquetRowStream } from "#parquet/streams";
+import { writeParquetFile } from "#parquet/writers";
+async function* readAll(inputs) {
+    for (const input of inputs) {
+        yield* openParquetRowStream(input);
+    }
+}
+/**
+ * Merges the inputs through a windowed shuffle into one or more parquet files and reports their contents.
+ *
+ * @throws When the inputs contain more than one `source`, because a manifest
+ * entry records one source per file.
+ * The check runs after the stream ends, so files flushed before it remain on disk.
+ */
+export async function mergeSourceFiles(options) {
+    const inputs = options.inputs.map((input) => input.toString());
+    const random = mulberry32(options.seed ?? DEFAULT_SHUFFLE_SEED);
+    const rowsPerFile = options.rowsPerFile ?? ROWS_PER_FILE;
+    const stem = options.output.toString().replace(/\.parquet$/u, "");
+    const byCountry = {};
+    const bySource = {};
+    const outputs = [];
+    let pending = [];
+    let rows = 0;
+    const flush = async () => {
+        if (!pending.length)
+            return;
+        const output = `${stem}-${String(outputs.length).padStart(5, "0")}.parquet`;
+        await writeParquetFile(pending, output);
+        outputs.push(output);
+        pending = [];
+    };
+    for await (const row of shuffleWithinWindow(readAll(options.inputs), random, options.windowSize ?? DEFAULT_SHUFFLE_WINDOW)) {
+        pending.push(row);
+        rows++;
+        byCountry[row.country] = (byCountry[row.country] ?? 0) + 1;
+        bySource[row.source] = (bySource[row.source] ?? 0) + 1;
+        if (pending.length >= rowsPerFile) {
+            await flush();
+        }
+    }
+    const sources = Object.keys(bySource);
+    if (sources.length > 1) {
+        throw new Error(`the inputs carry ${sources.length} sources (${sources.join(", ")}), and a merged file's manifest entry names one. ` +
+            `Merge the files of one source at a time.`);
+    }
+    await flush();
+    return { inputs, outputs, rows, byCountry, bySource };
+}
+//# sourceMappingURL=merge-source.js.map

package/out/tools/overlay/merge-source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"merge-source.js","sourceRoot":"","sources":["../../../lib/tools/overlay/merge-source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAmB,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAChE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACpG,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAyDnD,KAAK,SAAS,CAAC,CAAC,OAAO,CAAC,MAAkC;IACzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,CAAC,CAAC,oBAAoB,CAAa,KAAK,CAAC,CAAA;IAC/C,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAA2B;IACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAA;IAC9D,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,oBAAoB,CAAC,CAAA;IAC/D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,aAAa,CAAA;IACxD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAA;IAEjE,MAAM,SAAS,GAA2B,EAAE,CAAA;IAC5C,MAAM,QAAQ,GAA2B,EAAE,CAAA;IAC3C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,IAAI,OAAO,GAAiB,EAAE,CAAA;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,OAAM;QAE3B,MAAM,MAAM,GAAG,GAAG,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAA;QAE3E,MAAM,gBAAgB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACvC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACpB,OAAO,GAAG,EAAE,CAAA;IACb,CAAC,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,mBAAmB,CAC1C,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EACvB,MAAM,EACN,OAAO,CAAC,UAAU,IAAI,sBAAsB,CAC5C,EAAE,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEjB,IAAI,EAAE,CAAA;QACN,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAC1D,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;YACnC,MAAM,KAAK,EAAE,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAErC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACd,oBAAoB,OAAO,CAAC,MAAM,aAAa,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD;YACnH,0CAA0C,CAC3C,CAAA;IACF,CAAC;IAED,MAAM,KAAK,EAAE,CAAA;IAEb,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAA;AACtD,CAAC"}

package/out/tools/overlay/split-slice.d.ts

@@ -0,0 +1,66 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Route an already-written overlay parquet through the holdout policy.
+ *
+ * A parquet row has no `components` map, so each component is read back from `raw` over the `span_starts` /
+ * `span_ends` / `span_tags` triple that `alignRow` wrote.
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { CanonicalRow } from "#types";
+import { type CountryHoldout, type SplitName } from "#utils/split";
+/**
+ * The fields {@linkcode holdoutComponents} reads, with the span triple optional
+ * so the absent-triple refusal is reachable from a test rather than only from a corrupt file.
+ */
+export interface SpannedRow {
+    raw: string;
+    source_id: string;
+    span_starts?: readonly number[];
+    span_ends?: readonly number[];
+    span_tags?: readonly string[];
+}
+/**
+ * Read the components a holdout decision needs back out of one labeled row.
+ *
+ * Raises on an absent or non-parallel span triple, because treating an unreadable row
+ * as one with no held-out component would put it in the train split.
+ */
+export declare function holdoutComponents(row: SpannedRow, index: number, input: string): CanonicalRow["components"];
+export interface SplitSliceOptions {
+    /**
+     * The parquet to route.
+     * Its rows are read once and held per split in memory.
+     */
+    input: PathBuilderLike;
+    /**
+     * Where the per-split parquets are written.
+     * Each filename combines the input and its split.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Defaults to `defaultHoldouts()`, which is the policy the base build applies.
+     */
+    holdouts?: Record<string, CountryHoldout>;
+}
+export interface SplitSliceResult {
+    input: string;
+    rows: number;
+    counts: Record<SplitName, number>;
+    /**
+     * The file written per split.
+     *
+     * A split with zero rows has no output file.
+     * An empty parquet would suggest that the writer lost its rows.
+     */
+    outputs: Partial<Record<SplitName, string>>;
+}
+/**
+ * Split one overlay parquet into up to three, by the same policy the base build applies.
+ *
+ * Each output filename combines the input name and split name.
+ * The function leaves the input file in place.
+ */
+export declare function splitOverlaySlice(options: SplitSliceOptions): Promise<SplitSliceResult>;
+//# sourceMappingURL=split-slice.d.ts.map

package/out/tools/overlay/split-slice.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split-slice.d.ts","sourceRoot":"","sources":["../../../lib/tools/overlay/split-slice.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAY,KAAK,eAAe,EAAe,MAAM,SAAS,CAAA;AAKrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AAC1C,OAAO,EAAE,KAAK,cAAc,EAAmB,KAAK,SAAS,EAAe,MAAM,cAAc,CAAA;AAUhG;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC/B,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7B,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7B;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,CA0B3G;AAED,MAAM,WAAW,iBAAiB;IACjC;;;OAGG;IACH,KAAK,EAAE,eAAe,CAAA;IACtB;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAA;IAC1B;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;CACzC;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC;;;;;OAKG;IACH,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;CAC3C;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAkC7F"}

package/out/tools/overlay/split-slice.js

@@ -0,0 +1,78 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Route an already-written overlay parquet through the holdout policy.
+ *
+ * A parquet row has no `components` map, so each component is read back from `raw` over the `span_starts` /
+ * `span_ends` / `span_tags` triple that `alignRow` wrote.
+ */
+import { basename, resolvePath } from "path-ts";
+import { openParquetRowStream } from "#parquet/streams";
+import { writeParquetFile } from "#parquet/writers";
+import { defaultHoldouts, splitForRow } from "#utils/split";
+/**
+ * The component tags the holdout policy reads.
+ *
+ * `splitForRow` consults `region`, `postcode` and `locality` and no other field,
+ * so reconstructing the rest would cost a string slice per span for no decision.
+ */
+const HOLDOUT_TAGS = new Set(["region", "postcode", "locality"]);
+/**
+ * Read the components a holdout decision needs back out of one labeled row.
+ *
+ * Raises on an absent or non-parallel span triple, because treating an unreadable row
+ * as one with no held-out component would put it in the train split.
+ */
+export function holdoutComponents(row, index, input) {
+    const { raw, span_starts: starts, span_ends: ends, span_tags: tags } = row;
+    if (!starts || !ends || !tags) {
+        throw new Error(`${input} row ${index} (${row.source_id}) carries no span triple, so its components cannot be read. ` +
+            `Re-emit the file through \`alignRow\` before splitting it.`);
+    }
+    if (starts.length !== ends.length || starts.length !== tags.length) {
+        throw new Error(`${input} row ${index} (${row.source_id}) has a span triple of ${starts.length}/${ends.length}/${tags.length}; ` +
+            `the three are parallel arrays and must agree in length.`);
+    }
+    const components = {};
+    for (const [span, tag] of tags.entries()) {
+        if (!HOLDOUT_TAGS.has(tag) || components[tag] !== undefined)
+            continue;
+        components[tag] = raw.slice(starts[span], ends[span]);
+    }
+    return components;
+}
+/**
+ * Split one overlay parquet into up to three, by the same policy the base build applies.
+ *
+ * Each output filename combines the input name and split name.
+ * The function leaves the input file in place.
+ */
+export async function splitOverlaySlice(options) {
+    const input = options.input.toString();
+    const holdouts = options.holdouts ?? defaultHoldouts();
+    const stem = basename(input).replace(/\.parquet$/u, "");
+    const buckets = { train: [], val: [], test: [] };
+    let index = 0;
+    for await (const row of openParquetRowStream(input)) {
+        const components = holdoutComponents(row, index, input);
+        const split = splitForRow({ source_id: row.source_id, country: row.country, components }, holdouts);
+        buckets[split].push(row);
+        index++;
+    }
+    const outputs = {};
+    for (const split of ["train", "val", "test"]) {
+        if (!buckets[split].length)
+            continue;
+        const output = resolvePath(options.outputDir, `${stem}.${split}.parquet`);
+        await writeParquetFile(buckets[split], output);
+        outputs[split] = output;
+    }
+    return {
+        input,
+        rows: index,
+        counts: { train: buckets.train.length, val: buckets.val.length, test: buckets.test.length },
+        outputs,
+    };
+}
+//# sourceMappingURL=split-slice.js.map

package/out/tools/overlay/split-slice.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split-slice.js","sourceRoot":"","sources":["../../../lib/tools/overlay/split-slice.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,QAAQ,EAAwB,WAAW,EAAE,MAAM,SAAS,CAAA;AAGrE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEnD,OAAO,EAAuB,eAAe,EAAkB,WAAW,EAAE,MAAM,cAAc,CAAA;AAEhG;;;;;GAKG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAA;AAcxE;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAe,EAAE,KAAa,EAAE,KAAa;IAC9E,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,GAAG,CAAA;IAE1E,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,QAAQ,KAAK,KAAK,GAAG,CAAC,SAAS,8DAA8D;YACpG,4DAA4D,CAC7D,CAAA;IACF,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QACpE,MAAM,IAAI,KAAK,CACd,GAAG,KAAK,QAAQ,KAAK,KAAK,GAAG,CAAC,SAAS,0BAA0B,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI;YAC/G,yDAAyD,CAC1D,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAA2B,EAAE,CAAA;IAE7C,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,SAAQ;QAErE,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAE,EAAE,IAAI,CAAC,IAAI,CAAE,CAAC,CAAA;IACxD,CAAC;IAED,OAAO,UAAmD,CAAA;AAC3D,CAAC;AAgCD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,OAA0B;IACjE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAA;IACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACtD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAA;IAEvD,MAAM,OAAO,GAAoC,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;IACjF,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAAa,KAAK,CAAC,EAAE,CAAC;QACjE,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;QACvD,MAAM,KAAK,GAAG,WAAW,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,QAAQ,CAAC,CAAA;QAEnG,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAExB,KAAK,EAAE,CAAA;IACR,CAAC;IAED,MAAM,OAAO,GAAuC,EAAE,CAAA;IAEtD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM;YAAE,SAAQ;QAEpC,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,IAAI,KAAK,UAAU,CAAC,CAAA;QAEzE,MAAM,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,CAAA;QAC9C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;IACxB,CAAC;IAED,OAAO;QACN,KAAK;QACL,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE;QAC3F,OAAO;KACP,CAAA;AACF,CAAC"}

package/out/tools/overlay/translit.d.ts

@@ -0,0 +1,54 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet files from the DeepSeek-generated transliteration jsonl and emit the
+ *   corpus-v0.4.0 manifest that combines them with the existing kryptonite + v0.3.0 files.
+ *
+ *   This module sits beside `kryptonite.ts`. Both modules use the same composition pattern: take a
+ *   base manifest, append new parquet files, write a combined manifest. Differences specific to
+ *   transliteration:
+ *
+ *   - One jsonl contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one parquet file per script so `audit.ts` can attribute each file to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each file is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 file paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined manifest so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for prompts, model and the
+ *   reproducibility interface.
+ *
+ *   Invoke via `mailwoman corpus slice translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/manifest.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { type PathBuilderLike } from "path-ts";
+export interface TranslitOverlayOptions {
+    jsonl: string;
+    baseManifest: string;
+    outDir: PathBuilderLike;
+    /**
+     * Default `"0.4.0"`.
+     */
+    corpusVersion?: string;
+    /**
+     * Default `"/data/"`.
+     */
+    canonicalPathPrefix?: string;
+    /**
+     * Prefix currently used in the base manifest's file paths, to be rewritten
+     * to {@link TranslitOverlayOptions.canonicalPathPrefix}.
+     *
+     * Defaults to `dataRootPath()` with a trailing slash — the root that wrote those paths.
+     * Pass it explicitly when translating a manifest generated under a different
+     * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
+     */
+    legacyPathPrefix?: string;
+}
+export declare function buildTranslitOverlay(options: TranslitOverlayOptions, report?: (line: string) => void): Promise<void>;
+//# sourceMappingURL=translit.d.ts.map

package/out/tools/overlay/translit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"translit.d.ts","sourceRoot":"","sources":["../../../lib/tools/overlay/translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAOH,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAU3D,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,eAAe,CAAA;IACvB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAkED,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CAkGf"}