@mailwoman/corpus 5.9.0 → 5.10.0

package/out/src/tools/shard-kryptonite.js

@@ -0,0 +1,101 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet shard from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
+ *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every shard from
+ *   v0.3.0 plus the new kryptonite shard(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *
+ *   Invoke via `mailwoman corpus shard kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { iterateJSONL } from "@mailwoman/core/utils";
+import { alignRow } from "../align.js";
+import { PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "../parquet.js";
+async function* canonicalRows(jsonl, corpusVersion) {
+    for await (const raw of iterateJSONL(jsonl)) {
+        // Strip sidecar underscore-prefixed fields the generator left behind for debugging.
+        const components = raw["components"];
+        yield {
+            raw: raw["raw"],
+            components,
+            country: raw["country"] ?? "US",
+            locale: raw["locale"] ?? undefined,
+            source: raw["source"] ?? "deepseek-kryptonite",
+            source_id: raw["source_id"],
+            corpus_version: corpusVersion,
+            license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+            synth: raw["synth"],
+        };
+    }
+}
+async function* labeledRows(jsonl, corpusVersion, quarantineLog) {
+    for await (const row of canonicalRows(jsonl, corpusVersion)) {
+        const result = alignRow(row);
+        if (result.kind === "labeled") {
+            yield result.row;
+        }
+        else {
+            quarantineLog.push(`${row.source_id}\t${result.row.reason}`);
+        }
+    }
+}
+export async function buildKryptoniteShard(options, report) {
+    const corpusVersion = options.corpusVersion ?? "0.4.0";
+    const source = options.source ?? "deepseek-kryptonite";
+    if (!existsSync(options.jsonl))
+        throw new Error(`jsonl not found: ${options.jsonl}`);
+    if (!existsSync(options.baseManifest))
+        throw new Error(`base-manifest not found: ${options.baseManifest}`);
+    await mkdir(options.outDir, { recursive: true });
+    const quarantine = [];
+    const newManifest = await writeShards({ train: labeledRows(options.jsonl, corpusVersion, quarantine) }, { outputDir: options.outDir, corpusVersion });
+    report?.(`wrote ${newManifest.total_rows} rows into ${newManifest.shards.length} shard(s); ` +
+        `quarantined ${quarantine.length}`);
+    if (quarantine.length > 0) {
+        const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv");
+        writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8");
+        report?.(`quarantine log → ${qPath}`);
+    }
+    // Stamp the new shard's source field for audit.ts (which prefers shard.source over
+    // first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
+    // to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
+    for (const sh of newManifest.shards) {
+        ;
+        sh.source = source;
+    }
+    // Compose the final corpus-v0.4.0 manifest: every shard from base + the new shard(s).
+    const base = JSON.parse(readFileSync(options.baseManifest, "utf8"));
+    const combined = {
+        corpus_version: corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_shard: base.rows_per_shard,
+        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+        shards: [...base.shards, ...newManifest.shards],
+        counts: {
+            train: base.counts.train + (newManifest.counts.train ?? 0),
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newManifest.total_rows,
+    };
+    // Stamp source on the legacy v0.3.0 shards too, so audit's shard.source path is the
+    // authoritative one. v0.3.0 shards mix sources; we use the first_source_id-prefix
+    // inference for them (audit.ts will re-derive on its own when shard.source is absent).
+    const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json");
+    writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8");
+    report?.(`wrote combined manifest → ${combinedPath}`);
+    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`);
+    report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newManifest.shards.length})`);
+    report?.(`  compression=${SHARD_COMPRESSION}`);
+}
+//# sourceMappingURL=shard-kryptonite.js.map

package/out/src/tools/shard-kryptonite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shard-kryptonite.js","sourceRoot":"","sources":["../../../src/tools/shard-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAa/F,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,YAAY,CAA0B,KAAK,CAAC,EAAE,CAAC;QACtE,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAC9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;SAC5C,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEhD,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,CAC5C,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,aAAa;QAClF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,2BAA2B,CAAC,CAAA;QAC3F,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,mFAAmF;IACnF,sFAAsF;IACtF,yFAAyF;IACzF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACrC,CAAC;QAAC,EAAoC,CAAC,MAAM,GAAG,MAAM,CAAA;IACvD,CAAC;IAED,sFAAsF;IACtF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAkB,CAAA;IACpF,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IACD,oFAAoF;IACpF,kFAAkF;IAClF,uFAAuF;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,eAAe,CAAC,CAAA;IACtF,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;AAC/C,CAAC"}

package/out/src/tools/shard-translit.d.ts

@@ -0,0 +1,42 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet shards from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 shards.
+ *
+ *   Sibling to `shard-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new shards, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one shard per script so `audit.ts` can attribute each shard to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each shard is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 shard paths are
+ *       rewritten from `/mnt/playpen/mailwoman-data/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus shard translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+export interface ShardTranslitOptions {
+    jsonl: string;
+    baseManifest: string;
+    outDir: string;
+    /** Default `"0.4.0"`. */
+    corpusVersion?: string;
+    /** Default `"/data/"`. */
+    canonicalPathPrefix?: string;
+    /** Default `"/mnt/playpen/mailwoman-data/"`. */
+    legacyPathPrefix?: string;
+}
+export declare function buildTranslitShard(options: ShardTranslitOptions, report?: (line: string) => void): Promise<void>;
+//# sourceMappingURL=shard-translit.d.ts.map

package/out/src/tools/shard-translit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shard-translit.d.ts","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAcH,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,yBAAyB;IACzB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,0BAA0B;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAmGD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyFf"}

package/out/src/tools/shard-translit.js

@@ -0,0 +1,196 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet shards from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 shards.
+ *
+ *   Sibling to `shard-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new shards, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one shard per script so `audit.ts` can attribute each shard to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each shard is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 shard paths are
+ *       rewritten from `/mnt/playpen/mailwoman-data/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus shard translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { iterateJSONL, sha256File } from "@mailwoman/core/utils";
+import { alignRow } from "../align.js";
+import { ParquetWriter } from "../parquet-wrapper/index.js";
+import { LABELED_ROW_SCHEMA, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SHARD_COMPRESSION } from "../parquet.js";
+function toCanonicalRow(raw, corpusVersion) {
+    return {
+        raw: raw["raw"],
+        components: raw["components"],
+        country: raw["country"] ?? "US",
+        locale: raw["locale"] ?? undefined,
+        source: raw["source"],
+        source_id: raw["source_id"],
+        corpus_version: corpusVersion,
+        license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+        synth: raw["synth"],
+    };
+}
+function appendShape(row) {
+    const out = {
+        raw: row.raw,
+        tokens: row.tokens,
+        labels: row.labels,
+        country: row.country,
+        source: row.source,
+        source_id: row.source_id,
+        corpus_version: row.corpus_version,
+        license: row.license,
+    };
+    if (row.locale !== null) {
+        out.locale = row.locale;
+    }
+    if (row.synth_method !== null) {
+        out.synth_method = row.synth_method;
+    }
+    if (row.synth_base_id !== null) {
+        out.synth_base_id = row.synth_base_id;
+    }
+    return out;
+}
+/**
+ * Write one shard for a single source slug. Returns the populated ShardDescriptor + a list of quarantine reasons for
+ * rows that failed alignment.
+ */
+async function writeOneShard(rows, outPath, source, corpusVersion) {
+    const writer = await ParquetWriter.openFile(LABELED_ROW_SCHEMA, outPath, {
+        rowGroupSize: ROW_GROUP_SIZE,
+    });
+    writer.setMetadata("mailwoman.corpus_version", corpusVersion);
+    writer.setMetadata("mailwoman.split", "train");
+    writer.setMetadata("mailwoman.shard_source", source);
+    let firstSourceID = "";
+    let lastSourceID = "";
+    for (const row of rows) {
+        const pq = rowToParquet(row);
+        await writer.appendRow(appendShape(pq));
+        if (firstSourceID === "") {
+            firstSourceID = row.source_id;
+        }
+        lastSourceID = row.source_id;
+    }
+    await writer.close();
+    const fileStat = await stat(outPath);
+    const sha256 = await sha256File(outPath);
+    return {
+        split: "train",
+        path: outPath,
+        format: "parquet",
+        compression: SHARD_COMPRESSION,
+        rows: rows.length,
+        bytes: fileStat.size,
+        sha256,
+        first_source_id: firstSourceID,
+        last_source_id: lastSourceID,
+        // Stamp source so audit.ts attributes the shard without falling back to filename-prefix
+        // inference. Cast widens ShardDescriptor; the field is read by audit.ts.
+        ...{ source },
+    };
+}
+function canonicalizeShardPath(path, legacyPrefix, canonicalPrefix) {
+    if (path.startsWith(legacyPrefix))
+        return canonicalPrefix + path.slice(legacyPrefix.length);
+    return path;
+}
+export async function buildTranslitShard(options, report) {
+    const corpusVersion = options.corpusVersion ?? "0.4.0";
+    const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/";
+    const legacyPathPrefix = options.legacyPathPrefix ?? "/mnt/playpen/mailwoman-data/";
+    if (!existsSync(options.jsonl))
+        throw new Error(`jsonl not found: ${options.jsonl}`);
+    if (!existsSync(options.baseManifest))
+        throw new Error(`base-manifest not found: ${options.baseManifest}`);
+    const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`);
+    const trainDir = join(corpusDir, "train");
+    await mkdir(trainDir, { recursive: true });
+    // Bucket canonical rows by source. Quarantined rows are logged.
+    const buckets = new Map();
+    const quarantine = [];
+    let totalIn = 0;
+    for await (const raw of iterateJSONL(options.jsonl)) {
+        totalIn++;
+        const canon = toCanonicalRow(raw, corpusVersion);
+        const result = alignRow(canon);
+        if (result.kind !== "labeled") {
+            quarantine.push(`${canon.source_id}\t${result.row.reason}`);
+            continue;
+        }
+        const bucket = buckets.get(canon.source);
+        if (bucket) {
+            bucket.push(result.row);
+        }
+        else {
+            buckets.set(canon.source, [result.row]);
+        }
+    }
+    report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`);
+    const newShards = [];
+    const sortedKeys = [...buckets.keys()].sort();
+    for (const source of sortedKeys) {
+        const rows = buckets.get(source);
+        const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source;
+        const outPath = join(trainDir, `part-translit-${slug}.parquet`);
+        const descriptor = await writeOneShard(rows, outPath, source, corpusVersion);
+        newShards.push(descriptor);
+        report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`);
+    }
+    if (quarantine.length > 0) {
+        const qPath = join(corpusDir, "quarantine-transliteration.tsv");
+        writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8");
+        report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`);
+    }
+    // Compose final MANIFEST: rewrite base.shards paths from /mnt/playpen/... → /data/... and append
+    // the new translit shards. Kryptonite shard already lives in the base manifest (it was written
+    // there by Thread B).
+    const base = JSON.parse(readFileSync(options.baseManifest, "utf8"));
+    const rewrittenBase = base.shards.map((sh) => ({
+        ...sh,
+        path: canonicalizeShardPath(sh.path, legacyPathPrefix, canonicalPathPrefix),
+    }));
+    const newTrainRows = newShards.reduce((sum, sh) => sum + sh.rows, 0);
+    const combined = {
+        corpus_version: corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_shard: base.rows_per_shard,
+        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+        shards: [...rewrittenBase, ...newShards],
+        counts: {
+            train: base.counts.train + newTrainRows,
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newTrainRows,
+    };
+    const combinedPath = join(corpusDir, "MANIFEST.json");
+    writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8");
+    report?.(`wrote combined manifest → ${combinedPath}`);
+    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`);
+    report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newShards.length})`);
+    report?.(`  compression=${SHARD_COMPRESSION}`);
+    const pathFix = rewrittenBase.filter((s, i) => s.path !== base.shards[i].path).length;
+    if (pathFix > 0) {
+        report?.(`  path-canonicalized base shards: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`);
+    }
+}
+//# sourceMappingURL=shard-translit.js.map

package/out/src/tools/shard-translit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shard-translit.js","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAEhE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAE3D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAepH,SAAS,cAAc,CAAC,GAA4B,EAAE,aAAqB;IAC1E,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;QACzB,UAAU,EAAE,GAAG,CAAC,YAAY,CAA2B;QACvD,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;QAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;QAC9C,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;QACrC,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;QACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;KAC5C,CAAA;AACF,CAAC;AAED,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACzB,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IACxB,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QAC/B,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IACpC,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QAChC,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IACtC,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC3B,IAA2B,EAC3B,OAAe,EACf,MAAc,EACd,aAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,OAAO,EAAE;QACpF,YAAY,EAAE,cAAc;KAC5B,CAAC,CAAA;IACF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,aAAa,CAAC,CAAA;IAC7D,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;IAC9C,MAAM,CAAC,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAA;IAEpD,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC5B,MAAM,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;QAEhE,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YAC1B,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;QAC9B,CAAC;QACD,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;IAC7B,CAAC;IACD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IAEpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,OAAO;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI;QACpB,MAAM;QACN,eAAe,EAAE,aAAa;QAC9B,cAAc,EAAE,YAAY;QAC5B,wFAAwF;QACxF,yEAAyE;QACzE,GAAI,EAAE,MAAM,EAA6B;KACzC,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,YAAoB,EAAE,eAAuB;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3F,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,QAAQ,CAAA;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,8BAA8B,CAAA;IAEnF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE1C,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC/C,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,YAAY,CAA0B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9E,OAAO,EAAE,CAAA;QACT,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YAC3D,SAAQ;QACT,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAExC,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IACD,MAAM,EAAE,CAAC,QAAQ,OAAO,UAAU,UAAU,CAAC,MAAM,iBAAiB,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAA;IAElG,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAE7C,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,IAAI,UAAU,CAAC,CAAA;QAC/D,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;QAC5E,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC1B,MAAM,EAAE,CAAC,KAAK,MAAM,KAAK,UAAU,CAAC,IAAI,WAAW,OAAO,KAAK,UAAU,CAAC,KAAK,SAAS,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAA;QAC/D,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,KAAK,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,sBAAsB;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAkB,CAAA;IACpF,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,GAAG,EAAE;QACL,IAAI,EAAE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,mBAAmB,CAAC;KAC3E,CAAC,CAAC,CAAA;IACH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IACpE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,YAAY;YACvC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,YAAY;KAC1C,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACrD,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,YAAY,GAAG,CAAC,CAAA;IAChG,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtG,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAEtF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,qCAAqC,OAAO,aAAa,gBAAgB,QAAQ,mBAAmB,IAAI,CAAC,CAAA;IACnH,CAAC;AACF,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/corpus",
-  "version": "5.9.0",
+  "version": "5.10.0",
   "description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -17,18 +17,74 @@
   "type": "module",
   "exports": {
     "./package.json": "./package.json",
-    ".": "./out/src/index.js",
-    "./types": "./out/src/types.js",
-    "./format": "./out/src/format.js"
+    ".": {
+      "types": "./out/src/index.d.ts",
+      "default": "./out/src/index.js"
+    },
+    "./types": {
+      "types": "./out/src/types.d.ts",
+      "default": "./out/src/types.js"
+    },
+    "./format": {
+      "types": "./out/src/format.d.ts",
+      "default": "./out/src/format.js"
+    },
+    "./license": {
+      "types": "./out/src/license.d.ts",
+      "default": "./out/src/license.js"
+    },
+    "./build": {
+      "types": "./out/src/build.d.ts",
+      "default": "./out/src/build.js"
+    },
+    "./adapters": {
+      "types": "./out/src/adapters/index.d.ts",
+      "default": "./out/src/adapters/index.js"
+    },
+    "./tools": {
+      "types": "./out/src/tools/index.d.ts",
+      "default": "./out/src/tools/index.js"
+    }
   },
   "publishConfig": {
+    "exports": {
+      "./package.json": "./package.json",
+      ".": {
+        "types": "./out/src/index.d.ts",
+        "default": "./out/src/index.js"
+      },
+      "./types": {
+        "types": "./out/src/types.d.ts",
+        "default": "./out/src/types.js"
+      },
+      "./format": {
+        "types": "./out/src/format.d.ts",
+        "default": "./out/src/format.js"
+      },
+      "./license": {
+        "types": "./out/src/license.d.ts",
+        "default": "./out/src/license.js"
+      },
+      "./build": {
+        "types": "./out/src/build.d.ts",
+        "default": "./out/src/build.js"
+      },
+      "./adapters": {
+        "types": "./out/src/adapters/index.d.ts",
+        "default": "./out/src/adapters/index.js"
+      },
+      "./tools": {
+        "types": "./out/src/tools/index.d.ts",
+        "default": "./out/src/tools/index.js"
+      }
+    },
     "access": "public"
   },
   "dependencies": {
     "@dsnp/parquetjs": "1.8.7",
-    "@mailwoman/codex": "5.9.0",
-    "@mailwoman/core": "5.9.0",
-    "@mailwoman/formatter": "5.9.0",
+    "@mailwoman/codex": "5.10.0",
+    "@mailwoman/core": "5.10.0",
+    "@mailwoman/formatter": "5.10.0",
     "csv-parse": "^5.6.0",
     "fast-glob": "^3.3.3",
     "fastest-levenshtein": "^1.0.16",