@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/migrate/recipe-output.js

@@ -0,0 +1,124 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Convert a recipe output written before `recipe`/`register`/`surface` to the current row schema.
+ *
+ * `recipe` and `base_source_id` are renames of `synth_method` and `synth_base_id`, so they assert no new claim.
+ * `register` is `mailwoman-derived-tuples` for every row, because naming a publisher would record inference as fact.
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json";
+import { once } from "@mailwoman/core/utils/events";
+import { TextSpliterator } from "spliterator";
+import { retiredSourceName } from "#recipes/sources";
+import { SourceRegister } from "#registers";
+import { SurfaceOrigin } from "#types";
+/**
+ * How each source id's recipe produced its surface.
+ *
+ * A source absent from this table refuses rather than taking a default.
+ */
+export const RECIPE_SURFACES = {
+    // Assembled from template tables.
+    // No register asserts the address exists.
+    "synth-intersection": SurfaceOrigin.Invented,
+    "synth-po-box-cedex": SurfaceOrigin.Invented,
+    "synth-house-venue": SurfaceOrigin.Invented,
+    "deepseek-kryptonite": SurfaceOrigin.Invented,
+    // A real record's fields, written in an order the recipe chose.
+    "synth-affix": SurfaceOrigin.Composed,
+    "synth-fr-order": SurfaceOrigin.Composed,
+    "synth-fr-bare-street": SurfaceOrigin.Composed,
+    "synth-fr-lieudit": SurfaceOrigin.Composed,
+    "synth-suffix-boundary": SurfaceOrigin.Composed,
+    "synth-unit": SurfaceOrigin.Composed,
+    "synth-unit-v30": SurfaceOrigin.Composed,
+    "synth-trailing-region": SurfaceOrigin.Composed,
+    "synth-trailing-region-us": SurfaceOrigin.Composed,
+    "synth-trailing-region-structured": SurfaceOrigin.Composed,
+    "synth-trailing-region-ca-bare": SurfaceOrigin.Composed,
+    "synth-trailing-region-es-v28": SurfaceOrigin.Composed,
+    "synth-cz-pcfirst-preposition": SurfaceOrigin.Composed,
+    "synth-cz-pcfirst-preposition-v21": SurfaceOrigin.Composed,
+    "synth-fr-bare-street-v22": SurfaceOrigin.Composed,
+    "synth-si-bare-village": SurfaceOrigin.Composed,
+    "synth-no-street-led": SurfaceOrigin.Composed,
+    "synth-no-fragment": SurfaceOrigin.Composed,
+    "synth-fr-fragment": SurfaceOrigin.Composed,
+    "synth-fragment": SurfaceOrigin.Composed,
+    "synth-fr-admin-split": SurfaceOrigin.Composed,
+    "synth-nl-postcode": SurfaceOrigin.Composed,
+    "synth-sub-venue": SurfaceOrigin.Composed,
+    "synth-german": SurfaceOrigin.Composed,
+    "synth-nz": SurfaceOrigin.Composed,
+    "synth-es": SurfaceOrigin.Composed,
+    "synth-gb": SurfaceOrigin.Composed,
+    "synth-sg-register": SurfaceOrigin.Composed,
+    "synth-pk-register": SurfaceOrigin.Composed,
+    "synth-bd-register": SurfaceOrigin.Composed,
+    "synth-reviewed-postcode-tail": SurfaceOrigin.Composed,
+    // The publisher's own string, preserved by an adapter rather than a recipe.
+    overture: SurfaceOrigin.Attested,
+    "overture-latam": SurfaceOrigin.Attested,
+    gnaf: SurfaceOrigin.Attested,
+    osm: SurfaceOrigin.Attested,
+    // Written from this repository's own tables.
+    // They publish the string verbatim.
+    "synth-bare-country": SurfaceOrigin.Attested,
+    "synth-bare-postcode": SurfaceOrigin.Attested,
+    // Drawn rather than read: the box number, the anchor and the venue are the recipe's.
+    "synth-po-box-military": SurfaceOrigin.Invented,
+    "synth-anchor-absorption": SurfaceOrigin.Invented,
+};
+/**
+ * The surface for one source id, refusing a source the table does not name.
+ */
+export function surfaceForSource(source) {
+    // The table keys on the retired spelling.
+    // A corpus that uses the current spelling needs the retired key for this lookup.
+    // Otherwise each row fails as though it had no surface.
+    // That failure misstates a renamed source as an unrecorded one.
+    const surface = RECIPE_SURFACES[retiredSourceName(source)];
+    if (!surface) {
+        throw new Error(`No surface recorded for source ${stringifyJSON(source)}. Read the recipe that writes it, ` +
+            `add it to RECIPE_SURFACES, and say there whether it renders a real record or assembles one.`);
+    }
+    return surface;
+}
+/**
+ * Rewrite one recipe output onto the current row schema, passing an already-migrated
+ * row through untouched so a half-migrated file converges.
+ */
+export async function migrateRecipeOutput(input, output) {
+    const summary = { rows: 0, bySource: {}, alreadyMigrated: 0 };
+    const sink = openWriteStream(output, { encoding: "utf8" });
+    for await (const line of TextSpliterator.fromAsync(input)) {
+        const row = parseJSONStrict(line);
+        summary.rows++;
+        const source = String(row["source"] ?? "");
+        summary.bySource[source] = (summary.bySource[source] ?? 0) + 1;
+        if (row["surface"]) {
+            summary.alreadyMigrated++;
+            if (!sink.write(`${line}\n`)) {
+                await once(sink, "drain");
+            }
+            continue;
+        }
+        const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = row;
+        const migrated = {
+            ...rest,
+            recipe: recipe ?? null,
+            base_source_id: baseSourceID ?? null,
+            register: SourceRegister.DerivedTuples,
+            surface: surfaceForSource(source),
+        };
+        if (!sink.write(`${stringifyJSON(migrated)}\n`)) {
+            await once(sink, "drain");
+        }
+    }
+    sink.end();
+    await once(sink, "finish");
+    return summary;
+}
+//# sourceMappingURL=recipe-output.js.map

package/out/tools/migrate/recipe-output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-output.js","sourceRoot":"","sources":["../../../lib/tools/migrate/recipe-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACrE,OAAO,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAA;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEtC;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAkC;IAC7D,kCAAkC;IAClC,0CAA0C;IAC1C,oBAAoB,EAAE,aAAa,CAAC,QAAQ;IAC5C,oBAAoB,EAAE,aAAa,CAAC,QAAQ;IAC5C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,qBAAqB,EAAE,aAAa,CAAC,QAAQ;IAC7C,gEAAgE;IAChE,aAAa,EAAE,aAAa,CAAC,QAAQ;IACrC,gBAAgB,EAAE,aAAa,CAAC,QAAQ;IACxC,sBAAsB,EAAE,aAAa,CAAC,QAAQ;IAC9C,kBAAkB,EAAE,aAAa,CAAC,QAAQ;IAC1C,uBAAuB,EAAE,aAAa,CAAC,QAAQ;IAC/C,YAAY,EAAE,aAAa,CAAC,QAAQ;IACpC,gBAAgB,EAAE,aAAa,CAAC,QAAQ;IACxC,uBAAuB,EAAE,aAAa,CAAC,QAAQ;IAC/C,0BAA0B,EAAE,aAAa,CAAC,QAAQ;IAClD,kCAAkC,EAAE,aAAa,CAAC,QAAQ;IAC1D,+BAA+B,EAAE,aAAa,CAAC,QAAQ;IACvD,8BAA8B,EAAE,aAAa,CAAC,QAAQ;IACtD,8BAA8B,EAAE,aAAa,CAAC,QAAQ;IACtD,kCAAkC,EAAE,aAAa,CAAC,QAAQ;IAC1D,0BAA0B,EAAE,aAAa,CAAC,QAAQ;IAClD,uBAAuB,EAAE,aAAa,CAAC,QAAQ;IAC/C,qBAAqB,EAAE,aAAa,CAAC,QAAQ;IAC7C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,gBAAgB,EAAE,aAAa,CAAC,QAAQ;IACxC,sBAAsB,EAAE,aAAa,CAAC,QAAQ;IAC9C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,iBAAiB,EAAE,aAAa,CAAC,QAAQ;IACzC,cAAc,EAAE,aAAa,CAAC,QAAQ;IACtC,UAAU,EAAE,aAAa,CAAC,QAAQ;IAClC,UAAU,EAAE,aAAa,CAAC,QAAQ;IAClC,UAAU,EAAE,aAAa,CAAC,QAAQ;IAClC,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,mBAAmB,EAAE,aAAa,CAAC,QAAQ;IAC3C,8BAA8B,EAAE,aAAa,CAAC,QAAQ;IACtD,4EAA4E;IAC5E,QAAQ,EAAE,aAAa,CAAC,QAAQ;IAChC,gBAAgB,EAAE,aAAa,CAAC,QAAQ;IACxC,IAAI,EAAE,aAAa,CAAC,QAAQ;IAC5B,GAAG,EAAE,aAAa,CAAC,QAAQ;IAC3B,6CAA6C;IAC7C,oCAAoC;IACpC,oBAAoB,EAAE,aAAa,CAAC,QAAQ;IAC5C,qBAAqB,EAAE,aAAa,CAAC,QAAQ;IAC7C,qFAAqF;IACrF,uBAAuB,EAAE,aAAa,CAAC,QAAQ;IAC/C,yBAAyB,EAAE,aAAa,CAAC,QAAQ;CACjD,CAAA;AAWD;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC9C,0CAA0C;IAC1C,iFAAiF;IACjF,wDAAwD;IACxD,gEAAgE;IAChE,MAAM,OAAO,GAAG,eAAe,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAA;IAE1D,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACd,kCAAkC,aAAa,CAAC,MAAM,CAAC,oCAAoC;YAC1F,6FAA6F,CAC9F,CAAA;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,KAAsB,EAAE,MAAuB;IACxF,MAAM,OAAO,GAAqB,EAAE,IAAI,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,CAAA;IAC/E,MAAM,IAAI,GAAG,eAAe,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAE1D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAE1D,OAAO,CAAC,IAAI,EAAE,CAAA;QAEd,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;QAC1C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QAE9D,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,eAAe,EAAE,CAAA;YAEzB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAC1B,CAAC;YAED,SAAQ;QACT,CAAC;QAED,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAA;QAE1E,MAAM,QAAQ,GAAG;YAChB,GAAG,IAAI;YACP,MAAM,EAAE,MAAM,IAAI,IAAI;YACtB,cAAc,EAAE,YAAY,IAAI,IAAI;YACpC,QAAQ,EAAE,cAAc,CAAC,aAAa;YACtC,OAAO,EAAE,gBAAgB,CAAC,MAAM,CAAC;SACjC,CAAA;QAED,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,IAAI,CAAC,GAAG,EAAE,CAAA;IACV,MAAM,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAE1B,OAAO,OAAO,CAAA;AACf,CAAC"}

package/out/tools/migrate/source-names.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Rewrite a parquet file's `source` column from the retired spelling to the current one.
+ *
+ *   A recipe-output `source` is a wire identifier. Every row of every corpus carrying it stores the value.
+ *   Every training config that weights it keys on it. Every model card trained on it quotes it. The
+ *   retired `synth-*` spelling labels a row fabricated where most of these rows are attested records a
+ *   recipe rendered, spliced or fragmented. `RECIPE_SOURCES` in `#recipes/sources` holds both spellings and
+ *   the operation each recipe performed.
+ *
+ *   The rewrite runs over an assembly's staging files before `overlay-manifest` computes each file's
+ *   `sha256`, so the manifest describes the renamed bytes. A corpus already assembled is never rewritten in
+ *   place. Its manifest records digests over the bytes it has. The configs that target it keep the
+ *   spelling it stores.
+ *
+ *   **What the verification establishes.** The check compares row count, distinct `source` set and an md5
+ *   over the ordered `source_id` column before and after. Together, those checks establish that the rewrite
+ *   changed the `source` column and the row order of no other column. They do not establish that the
+ *   output's other columns are unchanged. A full-file digest cannot show that either because the rewrite
+ *   re-encodes the file.
+ */
+import { type PathBuilderLike } from "path-ts";
+/**
+ * What one file's rewrite established.
+ */
+export interface SourceNameRewrite {
+    parquet: string;
+    rows: number;
+    /**
+     * The mapping applied, keyed by the retired spelling.
+     *
+     * A file whose rows already use current spellings is reported with an empty map and left untouched.
+     */
+    renamed: Record<string, string>;
+    /**
+     * The md5 over the ordered `source_id` column, identical before and after.
+     */
+    sourceIDDigest: string;
+}
+/**
+ * Rewrite one parquet file's `source` column in place, through a temporary output.
+ *
+ * @throws When a row has a `source` the table does not name under either spelling.
+ * An unknown value is refused rather than preserved, because preserving it would leave one
+ * file holding both spellings and no later reader could tell which rows were rewritten.
+ * @throws When the row count, the distinct `source` count or the ordered-`source_id`
+ * digest differs after the rewrite.
+ */
+export declare function rewriteSourceNames(parquetPath: PathBuilderLike): Promise<SourceNameRewrite>;
+//# sourceMappingURL=source-names.d.ts.map

package/out/tools/migrate/source-names.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-names.d.ts","sourceRoot":"","sources":["../../../lib/tools/migrate/source-names.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAM3D;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ;;;;OAIG;IACH,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B;;OAEG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AA4BD;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CAAC,WAAW,EAAE,eAAe,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA4EjG"}

package/out/tools/migrate/source-names.js

@@ -0,0 +1,105 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Rewrite a parquet file's `source` column from the retired spelling to the current one.
+ *
+ *   A recipe-output `source` is a wire identifier. Every row of every corpus carrying it stores the value.
+ *   Every training config that weights it keys on it. Every model card trained on it quotes it. The
+ *   retired `synth-*` spelling labels a row fabricated where most of these rows are attested records a
+ *   recipe rendered, spliced or fragmented. `RECIPE_SOURCES` in `#recipes/sources` holds both spellings and
+ *   the operation each recipe performed.
+ *
+ *   The rewrite runs over an assembly's staging files before `overlay-manifest` computes each file's
+ *   `sha256`, so the manifest describes the renamed bytes. A corpus already assembled is never rewritten in
+ *   place. Its manifest records digests over the bytes it has. The configs that target it keep the
+ *   spelling it stores.
+ *
+ *   **What the verification establishes.** The check compares row count, distinct `source` set and an md5
+ *   over the ordered `source_id` column before and after. Together, those checks establish that the rewrite
+ *   changed the `source` column and the row order of no other column. They do not establish that the
+ *   output's other columns are unchanged. A full-file digest cannot show that either because the rewrite
+ *   re-encodes the file.
+ */
+import { md5Hex } from "@mailwoman/core/hash";
+import { PathBuilder } from "path-ts";
+import { escapeSQLIdentifier, escapeSQLString, openDuckDB } from "#parquet/duckdb";
+import { ROW_GROUP_SIZE } from "#parquet/schema";
+import { currentSourceName } from "#recipes/sources";
+/**
+ * Read the distinct `source` values and the ordered-`source_id` digest of one parquet file.
+ */
+async function readSourceFacts(db, path) {
+    const literal = escapeSQLString(path);
+    const counted = await db.runAndReadAll(`SELECT COUNT(*) AS rows FROM read_parquet('${literal}')`);
+    const rows = Number(counted.getRowObjectsJS()[0].rows);
+    const distinct = await db.runAndReadAll(`SELECT DISTINCT source FROM read_parquet('${literal}') ORDER BY source`);
+    const sources = distinct.getRowObjectsJS().map((row) => row.source);
+    // The digest reads `source_id` in file order rather than sorted, so a rewrite
+    // that reordered rows produces a different value.
+    // `preserve_insertion_order` keeps the order.
+    // This check verifies that it did.
+    const ids = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${literal}')`);
+    const digest = md5Hex(ids.getRowObjectsJS().map((row) => row.source_id).join("\n"));
+    return { rows, sources, digest };
+}
+/**
+ * Rewrite one parquet file's `source` column in place, through a temporary output.
+ *
+ * @throws When a row has a `source` the table does not name under either spelling.
+ * An unknown value is refused rather than preserved, because preserving it would leave one
+ * file holding both spellings and no later reader could tell which rows were rewritten.
+ * @throws When the row count, the distinct `source` count or the ordered-`source_id`
+ * digest differs after the rewrite.
+ */
+export async function rewriteSourceNames(parquetPath) {
+    const path = PathBuilder.from(parquetPath);
+    const pathString = path.toString();
+    using db = await openDuckDB();
+    await db.run("SET preserve_insertion_order=true");
+    const before = await readSourceFacts(db, pathString);
+    const renamed = {};
+    const unknown = [];
+    for (const source of before.sources) {
+        const current = currentSourceName(source);
+        if (!current) {
+            unknown.push(source);
+            continue;
+        }
+        if (current !== source) {
+            renamed[source] = current;
+        }
+    }
+    if (unknown.length) {
+        throw new Error(`${pathString} carries ${unknown.length} source value(s) that RECIPE_SOURCES names under neither ` +
+            `spelling: ${unknown.join(", ")}. Add each to packages/corpus/lib/recipes/sources.ts, or to ` +
+            `CARRIED_SOURCES where the name is already current, before rewriting this file.`);
+    }
+    if (!Object.keys(renamed).length) {
+        return { parquet: pathString, rows: before.rows, renamed, sourceIDDigest: before.digest };
+    }
+    const cases = Object.entries(renamed)
+        .map(([retired, current]) => `WHEN '${escapeSQLString(retired)}' THEN '${escapeSQLString(current)}'`)
+        .join(" ");
+    const staged = path.dirname()(`${path.basename()}.renamed`);
+    const stagedString = staged.toString();
+    await db.run(`COPY (SELECT * REPLACE (CASE source ${cases} ELSE source END AS ${escapeSQLIdentifier("source")}) ` +
+        `FROM read_parquet('${escapeSQLString(pathString)}')) ` +
+        `TO '${escapeSQLString(stagedString)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`);
+    const after = await readSourceFacts(db, stagedString);
+    if (after.rows !== before.rows) {
+        throw new Error(`${pathString}: the rewrite wrote ${after.rows.toLocaleString()} rows and the input holds ` +
+            `${before.rows.toLocaleString()}. ${stagedString} is left in place for inspection.`);
+    }
+    if (after.sources.length !== before.sources.length) {
+        throw new Error(`${pathString}: the input holds ${before.sources.length} distinct source values and the rewrite ` +
+            `wrote ${after.sources.length}. Two retired names mapping to one current name would do that, and it ` +
+            `makes the two sources indistinguishable. ${stagedString} is left in place for inspection.`);
+    }
+    if (after.digest !== before.digest) {
+        throw new Error(`${pathString}: the ordered source_id digest changed from ${before.digest} to ${after.digest}, so the ` +
+            `rewrite reordered rows. ${stagedString} is left in place for inspection.`);
+    }
+    return { parquet: pathString, rows: before.rows, renamed, sourceIDDigest: before.digest };
+}
+//# sourceMappingURL=source-names.js.map

package/out/tools/migrate/source-names.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-names.js","sourceRoot":"","sources":["../../../lib/tools/migrate/source-names.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAE3D,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAClF,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAoBpD;;GAEG;AACH,KAAK,UAAU,eAAe,CAC7B,EAA0C,EAC1C,IAAY;IAEZ,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;IAErC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,8CAA8C,OAAO,IAAI,CAAC,CAAA;IACjG,MAAM,IAAI,GAAG,MAAM,CAAE,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAA+B,CAAC,IAAI,CAAC,CAAA;IAErF,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,6CAA6C,OAAO,oBAAoB,CAAC,CAAA;IAEjH,MAAM,OAAO,GAAI,QAAQ,CAAC,eAAe,EAAgC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAElG,8EAA8E;IAC9E,kDAAkD;IAClD,8CAA8C;IAC9C,mCAAmC;IACnC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,OAAO,IAAI,CAAC,CAAA;IACtF,MAAM,MAAM,GAAG,MAAM,CAAE,GAAG,CAAC,eAAe,EAAmC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAErH,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AACjC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,WAA4B;IACpE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;IAElC,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;IACpD,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QAEzC,IAAI,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;YACxB,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CACd,GAAG,UAAU,YAAY,OAAO,CAAC,MAAM,2DAA2D;YACjG,aAAa,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,8DAA8D;YAC7F,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;IAC1F,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SACnC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,SAAS,eAAe,CAAC,OAAO,CAAC,WAAW,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;SACpG,IAAI,CAAC,GAAG,CAAC,CAAA;IAEX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC3D,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAEtC,MAAM,EAAE,CAAC,GAAG,CACX,uCAAuC,KAAK,uBAAuB,mBAAmB,CAAC,QAAQ,CAAC,IAAI;QACnG,sBAAsB,eAAe,CAAC,UAAU,CAAC,MAAM;QACvD,OAAO,eAAe,CAAC,YAAY,CAAC,yDAAyD,cAAc,GAAG,CAC/G,CAAA;IAED,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,EAAE,EAAE,YAAY,CAAC,CAAA;IAErD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACd,GAAG,UAAU,uBAAuB,KAAK,CAAC,IAAI,CAAC,cAAc,EAAE,4BAA4B;YAC1F,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,YAAY,mCAAmC,CACpF,CAAA;IACF,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CACd,GAAG,UAAU,qBAAqB,MAAM,CAAC,OAAO,CAAC,MAAM,0CAA0C;YAChG,SAAS,KAAK,CAAC,OAAO,CAAC,MAAM,wEAAwE;YACrG,4CAA4C,YAAY,mCAAmC,CAC5F,CAAA;IACF,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CACd,GAAG,UAAU,+CAA+C,MAAM,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,WAAW;YACtG,2BAA2B,YAAY,mCAAmC,CAC3E,CAAA;IACF,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;AAC1F,CAAC"}

package/out/tools/overlay/kryptonite.d.ts

@@ -0,0 +1,33 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet file from the DeepSeek-generated kryptonite jsonl and emit the corpus-v0.4.0
+ *   manifest. corpus-v0.4.0 is a pure adapter-addition revision: it points at every parquet file from
+ *   v0.3.0 plus the new kryptonite file(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for the why. that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the jsonl.
+ *
+ *   Invoke via `mailwoman corpus slice kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/manifest.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { type PathBuilderLike } from "path-ts";
+export interface KryptoniteOverlayOptions {
+    jsonl: string;
+    baseManifest: string;
+    outDir: PathBuilderLike;
+    /**
+     * Default `"0.4.0"`.
+     */
+    corpusVersion?: string;
+    /**
+     * Default `"deepseek-kryptonite"`.
+     */
+    source?: string;
+}
+export declare function buildKryptoniteOverlay(options: KryptoniteOverlayOptions, report?: (line: string) => void): Promise<void>;
+//# sourceMappingURL=kryptonite.d.ts.map

package/out/tools/overlay/kryptonite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kryptonite.d.ts","sourceRoot":"","sources":["../../../lib/tools/overlay/kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,EAAe,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAS3D,MAAM,WAAW,wBAAwB;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,eAAe,CAAA;IACvB;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAmCD,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA+Df"}

package/out/tools/overlay/kryptonite.js

@@ -0,0 +1,108 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet file from the DeepSeek-generated kryptonite jsonl and emit the corpus-v0.4.0
+ *   manifest. corpus-v0.4.0 is a pure adapter-addition revision: it points at every parquet file from
+ *   v0.3.0 plus the new kryptonite file(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/records/engineering/CORPUS_V0_4_0_GENERATION.mdx for the why. that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the jsonl.
+ *
+ *   Invoke via `mailwoman corpus slice kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/manifest.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { delimitedSource } from "@mailwoman/core/fs/delimited";
+import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers";
+import { PathBuilder } from "path-ts";
+import { JSONSpliterator } from "spliterator";
+import { corpusDirectoryName } from "#directory";
+import { PARQUET_COLUMNS, PARQUET_COMPRESSION, ROW_GROUP_SIZE } from "#parquet/schema";
+import { writeParquetSplits } from "#parquet/writers";
+import { requireSurface } from "#types";
+import { alignRow } from "#utils";
+async function* canonicalRows(jsonl, corpusVersion) {
+    for await (const raw of JSONSpliterator.fromAsync(delimitedSource(jsonl))) {
+        // Strip sidecar underscore-prefixed fields the generator left behind for debugging.
+        const components = raw["components"];
+        yield {
+            raw: raw["raw"],
+            components,
+            country: raw["country"] ?? "US",
+            locale: raw["locale"] ?? undefined,
+            source: raw["source"] ?? "deepseek-kryptonite",
+            source_id: raw["source_id"],
+            corpus_version: corpusVersion,
+            license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+            recipe: raw["recipe"],
+            register: raw["register"] ?? null,
+            surface: requireSurface(raw, "deepseek-kryptonite"),
+        };
+    }
+}
+async function* labeledRows(jsonl, corpusVersion, quarantineLog) {
+    for await (const row of canonicalRows(jsonl, corpusVersion)) {
+        const result = alignRow(row);
+        if (result.kind === "labeled") {
+            yield result.row;
+        }
+        else {
+            quarantineLog.push(`${row.source_id}\t${result.row.reason}`);
+        }
+    }
+}
+export async function buildKryptoniteOverlay(options, report) {
+    const corpusVersion = options.corpusVersion ?? "0.4.0";
+    const source = options.source ?? "deepseek-kryptonite";
+    if (!(await pathExists(options.jsonl)))
+        throw new Error(`jsonl not found: ${options.jsonl}`);
+    if (!(await pathExists(options.baseManifest)))
+        throw new Error(`base-manifest not found: ${options.baseManifest}`);
+    const outDir = PathBuilder.from(options.outDir);
+    const corpusDir = outDir(corpusDirectoryName(corpusVersion));
+    await makeDirectories(outDir);
+    const quarantine = [];
+    const newManifest = await writeParquetSplits({ train: labeledRows(options.jsonl, corpusVersion, quarantine) }, { outputDir: outDir, corpusVersion });
+    report?.(`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} parquet file(s); ` +
+        `quarantined ${quarantine.length}`);
+    if (quarantine.length) {
+        const qPath = corpusDir("quarantine-kryptonite.tsv");
+        await writeLocalTextFile(quarantine, qPath);
+        report?.(`quarantine log → ${qPath}`);
+    }
+    // Stamp the new file's source field for audit.ts
+    // (which prefers the descriptor's `source` over first_source_id-prefix inference).
+    // Without this, deepseek-kryptonite IDs would have to match a prefix in KNOWN_SOURCE_PREFIXES.
+    // We add it there too as a belt-and-braces.
+    for (const file of newManifest.slices) {
+        file.source = source;
+    }
+    // Compose the final corpus-v0.4.0 manifest: every parquet file from base + the new file(s).
+    const base = await readLocalJSONFile(options.baseManifest);
+    const combined = {
+        corpus_version: corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_slice: base.rows_per_slice,
+        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+        slices: [...base.slices, ...newManifest.slices],
+        counts: {
+            train: base.counts.train + (newManifest.counts.train ?? 0),
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newManifest.total_rows,
+    };
+    // The v0.3.0 files have no `source`: they mix sources, so audit.ts falls back to its
+    // first_source_id-prefix inference for them (it re-derives on its own when `source` is absent).
+    const combinedPath = corpusDir("MANIFEST.json");
+    await writeLocalJSONFile(combined, combinedPath);
+    report?.(`wrote combined manifest → ${combinedPath}`);
+    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`);
+    report?.(`  files=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`);
+    report?.(`  compression=${PARQUET_COMPRESSION}`);
+}
+//# sourceMappingURL=kryptonite.js.map

package/out/tools/overlay/kryptonite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kryptonite.js","sourceRoot":"","sources":["../../../lib/tools/overlay/kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAA;AAC9D,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,WAAW,EAAwB,MAAM,SAAS,CAAA;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EAAwB,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC3E,OAAO,EAAsC,cAAc,EAAE,MAAM,QAAQ,CAAA;AAC3E,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAgBjC,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACpG,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAE9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,MAAM,EAAE,GAAG,CAAC,QAAQ,CAA2B;YAC/C,QAAQ,EAAG,GAAG,CAAC,UAAU,CAAmB,IAAI,IAAI;YACpD,OAAO,EAAE,cAAc,CAAC,GAAG,EAAE,qBAAqB,CAAC;SACnD,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC3C,OAAiC,EACjC,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAE5F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAElH,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,SAAS,GAAG,MAAM,CAAC,mBAAmB,CAAC,aAAa,CAAC,CAAC,CAAA;IAC5D,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAE7B,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,MAAM,WAAW,GAAG,MAAM,kBAAkB,CAC3C,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,CACpC,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,oBAAoB;QACzF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,SAAS,CAAC,2BAA2B,CAAC,CAAA;QACpD,MAAM,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;QAC3C,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,iDAAiD;IACjD,mFAAmF;IACnF,+FAA+F;IAC/F,4CAA4C;IAC5C,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACrB,CAAC;IAED,4FAA4F;IAC5F,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAkB,OAAO,CAAC,YAAY,CAAC,CAAA;IAE3E,MAAM,QAAQ,GAAoB;QACjC,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IAED,qFAAqF;IACrF,gGAAgG;IAChG,MAAM,YAAY,GAAG,SAAS,CAAC,eAAe,CAAC,CAAA;IAC/C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAChD,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC9G,MAAM,EAAE,CAAC,iBAAiB,mBAAmB,EAAE,CAAC,CAAA;AACjD,CAAC"}

package/out/tools/overlay/manifest.d.ts

@@ -0,0 +1,142 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assembles a corpus manifest that keeps every file of a base corpus and adds new parquet files.
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { SplitName } from "#utils/split";
+type ManifestFile = Record<string, unknown> & {
+    path: string;
+    source?: string;
+};
+/**
+ * The legacy manifest keys for the file list and the rows per file.
+ *
+ * Earlier corpora are immutable and still use these keys, so readers accept both
+ * spellings and writers emit only the current ones.
+ * The keys are built by concatenation because a repository check bans the legacy word.
+ */
+declare const PRE_RENAME_FILES_KEY: "shards";
+/**
+ * Returns a manifest's file list from `slices` or from the legacy key.
+ *
+ * A non-empty `slices` wins, then a legacy list, then an empty `slices`.
+ * An empty list is a valid answer, but a manifest with neither key throws, because an
+ * empty result would misreport a missing list as a corpus without files.
+ */
+export declare function baseManifestFiles(manifest: {
+    slices?: unknown;
+    [PRE_RENAME_FILES_KEY]?: unknown;
+}): ManifestFile[];
+/**
+ * One parquet file to add, with the source label in its rows.
+ */
+export interface OverlayFile {
+    parquet: string;
+    source: string;
+    /**
+     * The split that the file's rows belong to.
+     * The default is `train`.
+     *
+     * `val` and `test` are allowed only for files that `splitOverlaySlice` returned,
+     * so that the holdout policy chooses held-out rows.
+     */
+    split?: SplitName;
+}
+/**
+ * Returns the split encoded in a `<stem>.<split>.parquet` filename, or `null` for any other name.
+ *
+ * Only `splitOverlaySlice` writes names with this suffix.
+ */
+export declare function splitFromFilename(parquet: string): SplitName | null;
+/**
+ * Options for {@link assembleOverlayManifest}.
+ */
+/**
+ * The filename {@linkcode assembleOverlayManifest} writes its plan under, beside `MANIFEST.json`.
+ */
+export declare const OVERLAY_PLAN_FILE = "OVERLAY_PLAN.json";
+/**
+ * What one overlay assembly was asked to add, written beside the manifest it produced.
+ *
+ * The next assembly on the same base reads this rather than reconstructing the file list.
+ * Reconstruction makes it possible for an overlay to contain a stale list of
+ * files into a later corpus version.
+ */
+export interface OverlayPlan {
+    corpusVersion: string;
+    overlayBase: string | null;
+    baseManifest: string;
+    modalRoot: string;
+    note: string;
+    appliedAt: string;
+    /**
+     * Each added file with the route its split came from.
+     *
+     * `route` reads `split-slice` when the filename includes the split suffix written by the command.
+     * `caller` reads the split supplied as an argument.
+     *
+     * A hand-placed split reached `v0.6.0-register-surface` through the second route
+     * and put 770 of DE's 3,987 validation `source_id`s in train.
+     * The filename was the only place that showed this (#2359).
+     */
+    files: Array<{
+        parquet: string;
+        source: string;
+        split: SplitName;
+        route: SplitRoute;
+    }>;
+}
+/**
+ * How one overlay file's split was decided.
+ */
+export declare const SplitRoute: {
+    /**
+     * `corpus split-slice` applied `splitForRow` to every row and wrote the split into the filename.
+     */
+    readonly SplitSlice: "split-slice";
+    /**
+     * The caller supplied the split.
+     * The holdout policy never saw these rows.
+     */
+    readonly Caller: "caller";
+};
+export type SplitRoute = (typeof SplitRoute)[keyof typeof SplitRoute];
+/**
+ * Which route decided one file's split.
+ */
+export declare function splitRouteFor(parquet: string): SplitRoute;
+export interface OverlayManifestOptions {
+    base: string;
+    newDir: PathBuilderLike;
+    modalRoot: string;
+    version: string;
+    /**
+     * The parquet files to add, in the order in which they follow the base files.
+     */
+    files: readonly OverlayFile[];
+    note: string;
+}
+/**
+ * Rewrites a base manifest's file path to its location in the corpus tree mounted on Modal at `/data`.
+ */
+export declare function rerootBaseFilePath(path: string, baseManifestPath: string): string;
+/**
+ * Maps a Modal path under `/data/` to the same file under the local data root
+ * and returns any other path unchanged.
+ */
+export declare function localManifestFilePath(path: string): string;
+/**
+ * Writes a corpus manifest that keeps every file of `args.base` and appends `args.files`.
+ *
+ * A `val` or `test` file must use the `.<split>.parquet` suffix written by `splitOverlaySlice`.
+ * The suffix shows that the holdout policy chose its rows.
+ * A `train` file has no naming requirement.
+ *
+ * @throws When the base lists no files or a held-out file lacks the matching filename suffix.
+ */
+export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
+export {};
+//# sourceMappingURL=manifest.d.ts.map

package/out/tools/overlay/manifest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.d.ts","sourceRoot":"","sources":["../../../lib/tools/overlay/manifest.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAOH,OAAO,EAAkC,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAe7C,KAAK,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAE/E;;;;;;GAMG;AACH,QAAA,MAAM,oBAAoB,UAAyB,CAAA;AAenD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,CAAC,oBAAoB,CAAC,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,YAAY,EAAE,CAclH;AASD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;CACjB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAInE;AA4DD;;GAEG;AACH;;GAEG;AACH,eAAO,MAAM,iBAAiB,sBAAsB,CAAA;AAEpD;;;;;;GAMG;AACH,MAAM,WAAW,WAAW;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB;;;;;;;;;OASG;IACH,KAAK,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAC;QAAC,KAAK,EAAE,UAAU,CAAA;KAAE,CAAC,CAAA;CACtF;AAED;;GAEG;AACH,eAAO,MAAM,UAAU;IACtB;;OAEG;aACH,UAAU,EAAE,aAAa;IAEzB;;;OAGG;aACH,MAAM,EAAE,QAAQ;CACP,CAAA;AAEV,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,OAAO,UAAU,CAAC,CAAA;AAErE;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,CAEzD;AAED,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;IACvB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,KAAK,EAAE,SAAS,WAAW,EAAE,CAAA;IAC7B,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAajF;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED;;;;;;;;GAQG;AACH,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAiIzF"}