@mailwoman/corpus 9.4.0 → 10.0.0

package/out/tools/postcode-triples.js

@@ -40,12 +40,14 @@
  *   tuple that does not carry its placement teaches whichever convention the recipe happens to default to — see the
  *   recipe's header for the measurement.
  */
+import { officialLanguagesAlpha3, regionLanguagesAlpha3 } from "@mailwoman/codex/country";
+import { foldName } from "@mailwoman/codex/normalize";
 import { dataRootPath } from "@mailwoman/core/data-root";
 import { pathExists } from "@mailwoman/core/fs/readers";
 import { DatabaseClient } from "@mailwoman/sqlite/client";
 import { join } from "path-ts";
 import { TSVSpliterator } from "spliterator";
-import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames-postal/adapter";
+import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames/postal/adapter";
 import { escapeSQLString } from "#utils/parquet";
 /**
  * Where a country writes the postcode, and the locale tag its rows carry.
@@ -130,6 +132,40 @@ export function applyCountryBudget(triples, budget) {
     }
     return kept;
 }
+/**
+ * The surfaces a REGION is written as: its preferred names in the official language(s), then in the region's
+ * co-official languages, then the gazetteer's own `spr.name` (the English exonym). Deduplicated, order kept.
+ *
+ * `spr.name` is the English-preferred, diacritic-stripped label, and reading it alone is the defect #1673 measured:
+ * 53,078 ES rows teaching `Balearic Islands` (2,872 `Andalusia`, 5,680 `Castile and Leon`) against 4 rows of `Illes
+ * Balears`, and every province with its accent gone (`Cordoba`, `Leon`). A Spanish user writes `Islas Baleares` or
+ * `Illes Balears`; the exonym stays as ONE surface among them because a user may write it too.
+ */
+export function regionWrittenForms(sprName, names) {
+    const out = [];
+    for (const raw of [...names.official, ...names.coOfficial, sprName]) {
+        const name = raw.replace(PROVINCE_GENERIC, "");
+        if (name && !out.includes(name)) {
+            out.push(name);
+        }
+    }
+    return out;
+}
+/**
+ * The provincial generic Who's On First keeps in a province's Catalan and Asturian preferred names (`Província de
+ * Barcelona`, `Província d'A Coruña`). An envelope carries the name alone, so the generic is dropped and the surface
+ * dedupes against the Castilian one.
+ */
+const PROVINCE_GENERIC = /^prov[ií]ncia (?:de |d')/iu;
+/**
+ * The surface a LOCALITY is written as: the official-language preferred name that is `spr.name` with its diacritics
+ * restored (`Cordoba` → `Córdoba`), else the first official-language preferred name, else `spr.name`. One surface, not
+ * a fan-out: the region carries the multiplicity and the locality would multiply it.
+ */
+export function localityWrittenForm(sprName, names) {
+    const folded = foldName(sprName);
+    return names.official.find((name) => foldName(name) === folded) ?? names.official[0] ?? sprName;
+}
 /**
  * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
  * place's ancestry to a region.
@@ -137,6 +173,12 @@ export function applyCountryBudget(triples, budget) {
  * A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
  * a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
  * actually reachable.
+ *
+ * One triple per REGION SURFACE (see {@link regionWrittenForms}): a Balearic postcode yields `Islas Baleares`, `Illes
+ * Balears` and `Balearic Islands` rows, a Zamora one `Zamora` alone. The languages come from the codex — the country's
+ * official languages plus the province's co-official ones — and never from the names table's own language list, whose
+ * "preferred" name in a language not spoken in the province is often the parent community's (`Zamora` → `Castella i
+ * Lleó`).
  */
 export async function readTriplesFromParentJoin(countries, options = {}) {
     const postcodeDB = options.postcodeDB ?? String(dataRootPath("wof", "postalcode-intl.db"));
@@ -146,7 +188,8 @@ export async function readTriplesFromParentJoin(countries, options = {}) {
     using db = new DatabaseClient(adminDB, { readOnly: true });
     db.exec(`ATTACH DATABASE '${escapeSQLString(postcodeDB)}' AS pc`);
     const statement = db.prepare(`
-		SELECT p.name AS postcode, a.name AS locality, r.name AS region, c.name AS country, p.country AS cc
+		SELECT p.name AS postcode, a.id AS locality_id, a.name AS locality, r.id AS region_id, r.name AS region,
+		       c.name AS country, p.country AS cc
 		FROM pc.spr p
 		JOIN spr a ON a.id = p.parent_id AND a.placetype IN ('locality', 'localadmin')
 		JOIN ancestors anc ON anc.id = a.id AND anc.ancestor_placetype = 'region'
@@ -156,23 +199,59 @@ export async function readTriplesFromParentJoin(countries, options = {}) {
 		WHERE p.country = ? AND p.parent_id > 0
 		ORDER BY p.id
 	`);
+    const preferredStatement = db.prepare(`SELECT language, name FROM names WHERE id = ? AND privateuse = 'preferred' AND language IS NOT NULL ORDER BY language, name`);
+    const preferredByID = new Map();
+    /**
+     * Preferred names of one place, grouped by language, read once per id.
+     */
+    const preferredNames = (id) => {
+        let byLanguage = preferredByID.get(id);
+        if (!byLanguage) {
+            byLanguage = new Map();
+            for (const row of preferredStatement.all(id)) {
+                const list = byLanguage.get(row.language) ?? [];
+                list.push(row.name);
+                byLanguage.set(row.language, list);
+            }
+            preferredByID.set(id, byLanguage);
+        }
+        return byLanguage;
+    };
+    const namesIn = (byLanguage, languages) => languages.flatMap((language) => byLanguage.get(language) ?? []);
     const out = [];
     for (const cc of countries) {
         const convention = POSTCODE_CONVENTIONS.get(cc);
         if (!convention)
             continue;
+        const officialLanguages = officialLanguagesAlpha3(cc);
         for (const row of statement.all(cc)) {
-            if (!row["postcode"] || !row["locality"] || !row["region"])
+            if (!row.postcode || !row.locality || !row.region)
                 continue;
-            out.push({
-                postcode: row["postcode"],
-                locality: row["locality"],
-                region: row["region"],
-                country: row["country"] ?? "",
-                cc,
-                locale: convention.locale,
-                postcodePlacement: convention.placement,
-            });
+            const regionOfficial = namesIn(preferredNames(row.region_id), officialLanguages);
+            // The co-official table is keyed by the region's name in the first official language; a region the names
+            // table has no such name for is looked up by its `spr.name`, which for a monolingual country is the same string.
+            const regionLanguages = regionLanguagesAlpha3(cc, regionOfficial[0] ?? row.region);
+            const regionCoOfficialLanguages = regionLanguages.filter((language) => !officialLanguages.includes(language));
+            const regionNames = {
+                official: regionOfficial,
+                coOfficial: namesIn(preferredNames(row.region_id), regionCoOfficialLanguages),
+            };
+            const localityNames = {
+                official: namesIn(preferredNames(row.locality_id), officialLanguages),
+                coOfficial: [],
+            };
+            const locality = localityWrittenForm(row.locality, localityNames);
+            for (const region of regionWrittenForms(row.region, regionNames)) {
+                out.push({
+                    postcode: row.postcode,
+                    locality,
+                    region,
+                    country: row.country ?? "",
+                    cc,
+                    locale: convention.locale,
+                    postcodePlacement: convention.placement,
+                });
+            }
         }
     }
     return out;

package/out/tools/postcode-triples.js.map

@@ -1 +1 @@
-{"version":3,"file":"postcode-triples.js","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAA;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAsBhD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA0E,IAAI,GAAG,CAAC;IAClH,kHAAkH;IAClH,iGAAiG;IACjG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,mHAAmH;IACnH,2GAA2G;IAC3G,+CAA+C;IAC/C,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACxD,mHAAmH;IACnH,sGAAsG;IACtG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;CACtD,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAExC;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,QAAgB,sBAAsB;IAEtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAA;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI,KAAK;YAAE,SAAQ;QAExB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,MAA4C;IAE5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAEvE,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAE/B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAEnC,IAAI,CAAC,IAAI,GAAG;YAAE,SAAQ;QAEtB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,SAA4B,EAC5B,UAAqD,EAAE;IAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC1F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAErG,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAE9E,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvE,EAAE,CAAC,IAAI,CAAC,oBAAoB,eAAe,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;IAEjE,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;EAU5B,CAAC,CAAA;IAEF,MAAM,GAAG,GAAqB,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE/C,IAAI,CAAC,UAAU;YAAE,SAAQ;QAEzB,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAkC,EAAE,CAAC;YACtE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAAE,SAAQ;YAEpE,GAAG,CAAC,IAAI,CAAC;gBACR,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAC;gBACzB,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAC;gBACzB,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE;gBAC7B,EAAE;gBACF,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;aACvC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAAe,EAAE,OAAgB;IAC/E,MAAM,IAAI,GAAG,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAE1F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;IAEhD,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEpE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,MAAM,GAAG,IAAI,EAAE;SAClB,OAAO,CAAC,oFAAoF,CAAC;SAC7F,GAAG,CAAC,OAAO,CAAmC,EAAE,CAAC;QAClD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACd,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QAClC,CAAC;IACF,CAAC;IAED,iHAAiH;IACjH,gGAAgG;IAChG,IAAI,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;IAElC,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;AACvD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,OAAe,EACf,IAAY,EACZ,WAAmB,EACnB,UAA2D,EAAE;IAE7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAEpD,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAEvD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,CAAC,MAAM,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAA;IAC5F,MAAM,GAAG,GAAqB,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAA4B,EAAE,CAAC;QACxG,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACvE,MAAM,iBAAiB,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7E,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACzE,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAEvE,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM;YAAE,SAAQ;QAE/C,iHAAiH;QACjH,0EAA0E;QAC1E,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAExC,iGAAiG;QACjG,MAAM,GAAG,GAAG,iBAAiB,IAAI,iBAAiB,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;QAExF,yGAAyG;QACzG,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAA;QAEhE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAEb,GAAG,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,QAAQ;YACR,MAAM;YACN,OAAO,EAAE,WAAW;YACpB,EAAE,EAAE,OAAO;YACX,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;SACvC,CAAC,CAAA;IACH,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,WAAoB;IACvE,MAAM,IAAI,GAAG,WAAW,IAAI,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;IAErE,OAAO,IAAI,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;AACrE,CAAC"}
+{"version":3,"file":"postcode-triples.js","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AACzF,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,mCAAmC,CAAA;AAE3E,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAsBhD;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA0E,IAAI,GAAG,CAAC;IAClH,kHAAkH;IAClH,iGAAiG;IACjG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,mHAAmH;IACnH,2GAA2G;IAC3G,+CAA+C;IAC/C,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACxD,mHAAmH;IACnH,sGAAsG;IACtG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;CACtD,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAExC;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,KAAK,GAAW,sBAAsB;IAEtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAA;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI,KAAK;YAAE,SAAQ;QAExB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,MAA4C;IAE5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAEvE,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAE/B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAEnC,IAAI,CAAC,IAAI,GAAG;YAAE,SAAQ;QAEtB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAgBD;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,KAAqB;IACxE,MAAM,GAAG,GAAa,EAAE,CAAA;IAExB,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAA;QAE9C,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACf,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;GAIG;AACH,MAAM,gBAAgB,GAAG,4BAA4B,CAAA;AAErD;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe,EAAE,KAAqB;IACzE,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAA;IAEhC,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,OAAO,CAAA;AAChG,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC9C,SAA4B,EAC5B,OAAO,GAA8C,EAAE;IAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC1F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAErG,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAE9E,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvE,EAAE,CAAC,IAAI,CAAC,oBAAoB,eAAe,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;IAEjE,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;EAW5B,CAAC,CAAA;IAEF,MAAM,kBAAkB,GAAG,EAAE,CAAC,OAAO,CACpC,6HAA6H,CAC7H,CAAA;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAAiC,CAAA;IAE9D;;OAEG;IACH,MAAM,cAAc,GAAG,CAAC,EAAU,EAAyB,EAAE;QAC5D,IAAI,UAAU,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEtC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,UAAU,GAAG,IAAI,GAAG,EAAE,CAAA;YAEtB,KAAK,MAAM,GAAG,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAA8C,EAAE,CAAC;gBAC3F,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAE/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;gBACnB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;YACnC,CAAC;YAED,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAA;QAClC,CAAC;QAED,OAAO,UAAU,CAAA;IAClB,CAAC,CAAA;IAED,MAAM,OAAO,GAAG,CAAC,UAAiC,EAAE,SAA4B,EAAY,EAAE,CAC7F,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAA;IAEhE,MAAM,GAAG,GAAqB,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE/C,IAAI,CAAC,UAAU;YAAE,SAAQ;QAEzB,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,EAAE,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAOhC,EAAE,CAAC;YACJ,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,MAAM;gBAAE,SAAQ;YAE3D,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,iBAAiB,CAAC,CAAA;YAChF,yGAAyG;YACzG,iHAAiH;YACjH,MAAM,eAAe,GAAG,qBAAqB,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAA;YAClF,MAAM,yBAAyB,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;YAE7G,MAAM,WAAW,GAAmB;gBACnC,QAAQ,EAAE,cAAc;gBACxB,UAAU,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,yBAAyB,CAAC;aAC7E,CAAA;YAED,MAAM,aAAa,GAAmB;gBACrC,QAAQ,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC;gBACrE,UAAU,EAAE,EAAE;aACd,CAAA;YAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,GAAG,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;YAEjE,KAAK,MAAM,MAAM,IAAI,kBAAkB,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,CAAC;gBAClE,GAAG,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,QAAQ;oBACR,MAAM;oBACN,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;oBAC1B,EAAE;oBACF,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;iBACvC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAAe,EAAE,OAAgB;IAC/E,MAAM,IAAI,GAAG,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAE1F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;IAEhD,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAEpE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,MAAM,GAAG,IAAI,EAAE;SAClB,OAAO,CAAC,oFAAoF,CAAC;SAC7F,GAAG,CAAC,OAAO,CAAmC,EAAE,CAAC;QAClD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACd,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;QAClC,CAAC;IACF,CAAC;IAED,iHAAiH;IACjH,gGAAgG;IAChG,IAAI,CAAC,KAAK,CAAC,IAAI;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;IAElC,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;AACvD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,OAAe,EACf,IAAY,EACZ,WAAmB,EACnB,OAAO,GAAoD,EAAE;IAE7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAEpD,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAEvD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,CAAC,MAAM,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAA;IAC5F,MAAM,GAAG,GAAqB,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAA4B,EAAE,CAAC;QACxG,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACvE,MAAM,iBAAiB,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC7E,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QACzE,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAEvE,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM;YAAE,SAAQ;QAE/C,iHAAiH;QACjH,0EAA0E;QAC1E,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAExC,iGAAiG;QACjG,MAAM,GAAG,GAAG,iBAAiB,IAAI,iBAAiB,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;QAExF,yGAAyG;QACzG,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAA;QAEhE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAEb,GAAG,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,QAAQ;YACR,MAAM;YACN,OAAO,EAAE,WAAW;YACpB,EAAE,EAAE,OAAO;YACX,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;SACvC,CAAC,CAAA;IACH,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,WAAoB;IACvE,MAAM,IAAI,GAAG,WAAW,IAAI,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;IAErE,OAAO,IAAI,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;AACrE,CAAC"}

package/out/tools/slice/kryptonite.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
+ *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
+ *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *
+ *   Invoke via `mailwoman corpus slice kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+export interface SliceKryptoniteOptions {
+    jsonl: string;
+    baseManifest: string;
+    outDir: string;
+    /**
+     * Default `"0.4.0"`.
+     */
+    corpusVersion?: string;
+    /**
+     * Default `"deepseek-kryptonite"`.
+     */
+    source?: string;
+}
+export declare function buildKryptoniteSlice(options: SliceKryptoniteOptions, report?: (line: string) => void): Promise<void>;
+//# sourceMappingURL=kryptonite.d.ts.map

package/out/tools/slice/kryptonite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"kryptonite.d.ts","sourceRoot":"","sources":["../../../lib/tools/slice/kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAWH,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAiCD,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Df"}

package/out/tools/slice/kryptonite.js

@@ -0,0 +1,99 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet slice from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
+ *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every slice from
+ *   v0.3.0 plus the new kryptonite slice(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *
+ *   Invoke via `mailwoman corpus slice kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { pathExists, readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers";
+import { join } from "path-ts";
+import { JSONSpliterator } from "spliterator";
+import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SLICE_COMPRESSION, writeSlices } from "#utils";
+async function* canonicalRows(jsonl, corpusVersion) {
+    for await (const raw of JSONSpliterator.fromAsync(jsonl)) {
+        // Strip sidecar underscore-prefixed fields the generator left behind for debugging.
+        const components = raw["components"];
+        yield {
+            raw: raw["raw"],
+            components,
+            country: raw["country"] ?? "US",
+            locale: raw["locale"] ?? undefined,
+            source: raw["source"] ?? "deepseek-kryptonite",
+            source_id: raw["source_id"],
+            corpus_version: corpusVersion,
+            license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+            synth: raw["synth"],
+        };
+    }
+}
+async function* labeledRows(jsonl, corpusVersion, quarantineLog) {
+    for await (const row of canonicalRows(jsonl, corpusVersion)) {
+        const result = alignRow(row);
+        if (result.kind === "labeled") {
+            yield result.row;
+        }
+        else {
+            quarantineLog.push(`${row.source_id}\t${result.row.reason}`);
+        }
+    }
+}
+export async function buildKryptoniteSlice(options, report) {
+    const corpusVersion = options.corpusVersion ?? "0.4.0";
+    const source = options.source ?? "deepseek-kryptonite";
+    if (!(await pathExists(options.jsonl)))
+        throw new Error(`jsonl not found: ${options.jsonl}`);
+    if (!(await pathExists(options.baseManifest)))
+        throw new Error(`base-manifest not found: ${options.baseManifest}`);
+    await makeDirectories(options.outDir);
+    const quarantine = [];
+    const newManifest = await writeSlices({ train: labeledRows(options.jsonl, corpusVersion, quarantine) }, { outputDir: options.outDir, corpusVersion });
+    report?.(`wrote ${newManifest.total_rows} rows into ${newManifest.slices.length} slice(s); ` +
+        `quarantined ${quarantine.length}`);
+    if (quarantine.length) {
+        const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv");
+        await writeLocalTextFile(quarantine.join("\n") + "\n", qPath);
+        report?.(`quarantine log → ${qPath}`);
+    }
+    // Stamp the new slice's source field for audit.ts (which prefers slice.source over
+    // first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
+    // to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
+    for (const sh of newManifest.slices) {
+        sh.source = source;
+    }
+    // Compose the final corpus-v0.4.0 manifest: every slice from base + the new slice(s).
+    const base = await readLocalJSONFile(options.baseManifest);
+    const combined = {
+        corpus_version: corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_slice: base.rows_per_slice,
+        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+        slices: [...base.slices, ...newManifest.slices],
+        counts: {
+            train: base.counts.train + (newManifest.counts.train ?? 0),
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newManifest.total_rows,
+    };
+    // Stamp source on the legacy v0.3.0 slices too, so audit's slice.source path is the
+    // authoritative one. v0.3.0 slices mix sources; we use the first_source_id-prefix
+    // inference for them (audit.ts will re-derive on its own when slice.source is absent).
+    const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json");
+    await writeLocalJSONFile(combined, combinedPath);
+    report?.(`wrote combined manifest → ${combinedPath}`);
+    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`);
+    report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newManifest.slices.length})`);
+    report?.(`  compression=${SLICE_COMPRESSION}`);
+}
+//# sourceMappingURL=kryptonite.js.map

package/out/tools/slice/kryptonite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kryptonite.js","sourceRoot":"","sources":["../../../lib/tools/slice/kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAiBlG,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,KAAK,CAAC,EAAE,CAAC;QACnF,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAE9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;SAC5C,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAE5F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAElH,MAAM,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAErC,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,CAC5C,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,aAAa;QAClF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,2BAA2B,CAAC,CAAA;QAC3F,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAA;QAC7D,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,mFAAmF;IACnF,sFAAsF;IACtF,yFAAyF;IACzF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACrC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAA;IACnB,CAAC;IAED,sFAAsF;IACtF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAgB,OAAO,CAAC,YAAY,CAAC,CAAA;IAEzE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IAED,oFAAoF;IACpF,kFAAkF;IAClF,uFAAuF;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,eAAe,CAAC,CAAA;IACtF,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAChD,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;AAC/C,CAAC"}

package/out/tools/slice/translit.d.ts

@@ -0,0 +1,51 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet slices from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 slices.
+ *
+ *   Sibling to `slice-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new slices, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one slice per script so `audit.ts` can attribute each slice to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each slice is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 slice paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus slice translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+export interface SliceTranslitOptions {
+    jsonl: string;
+    baseManifest: string;
+    outDir: string;
+    /**
+     * Default `"0.4.0"`.
+     */
+    corpusVersion?: string;
+    /**
+     * Default `"/data/"`.
+     */
+    canonicalPathPrefix?: string;
+    /**
+     * Prefix the base manifest's slice paths currently carry, to be rewritten to
+     * {@link SliceTranslitOptions.canonicalPathPrefix}. Defaults to `mailwomanDataRoot()` with a trailing slash — the
+     * root that WROTE those paths. Pass it explicitly when translating a manifest generated under a different
+     * `$MAILWOMAN_DATA_ROOT` than the one you are running with.
+     */
+    legacyPathPrefix?: string;
+}
+export declare function buildTranslitSlice(options: SliceTranslitOptions, report?: (line: string) => void): Promise<void>;
+//# sourceMappingURL=translit.d.ts.map

package/out/tools/slice/translit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"translit.d.ts","sourceRoot":"","sources":["../../../lib/tools/slice/translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAaH,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AA0DD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CAgGf"}

package/out/tools/slice/translit.js

@@ -0,0 +1,166 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet slices from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 slices.
+ *
+ *   Sibling to `slice-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new slices, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one slice per script so `audit.ts` can attribute each slice to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each slice is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 slice paths are
+ *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus slice translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+import { mailwomanDataRoot } from "@mailwoman/core/data-root";
+import { pathExists, readLocalJSONFile, tryStat } from "@mailwoman/core/fs/readers";
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { join } from "path-ts";
+import { JSONSpliterator } from "spliterator";
+import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SLICE_COMPRESSION, writeParquetRows } from "#utils";
+function toCanonicalRow(raw, corpusVersion) {
+    return {
+        raw: raw["raw"],
+        components: raw["components"],
+        country: raw["country"] ?? "US",
+        locale: raw["locale"] ?? undefined,
+        source: raw["source"],
+        source_id: raw["source_id"],
+        corpus_version: corpusVersion,
+        license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+        synth: raw["synth"],
+    };
+}
+/**
+ * Write one slice for a single source slug. Returns the populated SliceDescriptor + a list of quarantine reasons for
+ * rows that failed alignment.
+ */
+async function writeOneSlice(rows, outPath, source) {
+    let firstSourceID = "";
+    let lastSourceID = "";
+    for (const row of rows) {
+        if (firstSourceID === "") {
+            firstSourceID = row.source_id;
+        }
+        lastSourceID = row.source_id;
+    }
+    await writeParquetRows(rows.map(rowToParquet), outPath);
+    const fileStat = await tryStat(outPath);
+    const sha256 = await sha256File(outPath);
+    return {
+        split: "train",
+        path: outPath,
+        format: "parquet",
+        compression: SLICE_COMPRESSION,
+        rows: rows.length,
+        bytes: fileStat?.size ?? 0,
+        sha256,
+        first_source_id: firstSourceID,
+        last_source_id: lastSourceID,
+        // Stamp source so audit.ts attributes the slice without falling back to filename-prefix inference.
+        source,
+    };
+}
+function canonicalizeSlicePath(path, legacyPrefix, canonicalPrefix) {
+    if (path.startsWith(legacyPrefix))
+        return canonicalPrefix + path.slice(legacyPrefix.length);
+    return path;
+}
+export async function buildTranslitSlice(options, report) {
+    const corpusVersion = options.corpusVersion ?? "0.4.0";
+    // This is a portable manifest namespace, not a host filesystem default.
+    const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/";
+    const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`;
+    if (!(await pathExists(options.jsonl)))
+        throw new Error(`jsonl not found: ${options.jsonl}`);
+    if (!(await pathExists(options.baseManifest)))
+        throw new Error(`base-manifest not found: ${options.baseManifest}`);
+    const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`);
+    const trainDir = join(corpusDir, "train");
+    await makeDirectories(trainDir);
+    // Bucket canonical rows by source. Quarantined rows are logged.
+    const buckets = new Map();
+    const quarantine = [];
+    let totalIn = 0;
+    for await (const raw of JSONSpliterator.fromAsync(options.jsonl)) {
+        totalIn++;
+        const canon = toCanonicalRow(raw, corpusVersion);
+        const result = alignRow(canon);
+        if (result.kind !== "labeled") {
+            quarantine.push(`${canon.source_id}\t${result.row.reason}`);
+            continue;
+        }
+        const bucket = buckets.get(canon.source);
+        if (bucket) {
+            bucket.push(result.row);
+        }
+        else {
+            buckets.set(canon.source, [result.row]);
+        }
+    }
+    report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`);
+    const newSlices = [];
+    const sortedKeys = [...buckets.keys()].toSorted();
+    for (const source of sortedKeys) {
+        const rows = buckets.get(source);
+        const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source;
+        const outPath = join(trainDir, `part-translit-${slug}.parquet`);
+        const descriptor = await writeOneSlice(rows, outPath, source);
+        newSlices.push(descriptor);
+        report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`);
+    }
+    if (quarantine.length) {
+        const qPath = join(corpusDir, "quarantine-transliteration.tsv");
+        await writeLocalTextFile(quarantine.join("\n") + "\n", qPath);
+        report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`);
+    }
+    // Compose final MANIFEST: rewrite base.slices paths from /mnt/playpen/... → /data/... and append
+    // the new translit slices. Kryptonite slice already lives in the base manifest (it was written
+    // there by Thread B).
+    const base = await readLocalJSONFile(options.baseManifest);
+    const rewrittenBase = base.slices.map((sh) => ({
+        ...sh,
+        path: canonicalizeSlicePath(sh.path, legacyPathPrefix, canonicalPathPrefix),
+    }));
+    const newTrainRows = newSlices.reduce((sum, sh) => sum + sh.rows, 0);
+    const combined = {
+        corpus_version: corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_slice: base.rows_per_slice,
+        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+        slices: [...rewrittenBase, ...newSlices],
+        counts: {
+            train: base.counts.train + newTrainRows,
+            val: base.counts.val,
+            test: base.counts.test,
+        },
+        total_rows: base.total_rows + newTrainRows,
+    };
+    const combinedPath = join(corpusDir, "MANIFEST.json");
+    await writeLocalJSONFile(combined, combinedPath);
+    report?.(`wrote combined manifest → ${combinedPath}`);
+    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`);
+    report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newSlices.length})`);
+    report?.(`  compression=${SLICE_COMPRESSION}`);
+    const pathFix = rewrittenBase.filter((s, i) => s.path !== base.slices[i].path).length;
+    if (pathFix > 0) {
+        report?.(`  path-canonicalized base slices: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`);
+    }
+}
+//# sourceMappingURL=translit.js.map

package/out/tools/slice/translit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"translit.js","sourceRoot":"","sources":["../../../lib/tools/slice/translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACnF,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAA;AAwBrH,SAAS,cAAc,CAAC,GAA4B,EAAE,aAAqB;IAC1E,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;QACzB,UAAU,EAAE,GAAG,CAAC,YAAY,CAA2B;QACvD,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;QAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;QAC9C,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;QACrC,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;QACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;KAC5C,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAAC,IAA2B,EAAE,OAAe,EAAE,MAAc;IACxF,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YAC1B,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;QAC9B,CAAC;QAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;IAC7B,CAAC;IAED,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAA;IAEvD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,OAAO;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;QAC1B,MAAM;QACN,eAAe,EAAE,aAAa;QAC9B,cAAc,EAAE,YAAY;QAC5B,mGAAmG;QACnG,MAAM;KACN,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,YAAoB,EAAE,eAAuB;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3F,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,wEAAwE;IACxE,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,QAAQ,CAAA;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,iBAAiB,EAAE,GAAG,CAAA;IAE9E,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAE5F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAElH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAE/B,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC/C,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,OAAO,EAAE,CAAA;QACT,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YAE3D,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAExC,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,QAAQ,OAAO,UAAU,UAAU,CAAC,MAAM,iBAAiB,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAA;IAElG,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEjD,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,IAAI,UAAU,CAAC,CAAA;QAC/D,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC7D,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC1B,MAAM,EAAE,CAAC,KAAK,MAAM,KAAK,UAAU,CAAC,IAAI,WAAW,OAAO,KAAK,UAAU,CAAC,KAAK,SAAS,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAA;QAC/D,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAA;QAC7D,MAAM,EAAE,CAAC,oBAAoB,KAAK,KAAK,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,sBAAsB;IACtB,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAgB,OAAO,CAAC,YAAY,CAAC,CAAA;IAEzE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,GAAG,EAAE;QACL,IAAI,EAAE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,mBAAmB,CAAC;KAC3E,CAAC,CAAC,CAAA;IAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,YAAY;YACvC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,YAAY;KAC1C,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACrD,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAChD,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,YAAY,GAAG,CAAC,CAAA;IAChG,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtG,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAEtF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,qCAAqC,OAAO,aAAa,gBAAgB,QAAQ,mBAAmB,IAAI,CAAC,CAAA;IACnH,CAAC;AACF,CAAC"}