@mailwoman/corpus 9.0.0 → 9.2.0

package/out/src/tools/postcode-triples.js

@@ -0,0 +1,304 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Extract `(postcode, locality, region, country)` tuples for the `trailing-region` shard.
+ *
+ *   This step used to be a one-off. Its output survived — 17,908 rows at
+ *   `$MAILWOMAN_DATA_ROOT/corpus/tuples/trailing-region-structured-tuples.jsonl` — but the code that produced it did
+ *   not, so a note describing the join was the only record and it did not match what the databases contain. That is the
+ *   reason this file exists: the shard is unbuildable for a new country without it.
+ *
+ *   ## Two sources, because one route does not reach every country
+ *
+ *   - **`postalcode-intl.db`** carries a real `parent_id` that resolves in the admin gazetteer. Measured share of rows
+ *     with a parent: NL 97.5%, FR 90.7%, DE 66.1%, ES 34.9%, IT 27.4%; of those, 93.8–100% land on a `locality` or
+ *     `localadmin`, and the region comes from that place's own ancestry. It is the ONLY shard with this — every
+ *     `postalcode-geonames-*` and `postalcode-<cc>-overture.db` row reads `parent_id = 0`.
+ *   - **GeoNames postal exports** carry the place and admin1 NAMES in columns 3 and 4, so there is nothing to join.
+ *     `mailwoman corpus fetch geonames-postal` puts them on disk.
+ *
+ *   A nearest-locality-centroid join was measured as the general fallback and REJECTED: scored against the `parent_id`
+ *   truth, it agreed NL 81.1% / DE 44.7% / ES 35.1% / FR 29.0% / IT 13.5%. A locality's centroid sits at its middle, so
+ *   an edge postcode is routinely nearer a neighbouring town's centroid. Do not reach for it again.
+ *
+ *   ## The hub cap is PER COUNTRY, because a pooled one is a mixture
+ *
+ *   A few localities act as catch-all parents — `Schwedt/Oder` claims 9,222 DE postcodes against a DE median of 1. Left
+ *   in, a handful of places dominate the shard. But the distribution differs so much by country that one threshold is
+ *   not one rule: a p99 pooled across the five countries lands at 522, which keeps 100% of ES and IT, 83% of FR, 49% of
+ *   NL and 19% of DE.
+ *
+ *   So the bound is a QUOTA and not a threshold. A threshold DELETES a locality that exceeds it, which removes exactly
+ *   the largest cities — the places a parser most needs to have seen. A quota keeps every locality and bounds how many
+ *   of its postcodes ride along, which is the balance the cap was reaching for without the deletion.
+ *
+ *   ## Placement is data, not a formatting choice
+ *
+ *   Each tuple is stamped with its country's {@link PostcodePlacement}. The same digits change tag with position, so a
+ *   tuple that does not carry its placement teaches whichever convention the recipe happens to default to — see the
+ *   recipe's header for the measurement.
+ */
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+import { dataRootPath } from "@mailwoman/core/utils";
+import { TSVSpliterator } from "spliterator";
+/**
+ * Where a country writes the postcode, and the locale tag its rows carry.
+ *
+ * A country is in this table only when a gauntlet board row ATTESTS its surface. An absent country is not an oversight
+ * to be filled in by guessing: extracting it with the wrong placement teaches a convention that country does not use,
+ * which is worse than not teaching it at all. `LEADING_POSTCODE_COUNTRIES` in `@mailwoman/neural`'s
+ * `placetype-pair-prior.ts` draws the same line for the same reason.
+ *
+ * AU and ZA are the worked examples of the bar. Both look like obvious additions and neither qualifies: the board's AU
+ * rows are bare-city (`Melbourne`, `Sydney, Australia`) and carry no postcode at all, so nothing here says where AU
+ * writes it; and ZA's `14 Long St, Green Point, Cape Town, 8001` carries no REGION, which this shard requires — a fact
+ * its GeoNames export agrees with, at 100% place and 0% admin1.
+ */
+export const POSTCODE_CONVENTIONS = new Map([
+    // `Rue de l'Église, 3, 29217 Plougonvelin, Bretagne, France` and its siblings — `fr_structured`, `de_structured`,
+    // `es_structured`, `it_structured`, `pt_structured`, `mx_supermanzana`, `nl-op4-p-r-sloterdijk`.
+    ["FR", { placement: "leading", locale: "fr-FR" }],
+    ["DE", { placement: "leading", locale: "de-DE" }],
+    ["ES", { placement: "leading", locale: "es-ES" }],
+    ["IT", { placement: "leading", locale: "it-IT" }],
+    ["NL", { placement: "leading", locale: "nl-NL" }],
+    ["PT", { placement: "leading", locale: "pt-PT" }],
+    ["MX", { placement: "leading", locale: "es-MX" }],
+    // `…, Barcelona 6001, Anzoátegui, Venezuela` — the four `ve_city_postcode_trailing_state` rows. No postcode source
+    // on disk and GeoNames does not publish VE, so this entry currently yields nothing; it is here because the
+    // placement is what makes the absence legible.
+    ["VE", { placement: "after_locality", locale: "es-VE" }],
+    // `12 MG Road, Indiranagar, Bengaluru, Karnataka 560038, India` — three `in_*` rows, and `AGENTS.md` says the same
+    // ("en-IN is absent BECAUSE the PIN goes last"). The one trailing placement with real data behind it.
+    ["IN", { placement: "after_region", locale: "en-IN" }],
+]);
+/**
+ * How many postcodes one locality may contribute.
+ *
+ * A quota rather than a cut-off — see the header. It bounds repetition without deleting a locality: a city with 9,222
+ * postcodes contributes this many and stays in the shard.
+ */
+export const DEFAULT_LOCALITY_QUOTA = 24;
+/**
+ * Take at most `quota` tuples per locality, in the order they arrive.
+ *
+ * Order matters and is the caller's to choose: both readers below walk their source in id / file order, which is stable
+ * across runs, so the same quota selects the same rows.
+ */
+export function applyLocalityQuota(triples, quota = DEFAULT_LOCALITY_QUOTA) {
+    const seen = new Map();
+    const kept = [];
+    for (const triple of triples) {
+        const key = `${triple.cc} ${triple.locality}`;
+        const n = seen.get(key) ?? 0;
+        if (n >= quota)
+            continue;
+        seen.set(key, n + 1);
+        kept.push(triple);
+    }
+    return kept;
+}
+/**
+ * Take at most `budget` tuples per COUNTRY, in the order they arrive.
+ *
+ * A per-locality quota bounds how often one place repeats; it cannot bound a country. IN has 128,152 distinct
+ * localities, so even at a quota of ONE it contributes 63,533 rows against 39,790 from the other seven combined — the
+ * shard would teach the trailing surface as an Indian fact rather than a general one, and at 103,323 rows it would take
+ * 30% of an 8,000-step run's sample budget at three reps per row.
+ *
+ * Applied AFTER {@link applyLocalityQuota}, so a country's budget is spent on breadth (many localities) rather than on
+ * one city's postcode list.
+ */
+export function applyCountryBudget(triples, budget) {
+    const spent = new Map();
+    const kept = [];
+    for (const triple of triples) {
+        const cap = typeof budget === "number" ? budget : budget.get(triple.cc);
+        if (cap === undefined)
+            continue;
+        const n = spent.get(triple.cc) ?? 0;
+        if (n >= cap)
+            continue;
+        spent.set(triple.cc, n + 1);
+        kept.push(triple);
+    }
+    return kept;
+}
+/**
+ * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
+ * place's ancestry to a region.
+ *
+ * A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
+ * a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
+ * actually reachable.
+ */
+export function readTriplesFromParentJoin(countries, options = {}) {
+    const postcodeDB = options.postcodeDB ?? String(dataRootPath("wof", "postalcode-intl.db"));
+    const adminDB = options.adminDB ?? String(dataRootPath("wof", "admin-global-priority-importance.db"));
+    if (!existsSync(postcodeDB) || !existsSync(adminDB))
+        return [];
+    const db = new DatabaseSync(adminDB, { readOnly: true });
+    try {
+        db.exec(`ATTACH DATABASE '${postcodeDB.replaceAll("'", "''")}' AS pc`);
+        const statement = db.prepare(`
+			SELECT p.name AS postcode, a.name AS locality, r.name AS region, c.name AS country, p.country AS cc
+			FROM pc.spr p
+			JOIN spr a ON a.id = p.parent_id AND a.placetype IN ('locality', 'localadmin')
+			JOIN ancestors anc ON anc.id = a.id AND anc.ancestor_placetype = 'region'
+			JOIN spr r ON r.id = anc.ancestor_id
+			JOIN ancestors cnc ON cnc.id = a.id AND cnc.ancestor_placetype = 'country'
+			JOIN spr c ON c.id = cnc.ancestor_id
+			WHERE p.country = ? AND p.parent_id > 0
+			ORDER BY p.id
+		`);
+        const out = [];
+        for (const cc of countries) {
+            const convention = POSTCODE_CONVENTIONS.get(cc);
+            if (!convention)
+                continue;
+            for (const row of statement.all(cc)) {
+                if (!row["postcode"] || !row["locality"] || !row["region"])
+                    continue;
+                out.push({
+                    postcode: row["postcode"],
+                    locality: row["locality"],
+                    region: row["region"],
+                    country: row["country"] ?? "",
+                    cc,
+                    locale: convention.locale,
+                    postcodePlacement: convention.placement,
+                });
+            }
+        }
+        return out;
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * GeoNames postal columns (0-based), matching `@mailwoman/corpus`'s `geonames-postal` adapter.
+ */
+const GEONAMES_COL = { country: 0, postcode: 1, place: 2, admin1: 3, admin2: 5 };
+/**
+ * A predicate answering whether a name is a LOCALITY the admin gazetteer knows, for one country.
+ *
+ * The parent-join reader gets this for free — its query restricts the parent to `locality`/`localadmin`, so every name
+ * it emits is one by construction. The GeoNames reader has no such guarantee, and the gap is large enough to matter:
+ * sampling 400 rows per country against the gazetteer, the share of GeoNames place names that name a locality we know
+ * is PT 75%, IN 62%, **MX 41%**. The Mexican misses are colonias — `Zona Centro`, `San Fernando INFONAVIT`, `FOVISSSTE
+ * 3a Sección` — and a row teaching one of those as `locality` trains the locality/dependent_locality boundary in the
+ * wrong direction. Dropping the row instead costs coverage and teaches nothing false, which is the better of the two.
+ *
+ * Returns a predicate that answers `true` for everything when the gazetteer is not on disk, so a checkout without it
+ * builds the same rows it did before rather than silently emitting none.
+ */
+export function createKnownLocalityGate(country, adminDB) {
+    const path = adminDB ?? String(dataRootPath("wof", "admin-global-priority-importance.db"));
+    if (!existsSync(path))
+        return () => true;
+    const db = new DatabaseSync(path, { readOnly: true });
+    try {
+        const names = new Set();
+        for (const row of db
+            .prepare("SELECT name FROM spr WHERE country = ? AND placetype IN ('locality', 'localadmin')")
+            .all(country)) {
+            if (row.name) {
+                names.add(row.name.toLowerCase());
+            }
+        }
+        // An empty set means the gazetteer has no localities for this country at all, which is a coverage fact about the
+        // gazetteer rather than a verdict on the source — so gate nothing rather than drop everything.
+        if (!names.size)
+            return () => true;
+        return (name) => names.has(name.toLowerCase());
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Read triples straight out of a GeoNames `<CC>.txt` export — no join, the names are columns 3 and 4.
+ *
+ * COLUMN 3 IS NOT THE LOCALITY. It is the finest-grained named place for the code, and for `560001` that is `Mahatma
+ * Gandhi Road` — a STREET — while the city, `Bengaluru`, is column 5 (admin2). MX is the same shape (`Roma Norte` is a
+ * colonia inside `Cuauhtémoc`) and so is PT (`Abrigada` inside `Alenquer`). Reading column 3 as the locality is how the
+ * v4.8.0 shard came to teach street names as cities. So `admin2` is the locality, `admin1` the region, and column 3 the
+ * DEPENDENT locality — which is also the left context the shard needs.
+ *
+ * WHICH COUNTRIES THIS READER CAN SERVE. It needs `admin2` (the city) and `admin1` (the region), and a country can
+ * publish one without the other. Measured 2026-08-23 across the exports on disk:
+ *
+ * | country    | rows      | admin1 | admin2 | usable here                     |
+ * | ---------- | --------: | -----: | -----: | ------------------------------- |
+ * | PT, MX, IN | 145k–207k | 100%   | 100%   | yes                             |
+ * | BD, LK     | 1.3k–1.8k | 100%   | 100%   | yes                             |
+ * | PH         | 2,317     | 88%    | 88%    | yes, 88% of rows                |
+ * | PK         | 2,563     | 100%   | **0%** | NO — no city column             |
+ * | TH         | 903       | 100%   | **1%** | NO — effectively no city column |
+ * | ID         | 81,058    | **0%** | 0%     | NO — no region either           |
+ * | ZA         | 3,920     | **0%** | —      | NO                              |
+ *
+ * A country in the NO rows yields zero from this reader, and that is the correct outcome rather than a gap to route
+ * around: taking column 3 as the locality is what made the v4.8.0 shard train `Mahatma Gandhi Road` as a city. If one
+ * of them is wanted, it needs a city column from somewhere else, not a relaxed mapping.
+ *
+ * NOT PUBLISHED AT ALL by GeoNames, checked the same day: VE, VN, NP, MM, KH. Those are acquisition questions, and for
+ * VE specifically OpenAddresses 404s too — see the arc retrospective.
+ *
+ * Three source properties a caller cannot see from a row count, all handled here. Hyphen-format countries publish each
+ * code TWICE (`3750-000` and `3750000`, exactly 2.00× for PT and PL), so the first surface of a code wins and its twin
+ * is dropped. Some countries populate the place but not admin1 — ZA is 100% place, 0% region — which yields nothing
+ * this shard can use, so those rows are dropped rather than emitted with a blank region. And the "place name" is often
+ * a SUB-locality, which {@link createKnownLocalityGate} filters.
+ */
+export async function readTriplesFromGeonames(country, path, countryName, options = {}) {
+    const convention = POSTCODE_CONVENTIONS.get(country);
+    if (!convention || !existsSync(path))
+        return [];
+    const isKnownLocality = options.isKnownLocality ?? createKnownLocalityGate(country);
+    const out = [];
+    const seen = new Set();
+    for await (const cells of TSVSpliterator.fromAsync(path, { header: false })) {
+        const postcode = (cells[GEONAMES_COL.postcode] ?? "").trim();
+        const dependentLocality = (cells[GEONAMES_COL.place] ?? "").trim();
+        const locality = (cells[GEONAMES_COL.admin2] ?? "").trim();
+        const region = (cells[GEONAMES_COL.admin1] ?? "").trim();
+        if (!postcode || !locality || !region)
+            continue;
+        // The gate applies to the LOCALITY — admin2 — not to the fine-grained name, which is expected to be a street or
+        // a colonia and is emitted as the dependent locality rather than dropped.
+        if (!isKnownLocality(locality))
+            continue;
+        // A dependent locality that merely repeats its parent teaches a doubled segment, not a boundary.
+        const dep = dependentLocality && dependentLocality !== locality ? dependentLocality : "";
+        // The bare twin of a punctuated code carries no new fact, and keeping both doubles the country's weight.
+        const key = `${postcode.replaceAll("-", "")} ${locality} ${dep}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({
+            postcode,
+            ...(dep ? { dependentLocality: dep } : {}),
+            locality,
+            region,
+            country: countryName,
+            cc: country,
+            locale: convention.locale,
+            postcodePlacement: convention.placement,
+        });
+    }
+    return out;
+}
+/**
+ * Resolve a GeoNames export path under the standard fetch out-root.
+ */
+export function geonamesPostalPath(country, sourcesRoot) {
+    const root = sourcesRoot ?? String(dataRootPath("corpus", "sources"));
+    return join(root, "geonames-postal", `${country.toUpperCase()}.txt`);
+}
+//# sourceMappingURL=postcode-triples.js.map

package/out/src/tools/postcode-triples.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"postcode-triples.js","sourceRoot":"","sources":["../../../src/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAyB5C;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAA0E,IAAI,GAAG,CAAC;IAClH,kHAAkH;IAClH,iGAAiG;IACjG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACjD,mHAAmH;IACnH,2GAA2G;IAC3G,+CAA+C;IAC/C,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IACxD,mHAAmH;IACnH,sGAAsG;IACtG,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;CACtD,CAAC,CAAA;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAA;AAExC;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,QAAgB,sBAAsB;IAEtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAA;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI,KAAK;YAAE,SAAQ;QAExB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,kBAAkB,CACjC,OAAkC,EAClC,MAA4C;IAE5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IACvC,MAAM,IAAI,GAAqB,EAAE,CAAA;IAEjC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAEvE,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAE/B,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAEnC,IAAI,CAAC,IAAI,GAAG;YAAE,SAAQ;QAEtB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CACxC,SAA4B,EAC5B,UAAqD,EAAE;IAEvD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAC1F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAErG,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAA;IAE9D,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,IAAI,CAAC;QACJ,EAAE,CAAC,IAAI,CAAC,oBAAoB,UAAU,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAEtE,MAAM,SAAS,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;GAU5B,CAAC,CAAA;QAEF,MAAM,GAAG,GAAqB,EAAE,CAAA;QAEhC,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YAE/C,IAAI,CAAC,UAAU;gBAAE,SAAQ;YAEzB,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAkC,EAAE,CAAC;gBACtE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;oBAAE,SAAQ;gBAEpE,GAAG,CAAC,IAAI,CAAC;oBACR,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAC;oBACzB,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAC;oBACzB,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC;oBACrB,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE;oBAC7B,EAAE;oBACF,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;iBACvC,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAED,OAAO,GAAG,CAAA;IACX,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,KAAK,EAAE,CAAA;IACX,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,YAAY,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAW,CAAA;AAEzF;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAe,EAAE,OAAgB;IACxE,MAAM,IAAI,GAAG,OAAO,IAAI,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,qCAAqC,CAAC,CAAC,CAAA;IAE1F,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;IAExC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IAErD,IAAI,CAAC;QACJ,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;QAE/B,KAAK,MAAM,GAAG,IAAI,EAAE;aAClB,OAAO,CAAC,oFAAoF,CAAC;aAC7F,GAAG,CAAC,OAAO,CAAmC,EAAE,CAAC;YAClD,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,iHAAiH;QACjH,+FAA+F;QAC/F,IAAI,CAAC,KAAK,CAAC,IAAI;YAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAA;QAElC,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;IACvD,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,KAAK,EAAE,CAAA;IACX,CAAC;AACF,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC5C,OAAe,EACf,IAAY,EACZ,WAAmB,EACnB,UAA2D,EAAE;IAE7D,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAEpD,IAAI,CAAC,UAAU,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAA;IAE/C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAA;IACnF,MAAM,GAAG,GAAqB,EAAE,CAAA;IAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAA4B,EAAE,CAAC;QACxG,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC5D,MAAM,iBAAiB,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAClE,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAC1D,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;QAExD,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM;YAAE,SAAQ;QAE/C,gHAAgH;QAChH,0EAA0E;QAC1E,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAExC,iGAAiG;QACjG,MAAM,GAAG,GAAG,iBAAiB,IAAI,iBAAiB,KAAK,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAA;QAExF,yGAAyG;QACzG,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAA;QAEhE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAEb,GAAG,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,QAAQ;YACR,MAAM;YACN,OAAO,EAAE,WAAW;YACpB,EAAE,EAAE,OAAO;YACX,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,iBAAiB,EAAE,UAAU,CAAC,SAAS;SACvC,CAAC,CAAA;IACH,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,WAAoB;IACvE,MAAM,IAAI,GAAG,WAAW,IAAI,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;IAErE,OAAO,IAAI,CAAC,IAAI,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;AACrE,CAAC"}

package/out/src/tools/shard-kryptonite.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shard-kryptonite.d.ts","sourceRoot":"","sources":["../../../src/tools/shard-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAcH,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAiCD,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Df"}
+{"version":3,"file":"shard-kryptonite.d.ts","sourceRoot":"","sources":["../../../src/tools/shard-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAaH,MAAM,WAAW,sBAAsB;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAiCD,wBAAsB,oBAAoB,CACzC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA6Df"}

package/out/src/tools/shard-kryptonite.js

@@ -20,8 +20,7 @@ import { mkdir } from "node:fs/promises";
 import { join } from "node:path";
 import { parseJSONStrict } from "@mailwoman/core/objects";
 import { JSONSpliterator } from "spliterator";
-import { alignRow } from "../align.js";
-import { PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "../parquet.js";
+import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "#utils";
 async function* canonicalRows(jsonl, corpusVersion) {
     for await (const raw of JSONSpliterator.fromAsync(jsonl)) {
         // Strip sidecar underscore-prefixed fields the generator left behind for debugging.
@@ -71,7 +70,6 @@ export async function buildKryptoniteShard(options, report) {
     // first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
     // to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
     for (const sh of newManifest.shards) {
-        ;
         sh.source = source;
     }
     // Compose the final corpus-v0.4.0 manifest: every shard from base + the new shard(s).

package/out/src/tools/shard-kryptonite.js.map

@@ -1 +1 @@
-{"version":3,"file":"shard-kryptonite.js","sourceRoot":"","sources":["../../../src/tools/shard-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAiB/F,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,KAAK,CAAC,EAAE,CAAC;QACnF,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAE9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;SAC5C,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEhD,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,CAC5C,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,aAAa;QAClF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,2BAA2B,CAAC,CAAA;QAC3F,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,mFAAmF;IACnF,sFAAsF;IACtF,yFAAyF;IACzF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACrC,CAAC;QAAC,EAAoC,CAAC,MAAM,GAAG,MAAM,CAAA;IACvD,CAAC;IAED,sFAAsF;IACtF,MAAM,IAAI,GAAG,eAAe,CAAgB,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEvF,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IAED,oFAAoF;IACpF,kFAAkF;IAClF,uFAAuF;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,eAAe,CAAC,CAAA;IACtF,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;AAC/C,CAAC"}
+{"version":3,"file":"shard-kryptonite.js","sourceRoot":"","sources":["../../../src/tools/shard-kryptonite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAA;AAiBlG,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,KAAa,EAAE,aAAqB;IACjE,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,KAAK,CAAC,EAAE,CAAC;QACnF,oFAAoF;QACpF,MAAM,UAAU,GAAG,GAAG,CAAC,YAAY,CAA2B,CAAA;QAE9D,MAAM;YACL,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;YACzB,UAAU;YACV,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;YAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;YAC9C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,qBAAqB;YAC1D,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;YACrC,cAAc,EAAE,aAAa;YAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;YACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;SAC5C,CAAA;IACF,CAAC;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAC,KAAa,EAAE,aAAqB,EAAE,aAAuB;IACxF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,aAAa,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,MAAM,CAAC,GAAG,CAAA;QACjB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QAC7D,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAA;IAEtD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEhD,MAAM,UAAU,GAAa,EAAE,CAAA;IAE/B,MAAM,WAAW,GAAG,MAAM,WAAW,CACpC,EAAE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,EAChE,EAAE,SAAS,EAAE,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,CAC5C,CAAA;IAED,MAAM,EAAE,CACP,SAAS,WAAW,CAAC,UAAU,cAAc,WAAW,CAAC,MAAM,CAAC,MAAM,aAAa;QAClF,eAAe,UAAU,CAAC,MAAM,EAAE,CACnC,CAAA;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,2BAA2B,CAAC,CAAA;QAC3F,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,EAAE,CAAC,CAAA;IACtC,CAAC;IAED,mFAAmF;IACnF,sFAAsF;IACtF,yFAAyF;IACzF,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACrC,EAAE,CAAC,MAAM,GAAG,MAAM,CAAA;IACnB,CAAC;IAED,sFAAsF;IACtF,MAAM,IAAI,GAAG,eAAe,CAAgB,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEvF,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;YAC1D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU;KACpD,CAAA;IAED,oFAAoF;IACpF,kFAAkF;IAClF,uFAAuF;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,EAAE,eAAe,CAAC,CAAA;IACtF,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,WAAW,CAAC,UAAU,GAAG,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;AAC/C,CAAC"}

package/out/src/tools/shard-translit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shard-translit.d.ts","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAgBH,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AAsGD,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CA+Ff"}
+{"version":3,"file":"shard-translit.d.ts","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAwBH,MAAM,WAAW,oBAAoB;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AA0ED,wBAAsB,kBAAkB,CACvC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,IAAI,CAAC,CAgGf"}

package/out/src/tools/shard-translit.js

@@ -33,9 +33,8 @@ import { join } from "node:path";
 import { parseJSONStrict } from "@mailwoman/core/objects";
 import { mailwomanDataRoot, sha256File } from "@mailwoman/core/utils";
 import { JSONSpliterator } from "spliterator";
-import { alignRow } from "../align.js";
+import { alignRow, appendShape, LABELED_ROW_SCHEMA, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SHARD_COMPRESSION, } from "#utils";
 import { ParquetWriter } from "../parquet-wrapper/index.js";
-import { LABELED_ROW_SCHEMA, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SHARD_COMPRESSION } from "../parquet.js";
 function toCanonicalRow(raw, corpusVersion) {
     return {
         raw: raw["raw"],
@@ -49,28 +48,6 @@ function toCanonicalRow(raw, corpusVersion) {
         synth: raw["synth"],
     };
 }
-function appendShape(row) {
-    const out = {
-        raw: row.raw,
-        tokens: row.tokens,
-        labels: row.labels,
-        country: row.country,
-        source: row.source,
-        source_id: row.source_id,
-        corpus_version: row.corpus_version,
-        license: row.license,
-    };
-    if (row.locale !== null) {
-        out.locale = row.locale;
-    }
-    if (row.synth_method !== null) {
-        out.synth_method = row.synth_method;
-    }
-    if (row.synth_base_id !== null) {
-        out.synth_base_id = row.synth_base_id;
-    }
-    return out;
-}
 /**
  * Write one shard for a single source slug. Returns the populated ShardDescriptor + a list of quarantine reasons for
  * rows that failed alignment.
@@ -105,9 +82,8 @@ async function writeOneShard(rows, outPath, source, corpusVersion) {
         sha256,
         first_source_id: firstSourceID,
         last_source_id: lastSourceID,
-        // Stamp source so audit.ts attributes the shard without falling back to filename-prefix
-        // inference. Cast widens ShardDescriptor; the field is read by audit.ts.
-        ...{ source },
+        // Stamp source so audit.ts attributes the shard without falling back to filename-prefix inference.
+        source,
     };
 }
 function canonicalizeShardPath(path, legacyPrefix, canonicalPrefix) {
@@ -117,6 +93,7 @@ function canonicalizeShardPath(path, legacyPrefix, canonicalPrefix) {
 }
 export async function buildTranslitShard(options, report) {
     const corpusVersion = options.corpusVersion ?? "0.4.0";
+    // This is a portable manifest namespace, not a host filesystem default.
     const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/";
     const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`;
     if (!existsSync(options.jsonl))

package/out/src/tools/shard-translit.js.map

@@ -1 +1 @@
-{"version":3,"file":"shard-translit.js","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAE3D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAwBpH,SAAS,cAAc,CAAC,GAA4B,EAAE,aAAqB;IAC1E,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;QACzB,UAAU,EAAE,GAAG,CAAC,YAAY,CAA2B;QACvD,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;QAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;QAC9C,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;QACrC,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;QACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;KAC5C,CAAA;AACF,CAAC;AAED,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QACzB,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IACxB,CAAC;IAED,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QAC/B,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IACpC,CAAC;IAED,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;QAChC,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IACtC,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC3B,IAA2B,EAC3B,OAAe,EACf,MAAc,EACd,aAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,OAAO,EAAE;QACpF,YAAY,EAAE,cAAc;KAC5B,CAAC,CAAA;IAEF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,aAAa,CAAC,CAAA;IAC7D,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;IAC9C,MAAM,CAAC,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAA;IAEpD,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC5B,MAAM,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;QAEhE,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YAC1B,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;QAC9B,CAAC;QAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;IAC7B,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IAEpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,OAAO;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI;QACpB,MAAM;QACN,eAAe,EAAE,aAAa;QAC9B,cAAc,EAAE,YAAY;QAC5B,wFAAwF;QACxF,yEAAyE;QACzE,GAAI,EAAE,MAAM,EAA6B;KACzC,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,YAAoB,EAAE,eAAuB;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3F,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,QAAQ,CAAA;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,iBAAiB,EAAE,GAAG,CAAA;IAE9E,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE1C,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC/C,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,OAAO,EAAE,CAAA;QACT,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YAE3D,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAExC,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,QAAQ,OAAO,UAAU,UAAU,CAAC,MAAM,iBAAiB,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAA;IAElG,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEjD,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,IAAI,UAAU,CAAC,CAAA;QAC/D,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;QAC5E,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC1B,MAAM,EAAE,CAAC,KAAK,MAAM,KAAK,UAAU,CAAC,IAAI,WAAW,OAAO,KAAK,UAAU,CAAC,KAAK,SAAS,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAA;QAC/D,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,KAAK,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,sBAAsB;IACtB,MAAM,IAAI,GAAG,eAAe,CAAgB,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEvF,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,GAAG,EAAE;QACL,IAAI,EAAE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,mBAAmB,CAAC;KAC3E,CAAC,CAAC,CAAA;IAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,YAAY;YACvC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,YAAY;KAC1C,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACrD,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,YAAY,GAAG,CAAC,CAAA;IAChG,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtG,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAEtF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,qCAAqC,OAAO,aAAa,gBAAgB,QAAQ,mBAAmB,IAAI,CAAC,CAAA;IACnH,CAAC;AACF,CAAC"}
+{"version":3,"file":"shard-translit.js","sourceRoot":"","sources":["../../../src/tools/shard-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EACN,QAAQ,EACR,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,YAAY,EACZ,iBAAiB,GACjB,MAAM,QAAQ,CAAA;AAGf,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAuB3D,SAAS,cAAc,CAAC,GAA4B,EAAE,aAAqB;IAC1E,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;QACzB,UAAU,EAAE,GAAG,CAAC,YAAY,CAA2B;QACvD,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;QAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;QAC9C,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;QACrC,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;QACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;KAC5C,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAC3B,IAA2B,EAC3B,OAAe,EACf,MAAc,EACd,aAAqB;IAErB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,OAAO,EAAE;QACpF,YAAY,EAAE,cAAc;KAC5B,CAAC,CAAA;IAEF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,aAAa,CAAC,CAAA;IAC7D,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAA;IAC9C,MAAM,CAAC,WAAW,CAAC,wBAAwB,EAAE,MAAM,CAAC,CAAA;IAEpD,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;QAC5B,MAAM,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAA;QAEvC,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YAC1B,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;QAC9B,CAAC;QAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;IAC7B,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IAEpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAA;IACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,OAAO;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,CAAC,IAAI;QACpB,MAAM;QACN,eAAe,EAAE,aAAa;QAC9B,cAAc,EAAE,YAAY;QAC5B,mGAAmG;QACnG,MAAM;KACN,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,YAAoB,EAAE,eAAuB;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3F,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,wEAAwE;IACxE,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,QAAQ,CAAA;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,iBAAiB,EAAE,GAAG,CAAA;IAE9E,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEpF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAE1G,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE1C,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC/C,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,OAAO,EAAE,CAAA;QACT,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YAE3D,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAExC,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,QAAQ,OAAO,UAAU,UAAU,CAAC,MAAM,iBAAiB,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAA;IAElG,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEjD,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,IAAI,UAAU,CAAC,CAAA;QAC/D,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;QAC5E,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC1B,MAAM,EAAE,CAAC,KAAK,MAAM,KAAK,UAAU,CAAC,IAAI,WAAW,OAAO,KAAK,UAAU,CAAC,KAAK,SAAS,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAA;QAC/D,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;QAC1D,MAAM,EAAE,CAAC,oBAAoB,KAAK,KAAK,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,sBAAsB;IACtB,MAAM,IAAI,GAAG,eAAe,CAAgB,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEvF,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,GAAG,EAAE;QACL,IAAI,EAAE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,mBAAmB,CAAC;KAC3E,CAAC,CAAC,CAAA;IAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,YAAY;YACvC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,YAAY;KAC1C,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACrD,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAA;IAC7E,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,YAAY,GAAG,CAAC,CAAA;IAChG,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtG,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAEtF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,qCAAqC,OAAO,aAAa,gBAAgB,QAAQ,mBAAmB,IAAI,CAAC,CAAA;IACnH,CAAC;AACF,CAAC"}

package/out/src/tools/sub-venue/harvest.d.ts

@@ -0,0 +1,100 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
+ *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
+ *   measured shape of each designator's identifier half.
+ *
+ *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
+ *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
+ *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
+ *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ *
+ *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
+ *   over raw OSM at all:
+ *
+ *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
+ *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
+ *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
+ *   designator token at all. So names are not harvested wholesale — a name contributes only when it
+ *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
+ *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
+ *
+ *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
+ *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
+ *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
+ *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
+ *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
+ *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
+ *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
+ */
+import { type SurfaceIndex } from "./surfaces.ts";
+import type { IdentifierShape, SubVenueSurface } from "./table.ts";
+/**
+ * One row of a harvestable source, as read back off JSONL or out of a layer database.
+ *
+ * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
+ * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
+ * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
+ * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ */
+export interface SubVenueHarvestRow {
+    /**
+     * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
+     * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
+     */
+    designatorID: string;
+    /**
+     * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
+     * read it; the sub-venue shard recipe fills its venue slot from it.
+     */
+    tier?: string;
+    name?: string | null;
+    ref?: string | null;
+    localizedNames?: Record<string, string>;
+}
+/**
+ * Classify an OSM `ref` into an {@link IdentifierShape} class.
+ *
+ * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
+ * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
+ * more than one stand: `16-18` and `0/1`.
+ */
+export declare function classifyIdentifier(ref: string): string;
+/**
+ * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
+ *
+ * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
+ */
+export interface HarvestOptions {
+    /**
+     * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
+     */
+    source?: string;
+    /**
+     * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
+     */
+    region?: string;
+}
+/**
+ * Harvest attested phrases and identifier shapes out of a source's rows.
+ *
+ * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
+ * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
+ * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
+ * own designator is recorded as `context`.
+ *
+ * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
+ */
+export declare function extractAttestedPhrases(rows: Iterable<SubVenueHarvestRow>, index: SurfaceIndex, options?: HarvestOptions): {
+    surfaces: SubVenueSurface[];
+    identifierShapes: IdentifierShape[];
+};
+/**
+ * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
+ * extract is a build output, and one malformed line should not cost the whole lexicon.
+ */
+export declare function readSubVenueJSONL(path: string): SubVenueHarvestRow[];
+//# sourceMappingURL=harvest.d.ts.map

package/out/src/tools/sub-venue/harvest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest.d.ts","sourceRoot":"","sources":["../../../../src/tools/sub-venue/harvest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAOH,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,eAAe,CAAA;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAElE;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACvC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CActD;AAYD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,IAAI,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAClC,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,cAAmB,GAC1B;IAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAAC,gBAAgB,EAAE,eAAe,EAAE,CAAA;CAAE,CAiFtE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE,CAmBpE"}