@mailwoman/corpus 9.1.0 → 9.3.0

package/out/recipes/bare-country.js

@@ -0,0 +1,87 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
+ *   country name, so the class is out of the model's training distribution and the tag is a coin
+ *   flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
+ *   `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
+ *   bare-country race (#1651) covers the answer either way; this slice closes the PARSE half so the
+ *   tag itself is right.
+ *
+ *   Source is the codex country table — every ISO canonical English name plus the curated surface
+ *   forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
+ *   `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
+ *   English words), and teaching it here would be an unmeasured claim. Case augmentation is the
+ *   loader's job, as everywhere.
+ */
+import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country";
+import { alignAndWrite, makeMulberry32, sliceSourceID } from "#recipes/scaffold";
+/**
+ * Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
+ */
+const MIN_NAME_LENGTH = 4;
+/**
+ * Every (surface, iso2) pair the slice emits: the ISO canonical names plus each country's curated surface forms,
+ * deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
+ */
+function* bareCountrySurfaces() {
+    const seen = new Set();
+    for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
+        for (const surface of forms) {
+            if (surface.length < MIN_NAME_LENGTH || seen.has(surface))
+                continue;
+            seen.add(surface);
+            yield { surface, iso2 };
+        }
+    }
+    for (const name of CountryNames) {
+        if (name.length < MIN_NAME_LENGTH || seen.has(name))
+            continue;
+        const matched = matchCountry(name);
+        if (!matched)
+            continue;
+        seen.add(name);
+        yield { surface: name, iso2: matched.iso2 };
+    }
+}
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const bareCountryRecipe = {
+    name: "bare-country",
+    description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
+    mode: "generate",
+    async run(opts, write) {
+        // Seeded for parity with the other recipes; the surfaces themselves drive the content.
+        makeMulberry32(opts.seed);
+        let read = 0;
+        let emitted = 0;
+        let skipped = 0;
+        for (const { surface, iso2 } of bareCountrySurfaces()) {
+            read++;
+            const components = { country: surface };
+            const source_id = sliceSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) });
+            const canonical = {
+                raw: surface,
+                components,
+                country: iso2,
+                locale: "und",
+                source: "synth-bare-country",
+                source_id,
+                corpus_version: "0.11.0",
+                license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
+            };
+            if (alignAndWrite(write, canonical, "bare-country")) {
+                emitted++;
+            }
+            else {
+                skipped++;
+            }
+        }
+        return { read, emitted, skipped };
+    },
+};
+//# sourceMappingURL=bare-country.js.map

package/out/recipes/bare-country.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bare-country.js","sourceRoot":"","sources":["../../lib/recipes/bare-country.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAE5F,OAAO,EAAE,aAAa,EAAE,cAAc,EAAqB,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEnG;;GAEG;AACH,MAAM,eAAe,GAAG,CAAC,CAAA;AAEzB;;;GAGG;AACH,QAAQ,CAAC,CAAC,mBAAmB;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACnE,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAQ;YACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YACjB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;QACxB,CAAC;IACF,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAA;IAC5C,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAiB;IAC9C,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,gGAAgG;IAC7G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,uFAAuF;QACvF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,mBAAmB,EAAE,EAAE,CAAC;YACvD,IAAI,EAAE,CAAA;YAEN,MAAM,UAAU,GAA2B,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,aAAa,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAEtG,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,OAAO;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB;gBAC5B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EAAE,4FAA4F;aACrG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}

package/out/recipes/boundary-stress.d.ts

@@ -0,0 +1,20 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `boundary-stress` slice recipe (#375) — the boundary-instability augmentation. Self-generates
+ *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
+ *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
+ *   change for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
+ *   scripts/build-boundary-stress-slice.mjs.
+ *
+ *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export declare const boundaryStressRecipe: CorpusRecipe;
+//# sourceMappingURL=boundary-stress.d.ts.map

package/out/recipes/boundary-stress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../lib/recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAkB,KAAK,YAAY,EAAiB,MAAM,mBAAmB,CAAA;AAoCpF;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,YA6ClC,CAAA"}

package/out/recipes/boundary-stress.js

@@ -0,0 +1,83 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `boundary-stress` slice recipe (#375) — the boundary-instability augmentation. Self-generates
+ *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
+ *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
+ *   change for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
+ *   scripts/build-boundary-stress-slice.mjs.
+ *
+ *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
+ */
+import { makeMulberry32, sliceSourceID } from "#recipes/scaffold";
+import { synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress";
+import { alignRow } from "#utils";
+/**
+ * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
+ * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
+ * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
+ * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is required — it drives the cumulative
+ * thresholds below.
+ */
+const WEIGHTS = {
+    "street-eats-affix": 0.22,
+    "comma-less-city-state": 0.22,
+    "fr-prefix": 0.18,
+    "bare-locality": 0.11,
+    "house-number-before-street": 0.189,
+    "house-number-after-street": 0.081,
+};
+const CUM = (() => {
+    let acc = 0;
+    return Object.entries(WEIGHTS).map(([t, w]) => [t, (acc += w)]);
+})();
+function pickTemplate(r) {
+    const x = r();
+    for (const [t, c] of CUM)
+        if (x <= c)
+            return t;
+    return CUM.at(-1)[0];
+}
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const boundaryStressRecipe = {
+    name: "boundary-stress",
+    description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
+    mode: "generate",
+    async run(opts, write) {
+        // Emit PRNG: the legacy build-boundary-stress-slice.mjs seeded mulberry32(opts.seed).
+        const random = makeMulberry32(opts.seed);
+        const count = opts.count ?? 20_000;
+        let emitted = 0;
+        let skipped = 0;
+        for (let i = 0; i < count; i++) {
+            const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) });
+            const country = row.locale.split("-")[1] ?? "US";
+            const source_id = sliceSourceID("synth-boundary-stress", { ...row.components, v: String(i) });
+            const canonical = {
+                raw: row.raw,
+                components: row.components,
+                country,
+                locale: row.locale,
+                source: "synth-boundary-stress",
+                source_id,
+                corpus_version: "0.6.0",
+                license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
+            };
+            const aligned = alignRow(canonical);
+            if (aligned.kind !== "labeled") {
+                skipped++;
+                continue;
+            }
+            // Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
+            write(JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n");
+            emitted++;
+        }
+        return { emitted, skipped };
+    },
+};
+//# sourceMappingURL=boundary-stress.js.map

package/out/recipes/boundary-stress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../lib/recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,cAAc,EAAqB,aAAa,EAAE,MAAM,mBAAmB,CAAA;AACpF,OAAO,EAA+B,2BAA2B,EAAE,MAAM,+BAA+B,CAAA;AACxG,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC;;;;;;GAMG;AACH,MAAM,OAAO,GAA2C;IACvD,mBAAmB,EAAE,IAAI;IACzB,uBAAuB,EAAE,IAAI;IAC7B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,KAAK;IACnC,2BAA2B,EAAE,KAAK;CAClC,CAAA;AAED,MAAM,GAAG,GAA4C,CAAC,GAAG,EAAE;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,OAAQ,MAAM,CAAC,OAAO,CAAC,OAAO,CAA6C,CAAC,GAAG,CAC9E,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAqC,CAC/D,CAAA;AACF,CAAC,CAAC,EAAE,CAAA;AAEJ,SAAS,YAAY,CAAC,CAAe;IACpC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IAEb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;IAE9C,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAiB;IACjD,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,sFAAsF;QACtF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAA;QAClC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,2BAA2B,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAE7F,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,iGAAiG;YACjG,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAC/G,CAAA;YAED,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}

package/out/recipes/cn-organizational-units.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
+ *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
+ *   character by character for the CJK sibling model.
+ *
+ *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
+ *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
+ *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
+ *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
+ *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
+ *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
+ *
+ *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
+ *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
+ *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
+ *   licence and the inference is stated in the `license` field rather than hidden behind it.
+ *
+ *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
+ *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
+ *   on this recipe's rows; its label set has no `locality_unit`.
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+/**
+ * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
+ * no organizational chain and so teaches nothing this recipe exists for.
+ */
+export declare function labelCNOrganizationalRow(raw: string): Record<string, string> | null;
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
+ * reading of a generic rather than a guess.
+ */
+export declare const cnOrganizationalUnitsRecipe: CorpusRecipe;
+//# sourceMappingURL=cn-organizational-units.d.ts.map

package/out/recipes/cn-organizational-units.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cn-organizational-units.d.ts","sourceRoot":"","sources":["../../lib/recipes/cn-organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,KAAK,YAAY,EAA6B,MAAM,mBAAmB,CAAA;AAmBhF;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAkDnF;AAmBD;;;GAGG;AACH,eAAO,MAAM,2BAA2B,EAAE,YA2EzC,CAAA"}

package/out/recipes/cn-organizational-units.js

@@ -0,0 +1,159 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
+ *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
+ *   character by character for the CJK sibling model.
+ *
+ *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
+ *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
+ *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
+ *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
+ *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
+ *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
+ *
+ *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
+ *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
+ *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
+ *   licence and the inference is stated in the `license` field rather than hidden behind it.
+ *
+ *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
+ *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
+ *   on this recipe's rows; its label set has no `locality_unit`.
+ */
+import { splitCNUnitChain } from "@mailwoman/core/locale/zh-cn-units";
+import { readTuples, sliceSourceID } from "#recipes/scaffold";
+import { alignRow } from "#utils/align";
+import { cjkAwareTokenizer } from "#utils/tokenize";
+/**
+ * The leading run of Han characters (plus the digits an ordinal may be written with): the Chinese half of a row.
+ */
+const LEADING_HAN = /^[\p{Script=Han}〇\d]+/u;
+/**
+ * Admin prefixes a CJK address writes in front of the settlement, coarsest first. Each is matched at the START of what
+ * remains, so `云南省临沧市孟定农场三分场二队` peels `云南省` (province), `临沧市` (city), and hands `孟定农场三分场二队` to the unit reader.
+ */
+const ADMIN_PREFIXES = [
+    [/^(.+?(?:省|自治区))/u, "region"],
+    [/^(.+?市)/u, "locality"],
+    [/^(.+?(?:自治县|县|旗))/u, "subregion"],
+];
+/**
+ * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
+ * no organizational chain and so teaches nothing this recipe exists for.
+ */
+export function labelCNOrganizationalRow(raw) {
+    const han = LEADING_HAN.exec(raw)?.[0];
+    if (!han)
+        return null;
+    const components = {};
+    let rest = han;
+    for (const [pattern, tag] of ADMIN_PREFIXES) {
+        const match = pattern.exec(rest);
+        // A prefix must leave something behind it, or the whole run was the admin name and there is no settlement.
+        if (match && match[1].length < rest.length) {
+            components[tag] = match[1];
+            rest = rest.slice(match[1].length);
+        }
+    }
+    const split = splitCNUnitChain(rest);
+    if (!split)
+        return null;
+    if (split.head) {
+        components.dependent_locality = split.head;
+    }
+    components.locality_unit = split.chain;
+    const tail = raw.slice(han.length).trim();
+    if (tail) {
+        // `, Heilongjiang, China` or `Hunan China` or `Inner Mongolia`: comma segments when there are commas, else the last
+        // word is the country when it says so and the rest is the admin1 name.
+        const segments = tail.includes(",")
+            ? tail
+                .split(",")
+                .map((segment) => segment.trim())
+                .filter((segment) => segment.length > 0)
+            : tailWithoutCommas(tail);
+        for (const segment of segments) {
+            if (/^china$/iu.test(segment)) {
+                components.country = segment;
+            }
+            else if (!components.region && !/^\d+$/u.test(segment)) {
+                components.region = segment;
+            }
+        }
+    }
+    return components;
+}
+/**
+ * A space-separated tail: `Hunan China` → [`Hunan`, `China`]; `Inner Mongolia` → [`Inner Mongolia`]; `Xinjiang Uyghur`
+ * → [`Xinjiang Uyghur`].
+ */
+function tailWithoutCommas(tail) {
+    const words = tail.split(/\s+/u).filter((word) => word.length > 0);
+    const last = words.at(-1);
+    if (last && /^china$/iu.test(last) && words.length > 1) {
+        return [words.slice(0, -1).join(" "), last];
+    }
+    return [words.join(" ")];
+}
+const SOURCE = "coarse-placer-cn-units";
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
+ * reading of a generic rather than a guess.
+ */
+export const cnOrganizationalUnitsRecipe = {
+    name: "cn-organizational-units",
+    description: "CN rows whose settlement is an organizational ladder (分场/队/连/组), labeled by the suffix grammar as one locality_unit span, per-character BIO",
+    mode: "tuples",
+    options: [],
+    async run(opts, write) {
+        if (!opts.input) {
+            throw new Error("cn-organizational-units: --input <rows.jsonl> is required ({ raw, country } rows)");
+        }
+        const tokenizer = cjkAwareTokenizer();
+        let emitted = 0;
+        let skipped = 0;
+        let quarantined = 0;
+        for await (const tuple of readTuples(opts.input)) {
+            const raw = typeof tuple["raw"] === "string" ? tuple["raw"].trim() : "";
+            const country = typeof tuple["country"] === "string" ? tuple["country"].toUpperCase() : "CN";
+            if (!raw || country !== "CN") {
+                skipped++;
+                continue;
+            }
+            const components = labelCNOrganizationalRow(raw);
+            if (!components) {
+                skipped++;
+                continue;
+            }
+            if (opts.golden) {
+                write(JSON.stringify({ raw, components, country: "CN", locale: "zh-CN" }) + "\n");
+                emitted++;
+                continue;
+            }
+            const canonical = {
+                raw,
+                components,
+                country: "CN",
+                locale: "zh-CN",
+                source: opts.sourceName ?? SOURCE,
+                source_id: sliceSourceID(opts.sourceName ?? SOURCE, { raw }),
+                corpus_version: "0.4.0",
+                license: "CC-BY-4.0 — GeoNames populated places, INFERRED from the `<name>, <admin1>, <country>` row shape; data/coarse-placer carries no per-row source",
+            };
+            // Verbatim only: every value above was sliced out of `raw`, so an edit-distance match would mean this file
+            // has a bug, not that the source spells something differently.
+            const aligned = alignRow(canonical, { tokenizer, maxEditDistance: 0 });
+            if (aligned.kind !== "labeled" || !aligned.row) {
+                quarantined++;
+                continue;
+            }
+            write(JSON.stringify({ ...aligned.row, synth_method: "cn-organizational-units", synth_base_id: null }) + "\n");
+            emitted++;
+        }
+        console.error(`  cn-organizational-units: ${emitted} labeled, ${skipped} without a chain, ${quarantined} quarantined`);
+        return { emitted, skipped, quarantined };
+    },
+};
+//# sourceMappingURL=cn-organizational-units.js.map

package/out/recipes/cn-organizational-units.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cn-organizational-units.js","sourceRoot":"","sources":["../../lib/recipes/cn-organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AAErE,OAAO,EAAqB,UAAU,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD;;GAEG;AACH,MAAM,WAAW,GAAG,wBAAwB,CAAA;AAE5C;;;GAGG;AACH,MAAM,cAAc,GAAwF;IAC3G,CAAC,kBAAkB,EAAE,QAAQ,CAAC;IAC9B,CAAC,UAAU,EAAE,UAAU,CAAC;IACxB,CAAC,oBAAoB,EAAE,WAAW,CAAC;CACnC,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAW;IACnD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAEtC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,MAAM,UAAU,GAA2B,EAAE,CAAA;IAC7C,IAAI,IAAI,GAAG,GAAG,CAAA;IAEd,KAAK,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEhC,2GAA2G;QAC3G,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAC3B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QACpC,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAEpC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAChB,UAAU,CAAC,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAA;IAC3C,CAAC;IAED,UAAU,CAAC,aAAa,GAAG,KAAK,CAAC,KAAK,CAAA;IAEtC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;IAEzC,IAAI,IAAI,EAAE,CAAC;QACV,oHAAoH;QACpH,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClC,CAAC,CAAC,IAAI;iBACH,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAChC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC1C,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;QAE1B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,UAAU,CAAC,OAAO,GAAG,OAAO,CAAA;YAC7B,CAAC;iBAAM,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,UAAU,CAAC,MAAM,GAAG,OAAO,CAAA;YAC5B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzB,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzB,CAAC;AAED,MAAM,MAAM,GAAG,wBAAwB,CAAA;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAiB;IACxD,IAAI,EAAE,yBAAyB;IAC/B,WAAW,EACV,6IAA6I;IAC9I,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,EAAE;IACX,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAA;QACrC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QAEnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;YACvE,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;YAE5F,IAAI,CAAC,GAAG,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC9B,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,CAAA;YAEhD,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAEjF,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,IAAI,CAAC,UAAU,IAAI,MAAM;gBACjC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC;gBAC5D,cAAc,EAAE,OAAO;gBACvB,OAAO,EACN,gJAAgJ;aACjJ,CAAA;YAED,2GAA2G;YAC3G,+DAA+D;YAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,EAAE,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAA;YAExG,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,WAAW,EAAE,CAAA;gBAEb,SAAQ;YACT,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,yBAAyB,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAE9G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,8BAA8B,OAAO,aAAa,OAAO,qBAAqB,WAAW,cAAc,CACvG,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IACzC,CAAC;CACD,CAAA"}

package/out/recipes/country-balanced.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `country-balanced` slice recipe — the BALANCED, MODEL-FIRST country-coverage slice (#464). The
+ *   shipped model is STARVED on `country` (P=R=F1=0 on the homograph eval), so this fills the void
+ *   the way the `unit` slice did, but built to AVOID over-firing "trailing token ⇒ country". Three
+ *   ingredients, ported faithfully from scripts/build-country-slice-balanced.mjs:
+ *
+ *   1. Breadth/recall — real OA skeletons (US/DE/FR/IT/NL) with a country token in a varied surface form
+ *        from `@mailwoman/codex/country` (canonical / endonym / ISO code), + ~30% country-ABSENT
+ *        negatives (teach O-emission, the precision floor).
+ *   2. Homograph CONTRAST pairs — each true country-name homograph (Georgia, Jordan, Lebanon, Mexico,
+ *        Peru, Turkey) rendered BOTH ways: as `country` (foreign-city context) AND as the US
+ *        `region`/`locality` (US-ZIP context). Teaches that the label is CONTEXTUAL, not
+ *        positional.
+ *   3. Code-as-region negatives — 2-letter codes that are both a US state abbrev and an ISO country code
+ *        (CA/GA/IN/MA/PA/AL) in US-ZIP context → must read as `region`, never `country`.
+ *
+ *   `--golden` emits a held-out synthetic val over the VT (US) + Berlin (DE) holdouts. This is a
+ *   `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
+ *   the input. The passed `random` (the framework LCG) is consumed in the exact call order the
+ *   legacy script used.
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export declare const countryBalancedRecipe: CorpusRecipe;
+//# sourceMappingURL=country-balanced.d.ts.map

package/out/recipes/country-balanced.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../lib/recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AASH,OAAO,EAAgC,KAAK,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAmXnF;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,YAoGnC,CAAA"}