@mailwoman/corpus 9.3.0 → 10.0.0

package/out/au/adapters/gnaf/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../lib/au/adapters/gnaf/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAOH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,eAAe,SAAS,CAAA;AACrC;;GAEG;AACH,eAAO,MAAM,oBAAoB,cAAc,CAAA;AA4B/C;;;GAGG;AACH,wBAAgB,iBAAiB,IAAI,aAAa,CAqEjD;AAED;;GAEG;AACH,eAAO,MAAM,WAAW,eAAsB,CAAA"}

package/out/au/adapters/gnaf/adapter.js

@@ -0,0 +1,127 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   G-NAF (Australia) corpus adapter — the parser-teaching half of #208.
+ *
+ *   The model mis-parses Australian addresses in their native postcode-first / house-number-last
+ *   order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street
+ *   ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap,
+ *   not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the
+ *   parse were order-robust). EU survives the same eval because its postcodes are
+ *   format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only
+ *   disambiguates by position.
+ *
+ *   So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real
+ *   AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first,
+ *   locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every
+ *   position across the slice. This is the exact mechanism that fixed #148's v1.9.0 order-overfit
+ *   for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0);
+ *   AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes
+ *   unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a
+ *   clean single-variable extension of the proven recipe + matches its source-mass structure. The
+ *   corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so
+ *   alignment lands).
+ *
+ *   Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
+ *   line). Open G-NAF licence — attribute "Geoscape Australia".
+ */
+import { componentsPresentIn } from "@mailwoman/codex/address-format";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { TextSpliterator } from "spliterator";
+import { stableSourceID } from "#adapters/utils";
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const GNAF_ADAPTER_ID = "gnaf";
+/**
+ * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
+ */
+export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0";
+/**
+ * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the
+ * two postcode-LEADING forms are the ones it fails, so they carry the change. We keep the canonical form too so the
+ * retrain doesn't forget it.
+ */
+function renderOrders(c) {
+    const region = c.region ? ` ${c.region}` : "";
+    return [
+        // real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
+        `${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
+        // postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
+        `${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
+        // locality-first: "Carlton, 2000, Barry Street 50"
+        `${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
+    ];
+}
+/**
+ * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned
+ * to AU regardless of `opts.country` (G-NAF is Australia-only).
+ */
+export function createGNAFAdapter() {
+    return {
+        id: GNAF_ADAPTER_ID,
+        defaultLicense: GNAF_DEFAULT_LICENSE,
+        description: "G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
+        async *rows(opts) {
+            let emitted = 0;
+            let idx = 0;
+            // Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
+            // loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
+            // gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
+            // The render order rotates (i % 3), matching v1.9.1's rerender.
+            for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+                if (opts.signal?.aborted)
+                    break;
+                if (opts.limit !== undefined && emitted >= opts.limit)
+                    break;
+                if (!line.trim())
+                    continue;
+                const t = tryParsingJSON(line);
+                if (t === null)
+                    continue;
+                if (!t.house_number || !t.street || !t.locality || !t.postcode)
+                    continue;
+                const orders = renderOrders(t);
+                const order = idx % orders.length;
+                idx++;
+                const raw = orders[order];
+                const components = {
+                    house_number: t.house_number,
+                    street: t.street,
+                    locality: t.locality,
+                    postcode: t.postcode,
+                };
+                // region (state) rides only the canonical render (order 0); the postcode-leading layouts
+                // omit it (matching the eval's serialization) so it never breaks verbatim alignment.
+                if (order === 0 && t.region) {
+                    components.region = t.region;
+                }
+                // `raw` here is one of three DELIBERATE word orders, two of which no layout prints — the postcode-leading
+                // forms this adapter exists to teach. So the question is containment against a string this adapter built,
+                // not what a layout would have printed.
+                const aligned = componentsPresentIn(components, raw);
+                if (!Object.keys(aligned).length)
+                    continue;
+                yield {
+                    raw,
+                    components: aligned,
+                    country: "AU",
+                    locale: "en-AU",
+                    source: GNAF_ADAPTER_ID,
+                    source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
+                    corpus_version: "",
+                    license: GNAF_DEFAULT_LICENSE,
+                };
+                emitted++;
+            }
+        },
+    };
+}
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const gnafAdapter = createGNAFAdapter();
+//# sourceMappingURL=adapter.js.map

package/out/au/adapters/gnaf/adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../lib/au/adapters/gnaf/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAA;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAGhD;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAA;AACrC;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAA;AAU/C;;;;GAIG;AACH,SAAS,YAAY,CAAC,CAAY;IACjC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAE7C,OAAO;QACN,yGAAyG;QACzG,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE;QACrE,yEAAyE;QACzE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,EAAE;QAC5D,mDAAmD;QACnD,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,EAAE;KAC7D,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB;IAChC,OAAO;QACN,EAAE,EAAE,eAAe;QACnB,cAAc,EAAE,oBAAoB;QACpC,WAAW,EACV,4KAA4K;QAE7K,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,GAAG,GAAG,CAAC,CAAA;YAEX,gGAAgG;YAChG,kGAAkG;YAClG,qGAAqG;YACrG,gEAAgE;YAChE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpE,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAQ;gBAE1B,MAAM,CAAC,GAAG,cAAc,CAAY,IAAI,CAAC,CAAA;gBAEzC,IAAI,CAAC,KAAK,IAAI;oBAAE,SAAQ;gBAExB,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ;oBAAE,SAAQ;gBAExE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;gBAC9B,MAAM,KAAK,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;gBAEjC,GAAG,EAAE,CAAA;gBACL,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAE,CAAA;gBAE1B,MAAM,UAAU,GAA+B;oBAC9C,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;iBACpB,CAAA;gBAED,yFAAyF;gBACzF,qFAAqF;gBACrF,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;oBAC7B,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;gBAC7B,CAAC;gBAED,0GAA0G;gBAC1G,0GAA0G;gBAC1G,wCAAwC;gBACxC,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAEpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;oBAAE,SAAQ;gBAE1C,MAAM;oBACL,GAAG;oBACH,UAAU,EAAE,OAAO;oBACnB,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,eAAe;oBACvB,SAAS,EAAE,GAAG,cAAc,CAAC,eAAe,EAAE,OAAO,CAAC,KAAK,KAAK,EAAE;oBAClE,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,oBAAoB;iBAC7B,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAA"}

package/out/au/adapters/gnaf/assemble.d.ts

@@ -0,0 +1,65 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
+ *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
+ *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
+ *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
+ *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
+ *   (ACT/NSW/…).
+ *
+ *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; header names key each row) — NOT
+ *   raw `read_csv` SQL, which a flat-file join doesn't need and
+ *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
+ *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
+ *   bounded (the OOM lesson from the Overture ingest).
+ *
+ *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
+ *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
+ *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
+ *   component labels, not lat/lon.
+ *
+ *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
+ *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
+ *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
+ */
+export interface GNAFAssembleOptions {
+    /**
+     * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
+     */
+    standardDir: string;
+    /**
+     * Target sample size (uniform reservoir → population-proportional across states).
+     */
+    sampleSize: number;
+    /**
+     * Output JSONL path.
+     */
+    out: string;
+    /**
+     * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
+     */
+    holdoutPath?: string;
+    /**
+     * Progress sink (the CLI passes a setter).
+     */
+    onProgress?: (message: string) => void;
+}
+export interface GNAFAssembleResult {
+    written: number;
+    seen: number;
+    heldOut: number;
+    byState: Record<string, number>;
+}
+/**
+ * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
+ */
+export declare function titlecase(s: string): string;
+/**
+ * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
+ */
+export declare function gnafHoldoutKey(street: string, locality: string, postcode: string): string;
+export declare function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult>;
+//# sourceMappingURL=assemble.d.ts.map

package/out/au/adapters/gnaf/assemble.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assemble.d.ts","sourceRoot":"","sources":["../../../../lib/au/adapters/gnaf/assemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAQH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACtC;AAED,MAAM,WAAW,kBAAkB;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAK3C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEzF;AA8CD,wBAAsB,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAmGzF"}

package/out/au/adapters/gnaf/assemble.js

@@ -0,0 +1,154 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
+ *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
+ *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
+ *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
+ *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
+ *   (ACT/NSW/…).
+ *
+ *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; header names key each row) — NOT
+ *   raw `read_csv` SQL, which a flat-file join doesn't need and
+ *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
+ *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
+ *   bounded (the OOM lesson from the Overture ingest).
+ *
+ *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
+ *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
+ *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
+ *   component labels, not lat/lon.
+ *
+ *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
+ *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
+ *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
+ */
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { join } from "path-ts";
+import { PSVSpliterator, TextSpliterator } from "spliterator";
+import { Globerator } from "spliterator/node/fs";
+/**
+ * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
+ */
+export function titlecase(s) {
+    return s
+        .toLowerCase()
+        .replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p, c) => p + c.toUpperCase())
+        .trim();
+}
+/**
+ * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
+ */
+export function gnafHoldoutKey(street, locality, postcode) {
+    return `${street}|${locality}|${postcode}`.toLowerCase();
+}
+async function* psvObjects(path) {
+    yield* PSVSpliterator.fromAsync(path);
+}
+/**
+ * Load a small lookup table fully into a Map keyed by `keyCol`.
+ */
+async function loadMap(paths, keyCol, pick) {
+    const m = new Map();
+    for (const p of paths) {
+        for await (const r of psvObjects(p)) {
+            const k = r[keyCol];
+            if (k != null && k !== "") {
+                m.set(String(k), pick(r));
+            }
+        }
+    }
+    return m;
+}
+/**
+ * Build the held-out key set from an eval JSONL whose rows carry a `components` object.
+ */
+async function loadHoldout(path) {
+    const keys = new Set();
+    for await (const line of TextSpliterator.fromAsync(path)) {
+        if (!line.trim())
+            continue;
+        const c = tryParsingJSON(line)?.components;
+        if (c?.street && c?.locality && c?.postcode) {
+            keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode));
+        }
+    }
+    return keys;
+}
+export async function assembleGNAF(opts) {
+    const progress = opts.onProgress ?? (() => { });
+    const files = await Globerator.from("*", { cwd: opts.standardDir, absolute: false }).toArray();
+    const pick = (re, exclude) => files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f));
+    // `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
+    const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/);
+    const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/);
+    const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/);
+    const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set();
+    if (opts.holdoutPath) {
+        progress(`held-out eval keys: ${holdout.size}`);
+    }
+    progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`);
+    const streetMap = await loadMap(streetPaths, "street_locality_pid", (r) => ({
+        name: String(r.street_name ?? ""),
+        type: String(r.street_type_code ?? ""),
+        suffix: String(r.street_suffix_code ?? ""),
+    }));
+    const localityMap = await loadMap(localityPaths, "locality_pid", (r) => String(r.locality_name ?? ""));
+    progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`);
+    const reservoir = [];
+    let seen = 0;
+    let heldOut = 0;
+    for (const p of addressPaths) {
+        const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? "";
+        for await (const r of psvObjects(p)) {
+            const numberFirst = String(r.number_first ?? "");
+            if (!numberFirst || r.date_retired || !r.postcode)
+                continue;
+            const st = streetMap.get(String(r.street_locality_pid ?? ""));
+            const suburbRaw = localityMap.get(String(r.locality_pid ?? ""));
+            if (!st?.name || !suburbRaw)
+                continue;
+            const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim();
+            const locality = titlecase(suburbRaw);
+            const postcode = String(r.postcode);
+            if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
+                heldOut++;
+                continue;
+            }
+            let house = numberFirst + (r.number_first_suffix ? String(r.number_first_suffix) : "");
+            if (r.number_last) {
+                house = `${house}-${String(r.number_last)}`;
+            }
+            if (r.flat_number) {
+                house = `${String(r.flat_number)}/${house}`;
+            }
+            const tuple = { house_number: house, street, locality, region: state, postcode };
+            seen++;
+            if (reservoir.length < opts.sampleSize) {
+                reservoir.push(tuple);
+            }
+            else {
+                const j = Math.floor(Math.random() * seen);
+                if (j < opts.sampleSize) {
+                    reservoir[j] = tuple;
+                }
+            }
+        }
+        progress(`${state}: ${seen.toLocaleString()} valid joinable seen`);
+    }
+    const out = openWriteStream(opts.out);
+    const byState = {};
+    for (const t of reservoir) {
+        out.write(JSON.stringify(t) + "\n");
+        byState[t.region] = (byState[t.region] ?? 0) + 1;
+    }
+    await new Promise((res) => {
+        out.end(res);
+    });
+    progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`);
+    return { written: reservoir.length, seen, heldOut, byState };
+}
+//# sourceMappingURL=assemble.js.map

package/out/au/adapters/gnaf/assemble.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assemble.js","sourceRoot":"","sources":["../../../../lib/au/adapters/gnaf/assemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAgChD;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,CAAS;IAClC,OAAO,CAAC;SACN,WAAW,EAAE;SACb,UAAU,CAAC,sBAAsB,EAAE,CAAC,EAAE,EAAE,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;SACrF,IAAI,EAAE,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;IAChF,OAAO,GAAG,MAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;AACzD,CAAC;AAID,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,IAAY;IACtC,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAuB,CAAA;AAC5D,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,OAAO,CAAI,KAAe,EAAE,MAAc,EAAE,IAAmB;IAC7E,MAAM,CAAC,GAAG,IAAI,GAAG,EAAa,CAAA;IAE9B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;YAEnB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC3B,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAE1B,MAAM,CAAC,GAAG,cAAc,CAA0C,IAAI,CAAC,EAAE,UAAU,CAAA;QAEnF,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,QAAQ,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAyB;IAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAC9C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAA;IAE9F,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,OAAgB,EAAE,EAAE,CAC7C,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;IAEvG,+FAA+F;IAC/F,MAAM,WAAW,GAAG,IAAI,CAAC,4BAA4B,CAAC,CAAA;IACtD,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,EAAE,4BAA4B,CAAC,CAAA;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,2BAA2B,CAAC,CAAA;IAEtD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAA;IAE1F,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACtB,QAAQ,CAAC,uBAAuB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAChD,CAAC;IAED,QAAQ,CAAC,4BAA4B,WAAW,CAAC,MAAM,uBAAuB,aAAa,CAAC,MAAM,IAAI,CAAC,CAAA;IAEvG,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3E,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;QACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACtC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,EAAE,CAAC;KAC1C,CAAC,CAAC,CAAA;IAEH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAA;IACtG,QAAQ,CAAC,WAAW,SAAS,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAEtG,MAAM,SAAS,GACd,EAAE,CAAA;IAEH,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAEnE,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;YAEhD,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,QAAQ;gBAAE,SAAQ;YAC3D,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,CAAA;YAC7D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAA;YAE/D,IAAI,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,SAAS;gBAAE,SAAQ;YACrC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAA;YACjH,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAEtF,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnB,KAAK,GAAG,GAAG,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAA;YAC5C,CAAC;YAED,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnB,KAAK,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,EAAE,CAAA;YAC5C,CAAC;YAED,MAAM,KAAK,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;YAEhF,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;gBAE1C,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBACzB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAE1C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3B,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAE3E,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;AAC7D,CAAC"}

package/out/cn/recipes/organizational-units.d.ts

@@ -0,0 +1,36 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
+ *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
+ *   character by character for the CJK sibling model.
+ *
+ *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
+ *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
+ *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
+ *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
+ *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
+ *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
+ *
+ *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
+ *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
+ *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
+ *   licence and the inference is stated in the `license` field rather than hidden behind it.
+ *
+ *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
+ *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
+ *   on this recipe's rows; its label set has no `locality_unit`.
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+/**
+ * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
+ * no organizational chain and so teaches nothing this recipe exists for.
+ */
+export declare function labelCNOrganizationalRow(raw: string): Record<string, string> | null;
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
+ * reading of a generic rather than a guess.
+ */
+export declare const cnOrganizationalUnitsRecipe: CorpusRecipe;
+//# sourceMappingURL=organizational-units.d.ts.map

package/out/cn/recipes/organizational-units.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"organizational-units.d.ts","sourceRoot":"","sources":["../../../lib/cn/recipes/organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,KAAK,YAAY,EAA6B,MAAM,mBAAmB,CAAA;AAmBhF;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAkDnF;AAmBD;;;GAGG;AACH,eAAO,MAAM,2BAA2B,EAAE,YA2EzC,CAAA"}

package/out/cn/recipes/organizational-units.js

@@ -0,0 +1,159 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
+ *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
+ *   character by character for the CJK sibling model.
+ *
+ *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
+ *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
+ *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
+ *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
+ *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
+ *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
+ *
+ *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
+ *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
+ *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
+ *   licence and the inference is stated in the `license` field rather than hidden behind it.
+ *
+ *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
+ *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
+ *   on this recipe's rows; its label set has no `locality_unit`.
+ */
+import { splitCNUnitChain } from "@mailwoman/core/locale/zh-cn-units";
+import { readTuples, sliceSourceID } from "#recipes/scaffold";
+import { alignRow } from "#utils/align";
+import { cjkAwareTokenizer } from "#utils/tokenize";
+/**
+ * The leading run of Han characters (plus the digits an ordinal may be written with): the Chinese half of a row.
+ */
+const LEADING_HAN = /^[\p{Script=Han}〇\d]+/u;
+/**
+ * Admin prefixes a CJK address writes in front of the settlement, coarsest first. Each is matched at the START of what
+ * remains, so `云南省临沧市孟定农场三分场二队` peels `云南省` (province), `临沧市` (city), and hands `孟定农场三分场二队` to the unit reader.
+ */
+const ADMIN_PREFIXES = [
+    [/^(.+?(?:省|自治区))/u, "region"],
+    [/^(.+?市)/u, "locality"],
+    [/^(.+?(?:自治县|县|旗))/u, "subregion"],
+];
+/**
+ * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
+ * no organizational chain and so teaches nothing this recipe exists for.
+ */
+export function labelCNOrganizationalRow(raw) {
+    const han = LEADING_HAN.exec(raw)?.[0];
+    if (!han)
+        return null;
+    const components = {};
+    let rest = han;
+    for (const [pattern, tag] of ADMIN_PREFIXES) {
+        const match = pattern.exec(rest);
+        // A prefix must leave something behind it, or the whole run was the admin name and there is no settlement.
+        if (match && match[1].length < rest.length) {
+            components[tag] = match[1];
+            rest = rest.slice(match[1].length);
+        }
+    }
+    const split = splitCNUnitChain(rest);
+    if (!split)
+        return null;
+    if (split.head) {
+        components.dependent_locality = split.head;
+    }
+    components.locality_unit = split.chain;
+    const tail = raw.slice(han.length).trim();
+    if (tail) {
+        // `, Heilongjiang, China` or `Hunan China` or `Inner Mongolia`: comma segments when there are commas, else the last
+        // word is the country when it says so and the rest is the admin1 name.
+        const segments = tail.includes(",")
+            ? tail
+                .split(",")
+                .map((segment) => segment.trim())
+                .filter((segment) => segment.length > 0)
+            : tailWithoutCommas(tail);
+        for (const segment of segments) {
+            if (/^china$/iu.test(segment)) {
+                components.country = segment;
+            }
+            else if (!components.region && !/^\d+$/u.test(segment)) {
+                components.region = segment;
+            }
+        }
+    }
+    return components;
+}
+/**
+ * A space-separated tail: `Hunan China` → [`Hunan`, `China`]; `Inner Mongolia` → [`Inner Mongolia`]; `Xinjiang Uyghur`
+ * → [`Xinjiang Uyghur`].
+ */
+function tailWithoutCommas(tail) {
+    const words = tail.split(/\s+/u).filter((word) => word.length > 0);
+    const last = words.at(-1);
+    if (last && /^china$/iu.test(last) && words.length > 1) {
+        return [words.slice(0, -1).join(" "), last];
+    }
+    return [words.join(" ")];
+}
+const SOURCE = "coarse-placer-cn-units";
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
+ * reading of a generic rather than a guess.
+ */
+export const cnOrganizationalUnitsRecipe = {
+    name: "cn-organizational-units",
+    description: "CN rows whose settlement is an organizational ladder (分场/队/连/组), labeled by the suffix grammar as one locality_unit span, per-character BIO",
+    mode: "tuples",
+    options: [],
+    async run(opts, write) {
+        if (!opts.input) {
+            throw new Error("cn-organizational-units: --input <rows.jsonl> is required ({ raw, country } rows)");
+        }
+        const tokenizer = cjkAwareTokenizer();
+        let emitted = 0;
+        let skipped = 0;
+        let quarantined = 0;
+        for await (const tuple of readTuples(opts.input)) {
+            const raw = typeof tuple["raw"] === "string" ? tuple["raw"].trim() : "";
+            const country = typeof tuple["country"] === "string" ? tuple["country"].toUpperCase() : "CN";
+            if (!raw || country !== "CN") {
+                skipped++;
+                continue;
+            }
+            const components = labelCNOrganizationalRow(raw);
+            if (!components) {
+                skipped++;
+                continue;
+            }
+            if (opts.golden) {
+                write(JSON.stringify({ raw, components, country: "CN", locale: "zh-CN" }) + "\n");
+                emitted++;
+                continue;
+            }
+            const canonical = {
+                raw,
+                components,
+                country: "CN",
+                locale: "zh-CN",
+                source: opts.sourceName ?? SOURCE,
+                source_id: sliceSourceID(opts.sourceName ?? SOURCE, { raw }),
+                corpus_version: "0.4.0",
+                license: "CC-BY-4.0 — GeoNames populated places, INFERRED from the `<name>, <admin1>, <country>` row shape; data/coarse-placer carries no per-row source",
+            };
+            // Verbatim only: every value above was sliced out of `raw`, so an edit-distance match would mean this file
+            // has a bug, not that the source spells something differently.
+            const aligned = alignRow(canonical, { tokenizer, maxEditDistance: 0 });
+            if (aligned.kind !== "labeled" || !aligned.row) {
+                quarantined++;
+                continue;
+            }
+            write(JSON.stringify({ ...aligned.row, synth_method: "cn-organizational-units", synth_base_id: null }) + "\n");
+            emitted++;
+        }
+        console.error(`  cn-organizational-units: ${emitted} labeled, ${skipped} without a chain, ${quarantined} quarantined`);
+        return { emitted, skipped, quarantined };
+    },
+};
+//# sourceMappingURL=organizational-units.js.map

package/out/cn/recipes/organizational-units.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"organizational-units.js","sourceRoot":"","sources":["../../../lib/cn/recipes/organizational-units.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAA;AAErE,OAAO,EAAqB,UAAU,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAChF,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD;;GAEG;AACH,MAAM,WAAW,GAAG,wBAAwB,CAAA;AAE5C;;;GAGG;AACH,MAAM,cAAc,GAAwF;IAC3G,CAAC,kBAAkB,EAAE,QAAQ,CAAC;IAC9B,CAAC,UAAU,EAAE,UAAU,CAAC;IACxB,CAAC,oBAAoB,EAAE,WAAW,CAAC;CACnC,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CAAC,GAAW;IACnD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAEtC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,MAAM,UAAU,GAA2B,EAAE,CAAA;IAC7C,IAAI,IAAI,GAAG,GAAG,CAAA;IAEd,KAAK,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEhC,2GAA2G;QAC3G,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7C,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAC3B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAA;QACpC,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAEpC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;QAChB,UAAU,CAAC,kBAAkB,GAAG,KAAK,CAAC,IAAI,CAAA;IAC3C,CAAC;IAED,UAAU,CAAC,aAAa,GAAG,KAAK,CAAC,KAAK,CAAA;IAEtC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;IAEzC,IAAI,IAAI,EAAE,CAAC;QACV,oHAAoH;QACpH,uEAAuE;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClC,CAAC,CAAC,IAAI;iBACH,KAAK,CAAC,GAAG,CAAC;iBACV,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAChC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC1C,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAA;QAE1B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC/B,UAAU,CAAC,OAAO,GAAG,OAAO,CAAA;YAC7B,CAAC;iBAAM,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,UAAU,CAAC,MAAM,GAAG,OAAO,CAAA;YAC5B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,UAAU,CAAA;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,IAAY;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzB,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzB,CAAC;AAED,MAAM,MAAM,GAAG,wBAAwB,CAAA;AAEvC;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAiB;IACxD,IAAI,EAAE,yBAAyB;IAC/B,WAAW,EACV,6IAA6I;IAC9I,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,EAAE;IACX,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,SAAS,GAAG,iBAAiB,EAAE,CAAA;QACrC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QAEnB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;YACvE,MAAM,OAAO,GAAG,OAAO,KAAK,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;YAE5F,IAAI,CAAC,GAAG,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC9B,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,UAAU,GAAG,wBAAwB,CAAC,GAAG,CAAC,CAAA;YAEhD,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAEjF,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,IAAI,CAAC,UAAU,IAAI,MAAM;gBACjC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC;gBAC5D,cAAc,EAAE,OAAO;gBACvB,OAAO,EACN,gJAAgJ;aACjJ,CAAA;YAED,2GAA2G;YAC3G,+DAA+D;YAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,EAAE,EAAE,SAAS,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAA;YAExG,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,WAAW,EAAE,CAAA;gBAEb,SAAQ;YACT,CAAC;YAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,yBAAyB,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAE9G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,8BAA8B,OAAO,aAAa,OAAO,qBAAqB,WAAW,cAAc,CACvG,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IACzC,CAAC;CACD,CAAA"}

package/out/cz/recipes/pcfirst-preposition.d.ts

@@ -0,0 +1,22 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `cz-pcfirst-preposition` — the Czech pc-first prepositional-locality slice, fourth orthography
+ *   of the #901 leading-name-boundary family. The #897 close-out read all 8 residual CZ rows as one
+ *   class: a LEADING postcode mis-assigned as house_number while the multi-word "nad/pod/u X"
+ *   locality shatters ("51244 Rokytnice nad Jizerou, Dolní Rokytnice 111" → street
+ *   'RokytnicenadJizerou' + house_number '51244'). That leading-5-digit confusion is the
+ *   anchor-pollution class whose decode-time OVERRIDE was correctly killed in #723 — this slice is
+ *   the model-first fix as DATA: real prepositional localities in the order that breaks, so the
+ *   model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
+ *   cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
+ */
+import { type CorpusRecipe } from "#recipes/scaffold";
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export declare const czPcFirstPrepositionRecipe: CorpusRecipe;
+//# sourceMappingURL=pcfirst-preposition.d.ts.map

package/out/cz/recipes/pcfirst-preposition.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pcfirst-preposition.d.ts","sourceRoot":"","sources":["../../../lib/cz/recipes/pcfirst-preposition.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EAA6B,KAAK,YAAY,EAAiB,MAAM,mBAAmB,CAAA;AAQ/F;;;GAGG;AACH,eAAO,MAAM,0BAA0B,EAAE,YAiFxC,CAAA"}