@mailwoman/corpus 7.5.0 → 7.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,28 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `fr-lieudit` shard recipe — FR lieu-dit (hamlet/place) `dependent_locality` coverage
7
+ * (`.superpowers/sdd/deploc-world-survey.md`, FR section, 2026-07-22). Streams every BAN
8
+ * `adresses-<dept>.csv` dump under `--ban-dir` through `@mailwoman/ban/sdk`'s
9
+ * `extractBANAddrPoints`, which now surfaces a cleaned `lieuDit` per record (junk/dup filtering
10
+ * lives in `ban/sdk/extract.ts`'s `cleanLieuDit`, not duplicated here). Only rows carrying a clean
11
+ * lieu-dit survive into the pool; the existing `ban`/`synth-fr` sources and their emitted rows are
12
+ * untouched — this recipe reads the SAME raw CSVs but emits under its own source name.
13
+ *
14
+ * Mapping: lieu-dit → `dependent_locality`, commune → `locality`. Rendered to match the formatter's
15
+ * FR `place`-slot convention (`fix(formatter): render dependent_locality for neither-slot templates`,
16
+ * b1edc1b7, verified via a `formatAddress` smoke call): house+street on line 1, the lieu-dit ALONE on
17
+ * line 2, postcode+commune on line 3 — French postal convention (La Poste's line 5).
18
+ *
19
+ * ~1.69M clean rows survive the filter nationally (26M total BAN rows, 1.81M raw `nom_ld` fills, ~6.6%
20
+ * junk/dup). The pool is read in full (small string tuples only — no coordinates needed) and
21
+ * Fisher-Yates shuffled with the seeded PRNG before slicing to `--count`, rather than sampled WITH
22
+ * replacement — at a `--count` a sizeable fraction of the pool, with-replacement draws would produce a
23
+ * large duplicate rate (birthday-paradox math: ~190k expected collisions at count=800k over a 1.69M
24
+ * pool).
25
+ */
26
+ import { type ShardRecipe } from "./scaffold.ts";
27
+ export declare const frLieuditRecipe: ShardRecipe;
28
+ //# sourceMappingURL=fr-lieudit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fr-lieudit.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-lieudit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAcH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA4HhE,eAAO,MAAM,eAAe,EAAE,WA+G7B,CAAA"}
@@ -0,0 +1,212 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `fr-lieudit` shard recipe — FR lieu-dit (hamlet/place) `dependent_locality` coverage
7
+ * (`.superpowers/sdd/deploc-world-survey.md`, FR section, 2026-07-22). Streams every BAN
8
+ * `adresses-<dept>.csv` dump under `--ban-dir` through `@mailwoman/ban/sdk`'s
9
+ * `extractBANAddrPoints`, which now surfaces a cleaned `lieuDit` per record (junk/dup filtering
10
+ * lives in `ban/sdk/extract.ts`'s `cleanLieuDit`, not duplicated here). Only rows carrying a clean
11
+ * lieu-dit survive into the pool; the existing `ban`/`synth-fr` sources and their emitted rows are
12
+ * untouched — this recipe reads the SAME raw CSVs but emits under its own source name.
13
+ *
14
+ * Mapping: lieu-dit → `dependent_locality`, commune → `locality`. Rendered to match the formatter's
15
+ * FR `place`-slot convention (`fix(formatter): render dependent_locality for neither-slot templates`,
16
+ * b1edc1b7, verified via a `formatAddress` smoke call): house+street on line 1, the lieu-dit ALONE on
17
+ * line 2, postcode+commune on line 3 — French postal convention (La Poste's line 5).
18
+ *
19
+ * ~1.69M clean rows survive the filter nationally (26M total BAN rows, 1.81M raw `nom_ld` fills, ~6.6%
20
+ * junk/dup). The pool is read in full (small string tuples only — no coordinates needed) and
21
+ * Fisher-Yates shuffled with the seeded PRNG before slicing to `--count`, rather than sampled WITH
22
+ * replacement — at a `--count` a sizeable fraction of the pool, with-replacement draws would produce a
23
+ * large duplicate rate (birthday-paradox math: ~190k expected collisions at count=800k over a 1.69M
24
+ * pool).
25
+ */
26
+ import { readdirSync } from "node:fs";
27
+ import { join } from "node:path";
28
+ import { extractBANAddrPoints } from "@mailwoman/ban/sdk";
29
+ import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country";
30
+ import { dataRootPath } from "@mailwoman/core/utils";
31
+ import { stableSourceID } from "../adapter.js";
32
+ import { decomposeFrStreet } from "../adapters/ban/street-decompose.js";
33
+ import { alignRow } from "../align.js";
34
+ import { makeMulberry32 } from "./scaffold.js";
35
+ const DEFAULT_LICENSE = "Licence Ouverte 2.0"; // matches the `ban` adapter's Tier-B election for BAN data
36
+ /**
37
+ * Enumerate `adresses-<dept>.csv[.gz]` files in `banDir`, ONE path per département. Excludes the `merged`/`france`
38
+ * aggregates (they duplicate the per-département rows) and, when both a `.csv` and a `.csv.gz` exist for the same dept
39
+ * (observed on disk for 13/2A/48/69/75 — a stale re-fetch artifact), prefers the uncompressed `.csv` — mirrors
40
+ * `ban/scripts/build-address-point-shard.ts`'s `departementFiles`, which hit and fixed this exact double-count trap
41
+ * first.
42
+ */
43
+ function departementFiles(banDir) {
44
+ const byDept = new Map();
45
+ for (const name of readdirSync(banDir).sort()) {
46
+ const m = /^adresses-(.+?)\.csv(\.gz)?$/.exec(name);
47
+ if (!m)
48
+ continue;
49
+ const dept = m[1];
50
+ if (dept === "merged" || dept === "france")
51
+ continue;
52
+ const existing = byDept.get(dept);
53
+ if (!existing || (existing.endsWith(".gz") && !name.endsWith(".gz"))) {
54
+ byDept.set(dept, join(banDir, name));
55
+ }
56
+ }
57
+ return [...byDept.keys()].sort().map((dept) => byDept.get(dept));
58
+ }
59
+ /** Stream every département file, keeping only rows with a clean `lieuDit` (junk/dup filtering lives in `ban/sdk`). */
60
+ async function readLieuDitPool(banDir) {
61
+ const files = departementFiles(banDir);
62
+ if (files.length === 0) {
63
+ throw new Error(`No BAN adresses-<dept>.csv files found in ${banDir} — fetch BAN first (\`mailwoman corpus fetch ban\`).`);
64
+ }
65
+ const pool = [];
66
+ let scanned = 0;
67
+ for (const path of files) {
68
+ let deptCount = 0;
69
+ for await (const rec of extractBANAddrPoints(path)) {
70
+ scanned++;
71
+ if (!rec.lieuDit || !rec.city)
72
+ continue;
73
+ pool.push({
74
+ numero: rec.numero,
75
+ rep: rec.rep,
76
+ street: rec.street,
77
+ postcode: rec.postcode,
78
+ locality: rec.city,
79
+ dependentLocality: rec.lieuDit,
80
+ });
81
+ deptCount++;
82
+ }
83
+ console.error(` ${path}: ${deptCount.toLocaleString()} clean lieu-dit rows`);
84
+ }
85
+ console.error(` scanned ${scanned.toLocaleString()} BAN rows across ${files.length} départements → pool=${pool.length.toLocaleString()}`);
86
+ return pool;
87
+ }
88
+ /** `house_number` = `numero` + folded `rep` ("10 bis"), matching the `ban` adapter's own composition. */
89
+ function composeHouseNumber(numero, rep) {
90
+ return rep ? `${numero} ${rep}` : numero;
91
+ }
92
+ /**
93
+ * Render the raw address string: house+street line, the lieu-dit ALONE on its own line, postcode+commune line — the
94
+ * exact shape `formatAddress` produces for FR's `place`-slot mapping (verified via a smoke call before this recipe was
95
+ * written; see the module docstring).
96
+ */
97
+ function composeRaw(house, street, dependentLocality, postcode, locality) {
98
+ const lines = [];
99
+ const streetLine = `${house} ${street}`.trim();
100
+ if (streetLine) {
101
+ lines.push(streetLine);
102
+ }
103
+ lines.push(dependentLocality);
104
+ const cityLine = [postcode, locality].filter(Boolean).join(" ").trim();
105
+ if (cityLine) {
106
+ lines.push(cityLine);
107
+ }
108
+ return lines.join("\n");
109
+ }
110
+ /** Fisher-Yates shuffle, in place, with the recipe's seeded PRNG — reproducible sampling without replacement. */
111
+ function shuffleInPlace(arr, random) {
112
+ for (let i = arr.length - 1; i > 0; i--) {
113
+ const j = Math.floor(random() * (i + 1));
114
+ [arr[i], arr[j]] = [arr[j], arr[i]];
115
+ }
116
+ }
117
+ export const frLieuditRecipe = {
118
+ name: "fr-lieudit",
119
+ description: "FR lieu-dit rows: BAN nom_ld → dependent_locality (commune → locality), lieu-dit on its own line",
120
+ mode: "generate",
121
+ options: [
122
+ {
123
+ flag: "--ban-dir <dir>",
124
+ description: "BAN adresses-<dept>.csv directory. Default $MAILWOMAN_DATA_ROOT/corpus/sources/ban",
125
+ },
126
+ {
127
+ flag: "--country-fraction <f>",
128
+ description: "Fraction of rows that append an explicit 'France' surface form + a `country` component. Default 0",
129
+ },
130
+ ],
131
+ async run(opts, write) {
132
+ const random = makeMulberry32(opts.seed);
133
+ const source = opts.sourceName ?? "synth-fr-lieudit";
134
+ const count = opts.count ?? 800_000;
135
+ const banDir = opts.banDir ?? dataRootPath("corpus", "sources", "ban");
136
+ const countryFraction = opts.countryFraction ?? 0;
137
+ if (!(countryFraction >= 0 && countryFraction <= 1)) {
138
+ throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`);
139
+ }
140
+ const pool = await readLieuDitPool(banDir);
141
+ if (pool.length === 0) {
142
+ throw new Error(`No clean lieu-dit rows found under ${banDir} — see ban/sdk/extract.ts's cleanLieuDit filter.`);
143
+ }
144
+ shuffleInPlace(pool, random);
145
+ const selected = pool.slice(0, Math.min(count, pool.length));
146
+ let emitted = 0;
147
+ let skipped = 0;
148
+ let countryAppended = 0;
149
+ for (const t of selected) {
150
+ const house = composeHouseNumber(t.numero, t.rep);
151
+ const decomposed = decomposeFrStreet(t.street);
152
+ const components = {
153
+ house_number: house,
154
+ dependent_locality: t.dependentLocality,
155
+ locality: t.locality,
156
+ };
157
+ if (decomposed.prefix) {
158
+ components.street_prefix = decomposed.prefix;
159
+ }
160
+ if (decomposed.street) {
161
+ components.street = decomposed.street;
162
+ }
163
+ if (t.postcode) {
164
+ components.postcode = t.postcode;
165
+ }
166
+ let raw = composeRaw(house, t.street, t.dependentLocality, t.postcode, t.locality);
167
+ if (!raw) {
168
+ skipped++;
169
+ continue;
170
+ }
171
+ // Country-append (the fr-admin-split #728 pattern, generalized): ~`countryFraction` of the time
172
+ // append an explicit "France" surface form onto the trailing (postcode+commune) line + a
173
+ // `country` component — the model relearns to emit country WHEN present without over-firing it
174
+ // on the (still-majority) country-less rows. `countryFraction <= 0` (the default) never draws
175
+ // from `random`, so the byte-stream is unaffected when the flag is unset.
176
+ if (countryFraction > 0 && random() < countryFraction) {
177
+ const forms = COUNTRY_SURFACE_FORMS.FR;
178
+ const form = forms[Math.floor(random() * forms.length)];
179
+ raw = `${raw}, ${form}`;
180
+ components.country = form;
181
+ countryAppended++;
182
+ }
183
+ const sourceID = stableSourceID(source, {
184
+ street: t.street,
185
+ house_number: house,
186
+ dependent_locality: t.dependentLocality,
187
+ locality: t.locality,
188
+ postcode: t.postcode ?? undefined,
189
+ });
190
+ const canonical = {
191
+ raw,
192
+ components,
193
+ country: "FR",
194
+ locale: "fr-FR",
195
+ source,
196
+ source_id: sourceID,
197
+ corpus_version: "",
198
+ license: DEFAULT_LICENSE,
199
+ };
200
+ const aligned = alignRow(canonical);
201
+ if (aligned.kind !== "labeled" || !aligned.row) {
202
+ skipped++;
203
+ continue;
204
+ }
205
+ write(JSON.stringify({ ...aligned.row, synth_method: source, synth_base_id: null }) + "\n");
206
+ emitted++;
207
+ }
208
+ console.error(` emitted=${emitted} skipped=${skipped} country-appended=${countryAppended} pool=${pool.length}`);
209
+ return { emitted, skipped };
210
+ },
211
+ };
212
+ //# sourceMappingURL=fr-lieudit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fr-lieudit.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-lieudit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAEhE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAEhE,MAAM,eAAe,GAAG,qBAAqB,CAAA,CAAC,2DAA2D;AAYzG;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEnD,IAAI,CAAC,CAAC;YAAE,SAAQ;QAEhB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;QAElB,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,QAAQ;YAAE,SAAQ;QAEpD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAEjC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAA;AAClE,CAAC;AAED,uHAAuH;AACvH,KAAK,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAEtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACd,6CAA6C,MAAM,sDAAsD,CACzG,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAmB,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI;gBAAE,SAAQ;YAEvC,IAAI,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,iBAAiB,EAAE,GAAG,CAAC,OAAO;aAC9B,CAAC,CAAA;YACF,SAAS,EAAE,CAAA;QACZ,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,SAAS,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAA;IAC9E,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,aAAa,OAAO,CAAC,cAAc,EAAE,oBAAoB,KAAK,CAAC,MAAM,wBAAwB,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,CAC3H,CAAA;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,yGAAyG;AACzG,SAAS,kBAAkB,CAAC,MAAc,EAAE,GAAkB;IAC7D,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAA;AACzC,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAClB,KAAa,EACb,MAAc,EACd,iBAAyB,EACzB,QAAuB,EACvB,QAAgB;IAEhB,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE,CAAA;IAE9C,IAAI,UAAU,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACvB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAC7B,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEtE,IAAI,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED,iHAAiH;AACjH,SAAS,cAAc,CAAI,GAAQ,EAAE,MAAoB;IACxD,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CACvC;QAAA,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IACvC,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAgB;IAC3C,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,kGAAkG;IAC/G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,oFAAoF;SACjG;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAA;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QACtE,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;QAEjD,IAAI,CAAC,CAAC,eAAe,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,eAAe,EAAE,CAAC,CAAA;QAChF,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;QAE1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,kDAAkD,CAAC,CAAA;QAChH,CAAC;QAED,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE5D,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,kBAAkB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YAE9C,MAAM,UAAU,GAA0C;gBACzD,YAAY,EAAE,KAAK;gBACnB,kBAAkB,EAAE,CAAC,CAAC,iBAAiB;gBACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACpB,CAAA;YAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;YAC7C,CAAC;YAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;YACtC,CAAC;YAED,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAChB,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAA;YACjC,CAAC;YAED,IAAI,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAA;YAElF,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,gGAAgG;YAChG,yFAAyF;YACzF,+FAA+F;YAC/F,8FAA8F;YAC9F,0EAA0E;YAC1E,IAAI,eAAe,GAAG,CAAC,IAAI,MAAM,EAAE,GAAG,eAAe,EAAE,CAAC;gBACvD,MAAM,KAAK,GAAG,qBAAqB,CAAC,EAAE,CAAA;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAE,CAAA;gBACxD,GAAG,GAAG,GAAG,GAAG,KAAK,IAAI,EAAE,CAAA;gBACvB,UAAU,CAAC,OAAO,GAAG,IAAI,CAAA;gBACzB,eAAe,EAAE,CAAA;YAClB,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,YAAY,EAAE,KAAK;gBACnB,kBAAkB,EAAE,CAAC,CAAC,iBAAiB;gBACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,SAAS;aACjC,CAAC,CAAA;YACF,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,EAAE;gBAClB,OAAO,EAAE,eAAe;aACxB,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC3F,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,aAAa,OAAO,YAAY,OAAO,qBAAqB,eAAe,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QAEhH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAoBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA6B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAqBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA8B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
@@ -14,6 +14,7 @@ import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.js";
14
14
  import { frAdminSplitRecipe } from "./fr-admin-split.js";
15
15
  import { frBareStreetRecipe } from "./fr-bare-street.js";
16
16
  import { frFragmentRecipe } from "./fr-fragment.js";
17
+ import { frLieuditRecipe } from "./fr-lieudit.js";
17
18
  import { frOrderRecipe } from "./fr-order.js";
18
19
  import { germanRecipe } from "./german.js";
19
20
  import { houseVenueRecipe } from "./house-venue.js";
@@ -48,6 +49,7 @@ const RECIPES = [
48
49
  frAdminSplitRecipe,
49
50
  frBareStreetRecipe,
50
51
  frFragmentRecipe,
52
+ frLieuditRecipe,
51
53
  czPcFirstPrepositionRecipe,
52
54
  nlPostcodeRecipe,
53
55
  noStreetLedRecipe,
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,eAAe;IACf,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
@@ -24,7 +24,7 @@
24
24
  * source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
25
25
  * country, so DE/FR emit streams are unchanged for a given seed.
26
26
  */
27
- import { type LocaleBaseTuple } from "../synthesize-german.ts";
27
+ import { type LocaleBaseTuple, type SynthesizedLocaleRow } from "../synthesize-german.ts";
28
28
  import { type ShardRecipe } from "./scaffold.ts";
29
29
  /**
30
30
  * One per-country OA source part: either a cached `zip` + `csv` member (streamed via `unzip -p`) or an extracted plain
@@ -43,6 +43,34 @@ export interface LocalePart {
43
43
  * rows). Without this, the default CITY→locality mapping wrongly trains the suburb as the locality.
44
44
  */
45
45
  districtAsLocality?: boolean;
46
+ /**
47
+ * ES pedanía part only — the header is the RAW (un-conformed) CNIG export schema (`numero`, `tipo_vial`,
48
+ * `nombre_via`, `poblacion`, `municipio`, `comunidad_autonoma`, `cod_postal`), NOT the standard OA
49
+ * NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE header every other part uses. Verified 2026-07-22 by exact- coordinate
50
+ * cross-check: the OA-conformed `extracted/es/countrywide.csv` collapses CITY to `municipio` and drops `poblacion`
51
+ * (Spain's below-municipio núcleo/pedanía name) entirely, so the pedanía signal survives ONLY in this raw export.
52
+ * `street` is reconstructed as `tipo_vial + " " + nombre_via` (verified byte-identical to the conformed STREET column
53
+ * for the same row). CITY-analog = `poblacion`, DISTRICT-analog = `municipio`.
54
+ */
55
+ cnigRaw?: boolean;
56
+ }
57
+ export interface LocaleCountrySource {
58
+ source: string;
59
+ parts: LocalePart[];
60
+ /**
61
+ * The `corpus_version` stamped on emitted rows. DE/FR keep the historical `0.4.0` (regenerating those shards must
62
+ * stay lineage-identical); ES/IT/NL are the #241 staging lineage (`v0.9.9-es-it-nl`).
63
+ */
64
+ corpusVersion: string;
65
+ /**
66
+ * An ALTERNATE part list, read instead of {@link parts} when the `--district-as-locality` override is explicitly
67
+ * `true` for this invocation (see `run()`). ES-only for now — the standard `parts` entry can't supply real
68
+ * dependent-locality signal (its OA-conformed CSV drops `poblacion`; see {@link LocalePart.cnigRaw}), so the pedanía
69
+ * build reads a wholly different raw source instead of flipping the standard CITY/DISTRICT columns. `undefined` for
70
+ * every other country — the override then just forces `districtAsLocality` on the normal `parts`, as GB/NZ already do
71
+ * per-part.
72
+ */
73
+ pedaniaParts?: LocalePart[];
46
74
  }
47
75
  /**
48
76
  * OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
@@ -78,5 +106,26 @@ export declare function cleanCityNoise(city: string): string | null;
78
106
  * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
79
107
  */
80
108
  export declare function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]>;
109
+ /**
110
+ * Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in
111
+ * place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a
112
+ * `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the
113
+ * (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away
114
+ * entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical
115
+ * to before this option existed. Exported for {@link locale.test.ts}.
116
+ */
117
+ export declare function applyCountryAppend(synth: SynthesizedLocaleRow, country: string, countryFraction: number, random: () => number): void;
118
+ /**
119
+ * Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged
120
+ * (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality`
121
+ * forced to that value for this invocation only. Exported for {@link locale.test.ts}.
122
+ */
123
+ export declare function applyDistrictAsLocalityOverride(part: LocalePart, override: boolean | undefined): LocalePart;
124
+ /**
125
+ * Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly
126
+ * `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other
127
+ * country/override combination. Exported for {@link locale.test.ts}.
128
+ */
129
+ export declare function resolveLocaleParts(countrySource: LocaleCountrySource, override: boolean | undefined): LocalePart[];
81
130
  export declare const localeRecipe: ShardRecipe;
82
131
  //# sourceMappingURL=locale.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAUH,OAAO,EAAE,KAAK,eAAe,EAAuB,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC5B;AAiFD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAWD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA2GhG;AAED,eAAO,MAAM,YAAY,EAAE,WAwH1B,CAAA"}
1
+ {"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAWH,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAuB,MAAM,yBAAyB,CAAA;AAC9G,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,UAAU,EAAE,CAAA;CAC3B;AA6FD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAgBD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAoJhG;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,MAAM,MAAM,GAClB,IAAI,CAgBN;AAED;;;;GAIG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAE3G;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,aAAa,EAAE,mBAAmB,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,EAAE,CAElH;AAED,eAAO,MAAM,YAAY,EAAE,WAgJ1B,CAAA"}
@@ -26,6 +26,7 @@
26
26
  */
27
27
  import { spawn } from "node:child_process";
28
28
  import { createReadStream } from "node:fs";
29
+ import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country";
29
30
  import { dataRootPath } from "@mailwoman/core/utils";
30
31
  import { CSVSpliterator } from "spliterator";
31
32
  import { stableSourceID } from "../adapter.js";
@@ -68,6 +69,18 @@ const COUNTRY_SOURCES = {
68
69
  source: "synth-es",
69
70
  corpusVersion: "0.9.9",
70
71
  parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
72
+ // Pedanía shard (`synth-es-pedania`, `--district-as-locality`). Reads the RAW (un-conformed) CNIG export
73
+ // cached at oa-cache/es__countrywide.zip — the `parts` CSV above lost `poblacion` in OA's own conform step
74
+ // (see {@link LocalePart.cnigRaw}). districtAsLocality is pinned true here (this part only exists to be
75
+ // read pedanía-style); the CLI override still applies harmlessly on top.
76
+ pedaniaParts: [
77
+ {
78
+ zip: dataRootPath("oa-cache", "es__countrywide.zip"),
79
+ csv: "es_addresses.csv",
80
+ cnigRaw: true,
81
+ districtAsLocality: true,
82
+ },
83
+ ],
71
84
  },
72
85
  NZ: {
73
86
  // LINZ-derived OA countrywide extract (2.12M rows). `districtAsLocality` inverts the CITY/DISTRICT
@@ -78,6 +91,16 @@ const COUNTRY_SOURCES = {
78
91
  // coord board; that split is a BUILD-TIME concern (scratchpad), so the committed recipe reads the full CSV.
79
92
  parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
80
93
  },
94
+ GB: {
95
+ // HM Land Registry Price Paid Data tuples (25.67M rows; see Task 2's ppd ingest). PPD's DISTRICT is the
96
+ // postal town (locality) and CITY is the dependent locality — legitimately EMPTY on the majority of rows
97
+ // (most GB addresses have no dependent locality). `districtAsLocality` maps DISTRICT→locality and, when
98
+ // present, CITY→dependent_locality; the `readTuples` gate above only drops a row when BOTH are empty, so
99
+ // the majority empty-CITY rows survive.
100
+ source: "synth-gb",
101
+ corpusVersion: "0.9.9",
102
+ parts: [{ path: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv"), districtAsLocality: true }],
103
+ },
81
104
  };
82
105
  /**
83
106
  * Per-part reservoir cap. Streaming + Algorithm-R reservoir sampling to this size keeps memory bounded regardless of
@@ -168,39 +191,79 @@ export async function readTuples(part, rng) {
168
191
  if (header === null) {
169
192
  header = cells.map((h) => h.trim().toLowerCase());
170
193
  const ix = (name) => header.indexOf(name);
171
- cols = {
172
- num: ix("number"),
173
- street: ix("street"),
174
- city: ix("city"),
175
- district: ix("district"),
176
- region: ix("region"),
177
- post: ix("postcode"),
178
- };
194
+ // `cnigRaw` (ES pedanía only): the RAW CNIG header has no NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE
195
+ // at all — `numero`/`nombre_via`/`poblacion`/`municipio`/`comunidad_autonoma`/`cod_postal` instead.
196
+ // See {@link LocalePart.cnigRaw}.
197
+ cols = part.cnigRaw
198
+ ? {
199
+ num: ix("numero"),
200
+ street: ix("nombre_via"),
201
+ tipoVial: ix("tipo_vial"),
202
+ city: ix("poblacion"),
203
+ district: ix("municipio"),
204
+ region: ix("comunidad_autonoma"),
205
+ post: ix("cod_postal"),
206
+ }
207
+ : {
208
+ num: ix("number"),
209
+ street: ix("street"),
210
+ tipoVial: -1,
211
+ city: ix("city"),
212
+ district: ix("district"),
213
+ region: ix("region"),
214
+ post: ix("postcode"),
215
+ };
179
216
  continue;
180
217
  }
181
218
  if (cols === null)
182
219
  continue;
183
- const street = get(cells, cols.street);
220
+ // cnigRaw: STREET is split into a road-type column (`tipo_vial`, e.g. "CARRETERA") and the name
221
+ // (`nombre_via`) — rejoin them exactly as OA's own conform step does (verified byte-identical to the
222
+ // conformed STREET column for the same source row). Every other part's header already carries the
223
+ // pre-joined STREET column, so `cols.tipoVial === -1` and this is a no-op there.
224
+ const street = cols.tipoVial >= 0
225
+ ? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(Boolean).join(" ")
226
+ : get(cells, cols.street);
184
227
  const rawCity = get(cells, cols.city);
185
- if (!street || !rawCity)
228
+ if (!street)
186
229
  continue;
187
- // Default: CITY → locality. NZ (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
230
+ // Default: CITY → locality. NZ/GB (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
188
231
  // (`Auckland`) and CITY holds the suburb (`Birkenhead`), so DISTRICT → locality and CITY →
189
232
  // dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
190
- // sub-locality. See {@link LocalePart.districtAsLocality}.
233
+ // sub-locality. GB PPD tuples flip which side is legitimately empty — on the MAJORITY of GB rows CITY
234
+ // (the dependent_locality) is empty and DISTRICT (the locality) is populated, so the gate below only
235
+ // drops a `districtAsLocality` row when BOTH are empty, not when CITY alone is (that used to silently
236
+ // drop most of the GB source — see the fixed `readTuples` gate below). See
237
+ // {@link LocalePart.districtAsLocality}.
191
238
  let locality;
192
239
  let dependent_locality;
193
240
  if (part.districtAsLocality) {
194
- const cleanedDistrict = cleanCityNoise(get(cells, cols.district));
241
+ const rawDistrict = get(cells, cols.district);
242
+ if (!rawCity && !rawDistrict)
243
+ continue;
244
+ const cleanedDistrict = cleanCityNoise(rawDistrict);
195
245
  if (cleanedDistrict) {
196
246
  locality = cleanedDistrict;
197
- dependent_locality = cleanCityNoise(rawCity) ?? undefined;
247
+ const cleanedCity = cleanCityNoise(rawCity);
248
+ // ES pedanía lesson (2026-07-22): the CNIG `poblacion` column is filled on ~93% of rows but
249
+ // EQUALS `municipio` on the majority of those (the address point sits in the municipio's own
250
+ // main town, not a below-municipio pedanía) — only ~32.6% of ES rows carry a genuinely
251
+ // DISTINCT poblacion. GB/NZ never hit this (CITY/DISTRICT name the same place only by rare
252
+ // coincidence), but the guard is general: a dependent_locality equal to its own locality is
253
+ // never a real sub-locality, so drop it rather than emit a same-value pair (would fail the
254
+ // dep_loc≠locality invariant every recipe otherwise upholds).
255
+ dependent_locality =
256
+ cleanedCity && cleanedCity.localeCompare(locality, undefined, { sensitivity: "base" }) !== 0
257
+ ? cleanedCity
258
+ : undefined;
198
259
  }
199
260
  else {
200
261
  locality = cleanCityNoise(rawCity);
201
262
  }
202
263
  }
203
264
  else {
265
+ if (!rawCity)
266
+ continue;
204
267
  locality = cleanCityNoise(rawCity);
205
268
  }
206
269
  if (!locality) {
@@ -235,13 +298,58 @@ export async function readTuples(part, rng) {
235
298
  console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`);
236
299
  return reservoir;
237
300
  }
301
+ /**
302
+ * Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in
303
+ * place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a
304
+ * `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the
305
+ * (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away
306
+ * entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical
307
+ * to before this option existed. Exported for {@link locale.test.ts}.
308
+ */
309
+ export function applyCountryAppend(synth, country, countryFraction, random) {
310
+ if (countryFraction > 0 && random() < countryFraction) {
311
+ const forms = COUNTRY_SURFACE_FORMS[country];
312
+ if (!forms?.length) {
313
+ // The BR/NZ lesson: a missing table entry must never silently no-op a requested fraction —
314
+ // it must raise so the gap is caught at build time, not discovered later as a 0% gate failure.
315
+ throw new Error(`No COUNTRY_SURFACE_FORMS entry for ${country} — add it to codex/country/country.ts before using --country-fraction`);
316
+ }
317
+ const form = forms[Math.floor(random() * forms.length)];
318
+ synth.raw = `${synth.raw}, ${form}`;
319
+ synth.components = { ...synth.components, country: form };
320
+ }
321
+ }
322
+ /**
323
+ * Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged
324
+ * (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality`
325
+ * forced to that value for this invocation only. Exported for {@link locale.test.ts}.
326
+ */
327
+ export function applyDistrictAsLocalityOverride(part, override) {
328
+ return override === undefined ? part : { ...part, districtAsLocality: override };
329
+ }
330
+ /**
331
+ * Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly
332
+ * `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other
333
+ * country/override combination. Exported for {@link locale.test.ts}.
334
+ */
335
+ export function resolveLocaleParts(countrySource, override) {
336
+ return override === true && countrySource.pedaniaParts ? countrySource.pedaniaParts : countrySource.parts;
337
+ }
238
338
  export const localeRecipe = {
239
339
  name: "locale",
240
- description: "Per-locale coverage rows (DE/FR/NL/IT/ES) from real OA tuples, both orders → synthesizeLocaleRow",
340
+ description: "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples, both orders → synthesizeLocaleRow",
241
341
  mode: "generate",
242
342
  options: [
243
- { flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ). Default DE" },
343
+ { flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ|GB). Default DE" },
244
344
  { flag: "--intl-fraction <f>", description: "Fraction rendered international order. Default 0.4" },
345
+ {
346
+ flag: "--country-fraction <f>",
347
+ description: "Fraction of rows that append an explicit country surface form (`, United Kingdom`) + a `country` component (fr-admin-split pattern). Default 0 — byte-identical to before when unset.",
348
+ },
349
+ {
350
+ flag: "--district-as-locality / --no-district-as-locality",
351
+ description: "Override the per-part districtAsLocality mapping for this run. Unset (default) leaves each COUNTRY_SOURCES part's own value untouched — every existing build stays byte-identical. ES additionally switches to the pedanía (poblacion→dependent_locality) source when passed as true — combine with --source-name synth-es-pedania.",
352
+ },
245
353
  ],
246
354
  async run(opts, write) {
247
355
  // Emit PRNG: the legacy build-locale-shard.mjs seeded mulberry32(opts.seed). The reservoir uses a
@@ -256,15 +364,27 @@ export const localeRecipe = {
256
364
  if (!(intlFraction >= 0 && intlFraction <= 1)) {
257
365
  throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`);
258
366
  }
367
+ // Default 0 → the `random() < countryFraction` draw below is short-circuited away entirely (never
368
+ // consumed), so every existing locale's emit stream is byte-identical to before this option existed.
369
+ const countryFraction = opts.countryFraction ?? 0;
370
+ if (!(countryFraction >= 0 && countryFraction <= 1)) {
371
+ throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`);
372
+ }
259
373
  const source = opts.sourceName ?? countrySource.source;
260
374
  const count = opts.count ?? 4000;
261
- const { parts } = countrySource;
375
+ // Tri-state: `undefined` (flag absent) touches nothing below — `parts` stays the default list and each
376
+ // part keeps its own `districtAsLocality`, so every existing locale build is byte-identical to before this
377
+ // option existed. `true` additionally selects `pedaniaParts` when the country registers one (ES); `false`
378
+ // forces the mapping off on every part read this run (a debugging escape hatch for GB/NZ).
379
+ const districtAsLocalityOverride = opts.districtAsLocality;
380
+ const parts = resolveLocaleParts(countrySource, districtAsLocalityOverride);
262
381
  const pool = [];
263
382
  for (let pi = 0; pi < parts.length; pi++) {
264
383
  // A reservoir PRNG per part, seeded but independent of the emit loop's `random`, so the sample is
265
384
  // reproducible without perturbing the synth/order draws.
266
385
  const reservoirRng = makeMulberry32((opts.seed ^ (0x9e3779b9 * (pi + 1))) >>> 0);
267
- const t = await readTuples(parts[pi], reservoirRng);
386
+ const effectivePart = applyDistrictAsLocalityOverride(parts[pi], districtAsLocalityOverride);
387
+ const t = await readTuples(effectivePart, reservoirRng);
268
388
  for (const x of t) {
269
389
  pool.push(x);
270
390
  } // NOT pool.push(...t) — spreading huge arrays overflows the stack
@@ -292,6 +412,7 @@ export const localeRecipe = {
292
412
  skipped++;
293
413
  continue;
294
414
  }
415
+ applyCountryAppend(synth, country, countryFraction, random);
295
416
  if (opts.golden) {
296
417
  // Golden rows must round-trip through alignRow exactly like training rows (#241 done-when): a
297
418
  // render that can't be BIO-labeled can't serve as a parser golden either. Consumes no RNG draw.