@mailwoman/corpus 7.5.0 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/shard-recipes/fr-lieudit.d.ts +28 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-lieudit.js +212 -0
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +2 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +50 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +139 -18
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +12 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +16 -0
- package/out/src/tools/fetch/ppd.d.ts.map +1 -0
- package/out/src/tools/fetch/ppd.js +89 -0
- package/out/src/tools/fetch/ppd.js.map +1 -0
- package/out/src/tools/gb-title-case.d.ts +2 -0
- package/out/src/tools/gb-title-case.d.ts.map +1 -0
- package/out/src/tools/gb-title-case.js +65 -0
- package/out/src/tools/gb-title-case.js.map +1 -0
- package/package.json +5 -4
- package/src/shard-recipes/fr-lieudit.ts +274 -0
- package/src/shard-recipes/index.ts +2 -0
- package/src/shard-recipes/locale.ts +181 -20
- package/src/shard-recipes/scaffold.ts +12 -0
- package/src/tools/fetch/ppd.ts +118 -0
- package/src/tools/gb-title-case.ts +68 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-lieudit` shard recipe — FR lieu-dit (hamlet/place) `dependent_locality` coverage
|
|
7
|
+
* (`.superpowers/sdd/deploc-world-survey.md`, FR section, 2026-07-22). Streams every BAN
|
|
8
|
+
* `adresses-<dept>.csv` dump under `--ban-dir` through `@mailwoman/ban/sdk`'s
|
|
9
|
+
* `extractBANAddrPoints`, which now surfaces a cleaned `lieuDit` per record (junk/dup filtering
|
|
10
|
+
* lives in `ban/sdk/extract.ts`'s `cleanLieuDit`, not duplicated here). Only rows carrying a clean
|
|
11
|
+
* lieu-dit survive into the pool; the existing `ban`/`synth-fr` sources and their emitted rows are
|
|
12
|
+
* untouched — this recipe reads the SAME raw CSVs but emits under its own source name.
|
|
13
|
+
*
|
|
14
|
+
* Mapping: lieu-dit → `dependent_locality`, commune → `locality`. Rendered to match the formatter's
|
|
15
|
+
* FR `place`-slot convention (`fix(formatter): render dependent_locality for neither-slot templates`,
|
|
16
|
+
* b1edc1b7, verified via a `formatAddress` smoke call): house+street on line 1, the lieu-dit ALONE on
|
|
17
|
+
* line 2, postcode+commune on line 3 — French postal convention (La Poste's line 5).
|
|
18
|
+
*
|
|
19
|
+
* ~1.69M clean rows survive the filter nationally (26M total BAN rows, 1.81M raw `nom_ld` fills, ~6.6%
|
|
20
|
+
* junk/dup). The pool is read in full (small string tuples only — no coordinates needed) and
|
|
21
|
+
* Fisher-Yates shuffled with the seeded PRNG before slicing to `--count`, rather than sampled WITH
|
|
22
|
+
* replacement — at a `--count` a sizeable fraction of the pool, with-replacement draws would produce a
|
|
23
|
+
* large duplicate rate (birthday-paradox math: ~190k expected collisions at count=800k over a 1.69M
|
|
24
|
+
* pool).
|
|
25
|
+
*/
|
|
26
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
27
|
+
export declare const frLieuditRecipe: ShardRecipe;
|
|
28
|
+
//# sourceMappingURL=fr-lieudit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-lieudit.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-lieudit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAcH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA4HhE,eAAO,MAAM,eAAe,EAAE,WA+G7B,CAAA"}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-lieudit` shard recipe — FR lieu-dit (hamlet/place) `dependent_locality` coverage
|
|
7
|
+
* (`.superpowers/sdd/deploc-world-survey.md`, FR section, 2026-07-22). Streams every BAN
|
|
8
|
+
* `adresses-<dept>.csv` dump under `--ban-dir` through `@mailwoman/ban/sdk`'s
|
|
9
|
+
* `extractBANAddrPoints`, which now surfaces a cleaned `lieuDit` per record (junk/dup filtering
|
|
10
|
+
* lives in `ban/sdk/extract.ts`'s `cleanLieuDit`, not duplicated here). Only rows carrying a clean
|
|
11
|
+
* lieu-dit survive into the pool; the existing `ban`/`synth-fr` sources and their emitted rows are
|
|
12
|
+
* untouched — this recipe reads the SAME raw CSVs but emits under its own source name.
|
|
13
|
+
*
|
|
14
|
+
* Mapping: lieu-dit → `dependent_locality`, commune → `locality`. Rendered to match the formatter's
|
|
15
|
+
* FR `place`-slot convention (`fix(formatter): render dependent_locality for neither-slot templates`,
|
|
16
|
+
* b1edc1b7, verified via a `formatAddress` smoke call): house+street on line 1, the lieu-dit ALONE on
|
|
17
|
+
* line 2, postcode+commune on line 3 — French postal convention (La Poste's line 5).
|
|
18
|
+
*
|
|
19
|
+
* ~1.69M clean rows survive the filter nationally (26M total BAN rows, 1.81M raw `nom_ld` fills, ~6.6%
|
|
20
|
+
* junk/dup). The pool is read in full (small string tuples only — no coordinates needed) and
|
|
21
|
+
* Fisher-Yates shuffled with the seeded PRNG before slicing to `--count`, rather than sampled WITH
|
|
22
|
+
* replacement — at a `--count` a sizeable fraction of the pool, with-replacement draws would produce a
|
|
23
|
+
* large duplicate rate (birthday-paradox math: ~190k expected collisions at count=800k over a 1.69M
|
|
24
|
+
* pool).
|
|
25
|
+
*/
|
|
26
|
+
import { readdirSync } from "node:fs";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
import { extractBANAddrPoints } from "@mailwoman/ban/sdk";
|
|
29
|
+
import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country";
|
|
30
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
31
|
+
import { stableSourceID } from "../adapter.js";
|
|
32
|
+
import { decomposeFrStreet } from "../adapters/ban/street-decompose.js";
|
|
33
|
+
import { alignRow } from "../align.js";
|
|
34
|
+
import { makeMulberry32 } from "./scaffold.js";
|
|
35
|
+
const DEFAULT_LICENSE = "Licence Ouverte 2.0"; // matches the `ban` adapter's Tier-B election for BAN data
|
|
36
|
+
/**
|
|
37
|
+
* Enumerate `adresses-<dept>.csv[.gz]` files in `banDir`, ONE path per département. Excludes the `merged`/`france`
|
|
38
|
+
* aggregates (they duplicate the per-département rows) and, when both a `.csv` and a `.csv.gz` exist for the same dept
|
|
39
|
+
* (observed on disk for 13/2A/48/69/75 — a stale re-fetch artifact), prefers the uncompressed `.csv` — mirrors
|
|
40
|
+
* `ban/scripts/build-address-point-shard.ts`'s `departementFiles`, which hit and fixed this exact double-count trap
|
|
41
|
+
* first.
|
|
42
|
+
*/
|
|
43
|
+
function departementFiles(banDir) {
|
|
44
|
+
const byDept = new Map();
|
|
45
|
+
for (const name of readdirSync(banDir).sort()) {
|
|
46
|
+
const m = /^adresses-(.+?)\.csv(\.gz)?$/.exec(name);
|
|
47
|
+
if (!m)
|
|
48
|
+
continue;
|
|
49
|
+
const dept = m[1];
|
|
50
|
+
if (dept === "merged" || dept === "france")
|
|
51
|
+
continue;
|
|
52
|
+
const existing = byDept.get(dept);
|
|
53
|
+
if (!existing || (existing.endsWith(".gz") && !name.endsWith(".gz"))) {
|
|
54
|
+
byDept.set(dept, join(banDir, name));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return [...byDept.keys()].sort().map((dept) => byDept.get(dept));
|
|
58
|
+
}
|
|
59
|
+
/** Stream every département file, keeping only rows with a clean `lieuDit` (junk/dup filtering lives in `ban/sdk`). */
|
|
60
|
+
async function readLieuDitPool(banDir) {
|
|
61
|
+
const files = departementFiles(banDir);
|
|
62
|
+
if (files.length === 0) {
|
|
63
|
+
throw new Error(`No BAN adresses-<dept>.csv files found in ${banDir} — fetch BAN first (\`mailwoman corpus fetch ban\`).`);
|
|
64
|
+
}
|
|
65
|
+
const pool = [];
|
|
66
|
+
let scanned = 0;
|
|
67
|
+
for (const path of files) {
|
|
68
|
+
let deptCount = 0;
|
|
69
|
+
for await (const rec of extractBANAddrPoints(path)) {
|
|
70
|
+
scanned++;
|
|
71
|
+
if (!rec.lieuDit || !rec.city)
|
|
72
|
+
continue;
|
|
73
|
+
pool.push({
|
|
74
|
+
numero: rec.numero,
|
|
75
|
+
rep: rec.rep,
|
|
76
|
+
street: rec.street,
|
|
77
|
+
postcode: rec.postcode,
|
|
78
|
+
locality: rec.city,
|
|
79
|
+
dependentLocality: rec.lieuDit,
|
|
80
|
+
});
|
|
81
|
+
deptCount++;
|
|
82
|
+
}
|
|
83
|
+
console.error(` ${path}: ${deptCount.toLocaleString()} clean lieu-dit rows`);
|
|
84
|
+
}
|
|
85
|
+
console.error(` scanned ${scanned.toLocaleString()} BAN rows across ${files.length} départements → pool=${pool.length.toLocaleString()}`);
|
|
86
|
+
return pool;
|
|
87
|
+
}
|
|
88
|
+
/** `house_number` = `numero` + folded `rep` ("10 bis"), matching the `ban` adapter's own composition. */
|
|
89
|
+
function composeHouseNumber(numero, rep) {
|
|
90
|
+
return rep ? `${numero} ${rep}` : numero;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Render the raw address string: house+street line, the lieu-dit ALONE on its own line, postcode+commune line — the
|
|
94
|
+
* exact shape `formatAddress` produces for FR's `place`-slot mapping (verified via a smoke call before this recipe was
|
|
95
|
+
* written; see the module docstring).
|
|
96
|
+
*/
|
|
97
|
+
function composeRaw(house, street, dependentLocality, postcode, locality) {
|
|
98
|
+
const lines = [];
|
|
99
|
+
const streetLine = `${house} ${street}`.trim();
|
|
100
|
+
if (streetLine) {
|
|
101
|
+
lines.push(streetLine);
|
|
102
|
+
}
|
|
103
|
+
lines.push(dependentLocality);
|
|
104
|
+
const cityLine = [postcode, locality].filter(Boolean).join(" ").trim();
|
|
105
|
+
if (cityLine) {
|
|
106
|
+
lines.push(cityLine);
|
|
107
|
+
}
|
|
108
|
+
return lines.join("\n");
|
|
109
|
+
}
|
|
110
|
+
/** Fisher-Yates shuffle, in place, with the recipe's seeded PRNG — reproducible sampling without replacement. */
|
|
111
|
+
function shuffleInPlace(arr, random) {
|
|
112
|
+
for (let i = arr.length - 1; i > 0; i--) {
|
|
113
|
+
const j = Math.floor(random() * (i + 1));
|
|
114
|
+
[arr[i], arr[j]] = [arr[j], arr[i]];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
export const frLieuditRecipe = {
|
|
118
|
+
name: "fr-lieudit",
|
|
119
|
+
description: "FR lieu-dit rows: BAN nom_ld → dependent_locality (commune → locality), lieu-dit on its own line",
|
|
120
|
+
mode: "generate",
|
|
121
|
+
options: [
|
|
122
|
+
{
|
|
123
|
+
flag: "--ban-dir <dir>",
|
|
124
|
+
description: "BAN adresses-<dept>.csv directory. Default $MAILWOMAN_DATA_ROOT/corpus/sources/ban",
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
flag: "--country-fraction <f>",
|
|
128
|
+
description: "Fraction of rows that append an explicit 'France' surface form + a `country` component. Default 0",
|
|
129
|
+
},
|
|
130
|
+
],
|
|
131
|
+
async run(opts, write) {
|
|
132
|
+
const random = makeMulberry32(opts.seed);
|
|
133
|
+
const source = opts.sourceName ?? "synth-fr-lieudit";
|
|
134
|
+
const count = opts.count ?? 800_000;
|
|
135
|
+
const banDir = opts.banDir ?? dataRootPath("corpus", "sources", "ban");
|
|
136
|
+
const countryFraction = opts.countryFraction ?? 0;
|
|
137
|
+
if (!(countryFraction >= 0 && countryFraction <= 1)) {
|
|
138
|
+
throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`);
|
|
139
|
+
}
|
|
140
|
+
const pool = await readLieuDitPool(banDir);
|
|
141
|
+
if (pool.length === 0) {
|
|
142
|
+
throw new Error(`No clean lieu-dit rows found under ${banDir} — see ban/sdk/extract.ts's cleanLieuDit filter.`);
|
|
143
|
+
}
|
|
144
|
+
shuffleInPlace(pool, random);
|
|
145
|
+
const selected = pool.slice(0, Math.min(count, pool.length));
|
|
146
|
+
let emitted = 0;
|
|
147
|
+
let skipped = 0;
|
|
148
|
+
let countryAppended = 0;
|
|
149
|
+
for (const t of selected) {
|
|
150
|
+
const house = composeHouseNumber(t.numero, t.rep);
|
|
151
|
+
const decomposed = decomposeFrStreet(t.street);
|
|
152
|
+
const components = {
|
|
153
|
+
house_number: house,
|
|
154
|
+
dependent_locality: t.dependentLocality,
|
|
155
|
+
locality: t.locality,
|
|
156
|
+
};
|
|
157
|
+
if (decomposed.prefix) {
|
|
158
|
+
components.street_prefix = decomposed.prefix;
|
|
159
|
+
}
|
|
160
|
+
if (decomposed.street) {
|
|
161
|
+
components.street = decomposed.street;
|
|
162
|
+
}
|
|
163
|
+
if (t.postcode) {
|
|
164
|
+
components.postcode = t.postcode;
|
|
165
|
+
}
|
|
166
|
+
let raw = composeRaw(house, t.street, t.dependentLocality, t.postcode, t.locality);
|
|
167
|
+
if (!raw) {
|
|
168
|
+
skipped++;
|
|
169
|
+
continue;
|
|
170
|
+
}
|
|
171
|
+
// Country-append (the fr-admin-split #728 pattern, generalized): ~`countryFraction` of the time
|
|
172
|
+
// append an explicit "France" surface form onto the trailing (postcode+commune) line + a
|
|
173
|
+
// `country` component — the model relearns to emit country WHEN present without over-firing it
|
|
174
|
+
// on the (still-majority) country-less rows. `countryFraction <= 0` (the default) never draws
|
|
175
|
+
// from `random`, so the byte-stream is unaffected when the flag is unset.
|
|
176
|
+
if (countryFraction > 0 && random() < countryFraction) {
|
|
177
|
+
const forms = COUNTRY_SURFACE_FORMS.FR;
|
|
178
|
+
const form = forms[Math.floor(random() * forms.length)];
|
|
179
|
+
raw = `${raw}, ${form}`;
|
|
180
|
+
components.country = form;
|
|
181
|
+
countryAppended++;
|
|
182
|
+
}
|
|
183
|
+
const sourceID = stableSourceID(source, {
|
|
184
|
+
street: t.street,
|
|
185
|
+
house_number: house,
|
|
186
|
+
dependent_locality: t.dependentLocality,
|
|
187
|
+
locality: t.locality,
|
|
188
|
+
postcode: t.postcode ?? undefined,
|
|
189
|
+
});
|
|
190
|
+
const canonical = {
|
|
191
|
+
raw,
|
|
192
|
+
components,
|
|
193
|
+
country: "FR",
|
|
194
|
+
locale: "fr-FR",
|
|
195
|
+
source,
|
|
196
|
+
source_id: sourceID,
|
|
197
|
+
corpus_version: "",
|
|
198
|
+
license: DEFAULT_LICENSE,
|
|
199
|
+
};
|
|
200
|
+
const aligned = alignRow(canonical);
|
|
201
|
+
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
202
|
+
skipped++;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
write(JSON.stringify({ ...aligned.row, synth_method: source, synth_base_id: null }) + "\n");
|
|
206
|
+
emitted++;
|
|
207
|
+
}
|
|
208
|
+
console.error(` emitted=${emitted} skipped=${skipped} country-appended=${countryAppended} pool=${pool.length}`);
|
|
209
|
+
return { emitted, skipped };
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
//# sourceMappingURL=fr-lieudit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-lieudit.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-lieudit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAA;AAEhE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAEhE,MAAM,eAAe,GAAG,qBAAqB,CAAA,CAAC,2DAA2D;AAYzG;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,MAAc;IACvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,CAAC,GAAG,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAEnD,IAAI,CAAC,CAAC;YAAE,SAAQ;QAEhB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAA;QAElB,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,QAAQ;YAAE,SAAQ;QAEpD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAEjC,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,CAAA;AAClE,CAAC;AAED,uHAAuH;AACvH,KAAK,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,KAAK,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAEtC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACd,6CAA6C,MAAM,sDAAsD,CACzG,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAmB,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAA;QAEjB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;YACpD,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI;gBAAE,SAAQ;YAEvC,IAAI,CAAC,IAAI,CAAC;gBACT,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,QAAQ,EAAE,GAAG,CAAC,IAAI;gBAClB,iBAAiB,EAAE,GAAG,CAAC,OAAO;aAC9B,CAAC,CAAA;YACF,SAAS,EAAE,CAAA;QACZ,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,SAAS,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAA;IAC9E,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,aAAa,OAAO,CAAC,cAAc,EAAE,oBAAoB,KAAK,CAAC,MAAM,wBAAwB,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,CAC3H,CAAA;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,yGAAyG;AACzG,SAAS,kBAAkB,CAAC,MAAc,EAAE,GAAkB;IAC7D,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAA;AACzC,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAClB,KAAa,EACb,MAAc,EACd,iBAAyB,EACzB,QAAuB,EACvB,QAAgB;IAEhB,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE,CAAA;IAE9C,IAAI,UAAU,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACvB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAC7B,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEtE,IAAI,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED,iHAAiH;AACjH,SAAS,cAAc,CAAI,GAAQ,EAAE,MAAoB;IACxD,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CACvC;QAAA,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;IACvC,CAAC;AACF,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAgB;IAC3C,IAAI,EAAE,YAAY;IAClB,WAAW,EAAE,kGAAkG;IAC/G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,oFAAoF;SACjG;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAA;QACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,OAAO,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAA;QACtE,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;QAEjD,IAAI,CAAC,CAAC,eAAe,IAAI,CAAC,IAAI,eAAe,IAAI,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,6CAA6C,eAAe,EAAE,CAAC,CAAA;QAChF,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;QAE1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,kDAAkD,CAAC,CAAA;QAChH,CAAC;QAED,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;QAE5D,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,eAAe,GAAG,CAAC,CAAA;QAEvB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,kBAAkB,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;YACjD,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;YAE9C,MAAM,UAAU,GAA0C;gBACzD,YAAY,EAAE,KAAK;gBACnB,kBAAkB,EAAE,CAAC,CAAC,iBAAiB;gBACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACpB,CAAA;YAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;YAC7C,CAAC;YAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACvB,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;YACtC,CAAC;YAED,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAChB,UAAU,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAA;YACjC,CAAC;YAED,IAAI,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAA;YAElF,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,gGAAgG;YAChG,yFAAyF;YACzF,+FAA+F;YAC/F,8FAA8F;YAC9F,0EAA0E;YAC1E,IAAI,eAAe,GAAG,CAAC,IAAI,MAAM,EAAE,GAAG,eAAe,EAAE,CAAC;gBACvD,MAAM,KAAK,GAAG,qBAAqB,CAAC,EAAE,CAAA;gBACtC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAE,CAAA;gBACxD,GAAG,GAAG,GAAG,GAAG,KAAK,IAAI,EAAE,CAAA;gBACvB,UAAU,CAAC,OAAO,GAAG,IAAI,CAAA;gBACzB,eAAe,EAAE,CAAA;YAClB,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,YAAY,EAAE,KAAK;gBACnB,kBAAkB,EAAE,CAAC,CAAC,iBAAiB;gBACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,SAAS;aACjC,CAAC,CAAA;YACF,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,EAAE;gBAClB,OAAO,EAAE,eAAe;aACxB,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC3F,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,aAAa,OAAO,YAAY,OAAO,qBAAqB,eAAe,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QAEhH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAqBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA8B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
|
|
@@ -14,6 +14,7 @@ import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.js";
|
|
|
14
14
|
import { frAdminSplitRecipe } from "./fr-admin-split.js";
|
|
15
15
|
import { frBareStreetRecipe } from "./fr-bare-street.js";
|
|
16
16
|
import { frFragmentRecipe } from "./fr-fragment.js";
|
|
17
|
+
import { frLieuditRecipe } from "./fr-lieudit.js";
|
|
17
18
|
import { frOrderRecipe } from "./fr-order.js";
|
|
18
19
|
import { germanRecipe } from "./german.js";
|
|
19
20
|
import { houseVenueRecipe } from "./house-venue.js";
|
|
@@ -48,6 +49,7 @@ const RECIPES = [
|
|
|
48
49
|
frAdminSplitRecipe,
|
|
49
50
|
frBareStreetRecipe,
|
|
50
51
|
frFragmentRecipe,
|
|
52
|
+
frLieuditRecipe,
|
|
51
53
|
czPcFirstPrepositionRecipe,
|
|
52
54
|
nlPostcodeRecipe,
|
|
53
55
|
noStreetLedRecipe,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,eAAe;IACf,0BAA0B;IAC1B,gBAAgB;IAChB,iBAAiB;IACjB,gBAAgB;IAChB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
* source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
|
|
25
25
|
* country, so DE/FR emit streams are unchanged for a given seed.
|
|
26
26
|
*/
|
|
27
|
-
import { type LocaleBaseTuple } from "../synthesize-german.ts";
|
|
27
|
+
import { type LocaleBaseTuple, type SynthesizedLocaleRow } from "../synthesize-german.ts";
|
|
28
28
|
import { type ShardRecipe } from "./scaffold.ts";
|
|
29
29
|
/**
|
|
30
30
|
* One per-country OA source part: either a cached `zip` + `csv` member (streamed via `unzip -p`) or an extracted plain
|
|
@@ -43,6 +43,34 @@ export interface LocalePart {
|
|
|
43
43
|
* rows). Without this, the default CITY→locality mapping wrongly trains the suburb as the locality.
|
|
44
44
|
*/
|
|
45
45
|
districtAsLocality?: boolean;
|
|
46
|
+
/**
|
|
47
|
+
* ES pedanía part only — the header is the RAW (un-conformed) CNIG export schema (`numero`, `tipo_vial`,
|
|
48
|
+
* `nombre_via`, `poblacion`, `municipio`, `comunidad_autonoma`, `cod_postal`), NOT the standard OA
|
|
49
|
+
* NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE header every other part uses. Verified 2026-07-22 by exact- coordinate
|
|
50
|
+
* cross-check: the OA-conformed `extracted/es/countrywide.csv` collapses CITY to `municipio` and drops `poblacion`
|
|
51
|
+
* (Spain's below-municipio núcleo/pedanía name) entirely, so the pedanía signal survives ONLY in this raw export.
|
|
52
|
+
* `street` is reconstructed as `tipo_vial + " " + nombre_via` (verified byte-identical to the conformed STREET column
|
|
53
|
+
* for the same row). CITY-analog = `poblacion`, DISTRICT-analog = `municipio`.
|
|
54
|
+
*/
|
|
55
|
+
cnigRaw?: boolean;
|
|
56
|
+
}
|
|
57
|
+
export interface LocaleCountrySource {
|
|
58
|
+
source: string;
|
|
59
|
+
parts: LocalePart[];
|
|
60
|
+
/**
|
|
61
|
+
* The `corpus_version` stamped on emitted rows. DE/FR keep the historical `0.4.0` (regenerating those shards must
|
|
62
|
+
* stay lineage-identical); ES/IT/NL are the #241 staging lineage (`v0.9.9-es-it-nl`).
|
|
63
|
+
*/
|
|
64
|
+
corpusVersion: string;
|
|
65
|
+
/**
|
|
66
|
+
* An ALTERNATE part list, read instead of {@link parts} when the `--district-as-locality` override is explicitly
|
|
67
|
+
* `true` for this invocation (see `run()`). ES-only for now — the standard `parts` entry can't supply real
|
|
68
|
+
* dependent-locality signal (its OA-conformed CSV drops `poblacion`; see {@link LocalePart.cnigRaw}), so the pedanía
|
|
69
|
+
* build reads a wholly different raw source instead of flipping the standard CITY/DISTRICT columns. `undefined` for
|
|
70
|
+
* every other country — the override then just forces `districtAsLocality` on the normal `parts`, as GB/NZ already do
|
|
71
|
+
* per-part.
|
|
72
|
+
*/
|
|
73
|
+
pedaniaParts?: LocalePart[];
|
|
46
74
|
}
|
|
47
75
|
/**
|
|
48
76
|
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
@@ -78,5 +106,26 @@ export declare function cleanCityNoise(city: string): string | null;
|
|
|
78
106
|
* Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
|
|
79
107
|
*/
|
|
80
108
|
export declare function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]>;
|
|
109
|
+
/**
|
|
110
|
+
* Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in
|
|
111
|
+
* place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a
|
|
112
|
+
* `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the
|
|
113
|
+
* (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away
|
|
114
|
+
* entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical
|
|
115
|
+
* to before this option existed. Exported for {@link locale.test.ts}.
|
|
116
|
+
*/
|
|
117
|
+
export declare function applyCountryAppend(synth: SynthesizedLocaleRow, country: string, countryFraction: number, random: () => number): void;
|
|
118
|
+
/**
|
|
119
|
+
* Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged
|
|
120
|
+
* (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality`
|
|
121
|
+
* forced to that value for this invocation only. Exported for {@link locale.test.ts}.
|
|
122
|
+
*/
|
|
123
|
+
export declare function applyDistrictAsLocalityOverride(part: LocalePart, override: boolean | undefined): LocalePart;
|
|
124
|
+
/**
|
|
125
|
+
* Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly
|
|
126
|
+
* `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other
|
|
127
|
+
* country/override combination. Exported for {@link locale.test.ts}.
|
|
128
|
+
*/
|
|
129
|
+
export declare function resolveLocaleParts(countrySource: LocaleCountrySource, override: boolean | undefined): LocalePart[];
|
|
81
130
|
export declare const localeRecipe: ShardRecipe;
|
|
82
131
|
//# sourceMappingURL=locale.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;
|
|
1
|
+
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAWH,OAAO,EAAE,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAuB,MAAM,yBAAyB,CAAA;AAC9G,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;IAC5B;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,UAAU,EAAE,CAAA;CAC3B;AA6FD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAgBD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAoJhG;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,oBAAoB,EAC3B,OAAO,EAAE,MAAM,EACf,eAAe,EAAE,MAAM,EACvB,MAAM,EAAE,MAAM,MAAM,GAClB,IAAI,CAgBN;AAED;;;;GAIG;AACH,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,CAE3G;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,aAAa,EAAE,mBAAmB,EAAE,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,EAAE,CAElH;AAED,eAAO,MAAM,YAAY,EAAE,WAgJ1B,CAAA"}
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
import { spawn } from "node:child_process";
|
|
28
28
|
import { createReadStream } from "node:fs";
|
|
29
|
+
import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country";
|
|
29
30
|
import { dataRootPath } from "@mailwoman/core/utils";
|
|
30
31
|
import { CSVSpliterator } from "spliterator";
|
|
31
32
|
import { stableSourceID } from "../adapter.js";
|
|
@@ -68,6 +69,18 @@ const COUNTRY_SOURCES = {
|
|
|
68
69
|
source: "synth-es",
|
|
69
70
|
corpusVersion: "0.9.9",
|
|
70
71
|
parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
|
|
72
|
+
// Pedanía shard (`synth-es-pedania`, `--district-as-locality`). Reads the RAW (un-conformed) CNIG export
|
|
73
|
+
// cached at oa-cache/es__countrywide.zip — the `parts` CSV above lost `poblacion` in OA's own conform step
|
|
74
|
+
// (see {@link LocalePart.cnigRaw}). districtAsLocality is pinned true here (this part only exists to be
|
|
75
|
+
// read pedanía-style); the CLI override still applies harmlessly on top.
|
|
76
|
+
pedaniaParts: [
|
|
77
|
+
{
|
|
78
|
+
zip: dataRootPath("oa-cache", "es__countrywide.zip"),
|
|
79
|
+
csv: "es_addresses.csv",
|
|
80
|
+
cnigRaw: true,
|
|
81
|
+
districtAsLocality: true,
|
|
82
|
+
},
|
|
83
|
+
],
|
|
71
84
|
},
|
|
72
85
|
NZ: {
|
|
73
86
|
// LINZ-derived OA countrywide extract (2.12M rows). `districtAsLocality` inverts the CITY/DISTRICT
|
|
@@ -78,6 +91,16 @@ const COUNTRY_SOURCES = {
|
|
|
78
91
|
// coord board; that split is a BUILD-TIME concern (scratchpad), so the committed recipe reads the full CSV.
|
|
79
92
|
parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
|
|
80
93
|
},
|
|
94
|
+
GB: {
|
|
95
|
+
// HM Land Registry Price Paid Data tuples (25.67M rows; see Task 2's ppd ingest). PPD's DISTRICT is the
|
|
96
|
+
// postal town (locality) and CITY is the dependent locality — legitimately EMPTY on the majority of rows
|
|
97
|
+
// (most GB addresses have no dependent locality). `districtAsLocality` maps DISTRICT→locality and, when
|
|
98
|
+
// present, CITY→dependent_locality; the `readTuples` gate above only drops a row when BOTH are empty, so
|
|
99
|
+
// the majority empty-CITY rows survive.
|
|
100
|
+
source: "synth-gb",
|
|
101
|
+
corpusVersion: "0.9.9",
|
|
102
|
+
parts: [{ path: dataRootPath("ppd", "2026-07-22", "gb-tuples.csv"), districtAsLocality: true }],
|
|
103
|
+
},
|
|
81
104
|
};
|
|
82
105
|
/**
|
|
83
106
|
* Per-part reservoir cap. Streaming + Algorithm-R reservoir sampling to this size keeps memory bounded regardless of
|
|
@@ -168,39 +191,79 @@ export async function readTuples(part, rng) {
|
|
|
168
191
|
if (header === null) {
|
|
169
192
|
header = cells.map((h) => h.trim().toLowerCase());
|
|
170
193
|
const ix = (name) => header.indexOf(name);
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
194
|
+
// `cnigRaw` (ES pedanía only): the RAW CNIG header has no NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE
|
|
195
|
+
// at all — `numero`/`nombre_via`/`poblacion`/`municipio`/`comunidad_autonoma`/`cod_postal` instead.
|
|
196
|
+
// See {@link LocalePart.cnigRaw}.
|
|
197
|
+
cols = part.cnigRaw
|
|
198
|
+
? {
|
|
199
|
+
num: ix("numero"),
|
|
200
|
+
street: ix("nombre_via"),
|
|
201
|
+
tipoVial: ix("tipo_vial"),
|
|
202
|
+
city: ix("poblacion"),
|
|
203
|
+
district: ix("municipio"),
|
|
204
|
+
region: ix("comunidad_autonoma"),
|
|
205
|
+
post: ix("cod_postal"),
|
|
206
|
+
}
|
|
207
|
+
: {
|
|
208
|
+
num: ix("number"),
|
|
209
|
+
street: ix("street"),
|
|
210
|
+
tipoVial: -1,
|
|
211
|
+
city: ix("city"),
|
|
212
|
+
district: ix("district"),
|
|
213
|
+
region: ix("region"),
|
|
214
|
+
post: ix("postcode"),
|
|
215
|
+
};
|
|
179
216
|
continue;
|
|
180
217
|
}
|
|
181
218
|
if (cols === null)
|
|
182
219
|
continue;
|
|
183
|
-
|
|
220
|
+
// cnigRaw: STREET is split into a road-type column (`tipo_vial`, e.g. "CARRETERA") and the name
|
|
221
|
+
// (`nombre_via`) — rejoin them exactly as OA's own conform step does (verified byte-identical to the
|
|
222
|
+
// conformed STREET column for the same source row). Every other part's header already carries the
|
|
223
|
+
// pre-joined STREET column, so `cols.tipoVial === -1` and this is a no-op there.
|
|
224
|
+
const street = cols.tipoVial >= 0
|
|
225
|
+
? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(Boolean).join(" ")
|
|
226
|
+
: get(cells, cols.street);
|
|
184
227
|
const rawCity = get(cells, cols.city);
|
|
185
|
-
if (!street
|
|
228
|
+
if (!street)
|
|
186
229
|
continue;
|
|
187
|
-
// Default: CITY → locality. NZ (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
|
|
230
|
+
// Default: CITY → locality. NZ/GB (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
|
|
188
231
|
// (`Auckland`) and CITY holds the suburb (`Birkenhead`), so DISTRICT → locality and CITY →
|
|
189
232
|
// dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
|
|
190
|
-
// sub-locality.
|
|
233
|
+
// sub-locality. GB PPD tuples flip which side is legitimately empty — on the MAJORITY of GB rows CITY
|
|
234
|
+
// (the dependent_locality) is empty and DISTRICT (the locality) is populated, so the gate below only
|
|
235
|
+
// drops a `districtAsLocality` row when BOTH are empty, not when CITY alone is (that used to silently
|
|
236
|
+
// drop most of the GB source — see the fixed `readTuples` gate below). See
|
|
237
|
+
// {@link LocalePart.districtAsLocality}.
|
|
191
238
|
let locality;
|
|
192
239
|
let dependent_locality;
|
|
193
240
|
if (part.districtAsLocality) {
|
|
194
|
-
const
|
|
241
|
+
const rawDistrict = get(cells, cols.district);
|
|
242
|
+
if (!rawCity && !rawDistrict)
|
|
243
|
+
continue;
|
|
244
|
+
const cleanedDistrict = cleanCityNoise(rawDistrict);
|
|
195
245
|
if (cleanedDistrict) {
|
|
196
246
|
locality = cleanedDistrict;
|
|
197
|
-
|
|
247
|
+
const cleanedCity = cleanCityNoise(rawCity);
|
|
248
|
+
// ES pedanía lesson (2026-07-22): the CNIG `poblacion` column is filled on ~93% of rows but
|
|
249
|
+
// EQUALS `municipio` on the majority of those (the address point sits in the municipio's own
|
|
250
|
+
// main town, not a below-municipio pedanía) — only ~32.6% of ES rows carry a genuinely
|
|
251
|
+
// DISTINCT poblacion. GB/NZ never hit this (CITY/DISTRICT name the same place only by rare
|
|
252
|
+
// coincidence), but the guard is general: a dependent_locality equal to its own locality is
|
|
253
|
+
// never a real sub-locality, so drop it rather than emit a same-value pair (would fail the
|
|
254
|
+
// dep_loc≠locality invariant every recipe otherwise upholds).
|
|
255
|
+
dependent_locality =
|
|
256
|
+
cleanedCity && cleanedCity.localeCompare(locality, undefined, { sensitivity: "base" }) !== 0
|
|
257
|
+
? cleanedCity
|
|
258
|
+
: undefined;
|
|
198
259
|
}
|
|
199
260
|
else {
|
|
200
261
|
locality = cleanCityNoise(rawCity);
|
|
201
262
|
}
|
|
202
263
|
}
|
|
203
264
|
else {
|
|
265
|
+
if (!rawCity)
|
|
266
|
+
continue;
|
|
204
267
|
locality = cleanCityNoise(rawCity);
|
|
205
268
|
}
|
|
206
269
|
if (!locality) {
|
|
@@ -235,13 +298,58 @@ export async function readTuples(part, rng) {
|
|
|
235
298
|
console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`);
|
|
236
299
|
return reservoir;
|
|
237
300
|
}
|
|
301
|
+
/**
|
|
302
|
+
* Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in
|
|
303
|
+
* place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a
|
|
304
|
+
* `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the
|
|
305
|
+
* (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away
|
|
306
|
+
* entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical
|
|
307
|
+
* to before this option existed. Exported for {@link locale.test.ts}.
|
|
308
|
+
*/
|
|
309
|
+
export function applyCountryAppend(synth, country, countryFraction, random) {
|
|
310
|
+
if (countryFraction > 0 && random() < countryFraction) {
|
|
311
|
+
const forms = COUNTRY_SURFACE_FORMS[country];
|
|
312
|
+
if (!forms?.length) {
|
|
313
|
+
// The BR/NZ lesson: a missing table entry must never silently no-op a requested fraction —
|
|
314
|
+
// it must raise so the gap is caught at build time, not discovered later as a 0% gate failure.
|
|
315
|
+
throw new Error(`No COUNTRY_SURFACE_FORMS entry for ${country} — add it to codex/country/country.ts before using --country-fraction`);
|
|
316
|
+
}
|
|
317
|
+
const form = forms[Math.floor(random() * forms.length)];
|
|
318
|
+
synth.raw = `${synth.raw}, ${form}`;
|
|
319
|
+
synth.components = { ...synth.components, country: form };
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged
|
|
324
|
+
* (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality`
|
|
325
|
+
* forced to that value for this invocation only. Exported for {@link locale.test.ts}.
|
|
326
|
+
*/
|
|
327
|
+
export function applyDistrictAsLocalityOverride(part, override) {
|
|
328
|
+
return override === undefined ? part : { ...part, districtAsLocality: override };
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly
|
|
332
|
+
* `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other
|
|
333
|
+
* country/override combination. Exported for {@link locale.test.ts}.
|
|
334
|
+
*/
|
|
335
|
+
export function resolveLocaleParts(countrySource, override) {
|
|
336
|
+
return override === true && countrySource.pedaniaParts ? countrySource.pedaniaParts : countrySource.parts;
|
|
337
|
+
}
|
|
238
338
|
export const localeRecipe = {
|
|
239
339
|
name: "locale",
|
|
240
|
-
description: "Per-locale coverage rows (DE/FR/NL/IT/ES) from real OA tuples, both orders → synthesizeLocaleRow",
|
|
340
|
+
description: "Per-locale coverage rows (DE/FR/NL/IT/ES/NZ/GB) from real OA tuples, both orders → synthesizeLocaleRow",
|
|
241
341
|
mode: "generate",
|
|
242
342
|
options: [
|
|
243
|
-
{ flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ). Default DE" },
|
|
343
|
+
{ flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ|GB). Default DE" },
|
|
244
344
|
{ flag: "--intl-fraction <f>", description: "Fraction rendered international order. Default 0.4" },
|
|
345
|
+
{
|
|
346
|
+
flag: "--country-fraction <f>",
|
|
347
|
+
description: "Fraction of rows that append an explicit country surface form (`, United Kingdom`) + a `country` component (fr-admin-split pattern). Default 0 — byte-identical to before when unset.",
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
flag: "--district-as-locality / --no-district-as-locality",
|
|
351
|
+
description: "Override the per-part districtAsLocality mapping for this run. Unset (default) leaves each COUNTRY_SOURCES part's own value untouched — every existing build stays byte-identical. ES additionally switches to the pedanía (poblacion→dependent_locality) source when passed as true — combine with --source-name synth-es-pedania.",
|
|
352
|
+
},
|
|
245
353
|
],
|
|
246
354
|
async run(opts, write) {
|
|
247
355
|
// Emit PRNG: the legacy build-locale-shard.mjs seeded mulberry32(opts.seed). The reservoir uses a
|
|
@@ -256,15 +364,27 @@ export const localeRecipe = {
|
|
|
256
364
|
if (!(intlFraction >= 0 && intlFraction <= 1)) {
|
|
257
365
|
throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`);
|
|
258
366
|
}
|
|
367
|
+
// Default 0 → the `random() < countryFraction` draw below is short-circuited away entirely (never
|
|
368
|
+
// consumed), so every existing locale's emit stream is byte-identical to before this option existed.
|
|
369
|
+
const countryFraction = opts.countryFraction ?? 0;
|
|
370
|
+
if (!(countryFraction >= 0 && countryFraction <= 1)) {
|
|
371
|
+
throw new Error(`--country-fraction must be in [0, 1], got ${countryFraction}`);
|
|
372
|
+
}
|
|
259
373
|
const source = opts.sourceName ?? countrySource.source;
|
|
260
374
|
const count = opts.count ?? 4000;
|
|
261
|
-
|
|
375
|
+
// Tri-state: `undefined` (flag absent) touches nothing below — `parts` stays the default list and each
|
|
376
|
+
// part keeps its own `districtAsLocality`, so every existing locale build is byte-identical to before this
|
|
377
|
+
// option existed. `true` additionally selects `pedaniaParts` when the country registers one (ES); `false`
|
|
378
|
+
// forces the mapping off on every part read this run (a debugging escape hatch for GB/NZ).
|
|
379
|
+
const districtAsLocalityOverride = opts.districtAsLocality;
|
|
380
|
+
const parts = resolveLocaleParts(countrySource, districtAsLocalityOverride);
|
|
262
381
|
const pool = [];
|
|
263
382
|
for (let pi = 0; pi < parts.length; pi++) {
|
|
264
383
|
// A reservoir PRNG per part, seeded but independent of the emit loop's `random`, so the sample is
|
|
265
384
|
// reproducible without perturbing the synth/order draws.
|
|
266
385
|
const reservoirRng = makeMulberry32((opts.seed ^ (0x9e3779b9 * (pi + 1))) >>> 0);
|
|
267
|
-
const
|
|
386
|
+
const effectivePart = applyDistrictAsLocalityOverride(parts[pi], districtAsLocalityOverride);
|
|
387
|
+
const t = await readTuples(effectivePart, reservoirRng);
|
|
268
388
|
for (const x of t) {
|
|
269
389
|
pool.push(x);
|
|
270
390
|
} // NOT pool.push(...t) — spreading huge arrays overflows the stack
|
|
@@ -292,6 +412,7 @@ export const localeRecipe = {
|
|
|
292
412
|
skipped++;
|
|
293
413
|
continue;
|
|
294
414
|
}
|
|
415
|
+
applyCountryAppend(synth, country, countryFraction, random);
|
|
295
416
|
if (opts.golden) {
|
|
296
417
|
// Golden rows must round-trip through alignRow exactly like training rows (#241 done-when): a
|
|
297
418
|
// render that can't be BIO-labeled can't serve as a parser golden either. Consumes no RNG draw.
|