@mailwoman/corpus 4.16.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +23 -26
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +25 -28
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +10 -10
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts +2 -2
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +2 -2
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +2 -2
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +9 -9
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +5 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +3 -3
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -10
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +4 -5
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +9 -10
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +13 -14
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +8 -10
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +8 -8
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +6 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +6 -6
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +7 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +9 -9
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +6 -6
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts +2 -2
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +2 -2
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +15 -17
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +6 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -6
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +7 -7
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +12 -12
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +3 -3
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +6 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +17 -19
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +25 -28
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +8 -8
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +11 -11
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +14 -16
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +15 -16
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +13 -15
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -6
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -8
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +6 -8
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +1 -1
- package/out/src/golden.js.map +1 -1
- package/out/src/index.d.ts +1 -0
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/license.d.ts +5 -6
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +5 -6
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +2 -2
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +16 -18
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +22 -23
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +10 -10
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +8 -8
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +16 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -0
- package/out/src/shard-recipes/anchor-absorption.js +73 -0
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts +16 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -0
- package/out/src/shard-recipes/boundary-stress.js +77 -0
- package/out/src/shard-recipes/boundary-stress.js.map +1 -0
- package/out/src/shard-recipes/country-balanced.d.ts +28 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -0
- package/out/src/shard-recipes/country-balanced.js +370 -0
- package/out/src/shard-recipes/country-balanced.js.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts +34 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.js +175 -0
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts +24 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.js +76 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -0
- package/out/src/shard-recipes/fr-order.d.ts +34 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-order.js +238 -0
- package/out/src/shard-recipes/fr-order.js.map +1 -0
- package/out/src/shard-recipes/german.d.ts +19 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -0
- package/out/src/shard-recipes/german.js +170 -0
- package/out/src/shard-recipes/german.js.map +1 -0
- package/out/src/shard-recipes/house-venue.d.ts +14 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/house-venue.js +66 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts +18 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -0
- package/out/src/shard-recipes/index.js +58 -0
- package/out/src/shard-recipes/index.js.map +1 -0
- package/out/src/shard-recipes/intersection.d.ts +38 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -0
- package/out/src/shard-recipes/intersection.js +473 -0
- package/out/src/shard-recipes/intersection.js.map +1 -0
- package/out/src/shard-recipes/locale.d.ts +19 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -0
- package/out/src/shard-recipes/locale.js +257 -0
- package/out/src/shard-recipes/locale.js.map +1 -0
- package/out/src/shard-recipes/no-street.d.ts +14 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -0
- package/out/src/shard-recipes/no-street.js +66 -0
- package/out/src/shard-recipes/no-street.js.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts +34 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.js +686 -0
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -0
- package/out/src/shard-recipes/po-box.d.ts +13 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box.js +93 -0
- package/out/src/shard-recipes/po-box.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +104 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -0
- package/out/src/shard-recipes/scaffold.js +76 -0
- package/out/src/shard-recipes/scaffold.js.map +1 -0
- package/out/src/shard-recipes/street-affix.d.ts +31 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -0
- package/out/src/shard-recipes/street-affix.js +408 -0
- package/out/src/shard-recipes/street-affix.js.map +1 -0
- package/out/src/shard-recipes/street-bare.d.ts +21 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -0
- package/out/src/shard-recipes/street-bare.js +82 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -0
- package/out/src/shard-recipes/street.d.ts +11 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -0
- package/out/src/shard-recipes/street.js +63 -0
- package/out/src/shard-recipes/street.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +25 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -0
- package/out/src/shard-recipes/unit.js +255 -0
- package/out/src/shard-recipes/unit.js.map +1 -0
- package/out/src/split.d.ts +23 -29
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +22 -29
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +2 -3
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +2 -2
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -4
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +3 -4
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -19
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +14 -17
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts +2 -3
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +2 -3
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +2 -2
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +4 -4
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +5 -5
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +27 -23
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -17
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +27 -27
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +8 -9
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +3 -3
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +57 -66
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +56 -65
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +5 -6
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js +5 -6
- package/out/src/tokenize.js.map +1 -1
- package/out/src/types.d.ts +49 -51
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +32 -34
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +23 -25
- package/out/src/wof-json.js.map +1 -1
- package/package.json +14 -14
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-admin-split` shard recipe — the FR admin-split coverage shard (night 2026-06-19,
|
|
7
|
+
* surpass-v1.5.0). Teaches the model to SPLIT the département out of the locality on
|
|
8
|
+
* bare/space/comma-delimited French place rows — the admin-deciding failure class the pre-GPU
|
|
9
|
+
* self-validation proved moves the resolved coordinate (collision communes −61%; see
|
|
10
|
+
* docs/articles/evals/2026-06-19-fr-admin-split-prevalidation.md). Ported from
|
|
11
|
+
* scripts/build-fr-admin-split-shard.mjs.
|
|
12
|
+
*
|
|
13
|
+
* Failure shapes (the model currently mis-handles all of these):
|
|
14
|
+
*
|
|
15
|
+
* - `Thauron, Creuse` → région dropped to null (the comma+full-name miss)
|
|
16
|
+
* - `Montredon, Lozère` → région = "ère" (the diacritic subword split, #727)
|
|
17
|
+
* - (AU analog) `CANBERRA ACT` → the space-delimited admin fuse
|
|
18
|
+
*
|
|
19
|
+
* The département is the essential admin unit for FR postal geography and maps to the `region`
|
|
20
|
+
* component tag in our schema. We derive it DETERMINISTICALLY from the real postcode via codex
|
|
21
|
+
* `departementForCodePostal` (first two digits = département) — salvage-first, no re-derived
|
|
22
|
+
* table.
|
|
23
|
+
*
|
|
24
|
+
* Data (`--communes`, opts.communes): REAL BAN (Base Adresse Nationale) commune+postcode+coord
|
|
25
|
+
* tuples, one per line, TAB-separated `commune <TAB> postcode <TAB> lon <TAB> lat`. Build the
|
|
26
|
+
* input TSV once from the BAN staging CSV (see the legacy script header). Anchor-ON by
|
|
27
|
+
* construction: rows carry a REAL postcode token in `raw` + a `postcode` component, so the
|
|
28
|
+
* training loader paints the anchor feature onto that span automatically. The trailing-postcode
|
|
29
|
+
* anchor REINFORCES the FR split (FR postcode is trailing, unlike German PLZ-leading — the v0.9.2
|
|
30
|
+
* scar is positional, not universal).
|
|
31
|
+
*/
|
|
32
|
+
import { createReadStream } from "node:fs";
|
|
33
|
+
import { createInterface } from "node:readline";
|
|
34
|
+
import { departementForCodePostal } from "@mailwoman/codex/fr";
|
|
35
|
+
import { stableSourceID } from "../adapter.js";
|
|
36
|
+
import { alignRow } from "../align.js";
|
|
37
|
+
import { makeMulberry32 } from "./scaffold.js";
|
|
38
|
+
const DEFAULT_COMMUNES = "/tmp/reg/fr-communes.tsv";
|
|
39
|
+
const LICENSE = "BAN (Base Adresse Nationale) commune+postcode tuples, rendered admin-split — see ingest SOURCE";
|
|
40
|
+
/** Read the distinct commune TSV (commune, postcode, lon, lat); derive the département name. */
|
|
41
|
+
async function readCommunes(path) {
|
|
42
|
+
const rows = [];
|
|
43
|
+
const rl = createInterface({ input: createReadStream(path, { encoding: "utf8" }), crlfDelay: Infinity });
|
|
44
|
+
for await (const line of rl) {
|
|
45
|
+
if (!line)
|
|
46
|
+
continue;
|
|
47
|
+
const [commune, postcode, lon, lat] = line.split("\t");
|
|
48
|
+
if (!commune || !postcode)
|
|
49
|
+
continue;
|
|
50
|
+
const dep = departementForCodePostal(postcode);
|
|
51
|
+
if (!dep)
|
|
52
|
+
continue; // bad/unmappable postcode — skip (CEDEX, etc.)
|
|
53
|
+
// Substring invariant: a département whose name isn't a clean token (none are) or a commune
|
|
54
|
+
// containing the département name would confuse alignment — both are vanishingly rare here.
|
|
55
|
+
rows.push({ commune, postcode, departement: dep.name, lon, lat });
|
|
56
|
+
}
|
|
57
|
+
return rows;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Render one admin-split variant. The CORE teaching signal: the département, even as a full word after a comma or a
|
|
61
|
+
* space, is `region` — never folded into `locality`. Variants 1-3 are the failure class; 4-5 are canonical-FR
|
|
62
|
+
* preservation so the model doesn't over-fire region on every trailing token (and the bare commune still resolves).
|
|
63
|
+
*/
|
|
64
|
+
function render(random, c) {
|
|
65
|
+
const r = random();
|
|
66
|
+
const loc = random() < 0.1 ? c.commune.toUpperCase() : c.commune;
|
|
67
|
+
const dep = c.departement;
|
|
68
|
+
const pc = c.postcode;
|
|
69
|
+
let out;
|
|
70
|
+
if (r < 0.25) {
|
|
71
|
+
// 1. bare comma, NO postcode — the Thauron/#727 shape (anchor off)
|
|
72
|
+
out = { raw: `${loc}, ${dep}`, components: { locality: loc, region: dep }, order: "bare-comma" };
|
|
73
|
+
}
|
|
74
|
+
else if (r < 0.5) {
|
|
75
|
+
// 2. bare comma + postcode — anchor ON
|
|
76
|
+
out = {
|
|
77
|
+
raw: `${loc}, ${dep} ${pc}`,
|
|
78
|
+
components: { locality: loc, region: dep, postcode: pc },
|
|
79
|
+
order: "bare-comma-pc",
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
else if (r < 0.7) {
|
|
83
|
+
// 3. space-delimited admin (the AU `CANBERRA ACT` fuse applied to FR) — anchor ON
|
|
84
|
+
out = { raw: `${loc} ${dep} ${pc}`, components: { locality: loc, region: dep, postcode: pc }, order: "space-pc" };
|
|
85
|
+
}
|
|
86
|
+
else if (r < 0.85) {
|
|
87
|
+
// 4. canonical FR postcode-first (NO département) — preservation, anchor ON
|
|
88
|
+
out = { raw: `${pc} ${loc}`, components: { postcode: pc, locality: loc }, order: "canonical-pc-first" };
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
// 5. commune + postcode (NO département) — preservation, anchor ON
|
|
92
|
+
out = { raw: `${loc} ${pc}`, components: { locality: loc, postcode: pc }, order: "commune-pc" };
|
|
93
|
+
}
|
|
94
|
+
// fr.country preservation (the v1.8.0 #728 finding): the v1.8.0 shard's bare rows carried NO country
|
|
95
|
+
// token, so the model under-emitted country on FR (fr.country −3.5pp). ~20% of rows now append an
|
|
96
|
+
// explicit "France" + a `country` component — the model relearns to emit country WHEN the token is
|
|
97
|
+
// present without over-firing it on the (still-majority) country-less rows. Substring invariant holds.
|
|
98
|
+
if (random() < 0.2) {
|
|
99
|
+
out = {
|
|
100
|
+
raw: `${out.raw}, France`,
|
|
101
|
+
components: { ...out.components, country: "France" },
|
|
102
|
+
order: `${out.order}+fr`,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
107
|
+
export const frAdminSplitRecipe = {
|
|
108
|
+
name: "fr-admin-split",
|
|
109
|
+
description: "FR admin-split rows: BAN communes → split département into `region` (+ canonical-FR preservation)",
|
|
110
|
+
mode: "generate",
|
|
111
|
+
options: [
|
|
112
|
+
{ flag: "--communes <tsv>", description: "BAN commune+postcode+coord TSV. Default /tmp/reg/fr-communes.tsv" },
|
|
113
|
+
],
|
|
114
|
+
async run(opts, write) {
|
|
115
|
+
// Legacy build-fr-admin-split-shard.mjs seeded `mulberry32(opts.seed)`.
|
|
116
|
+
const random = makeMulberry32(opts.seed);
|
|
117
|
+
const count = opts.count ?? 60000;
|
|
118
|
+
const source = opts.sourceName ?? "synth-fr-admin-split";
|
|
119
|
+
const communesPath = opts.communes ?? DEFAULT_COMMUNES;
|
|
120
|
+
const pool = await readCommunes(communesPath);
|
|
121
|
+
console.error(` ${communesPath}: ${pool.length} communes with derived département`);
|
|
122
|
+
if (pool.length === 0) {
|
|
123
|
+
throw new Error("No communes — build the TSV from BAN first (see the recipe header).");
|
|
124
|
+
}
|
|
125
|
+
let emitted = 0;
|
|
126
|
+
let skipped = 0;
|
|
127
|
+
let guard = 0;
|
|
128
|
+
const orderCounts = {};
|
|
129
|
+
const N = pool.length;
|
|
130
|
+
while (emitted < count && guard++ < count * 12) {
|
|
131
|
+
const base = pool[Math.floor(random() * N)];
|
|
132
|
+
const { raw, components, order } = render(random, base);
|
|
133
|
+
// Alignment precondition: every component surface appears verbatim in raw.
|
|
134
|
+
const values = Object.values(components).filter((v) => Boolean(v));
|
|
135
|
+
if (!values.every((v) => raw.includes(v))) {
|
|
136
|
+
skipped++;
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (opts.golden) {
|
|
140
|
+
// Held-out eval slice for the centroid gate — carries the truth coordinate.
|
|
141
|
+
write(JSON.stringify({ raw, components, country: "FR", lat: Number(base.lat), lon: Number(base.lon) }) + "\n");
|
|
142
|
+
emitted++;
|
|
143
|
+
orderCounts[order] = (orderCounts[order] ?? 0) + 1;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const sourceID = stableSourceID(source, {
|
|
147
|
+
locality: components.locality,
|
|
148
|
+
region: components.region,
|
|
149
|
+
postcode: components.postcode,
|
|
150
|
+
});
|
|
151
|
+
const canonical = {
|
|
152
|
+
raw,
|
|
153
|
+
components,
|
|
154
|
+
country: "FR",
|
|
155
|
+
locale: "fr-FR",
|
|
156
|
+
source,
|
|
157
|
+
source_id: sourceID,
|
|
158
|
+
corpus_version: "0.5.0",
|
|
159
|
+
license: LICENSE,
|
|
160
|
+
};
|
|
161
|
+
const aligned = alignRow(canonical);
|
|
162
|
+
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
163
|
+
skipped++;
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
write(JSON.stringify({ ...aligned.row, synth_method: "fr-admin-split", synth_order: order, synth_base_id: null }) +
|
|
167
|
+
"\n");
|
|
168
|
+
emitted++;
|
|
169
|
+
orderCounts[order] = (orderCounts[order] ?? 0) + 1;
|
|
170
|
+
}
|
|
171
|
+
console.error(` emitted=${emitted} skipped=${skipped} order-mix=${JSON.stringify(orderCounts)}`);
|
|
172
|
+
return { emitted, skipped };
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
//# sourceMappingURL=fr-admin-split.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-admin-split.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-admin-split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAA;AAG9D,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAEhE,MAAM,gBAAgB,GAAG,0BAA0B,CAAA;AACnD,MAAM,OAAO,GAAG,gGAAgG,CAAA;AAkBhH,gGAAgG;AAChG,KAAK,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,IAAI,GAAiB,EAAE,CAAA;IAC7B,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAExG,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAEtD,IAAI,CAAC,OAAO,IAAI,CAAC,QAAQ;YAAE,SAAQ;QACnC,MAAM,GAAG,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAA;QAE9C,IAAI,CAAC,GAAG;YAAE,SAAQ,CAAC,+CAA+C;QAClE,4FAA4F;QAC5F,4FAA4F;QAC5F,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IAClE,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,MAAoB,EAAE,CAAa;IAClD,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAClB,MAAM,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;IAChE,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,CAAA;IACzB,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAA;IACrB,IAAI,GAAsB,CAAA;IAE1B,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QACd,mEAAmE;QACnE,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,KAAK,GAAG,EAAE,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAA;IACjG,CAAC;SAAM,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACpB,uCAAuC;QACvC,GAAG,GAAG;YACL,GAAG,EAAE,GAAG,GAAG,KAAK,GAAG,IAAI,EAAE,EAAE;YAC3B,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE;YACxD,KAAK,EAAE,eAAe;SACtB,CAAA;IACF,CAAC;SAAM,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACpB,kFAAkF;QAClF,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAA;IAClH,CAAC;SAAM,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QACrB,4EAA4E;QAC5E,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,GAAG,EAAE,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,CAAA;IACxG,CAAC;SAAM,CAAC;QACP,mEAAmE;QACnE,GAAG,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAA;IAChG,CAAC;IAED,qGAAqG;IACrG,kGAAkG;IAClG,mGAAmG;IACnG,uGAAuG;IACvG,IAAI,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QACpB,GAAG,GAAG;YACL,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,UAAU;YACzB,UAAU,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE;YACpD,KAAK,EAAE,GAAG,GAAG,CAAC,KAAK,KAAK;SACxB,CAAA;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAgB;IAC9C,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,mGAAmG;IAChH,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,kEAAkE,EAAE;KAC7G;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,wEAAwE;QACxE,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,sBAAsB,CAAA;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,IAAI,gBAAgB,CAAA;QAEtD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,YAAY,CAAC,CAAA;QAC7C,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,IAAI,CAAC,MAAM,oCAAoC,CAAC,CAAA;QAEpF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAA;QACvF,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,WAAW,GAA2B,EAAE,CAAA;QAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;YAEvD,2EAA2E;YAC3E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;YAE/E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3C,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,4EAA4E;gBAC5E,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAC9G,OAAO,EAAE,CAAA;gBACT,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;gBAClD,SAAQ;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,UAAU,CAAC,QAAQ;aAC7B,CAAC,CAAA;YACF,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,OAAO;aAChB,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;gBAC1G,IAAI,CACL,CAAA;YACD,OAAO,EAAE,CAAA;YACT,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,aAAa,OAAO,YAAY,OAAO,cAAc,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QAEjG,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-bare-street` shard recipe (#251) — the postcode-anchoring-imbalance lever. BAN (and every
|
|
7
|
+
* other comprehensive FR source) is postcode-COMPLETE, so the model learned the French
|
|
8
|
+
* street→locality boundary as "the token after the 5-digit postcode," never as "comma + city." Strip
|
|
9
|
+
* the postcode and it leaks the street's proper-noun tokens into the following locality ("Rue René
|
|
10
|
+
* Cassin, Paris" → street="Rue Ren", locality="Cassin"). This recipe mints the MISSING distribution:
|
|
11
|
+
* the BARE comma form, NO postcode, real `(street, number, city)` tuples from BAN (Licence Ouverte —
|
|
12
|
+
* permissive; the model stays clean of ODbL, unlike the opt-in OSM rooftop shards).
|
|
13
|
+
*
|
|
14
|
+
* Each tuple → `<n> <Rue/Avenue/…> <proper-noun name>, <City>` with the FR prefix split
|
|
15
|
+
* ({@link decomposeFrStreet}: "Rue" → street_prefix, the rest → street). Tuples whose street carries
|
|
16
|
+
* no recognized FR type word are skipped — the failing class is precisely the prefix-led street.
|
|
17
|
+
*
|
|
18
|
+
* ⚠ Convention loss-mask: this recipe TEACHES FR `street_prefix`. The conventions loss-mask forbids it
|
|
19
|
+
* for FR and will `-inf` these gold labels (the v1.6.0 ~7M-loss blow-up). Disable that mask for any
|
|
20
|
+
* run including this shard.
|
|
21
|
+
*/
|
|
22
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
23
|
+
export declare const frBareStreetRecipe: ShardRecipe;
|
|
24
|
+
//# sourceMappingURL=fr-bare-street.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-bare-street.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-bare-street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAE1G,eAAO,MAAM,kBAAkB,EAAE,WAwDhC,CAAA"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-bare-street` shard recipe (#251) — the postcode-anchoring-imbalance lever. BAN (and every
|
|
7
|
+
* other comprehensive FR source) is postcode-COMPLETE, so the model learned the French
|
|
8
|
+
* street→locality boundary as "the token after the 5-digit postcode," never as "comma + city." Strip
|
|
9
|
+
* the postcode and it leaks the street's proper-noun tokens into the following locality ("Rue René
|
|
10
|
+
* Cassin, Paris" → street="Rue Ren", locality="Cassin"). This recipe mints the MISSING distribution:
|
|
11
|
+
* the BARE comma form, NO postcode, real `(street, number, city)` tuples from BAN (Licence Ouverte —
|
|
12
|
+
* permissive; the model stays clean of ODbL, unlike the opt-in OSM rooftop shards).
|
|
13
|
+
*
|
|
14
|
+
* Each tuple → `<n> <Rue/Avenue/…> <proper-noun name>, <City>` with the FR prefix split
|
|
15
|
+
* ({@link decomposeFrStreet}: "Rue" → street_prefix, the rest → street). Tuples whose street carries
|
|
16
|
+
* no recognized FR type word are skipped — the failing class is precisely the prefix-led street.
|
|
17
|
+
*
|
|
18
|
+
* ⚠ Convention loss-mask: this recipe TEACHES FR `street_prefix`. The conventions loss-mask forbids it
|
|
19
|
+
* for FR and will `-inf` these gold labels (the v1.6.0 ~7M-loss blow-up). Disable that mask for any
|
|
20
|
+
* run including this shard.
|
|
21
|
+
*/
|
|
22
|
+
import { decomposeFrStreet } from "../adapters/ban/street-decompose.js";
|
|
23
|
+
import { alignAndWrite, makeMulberry32, readTuples, shardSourceID } from "./scaffold.js";
|
|
24
|
+
export const frBareStreetRecipe = {
|
|
25
|
+
name: "fr-bare-street",
|
|
26
|
+
description: "FR bare comma-form street+city, NO postcode (#251): '<n> Rue <name>, <City>' — the postcode-anchoring lever",
|
|
27
|
+
mode: "tuples",
|
|
28
|
+
async run(opts, write) {
|
|
29
|
+
// Seeded for parity with the other recipes; unused beyond reproducibility (the tuples drive the content).
|
|
30
|
+
makeMulberry32(opts.seed);
|
|
31
|
+
let read = 0;
|
|
32
|
+
let emitted = 0;
|
|
33
|
+
let skipped = 0;
|
|
34
|
+
for await (const t of readTuples(opts.input)) {
|
|
35
|
+
read++;
|
|
36
|
+
const fullStreet = String(t.street ?? "").trim();
|
|
37
|
+
const number = String(t.number ?? "").trim();
|
|
38
|
+
const locality = String(t.locality ?? "").trim();
|
|
39
|
+
if (!fullStreet || !number || !locality) {
|
|
40
|
+
skipped++;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
const { prefix, street } = decomposeFrStreet(fullStreet);
|
|
44
|
+
// The failing class is the prefix-led FR street; a no-prefix nom_voie ("La Ville Mois") isn't it.
|
|
45
|
+
if (!prefix || !street) {
|
|
46
|
+
skipped++;
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const components = {
|
|
50
|
+
house_number: number,
|
|
51
|
+
street_prefix: prefix,
|
|
52
|
+
street,
|
|
53
|
+
locality,
|
|
54
|
+
};
|
|
55
|
+
// BARE comma form, number-before (FR's dominant order), NO postcode — the whole point.
|
|
56
|
+
const raw = `${number} ${prefix} ${street}, ${locality}`;
|
|
57
|
+
const source_id = shardSourceID("synth-fr-bare-street", { ...components, v: String(read) });
|
|
58
|
+
const canonical = {
|
|
59
|
+
raw,
|
|
60
|
+
components,
|
|
61
|
+
country: "FR",
|
|
62
|
+
locale: "fr-FR",
|
|
63
|
+
source: "synth-fr-bare-street",
|
|
64
|
+
source_id,
|
|
65
|
+
corpus_version: "0.9.4",
|
|
66
|
+
license: "Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
|
|
67
|
+
};
|
|
68
|
+
if (alignAndWrite(write, canonical, "fr-bare-street"))
|
|
69
|
+
emitted++;
|
|
70
|
+
else
|
|
71
|
+
skipped++;
|
|
72
|
+
}
|
|
73
|
+
return { read, emitted, skipped };
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
//# sourceMappingURL=fr-bare-street.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-bare-street.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-bare-street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,MAAM,CAAC,MAAM,kBAAkB,GAAgB;IAC9C,IAAI,EAAE,gBAAgB;IACtB,WAAW,EACV,6GAA6G;IAC9G,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,0GAA0G;QAC1G,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAA;YACN,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAChD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEhD,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAA;YAExD,kGAAkG;YAClG,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,UAAU,GAA2B;gBAC1C,YAAY,EAAE,MAAM;gBACpB,aAAa,EAAE,MAAM;gBACrB,MAAM;gBACN,QAAQ;aACR,CAAA;YACD,uFAAuF;YACvF,MAAM,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAA;YACxD,MAAM,SAAS,GAAG,aAAa,CAAC,sBAAsB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC3F,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,sBAAsB;gBAC9B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EACN,uGAAuG;aACxG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,gBAAgB,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBAC3D,OAAO,EAAE,CAAA;QACf,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-order` shard recipe — French reversed-order coverage (#560). Reads REAL OpenAddresses FR
|
|
7
|
+
* tuples (`fr/countrywide.csv` from the cached zip), then for each row picks — by
|
|
8
|
+
* `--reversed-fraction` (default 0.5) — whether to render in CANONICAL French order
|
|
9
|
+
* (number-street, postcode-city) or one of four REVERSED / postcode-first variants the
|
|
10
|
+
* v4.4.0→v0.5.0 regression exposed (the model misses house_number in every reversed one):
|
|
11
|
+
*
|
|
12
|
+
* - A: "47110 Sainte-Livrade-sur-Lot, 69 Allée du Bugatel" (postcode city, HN street)
|
|
13
|
+
* - B: "Sainte-Livrade-sur-Lot, 47110, 619 Impasse de la Rose" (city, postcode, HN street)
|
|
14
|
+
* - C: "Sainte-Livrade-sur-Lot 59 bis Rue des Ecuries 47110" (city HN street postcode — NO commas)
|
|
15
|
+
* - D: "47110, 6 rue de la république, Sainte-Livrade-sur-Lot" (postcode, HN street, city)
|
|
16
|
+
*
|
|
17
|
+
* Sub-modes ride alongside order: `bis`/`ter`/`quater` ordinal suffixes in house_number, and
|
|
18
|
+
* ALL-CAPS locality. `--golden` emits a held-out reversed-order eval slice with a different
|
|
19
|
+
* seed.
|
|
20
|
+
*
|
|
21
|
+
* The inline synthesis (the OA-CSV reader, the ordinal/all-caps tables, the canonical + reversed
|
|
22
|
+
* renderers) is ported faithfully from scripts/build-fr-order-shard.mjs. This is a
|
|
23
|
+
* `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
|
|
24
|
+
* the input. The passed `random` (the framework LCG) is consumed in the exact call order the
|
|
25
|
+
* legacy script used.
|
|
26
|
+
*
|
|
27
|
+
* NOT ported (diagnostic-only, no effect on emitted bytes): the post-run `runSpanCheck` self-check
|
|
28
|
+
* (it reads the finished file back with a separate PRNG and prints to stderr; the recipe's output
|
|
29
|
+
* stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
|
|
30
|
+
* inlined the variant logic and never called it).
|
|
31
|
+
*/
|
|
32
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
33
|
+
export declare const frOrderRecipe: ShardRecipe;
|
|
34
|
+
//# sourceMappingURL=fr-order.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-order.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-order.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AASH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAwHhE,eAAO,MAAM,aAAa,EAAE,WAoH3B,CAAA"}
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `fr-order` shard recipe — French reversed-order coverage (#560). Reads REAL OpenAddresses FR
|
|
7
|
+
* tuples (`fr/countrywide.csv` from the cached zip), then for each row picks — by
|
|
8
|
+
* `--reversed-fraction` (default 0.5) — whether to render in CANONICAL French order
|
|
9
|
+
* (number-street, postcode-city) or one of four REVERSED / postcode-first variants the
|
|
10
|
+
* v4.4.0→v0.5.0 regression exposed (the model misses house_number in every reversed one):
|
|
11
|
+
*
|
|
12
|
+
* - A: "47110 Sainte-Livrade-sur-Lot, 69 Allée du Bugatel" (postcode city, HN street)
|
|
13
|
+
* - B: "Sainte-Livrade-sur-Lot, 47110, 619 Impasse de la Rose" (city, postcode, HN street)
|
|
14
|
+
* - C: "Sainte-Livrade-sur-Lot 59 bis Rue des Ecuries 47110" (city HN street postcode — NO commas)
|
|
15
|
+
* - D: "47110, 6 rue de la république, Sainte-Livrade-sur-Lot" (postcode, HN street, city)
|
|
16
|
+
*
|
|
17
|
+
* Sub-modes ride alongside order: `bis`/`ter`/`quater` ordinal suffixes in house_number, and
|
|
18
|
+
* ALL-CAPS locality. `--golden` emits a held-out reversed-order eval slice with a different
|
|
19
|
+
* seed.
|
|
20
|
+
*
|
|
21
|
+
* The inline synthesis (the OA-CSV reader, the ordinal/all-caps tables, the canonical + reversed
|
|
22
|
+
* renderers) is ported faithfully from scripts/build-fr-order-shard.mjs. This is a
|
|
23
|
+
* `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
|
|
24
|
+
* the input. The passed `random` (the framework LCG) is consumed in the exact call order the
|
|
25
|
+
* legacy script used.
|
|
26
|
+
*
|
|
27
|
+
* NOT ported (diagnostic-only, no effect on emitted bytes): the post-run `runSpanCheck` self-check
|
|
28
|
+
* (it reads the finished file back with a separate PRNG and prints to stderr; the recipe's output
|
|
29
|
+
* stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
|
|
30
|
+
* inlined the variant logic and never called it).
|
|
31
|
+
*/
|
|
32
|
+
import { spawnSync } from "node:child_process";
|
|
33
|
+
import { stableSourceID } from "../adapter.js";
|
|
34
|
+
import { alignRow } from "../align.js";
|
|
35
|
+
import { makeMulberry32 } from "./scaffold.js";
|
|
36
|
+
const SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" };
|
|
37
|
+
// Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
|
|
38
|
+
const ORDINAL_SUFFIXES = ["bis", "ter", "quater"];
|
|
39
|
+
// Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
|
|
40
|
+
const ORDINAL_PROB = 0.12;
|
|
41
|
+
// Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
|
|
42
|
+
const ALLCAPS_PROB = 0.1;
|
|
43
|
+
/** Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping). */
|
|
44
|
+
function splitCSV(line) {
|
|
45
|
+
const out = [];
|
|
46
|
+
let cur = "";
|
|
47
|
+
let inQ = false;
|
|
48
|
+
for (let i = 0; i < line.length; i++) {
|
|
49
|
+
const c = line[i];
|
|
50
|
+
if (inQ) {
|
|
51
|
+
if (c === '"') {
|
|
52
|
+
if (line[i + 1] === '"') {
|
|
53
|
+
cur += '"';
|
|
54
|
+
i++;
|
|
55
|
+
}
|
|
56
|
+
else
|
|
57
|
+
inQ = false;
|
|
58
|
+
}
|
|
59
|
+
else
|
|
60
|
+
cur += c;
|
|
61
|
+
}
|
|
62
|
+
else if (c === '"')
|
|
63
|
+
inQ = true;
|
|
64
|
+
else if (c === ",") {
|
|
65
|
+
out.push(cur);
|
|
66
|
+
cur = "";
|
|
67
|
+
}
|
|
68
|
+
else
|
|
69
|
+
cur += c;
|
|
70
|
+
}
|
|
71
|
+
out.push(cur);
|
|
72
|
+
return out;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
|
|
76
|
+
* string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
|
|
77
|
+
* reversed-order rendering to be meaningful).
|
|
78
|
+
*/
|
|
79
|
+
function readTuples(limit) {
|
|
80
|
+
const maxLines = Math.max(limit * 8, 40000) + 1;
|
|
81
|
+
const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
|
|
82
|
+
maxBuffer: 1024 * 1024 * 1024,
|
|
83
|
+
encoding: "buffer",
|
|
84
|
+
});
|
|
85
|
+
if (r.status !== 0) {
|
|
86
|
+
console.error(` WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`);
|
|
87
|
+
return [];
|
|
88
|
+
}
|
|
89
|
+
const lines = r.stdout.toString("utf8").split(/\r?\n/);
|
|
90
|
+
if (lines.length < 2)
|
|
91
|
+
return [];
|
|
92
|
+
const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
|
|
93
|
+
const idx = (name) => header.indexOf(name);
|
|
94
|
+
const iNum = idx("number"), iStreet = idx("street"), iCity = idx("city"), iPost = idx("postcode");
|
|
95
|
+
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
96
|
+
const tuples = [];
|
|
97
|
+
const seen = new Set();
|
|
98
|
+
for (let li = 1; li < lines.length && tuples.length < limit; li++) {
|
|
99
|
+
if (!lines[li])
|
|
100
|
+
continue;
|
|
101
|
+
const cells = splitCSV(lines[li]);
|
|
102
|
+
const street = get(cells, iStreet);
|
|
103
|
+
const locality = get(cells, iCity);
|
|
104
|
+
const house_number = get(cells, iNum);
|
|
105
|
+
const postcode = get(cells, iPost);
|
|
106
|
+
// Require all four fields: HN is the signal; postcode drives reversed-order variants.
|
|
107
|
+
if (!street || !locality || !house_number || !postcode)
|
|
108
|
+
continue;
|
|
109
|
+
const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase();
|
|
110
|
+
if (seen.has(key))
|
|
111
|
+
continue;
|
|
112
|
+
seen.add(key);
|
|
113
|
+
tuples.push({ house_number, street, locality, postcode });
|
|
114
|
+
}
|
|
115
|
+
return tuples;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Optionally augment a house_number with a French ordinal suffix ("59 bis", "4 ter"). Appended with a space so it forms
|
|
119
|
+
* one multi-token house_number string that alignRow can still locate verbatim.
|
|
120
|
+
*/
|
|
121
|
+
function maybeAddOrdinal(random, house_number) {
|
|
122
|
+
if (random() >= ORDINAL_PROB)
|
|
123
|
+
return house_number;
|
|
124
|
+
const suffix = ORDINAL_SUFFIXES[Math.floor(random() * ORDINAL_SUFFIXES.length)];
|
|
125
|
+
// Vary suffix case: "bis" (lower) vs "BIS" (upper) — a real-world split in the golden.
|
|
126
|
+
return `${house_number} ${random() < 0.5 ? suffix : suffix.toUpperCase()}`;
|
|
127
|
+
}
|
|
128
|
+
/** Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes". */
|
|
129
|
+
function renderCanonical(hn, street, postcode, locality) {
|
|
130
|
+
const raw = `${hn} ${street}, ${postcode} ${locality}`;
|
|
131
|
+
return { raw, components: { house_number: hn, street, postcode, locality } };
|
|
132
|
+
}
|
|
133
|
+
export const frOrderRecipe = {
|
|
134
|
+
name: "fr-order",
|
|
135
|
+
description: "French reversed-order rows (#560): real OA FR tuples rendered canonical + 4 postcode-first variants",
|
|
136
|
+
mode: "generate",
|
|
137
|
+
options: [
|
|
138
|
+
{ flag: "--reversed-fraction <p>", description: "Fraction rendered reversed-order. Default 0.5" },
|
|
139
|
+
{ flag: "--golden", description: "Emit the held-out reversed-order eval slice" },
|
|
140
|
+
],
|
|
141
|
+
async run(opts, write) {
|
|
142
|
+
if (opts.count == null)
|
|
143
|
+
throw new Error("fr-order recipe requires --count <N>");
|
|
144
|
+
const count = opts.count;
|
|
145
|
+
// Legacy build-fr-order-shard.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
|
|
146
|
+
// (The omitted diagnostic runSpanCheck used a separate mulberry32(opts.seed + 1) — not part of generation.)
|
|
147
|
+
const random = makeMulberry32(opts.seed);
|
|
148
|
+
const source = opts.sourceName ?? "synth-fr-order";
|
|
149
|
+
const reversedFraction = opts.reversedFraction ?? 0.5;
|
|
150
|
+
// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
|
|
151
|
+
const poolLimit = Math.max(count * 8, 40000);
|
|
152
|
+
const pool = readTuples(poolLimit);
|
|
153
|
+
console.error(` ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`);
|
|
154
|
+
if (pool.length === 0) {
|
|
155
|
+
throw new Error("No FR tuples found — is /tmp/oa-cache/fr__countrywide.zip present?");
|
|
156
|
+
}
|
|
157
|
+
let emitted = 0;
|
|
158
|
+
let skipped = 0;
|
|
159
|
+
let guard = 0;
|
|
160
|
+
const N = pool.length;
|
|
161
|
+
while (emitted < count && guard++ < count * 10) {
|
|
162
|
+
const base = pool[Math.floor(random() * N)];
|
|
163
|
+
const { street, postcode } = base;
|
|
164
|
+
const locality = random() < ALLCAPS_PROB ? base.locality.toUpperCase() : base.locality;
|
|
165
|
+
const house_number = maybeAddOrdinal(random, base.house_number);
|
|
166
|
+
// Pick canonical vs reversed by --reversed-fraction.
|
|
167
|
+
const isReversed = random() < reversedFraction;
|
|
168
|
+
let rendered;
|
|
169
|
+
if (isReversed) {
|
|
170
|
+
const variantRoll = random();
|
|
171
|
+
let raw;
|
|
172
|
+
if (variantRoll < 0.25) {
|
|
173
|
+
// Variant A: postcode+city as a unit, then HN+street
|
|
174
|
+
raw = `${postcode} ${locality}, ${house_number} ${street}`;
|
|
175
|
+
}
|
|
176
|
+
else if (variantRoll < 0.5) {
|
|
177
|
+
// Variant B: city, then postcode, then HN+street (comma-separated, postcode isolated)
|
|
178
|
+
raw = `${locality}, ${postcode}, ${house_number} ${street}`;
|
|
179
|
+
}
|
|
180
|
+
else if (variantRoll < 0.75) {
|
|
181
|
+
// Variant C: no commas — locality HN street postcode (the "run-together" format)
|
|
182
|
+
raw = `${locality} ${house_number} ${street} ${postcode}`;
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
// Variant D: postcode, HN+street, city (reversed top-to-bottom)
|
|
186
|
+
raw = `${postcode}, ${house_number} ${street}, ${locality}`;
|
|
187
|
+
}
|
|
188
|
+
rendered = { raw, components: { house_number, street, postcode, locality } };
|
|
189
|
+
}
|
|
190
|
+
else {
|
|
191
|
+
rendered = renderCanonical(house_number, street, postcode, locality);
|
|
192
|
+
}
|
|
193
|
+
const { raw, components } = rendered;
|
|
194
|
+
// Safety check: every component must appear verbatim in raw (alignment precondition).
|
|
195
|
+
const componentValues = Object.values(components).filter(Boolean);
|
|
196
|
+
if (!componentValues.every((v) => raw.includes(v))) {
|
|
197
|
+
skipped++;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
// --golden: emit per-locale-f1 eval rows ({raw, components, country:"FR"}).
|
|
201
|
+
if (opts.golden) {
|
|
202
|
+
write(JSON.stringify({ raw, components, country: "FR" }) + "\n");
|
|
203
|
+
emitted++;
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
const sourceID = stableSourceID(source, {
|
|
207
|
+
street: components.street,
|
|
208
|
+
house_number: components.house_number,
|
|
209
|
+
locality: components.locality,
|
|
210
|
+
postcode: components.postcode,
|
|
211
|
+
});
|
|
212
|
+
const canonical = {
|
|
213
|
+
raw,
|
|
214
|
+
components,
|
|
215
|
+
country: "FR",
|
|
216
|
+
locale: "fr-FR",
|
|
217
|
+
source,
|
|
218
|
+
source_id: sourceID,
|
|
219
|
+
corpus_version: "0.5.0",
|
|
220
|
+
license: "OpenAddresses FR countrywide tuples, rendered canonical + reversed-order — see ingest SOURCE",
|
|
221
|
+
};
|
|
222
|
+
const aligned = alignRow(canonical);
|
|
223
|
+
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
224
|
+
skipped++;
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
write(JSON.stringify({
|
|
228
|
+
...aligned.row,
|
|
229
|
+
synth_method: "fr-order",
|
|
230
|
+
synth_order: isReversed ? "reversed" : "canonical",
|
|
231
|
+
synth_base_id: null,
|
|
232
|
+
}) + "\n");
|
|
233
|
+
emitted++;
|
|
234
|
+
}
|
|
235
|
+
return { emitted, skipped };
|
|
236
|
+
},
|
|
237
|
+
};
|
|
238
|
+
//# sourceMappingURL=fr-order.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fr-order.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-order.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAI9C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAEhE,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,mCAAmC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAA;AAEtF,6FAA6F;AAC7F,MAAM,gBAAgB,GAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;AACpE,8FAA8F;AAC9F,MAAM,YAAY,GAAG,IAAI,CAAA;AACzB,6FAA6F;AAC7F,MAAM,YAAY,GAAG,GAAG,CAAA;AAUxB,yFAAyF;AACzF,SAAS,QAAQ,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,GAAG,GAAG,KAAK,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEjB,IAAI,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACzB,GAAG,IAAI,GAAG,CAAA;oBACV,CAAC,EAAE,CAAA;gBACJ,CAAC;;oBAAM,GAAG,GAAG,KAAK,CAAA;YACnB,CAAC;;gBAAM,GAAG,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,CAAC,KAAK,GAAG;YAAE,GAAG,GAAG,IAAI,CAAA;aAC3B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,GAAG,EAAE,CAAA;QACT,CAAC;;YAAM,GAAG,IAAI,CAAC,CAAA;IAChB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEb,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,KAAa;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;IAC/C,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,aAAa,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,GAAG,eAAe,QAAQ,EAAE,CAAC,EAAE;QACrG,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI;QAC7B,QAAQ,EAAE,QAAQ;KAClB,CAAC,CAAA;IAEF,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;QAE5E,OAAO,EAAE,CAAA;IACV,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAEtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IACrE,MAAM,GAAG,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,EACzB,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,EACvB,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EACnB,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAA;IACxB,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/G,MAAM,MAAM,GAAc,EAAE,CAAA;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,SAAQ;QACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC,CAAA;QAClC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAClC,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAElC,sFAAsF;QACtF,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChE,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,MAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;QAE7E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAC1D,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAoB,EAAE,YAAoB;IAClE,IAAI,MAAM,EAAE,IAAI,YAAY;QAAE,OAAO,YAAY,CAAA;IACjD,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAE,CAAA;IAEhF,uFAAuF;IACvF,OAAO,GAAG,YAAY,IAAI,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAA;AAC3E,CAAC;AAED,uFAAuF;AACvF,SAAS,eAAe,CACvB,EAAU,EACV,MAAc,EACd,QAAgB,EAChB,QAAgB;IAEhB,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,MAAM,KAAK,QAAQ,IAAI,QAAQ,EAAE,CAAA;IAEtD,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAA;AAC7E,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAgB;IACzC,IAAI,EAAE,UAAU;IAChB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,yBAAyB,EAAE,WAAW,EAAE,+CAA+C,EAAE;QACjG,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,6CAA6C,EAAE;KAChF;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QACxB,+GAA+G;QAC/G,4GAA4G;QAC5G,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,gBAAgB,CAAA;QAClD,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAA;QAErD,2EAA2E;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC5C,MAAM,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,CAAA;QAClC,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,GAAG,KAAK,IAAI,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAE5E,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAA;QACtF,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAA;YACjC,MAAM,QAAQ,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAA;YACtF,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAE/D,qDAAqD;YACrD,MAAM,UAAU,GAAG,MAAM,EAAE,GAAG,gBAAgB,CAAA;YAE9C,IAAI,QAA4E,CAAA;YAEhF,IAAI,UAAU,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAA;gBAC5B,IAAI,GAAW,CAAA;gBAEf,IAAI,WAAW,GAAG,IAAI,EAAE,CAAC;oBACxB,qDAAqD;oBACrD,GAAG,GAAG,GAAG,QAAQ,IAAI,QAAQ,KAAK,YAAY,IAAI,MAAM,EAAE,CAAA;gBAC3D,CAAC;qBAAM,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;oBAC9B,sFAAsF;oBACtF,GAAG,GAAG,GAAG,QAAQ,KAAK,QAAQ,KAAK,YAAY,IAAI,MAAM,EAAE,CAAA;gBAC5D,CAAC;qBAAM,IAAI,WAAW,GAAG,IAAI,EAAE,CAAC;oBAC/B,iFAAiF;oBACjF,GAAG,GAAG,GAAG,QAAQ,IAAI,YAAY,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAA;gBAC1D,CAAC;qBAAM,CAAC;oBACP,gEAAgE;oBAChE,GAAG,GAAG,GAAG,QAAQ,KAAK,YAAY,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAA;gBAC5D,CAAC;gBACD,QAAQ,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAA;YAC7E,CAAC;iBAAM,CAAC;gBACP,QAAQ,GAAG,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;YACrE,CAAC;YAED,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,QAAQ,CAAA;YAEpC,sFAAsF;YACtF,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,OAAO,CAAa,CAAA;YAE7E,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,4EAA4E;YAC5E,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAChE,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,UAAU,CAAC,YAAY;gBACrC,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;aAC7B,CAAC,CAAA;YACF,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,8FAA8F;aACvG,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC;gBACd,GAAG,OAAO,CAAC,GAAG;gBACd,YAAY,EAAE,UAAU;gBACxB,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW;gBAClD,aAAa,EAAE,IAAI;aACnB,CAAC,GAAG,IAAI,CACT,CAAA;YACD,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `german` shard recipe — German coverage rows from REAL OpenAddresses tuples (Berlin + Saxony,
|
|
7
|
+
* cached zips). Each sampled tuple is rendered via {@link synthesizeGermanRow} in BOTH orders —
|
|
8
|
+
* `--intl-fraction` (default 0.4) in international order (house-first / postcode-after-city), the
|
|
9
|
+
* rest in idiomatic German order — then aligned to BIO. Generate-mode: it builds a tuple pool
|
|
10
|
+
* from the cached zips, then draws `--count` rows from it with the passed `random` (so the emit
|
|
11
|
+
* stream matches the legacy reservoir-sample loop). Ported from scripts/build-german-shard.mjs.
|
|
12
|
+
*
|
|
13
|
+
* ORDER ROBUSTNESS (2026-06-06): mixing the two renderings stops a native-only shard from teaching
|
|
14
|
+
* German order so well it reads the US/feed-order eval as a "collapse". See
|
|
15
|
+
* docs/articles/evals/2026-06-06-anchor-pilot.md (the order-artifact correction).
|
|
16
|
+
*/
|
|
17
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
18
|
+
export declare const germanRecipe: ShardRecipe;
|
|
19
|
+
//# sourceMappingURL=german.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"german.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAOH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA4FhE,eAAO,MAAM,YAAY,EAAE,WAkF1B,CAAA"}
|