@mailwoman/corpus 5.1.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +18 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +76 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +6 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +39 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +142 -68
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +20 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -0
- package/out/src/shard-recipes/no-street-led.js +74 -0
- package/out/src/shard-recipes/no-street-led.js.map +1 -0
- package/out/src/shard-recipes/si-bare-village.d.ts +28 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -0
- package/out/src/shard-recipes/si-bare-village.js +82 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -0
- package/out/src/synthesize-german.d.ts +18 -0
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +20 -2
- package/out/src/synthesize-german.js.map +1 -1
- package/package.json +5 -5
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `cz-pcfirst-preposition` — the Czech pc-first prepositional-locality shard, fourth orthography
|
|
7
|
+
* of the #901 leading-name-boundary family. The #897 close-out read all 8 residual CZ rows as one
|
|
8
|
+
* class: a LEADING postcode mis-assigned as house_number while the multi-word "nad/pod/u X"
|
|
9
|
+
* locality shatters ("51244 Rokytnice nad Jizerou, Dolní Rokytnice 111" → street
|
|
10
|
+
* 'RokytnicenadJizerou' + house_number '51244'). That leading-5-digit confusion is the
|
|
11
|
+
* anchor-pollution class whose decode-time OVERRIDE was correctly killed in #723 — this shard is
|
|
12
|
+
* the model-first fix as DATA: real prepositional localities in the order that breaks, so the
|
|
13
|
+
* model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
|
|
14
|
+
* cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
|
|
15
|
+
*/
|
|
16
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
17
|
+
export declare const czPcFirstPrepositionRecipe: ShardRecipe;
|
|
18
|
+
//# sourceMappingURL=cz-pcfirst-preposition.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cz-pcfirst-preposition.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/cz-pcfirst-preposition.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAE1G,eAAO,MAAM,0BAA0B,EAAE,WA6DxC,CAAA"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `cz-pcfirst-preposition` — the Czech pc-first prepositional-locality shard, fourth orthography
|
|
7
|
+
* of the #901 leading-name-boundary family. The #897 close-out read all 8 residual CZ rows as one
|
|
8
|
+
* class: a LEADING postcode mis-assigned as house_number while the multi-word "nad/pod/u X"
|
|
9
|
+
* locality shatters ("51244 Rokytnice nad Jizerou, Dolní Rokytnice 111" → street
|
|
10
|
+
* 'RokytnicenadJizerou' + house_number '51244'). That leading-5-digit confusion is the
|
|
11
|
+
* anchor-pollution class whose decode-time OVERRIDE was correctly killed in #723 — this shard is
|
|
12
|
+
* the model-first fix as DATA: real prepositional localities in the order that breaks, so the
|
|
13
|
+
* model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
|
|
14
|
+
* cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
|
|
15
|
+
*/
|
|
16
|
+
import { alignAndWrite, makeMulberry32, readTuples, shardSourceID } from "./scaffold.js";
|
|
17
|
+
export const czPcFirstPrepositionRecipe = {
|
|
18
|
+
name: "cz-pcfirst-preposition",
|
|
19
|
+
description: "CZ pc-first + prepositional locality (#901 family): '«pc» «city nad X», «st» «n»' — the #723 class as data",
|
|
20
|
+
mode: "tuples",
|
|
21
|
+
async run(opts, write) {
|
|
22
|
+
makeMulberry32(opts.seed);
|
|
23
|
+
let read = 0;
|
|
24
|
+
let emitted = 0;
|
|
25
|
+
let skipped = 0;
|
|
26
|
+
for await (const t of readTuples(opts.input)) {
|
|
27
|
+
read++;
|
|
28
|
+
const street = String(t.street ?? "").trim();
|
|
29
|
+
const city = String(t.locality ?? "").trim();
|
|
30
|
+
const number = String(t.number ?? "").trim();
|
|
31
|
+
const postcode = String(t.postcode ?? "").trim();
|
|
32
|
+
if (!street || !city || !number || !postcode) {
|
|
33
|
+
skipped++;
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const order = read % 3;
|
|
37
|
+
let raw;
|
|
38
|
+
const components = {
|
|
39
|
+
street,
|
|
40
|
+
house_number: number,
|
|
41
|
+
postcode,
|
|
42
|
+
locality: city,
|
|
43
|
+
};
|
|
44
|
+
if (order === 0) {
|
|
45
|
+
raw = `${postcode} ${city}, ${street} ${number}`;
|
|
46
|
+
}
|
|
47
|
+
else if (order === 1) {
|
|
48
|
+
raw = `${street} ${number}, ${postcode} ${city}`;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
raw = `${city}, ${postcode}, ${street} ${number}`;
|
|
52
|
+
}
|
|
53
|
+
const source_id = shardSourceID("synth-cz-pcfirst-preposition", {
|
|
54
|
+
...components,
|
|
55
|
+
o: String(order),
|
|
56
|
+
v: String(read),
|
|
57
|
+
});
|
|
58
|
+
const canonical = {
|
|
59
|
+
raw,
|
|
60
|
+
components,
|
|
61
|
+
country: "CZ",
|
|
62
|
+
locale: "cs-CZ",
|
|
63
|
+
source: "synth-cz-pcfirst-preposition",
|
|
64
|
+
source_id,
|
|
65
|
+
corpus_version: "0.10.0",
|
|
66
|
+
license: "Synthetic — cz-pcfirst-preposition; (street, number, postcode, city) from OpenAddresses CZ (per-source attribution in the model card)",
|
|
67
|
+
};
|
|
68
|
+
if (alignAndWrite(write, canonical, "cz-pcfirst-preposition"))
|
|
69
|
+
emitted++;
|
|
70
|
+
else
|
|
71
|
+
skipped++;
|
|
72
|
+
}
|
|
73
|
+
return { read, emitted, skipped };
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
//# sourceMappingURL=cz-pcfirst-preposition.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cz-pcfirst-preposition.js","sourceRoot":"","sources":["../../../src/shard-recipes/cz-pcfirst-preposition.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,MAAM,CAAC,MAAM,0BAA0B,GAAgB;IACtD,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EACV,4GAA4G;IAC7G,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAA;YACN,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEhD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC9C,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAA;YACtB,IAAI,GAAW,CAAA;YACf,MAAM,UAAU,GAA2B;gBAC1C,MAAM;gBACN,YAAY,EAAE,MAAM;gBACpB,QAAQ;gBACR,QAAQ,EAAE,IAAI;aACd,CAAA;YAED,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,GAAG,GAAG,GAAG,QAAQ,IAAI,IAAI,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YACjD,CAAC;iBAAM,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACxB,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,IAAI,EAAE,CAAA;YACjD,CAAC;iBAAM,CAAC;gBACP,GAAG,GAAG,GAAG,IAAI,KAAK,QAAQ,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YAClD,CAAC;YACD,MAAM,SAAS,GAAG,aAAa,CAAC,8BAA8B,EAAE;gBAC/D,GAAG,UAAU;gBACb,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC;gBAChB,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC;aACf,CAAC,CAAA;YACF,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,8BAA8B;gBACtC,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EACN,uIAAuI;aACxI,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,wBAAwB,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBACnE,OAAO,EAAE,CAAA;QACf,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAiBH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAOhD,cAAc,eAAe,CAAA;AA0B7B,4BAA4B;AAC5B,eAAO,MAAM,aAAa,EAAE,WAAW,CAAC,MAAM,EAAE,WAAW,CAA4C,CAAA;AAEvG,+CAA+C;AAC/C,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAEpE;AAED,yCAAyC;AACzC,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD"}
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
import { anchorAbsorptionRecipe } from "./anchor-absorption.js";
|
|
11
11
|
import { boundaryStressRecipe } from "./boundary-stress.js";
|
|
12
12
|
import { countryBalancedRecipe } from "./country-balanced.js";
|
|
13
|
+
import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.js";
|
|
13
14
|
import { frAdminSplitRecipe } from "./fr-admin-split.js";
|
|
14
15
|
import { frBareStreetRecipe } from "./fr-bare-street.js";
|
|
15
16
|
import { frOrderRecipe } from "./fr-order.js";
|
|
@@ -17,9 +18,11 @@ import { germanRecipe } from "./german.js";
|
|
|
17
18
|
import { houseVenueRecipe } from "./house-venue.js";
|
|
18
19
|
import { intersectionRecipe } from "./intersection.js";
|
|
19
20
|
import { localeRecipe } from "./locale.js";
|
|
21
|
+
import { noStreetLedRecipe } from "./no-street-led.js";
|
|
20
22
|
import { noStreetRecipe } from "./no-street.js";
|
|
21
23
|
import { poBoxCedexRecipe } from "./po-box-cedex.js";
|
|
22
24
|
import { poBoxRecipe } from "./po-box.js";
|
|
25
|
+
import { siBareVillageRecipe } from "./si-bare-village.js";
|
|
23
26
|
import { streetAffixRecipe } from "./street-affix.js";
|
|
24
27
|
import { streetBareRecipe } from "./street-bare.js";
|
|
25
28
|
import { streetRecipe } from "./street.js";
|
|
@@ -41,6 +44,9 @@ const RECIPES = [
|
|
|
41
44
|
frOrderRecipe,
|
|
42
45
|
frAdminSplitRecipe,
|
|
43
46
|
frBareStreetRecipe,
|
|
47
|
+
czPcFirstPrepositionRecipe,
|
|
48
|
+
noStreetLedRecipe,
|
|
49
|
+
siBareVillageRecipe,
|
|
44
50
|
countryBalancedRecipe,
|
|
45
51
|
boundaryStressRecipe,
|
|
46
52
|
anchorAbsorptionRecipe,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/shard-recipes/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAA;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,0BAA0B,EAAE,MAAM,6BAA6B,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAA;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAA;AAEtC,cAAc,eAAe,CAAA;AAE7B,iDAAiD;AACjD,MAAM,OAAO,GAA2B;IACvC,YAAY;IACZ,gBAAgB;IAChB,iBAAiB;IACjB,cAAc;IACd,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,UAAU;IACV,kBAAkB;IAClB,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,kBAAkB;IAClB,kBAAkB;IAClB,0BAA0B;IAC1B,iBAAiB;IACjB,mBAAmB;IACnB,qBAAqB;IACrB,oBAAoB;IACpB,sBAAsB;CACtB,CAAA;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,aAAa,GAAqC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAEvG,+CAA+C;AAC/C,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,gBAAgB;IAC/B,OAAO,OAAO,CAAA;AACf,CAAC"}
|
|
@@ -7,13 +7,47 @@
|
|
|
7
7
|
* OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
|
|
8
8
|
* {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
|
|
9
9
|
* rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
|
|
10
|
-
* source CSV (`unzip -p
|
|
11
|
-
* countrywide work in bounded memory), then
|
|
12
|
-
* `random`. Ported from
|
|
10
|
+
* source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
|
|
11
|
+
* reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
|
|
12
|
+
* draws `--count` rows from the pool with the passed `random`. Ported from
|
|
13
|
+
* scripts/build-locale-shard.mjs.
|
|
13
14
|
*
|
|
14
|
-
* The reservoir uses its OWN seeded PRNG ({@link
|
|
15
|
-
* `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
15
|
+
* The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the
|
|
16
|
+
* emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
17
|
+
*
|
|
18
|
+
* SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the
|
|
19
|
+
* 2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed
|
|
20
|
+
* forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000
|
|
21
|
+
* eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse
|
|
22
|
+
* the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national
|
|
23
|
+
* convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued
|
|
24
|
+
* source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
|
|
25
|
+
* country, so DE/FR emit streams are unchanged for a given seed.
|
|
16
26
|
*/
|
|
17
27
|
import { type ShardRecipe } from "./scaffold.js";
|
|
28
|
+
/**
|
|
29
|
+
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
30
|
+
* the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop
|
|
31
|
+
* the tuple.
|
|
32
|
+
*
|
|
33
|
+
* Cleaned classes:
|
|
34
|
+
*
|
|
35
|
+
* 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of
|
|
36
|
+
* ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city.
|
|
37
|
+
* Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits.
|
|
38
|
+
* 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk
|
|
39
|
+
* (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names
|
|
40
|
+
* `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality
|
|
41
|
+
* labels.
|
|
42
|
+
*
|
|
43
|
+
* Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names):
|
|
44
|
+
*
|
|
45
|
+
* - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking
|
|
46
|
+
* `de`/`di` makes them full names, unlike the German glued-abbreviation class.
|
|
47
|
+
* - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim.
|
|
48
|
+
* - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation
|
|
49
|
+
* lever, not this shard's.
|
|
50
|
+
*/
|
|
51
|
+
export declare function cleanCityNoise(city: string): string | null;
|
|
18
52
|
export declare const localeRecipe: ShardRecipe;
|
|
19
53
|
//# sourceMappingURL=locale.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAWH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAqFhE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAwHD,eAAO,MAAM,YAAY,EAAE,WAsH1B,CAAA"}
|
|
@@ -7,54 +7,67 @@
|
|
|
7
7
|
* OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
|
|
8
8
|
* {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
|
|
9
9
|
* rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
|
|
10
|
-
* source CSV (`unzip -p
|
|
11
|
-
* countrywide work in bounded memory), then
|
|
12
|
-
* `random`. Ported from
|
|
10
|
+
* source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
|
|
11
|
+
* reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
|
|
12
|
+
* draws `--count` rows from the pool with the passed `random`. Ported from
|
|
13
|
+
* scripts/build-locale-shard.mjs.
|
|
13
14
|
*
|
|
14
|
-
* The reservoir uses its OWN seeded PRNG ({@link
|
|
15
|
-
* `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
15
|
+
* The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the
|
|
16
|
+
* emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
17
|
+
*
|
|
18
|
+
* SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the
|
|
19
|
+
* 2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed
|
|
20
|
+
* forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000
|
|
21
|
+
* eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse
|
|
22
|
+
* the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national
|
|
23
|
+
* convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued
|
|
24
|
+
* source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
|
|
25
|
+
* country, so DE/FR emit streams are unchanged for a given seed.
|
|
16
26
|
*/
|
|
17
27
|
import { spawn } from "node:child_process";
|
|
28
|
+
import { createReadStream } from "node:fs";
|
|
18
29
|
import { createInterface } from "node:readline";
|
|
30
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
19
31
|
import { stableSourceID } from "../adapter.js";
|
|
20
32
|
import { alignRow } from "../align.js";
|
|
21
33
|
import { synthesizeLocaleRow } from "../synthesize-german.js";
|
|
22
34
|
import { makeMulberry32 } from "./scaffold.js";
|
|
23
35
|
/**
|
|
24
|
-
* Per-country OA sources
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* carries a `
|
|
36
|
+
* Per-country OA sources + the source name used in the corpus. DE/FR still point at the legacy `/tmp/oa-cache` zips
|
|
37
|
+
* (their historical build inputs — materialize them there to regenerate). ES/NL read the extracted countrywide CSVs and
|
|
38
|
+
* IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is OA-conformed, so the old
|
|
39
|
+
* raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is empty; the
|
|
40
|
+
* international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad autónoma, IT =
|
|
41
|
+
* regione, NL = province).
|
|
28
42
|
*/
|
|
29
43
|
const COUNTRY_SOURCES = {
|
|
30
44
|
DE: {
|
|
31
45
|
source: "synth-german",
|
|
46
|
+
corpusVersion: "0.4.0",
|
|
32
47
|
parts: [
|
|
33
48
|
{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
|
|
34
49
|
{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
|
|
35
50
|
],
|
|
36
51
|
},
|
|
37
|
-
FR: {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
52
|
+
FR: {
|
|
53
|
+
source: "synth-fr",
|
|
54
|
+
corpusVersion: "0.4.0",
|
|
55
|
+
parts: [{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }],
|
|
56
|
+
},
|
|
57
|
+
NL: {
|
|
58
|
+
source: "synth-nl",
|
|
59
|
+
corpusVersion: "0.9.9",
|
|
60
|
+
parts: [{ path: dataRootPath("openaddresses", "extracted", "nl", "countrywide.csv") }],
|
|
61
|
+
},
|
|
62
|
+
IT: {
|
|
63
|
+
source: "synth-it",
|
|
64
|
+
corpusVersion: "0.9.9",
|
|
65
|
+
parts: [{ zip: dataRootPath("oa-cache", "it__countrywide.zip"), csv: "it/countrywide.csv" }],
|
|
66
|
+
},
|
|
43
67
|
ES: {
|
|
44
68
|
source: "synth-es",
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
zip: "/tmp/oa-cache/es__countrywide.zip",
|
|
48
|
-
csv: "es_addresses.csv",
|
|
49
|
-
conform: {
|
|
50
|
-
number: "numero",
|
|
51
|
-
street: ["tipo_vial", "nombre_via"],
|
|
52
|
-
city: "municipio",
|
|
53
|
-
region: "comunidad_autonoma",
|
|
54
|
-
postcode: "cod_postal",
|
|
55
|
-
},
|
|
56
|
-
},
|
|
57
|
-
],
|
|
69
|
+
corpusVersion: "0.9.9",
|
|
70
|
+
parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
|
|
58
71
|
},
|
|
59
72
|
};
|
|
60
73
|
/**
|
|
@@ -63,6 +76,47 @@ const COUNTRY_SOURCES = {
|
|
|
63
76
|
* sources (≤ ~1.2M) fit entirely, so they're sampled losslessly.
|
|
64
77
|
*/
|
|
65
78
|
const RESERVOIR_CAP = 1_200_000;
|
|
79
|
+
/**
|
|
80
|
+
* Fraction of NATIVE-order ES rows whose street→house join is space-collapsed (`CALLE MAYOR 12`) instead of the
|
|
81
|
+
* template's comma (`CALLE MAYOR, 12`). Both are real Spanish surfaces — the comma is the official convention, the
|
|
82
|
+
* space is what OA-derived feeds (and all 3,000 `openaddresses-es-sample.jsonl` rows) carry. 0.5 teaches both.
|
|
83
|
+
*/
|
|
84
|
+
const ES_SPACE_JOIN_FRACTION = 0.5;
|
|
85
|
+
/**
|
|
86
|
+
* Fraction of NL rows whose postcode keeps OA's glued shape (`1187LM`) instead of the spaced national convention (`1187
|
|
87
|
+
* LM`). The eval sample is 100% glued; the conventional spaced form is the `1012 LM` two-letter-suffix shape the model
|
|
88
|
+
* currently glues onto the city (#241). 0.5 teaches both.
|
|
89
|
+
*/
|
|
90
|
+
const NL_GLUED_POSTCODE_FRACTION = 0.5;
|
|
91
|
+
/**
|
|
92
|
+
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
93
|
+
* the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop
|
|
94
|
+
* the tuple.
|
|
95
|
+
*
|
|
96
|
+
* Cleaned classes:
|
|
97
|
+
*
|
|
98
|
+
* 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of
|
|
99
|
+
* ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city.
|
|
100
|
+
* Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits.
|
|
101
|
+
* 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk
|
|
102
|
+
* (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names
|
|
103
|
+
* `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality
|
|
104
|
+
* labels.
|
|
105
|
+
*
|
|
106
|
+
* Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names):
|
|
107
|
+
*
|
|
108
|
+
* - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking
|
|
109
|
+
* `de`/`di` makes them full names, unlike the German glued-abbreviation class.
|
|
110
|
+
* - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim.
|
|
111
|
+
* - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation
|
|
112
|
+
* lever, not this shard's.
|
|
113
|
+
*/
|
|
114
|
+
export function cleanCityNoise(city) {
|
|
115
|
+
if (/,|\d{4}/.test(city))
|
|
116
|
+
return null;
|
|
117
|
+
const stripped = city.replace(/\s*\(\p{L}{1,3}\)\s*$/u, "").trim();
|
|
118
|
+
return stripped || null;
|
|
119
|
+
}
|
|
66
120
|
/** Minimal RFC-4180-ish splitter (handles quoted fields). */
|
|
67
121
|
function splitCSV(line) {
|
|
68
122
|
const out = [];
|
|
@@ -95,57 +149,58 @@ function splitCSV(line) {
|
|
|
95
149
|
return out;
|
|
96
150
|
}
|
|
97
151
|
/**
|
|
98
|
-
* Stream real tuples out of
|
|
99
|
-
*
|
|
100
|
-
* from the emit loop's PRNG. NO global dedup (a
|
|
101
|
-
*
|
|
152
|
+
* Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV line-by-line
|
|
153
|
+
* — `unzip -p | readline` for zip parts, `createReadStream | readline` for extracted parts (both bounded memory) — and
|
|
154
|
+
* keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO global dedup (a
|
|
155
|
+
* 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the region falls
|
|
156
|
+
* back to `part.region` when the row's REGION cell is empty (DE).
|
|
102
157
|
*/
|
|
103
158
|
function readTuples(part, rng) {
|
|
104
159
|
return new Promise((resolve) => {
|
|
105
|
-
|
|
106
|
-
|
|
160
|
+
let input;
|
|
161
|
+
if (part.path) {
|
|
162
|
+
input = createReadStream(part.path, { encoding: "utf8" });
|
|
163
|
+
input.on("error", (err) => {
|
|
164
|
+
console.error(` WARN: read failed for ${part.path}: ${err.message}`);
|
|
165
|
+
resolve([]);
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
const child = spawn("unzip", ["-p", part.zip, part.csv]);
|
|
170
|
+
child.on("error", (err) => {
|
|
171
|
+
console.error(` WARN: unzip failed for ${part.zip}: ${err.message}`);
|
|
172
|
+
resolve([]);
|
|
173
|
+
});
|
|
174
|
+
input = child.stdout;
|
|
175
|
+
}
|
|
176
|
+
const rl = createInterface({ input, crlfDelay: Infinity });
|
|
107
177
|
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
108
178
|
const reservoir = [];
|
|
109
179
|
let cols = null;
|
|
110
180
|
let header = null;
|
|
111
181
|
let seen = 0;
|
|
182
|
+
let dropped = 0;
|
|
112
183
|
rl.on("line", (line) => {
|
|
113
184
|
if (!line)
|
|
114
185
|
return;
|
|
115
186
|
if (header === null) {
|
|
116
187
|
header = splitCSV(line).map((h) => h.trim().toLowerCase());
|
|
117
|
-
const ix = (name) => header.indexOf(
|
|
118
|
-
|
|
119
|
-
// upstream schema (ES — CNIG columns, street split across `tipo_vial` + `nombre_via`).
|
|
120
|
-
const c = part.conform;
|
|
121
|
-
cols = c
|
|
122
|
-
? {
|
|
123
|
-
num: ix(c.number),
|
|
124
|
-
streetParts: (Array.isArray(c.street) ? c.street : [c.street]).map(ix),
|
|
125
|
-
city: ix(c.city),
|
|
126
|
-
region: c.region ? ix(c.region) : -1,
|
|
127
|
-
post: ix(c.postcode),
|
|
128
|
-
}
|
|
129
|
-
: {
|
|
130
|
-
num: ix("number"),
|
|
131
|
-
streetParts: [ix("street")],
|
|
132
|
-
city: ix("city"),
|
|
133
|
-
region: ix("region"),
|
|
134
|
-
post: ix("postcode"),
|
|
135
|
-
};
|
|
188
|
+
const ix = (name) => header.indexOf(name);
|
|
189
|
+
cols = { num: ix("number"), street: ix("street"), city: ix("city"), region: ix("region"), post: ix("postcode") };
|
|
136
190
|
return;
|
|
137
191
|
}
|
|
138
192
|
if (cols === null)
|
|
139
193
|
return;
|
|
140
194
|
const cells = splitCSV(line);
|
|
141
|
-
const street = cols.
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
195
|
+
const street = get(cells, cols.street);
|
|
196
|
+
const rawCity = get(cells, cols.city);
|
|
197
|
+
if (!street || !rawCity)
|
|
198
|
+
return;
|
|
199
|
+
const locality = cleanCityNoise(rawCity);
|
|
200
|
+
if (!locality) {
|
|
201
|
+
dropped++;
|
|
148
202
|
return;
|
|
203
|
+
}
|
|
149
204
|
const tuple = {
|
|
150
205
|
house_number: get(cells, cols.num),
|
|
151
206
|
street,
|
|
@@ -165,13 +220,9 @@ function readTuples(part, rng) {
|
|
|
165
220
|
}
|
|
166
221
|
});
|
|
167
222
|
rl.on("close", () => {
|
|
168
|
-
console.error(` ${part.csv}: ${reservoir.length} sampled of ${seen} rows`);
|
|
223
|
+
console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`);
|
|
169
224
|
resolve(reservoir);
|
|
170
225
|
});
|
|
171
|
-
child.on("error", (err) => {
|
|
172
|
-
console.error(` WARN: unzip failed for ${part.zip}: ${err.message}`);
|
|
173
|
-
resolve([]);
|
|
174
|
-
});
|
|
175
226
|
});
|
|
176
227
|
}
|
|
177
228
|
export const localeRecipe = {
|
|
@@ -208,7 +259,7 @@ export const localeRecipe = {
|
|
|
208
259
|
pool.push(x); // NOT pool.push(...t) — spreading huge arrays overflows the stack
|
|
209
260
|
}
|
|
210
261
|
if (pool.length === 0) {
|
|
211
|
-
throw new Error(`No ${country} tuples found — are the
|
|
262
|
+
throw new Error(`No ${country} tuples found — are the source CSVs/zips present? (see COUNTRY_SOURCES)`);
|
|
212
263
|
}
|
|
213
264
|
let emitted = 0;
|
|
214
265
|
let skipped = 0;
|
|
@@ -217,12 +268,35 @@ export const localeRecipe = {
|
|
|
217
268
|
while (emitted < count && guard++ < count * 6) {
|
|
218
269
|
const base = pool[Math.floor(random() * N)];
|
|
219
270
|
const order = random() < intlFraction ? "international" : "native";
|
|
220
|
-
|
|
271
|
+
// Per-country surface-shape draws (#241) — consumed ONLY for that country, so the DE/FR emit
|
|
272
|
+
// streams for a given seed are unchanged by their existence.
|
|
273
|
+
const nativeHouseJoin = country === "ES" ? (random() < ES_SPACE_JOIN_FRACTION ? "space" : "template") : undefined;
|
|
274
|
+
const postcodeShape = country === "NL"
|
|
275
|
+
? random() < NL_GLUED_POSTCODE_FRACTION
|
|
276
|
+
? "as-source"
|
|
277
|
+
: "conventional"
|
|
278
|
+
: undefined;
|
|
279
|
+
const synth = synthesizeLocaleRow(base, country, { random, order, nativeHouseJoin, postcodeShape });
|
|
221
280
|
if (!synth) {
|
|
222
281
|
skipped++;
|
|
223
282
|
continue;
|
|
224
283
|
}
|
|
225
284
|
if (opts.golden) {
|
|
285
|
+
// Golden rows must round-trip through alignRow exactly like training rows (#241 done-when): a
|
|
286
|
+
// render that can't be BIO-labeled can't serve as a parser golden either. Consumes no RNG draw.
|
|
287
|
+
const goldenCanonical = {
|
|
288
|
+
raw: synth.raw,
|
|
289
|
+
components: synth.components,
|
|
290
|
+
country,
|
|
291
|
+
locale: synth.locale,
|
|
292
|
+
source,
|
|
293
|
+
source_id: "golden:align-check",
|
|
294
|
+
};
|
|
295
|
+
const goldenAligned = alignRow(goldenCanonical);
|
|
296
|
+
if (goldenAligned.kind !== "labeled" || !goldenAligned.row) {
|
|
297
|
+
skipped++;
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
226
300
|
write(JSON.stringify({ raw: synth.raw, components: synth.components, country, order }) + "\n");
|
|
227
301
|
emitted++;
|
|
228
302
|
continue;
|
|
@@ -240,7 +314,7 @@ export const localeRecipe = {
|
|
|
240
314
|
locale: synth.locale,
|
|
241
315
|
source,
|
|
242
316
|
source_id: sourceID,
|
|
243
|
-
corpus_version:
|
|
317
|
+
corpus_version: countrySource.corpusVersion,
|
|
244
318
|
license: `OpenAddresses ${country} tuples, rendered ${order}-order — see ingest SOURCES`,
|
|
245
319
|
};
|
|
246
320
|
const aligned = alignRow(canonical);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAwB,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAyBhE;;;;;;;GAOG;AACH,MAAM,eAAe,GAAwC;IAC5D,EAAE,EAAE;QACH,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE;YACN,EAAE,GAAG,EAAE,8BAA8B,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;YAC/E,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;SAC7F;KACD;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAChF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAC5F;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;CACD,CAAA;AAED;;;;GAIG;AACH,MAAM,aAAa,GAAG,SAAS,CAAA;AAE/B;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAElE,OAAO,QAAQ,IAAI,IAAI,CAAA;AACxB,CAAC;AAED,6DAA6D;AAC7D,SAAS,QAAQ,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,GAAG,GAAG,KAAK,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QAElB,IAAI,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACzB,GAAG,IAAI,GAAG,CAAA;oBACV,CAAC,EAAE,CAAA;gBACJ,CAAC;;oBAAM,GAAG,GAAG,KAAK,CAAA;YACnB,CAAC;;gBAAM,GAAG,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,CAAC,KAAK,GAAG;YAAE,GAAG,GAAG,IAAI,CAAA;aAC3B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,GAAG,EAAE,CAAA;QACT,CAAC;;YAAM,GAAG,IAAI,CAAC,CAAA;IAChB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEb,OAAO,GAAG,CAAA;AACX,CAAC;AAUD;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,IAAgB,EAAE,GAAiB;IACtD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,IAAI,KAA4B,CAAA;QAEhC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YACzD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAChC,OAAO,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrE,OAAO,CAAC,EAAE,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;QACH,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,GAAI,EAAE,IAAI,CAAC,GAAI,CAAC,CAAC,CAAA;YAC1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrE,OAAO,CAAC,EAAE,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,KAAK,GAAG,KAAK,CAAC,MAAO,CAAA;QACtB,CAAC;QACD,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC1D,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAC/G,MAAM,SAAS,GAAsB,EAAE,CAAA;QACvC,IAAI,IAAI,GAAuB,IAAI,CAAA;QACnC,IAAI,MAAM,GAAoB,IAAI,CAAA;QAClC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,CAAC,IAAI;gBAAE,OAAM;YAEjB,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;gBAC1D,MAAM,EAAE,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAC1D,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,CAAA;gBAEhH,OAAM;YACP,CAAC;YAED,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAM;YACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YACtC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;gBAAE,OAAM;YAC/B,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YAExC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO,EAAE,CAAA;gBAET,OAAM;YACP,CAAC;YACD,MAAM,KAAK,GAAoB;gBAC9B,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBAClC,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE;gBACpD,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;aAC/B,CAAA;YACD,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBACtC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;gBAElC,cAAc;gBACd,IAAI,CAAC,GAAG,aAAa;oBAAE,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;YAC5C,CAAC;QACF,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,OAAO,CAAC,KAAK,CACZ,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,MAAM,eAAe,IAAI,UAAU,OAAO,oBAAoB,CACvG,CAAA;YACD,OAAO,CAAC,SAAS,CAAC,CAAA;QACnB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAgB;IACxC,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,kGAAkG;IAC/G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAW,EAAE,6CAA6C,EAAE;QACtF,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,oDAAoD,EAAE;KAClG;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,kGAAkG;QAClG,0FAA0F;QAC1F,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QAE9C,IAAI,CAAC,aAAa,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,0CAA0C,OAAO,YAAY,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvG,CAAA;QACF,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;QAE7C,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;QAC1E,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,MAAM,CAAA;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAA;QAChC,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAA;QAE/B,MAAM,IAAI,GAAsB,EAAE,CAAA;QAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1C,kGAAkG;YAClG,yDAAyD;YACzD,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YAChF,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,EAAE,CAAE,EAAE,YAAY,CAAC,CAAA;YAEpD,KAAK,MAAM,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,kEAAkE;QACnG,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,MAAM,OAAO,yEAAyE,CAAC,CAAA;QACxG,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAA;YAClE,6FAA6F;YAC7F,6DAA6D;YAC7D,MAAM,eAAe,GACpB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAE,UAAoB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAChH,MAAM,aAAa,GAClB,OAAO,KAAK,IAAI;gBACf,CAAC,CAAC,MAAM,EAAE,GAAG,0BAA0B;oBACtC,CAAC,CAAE,WAAqB;oBACxB,CAAC,CAAE,cAAwB;gBAC5B,CAAC,CAAC,SAAS,CAAA;YACb,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAA;YAEnG,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,8FAA8F;gBAC9F,gGAAgG;gBAChG,MAAM,eAAe,GAAG;oBACvB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO;oBACP,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,oBAAoB;iBAC/B,CAAA;gBACD,MAAM,aAAa,GAAG,QAAQ,CAAC,eAAiD,CAAC,CAAA;gBAEjF,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC5D,OAAO,EAAE,CAAA;oBACT,SAAQ;gBACT,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAC9F,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC/B,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY;gBAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;aACnC,CAAC,CAAA;YACF,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,aAAa,CAAC,aAAa;gBAC3C,OAAO,EAAE,iBAAiB,OAAO,qBAAqB,KAAK,6BAA6B;aACxF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC/G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `no-street-led` — the Norwegian street-led shard, third orthography of the #901
|
|
7
|
+
* leading-name-boundary family. The post-#920 NO row-read measured the residual as
|
|
8
|
+
* ORDER-SENSITIVE decode: street-led forms ("Tangavegen 40, 5620 Tørvikbygd") carry a 30%
|
|
9
|
+
* residual vs pc-first's 7% — the diacritic street head (…vegen/…veien with ø/å/æ) is the same
|
|
10
|
+
* leading-name-before-comma boundary the FR bare-street and SI village forms exercise, in a
|
|
11
|
+
* third orthography. All three real orders cycle per tuple (balanced polarity — the v1.9.9
|
|
12
|
+
* lesson: a one-order gradient loses at convergence):
|
|
13
|
+
*
|
|
14
|
+
* 1. canonical "«st» «n», «pc» «city»" (the 30% residual class — the lesson)
|
|
15
|
+
* 2. city-first "«city», «pc», «st» «n»"
|
|
16
|
+
* 3. pc-first "«pc» «city», «st» «n»" (the 7% floor — the anchor the others converge to)
|
|
17
|
+
*/
|
|
18
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
19
|
+
export declare const noStreetLedRecipe: ShardRecipe;
|
|
20
|
+
//# sourceMappingURL=no-street-led.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"no-street-led.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/no-street-led.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAE1G,eAAO,MAAM,iBAAiB,EAAE,WAwD/B,CAAA"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `no-street-led` — the Norwegian street-led shard, third orthography of the #901
|
|
7
|
+
* leading-name-boundary family. The post-#920 NO row-read measured the residual as
|
|
8
|
+
* ORDER-SENSITIVE decode: street-led forms ("Tangavegen 40, 5620 Tørvikbygd") carry a 30%
|
|
9
|
+
* residual vs pc-first's 7% — the diacritic street head (…vegen/…veien with ø/å/æ) is the same
|
|
10
|
+
* leading-name-before-comma boundary the FR bare-street and SI village forms exercise, in a
|
|
11
|
+
* third orthography. All three real orders cycle per tuple (balanced polarity — the v1.9.9
|
|
12
|
+
* lesson: a one-order gradient loses at convergence):
|
|
13
|
+
*
|
|
14
|
+
* 1. canonical "«st» «n», «pc» «city»" (the 30% residual class — the lesson)
|
|
15
|
+
* 2. city-first "«city», «pc», «st» «n»"
|
|
16
|
+
* 3. pc-first "«pc» «city», «st» «n»" (the 7% floor — the anchor the others converge to)
|
|
17
|
+
*/
|
|
18
|
+
import { alignAndWrite, makeMulberry32, readTuples, shardSourceID } from "./scaffold.js";
|
|
19
|
+
export const noStreetLedRecipe = {
|
|
20
|
+
name: "no-street-led",
|
|
21
|
+
description: "NO street-led boundary form (#901 family): '«st» «n», «pc» «city»' — diacritic street heads",
|
|
22
|
+
mode: "tuples",
|
|
23
|
+
async run(opts, write) {
|
|
24
|
+
makeMulberry32(opts.seed);
|
|
25
|
+
let read = 0;
|
|
26
|
+
let emitted = 0;
|
|
27
|
+
let skipped = 0;
|
|
28
|
+
for await (const t of readTuples(opts.input)) {
|
|
29
|
+
read++;
|
|
30
|
+
const street = String(t.street ?? "").trim();
|
|
31
|
+
const city = String(t.locality ?? "").trim();
|
|
32
|
+
const number = String(t.number ?? "").trim();
|
|
33
|
+
const postcode = String(t.postcode ?? "").trim();
|
|
34
|
+
if (!street || !city || !number || !postcode) {
|
|
35
|
+
skipped++;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const order = read % 3;
|
|
39
|
+
let raw;
|
|
40
|
+
const components = {
|
|
41
|
+
street,
|
|
42
|
+
house_number: number,
|
|
43
|
+
postcode,
|
|
44
|
+
locality: city,
|
|
45
|
+
};
|
|
46
|
+
if (order === 0) {
|
|
47
|
+
raw = `${street} ${number}, ${postcode} ${city}`;
|
|
48
|
+
}
|
|
49
|
+
else if (order === 1) {
|
|
50
|
+
raw = `${city}, ${postcode}, ${street} ${number}`;
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
raw = `${postcode} ${city}, ${street} ${number}`;
|
|
54
|
+
}
|
|
55
|
+
const source_id = shardSourceID("synth-no-street-led", { ...components, o: String(order), v: String(read) });
|
|
56
|
+
const canonical = {
|
|
57
|
+
raw,
|
|
58
|
+
components,
|
|
59
|
+
country: "NO",
|
|
60
|
+
locale: "nb-NO",
|
|
61
|
+
source: "synth-no-street-led",
|
|
62
|
+
source_id,
|
|
63
|
+
corpus_version: "0.10.0",
|
|
64
|
+
license: "Synthetic — no-street-led; (street, number, postcode, city) from OpenAddresses NO (per-source attribution in the model card)",
|
|
65
|
+
};
|
|
66
|
+
if (alignAndWrite(write, canonical, "no-street-led"))
|
|
67
|
+
emitted++;
|
|
68
|
+
else
|
|
69
|
+
skipped++;
|
|
70
|
+
}
|
|
71
|
+
return { read, emitted, skipped };
|
|
72
|
+
},
|
|
73
|
+
};
|
|
74
|
+
//# sourceMappingURL=no-street-led.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"no-street-led.js","sourceRoot":"","sources":["../../../src/shard-recipes/no-street-led.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,MAAM,CAAC,MAAM,iBAAiB,GAAgB;IAC7C,IAAI,EAAE,eAAe;IACrB,WAAW,EAAE,6FAA6F;IAC1G,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAA;YACN,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEhD,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC9C,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAA;YACtB,IAAI,GAAW,CAAA;YACf,MAAM,UAAU,GAA2B;gBAC1C,MAAM;gBACN,YAAY,EAAE,MAAM;gBACpB,QAAQ;gBACR,QAAQ,EAAE,IAAI;aACd,CAAA;YAED,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,IAAI,EAAE,CAAA;YACjD,CAAC;iBAAM,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACxB,GAAG,GAAG,GAAG,IAAI,KAAK,QAAQ,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YAClD,CAAC;iBAAM,CAAC;gBACP,GAAG,GAAG,GAAG,QAAQ,IAAI,IAAI,KAAK,MAAM,IAAI,MAAM,EAAE,CAAA;YACjD,CAAC;YACD,MAAM,SAAS,GAAG,aAAa,CAAC,qBAAqB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC5G,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,qBAAqB;gBAC7B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EACN,8HAA8H;aAC/H,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,eAAe,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBAC1D,OAAO,EAAE,CAAA;QACf,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `si-bare-village` — the Slovenian no-street counter-shard (#901 run-2). Slovenia's rural
|
|
7
|
+
* addressing has NO street line: the village name is the street-level token and repeats as the
|
|
8
|
+
* locality ("Zabiče 8, 6250 Zabiče"). The v1.9.8 gate FALSIFIED the fr-bare-street shard alone on
|
|
9
|
+
* exactly this class (SI resolve −3.4pp; "Apače 108" split into street "Apače 10" + house "8") —
|
|
10
|
+
* the bare-street boundary lesson generalizes onto a form where the leading name must keep its
|
|
11
|
+
* number whole and the trailing mention must stay locality-bound. This shard is the paired
|
|
12
|
+
* counter-distribution: same lesson ("name before number, comma, then admin"), opposite polarity
|
|
13
|
+
* on the trailing mention.
|
|
14
|
+
*
|
|
15
|
+
* Three real-order templates cycle per tuple (mirrors the coord-golden orders so the eval and the
|
|
16
|
+
* training distribution agree):
|
|
17
|
+
*
|
|
18
|
+
* 1. canonical "«V» «n», «pc» «V»"
|
|
19
|
+
* 2. bare "«V» «n», «V»" (no postcode — the anchor-free form, the fr-bare lesson)
|
|
20
|
+
* 3. pc-first "«pc» «V», «V» «n»"
|
|
21
|
+
*
|
|
22
|
+
* Gold spans: leading «V» = street (matches OA ground truth — the village IS the address line),
|
|
23
|
+
* «n» = house_number (NEVER split mid-digits), «pc» = postcode (never swallowing the neighbor),
|
|
24
|
+
* trailing «V» = locality (the binding the resolver needs).
|
|
25
|
+
*/
|
|
26
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
27
|
+
export declare const siBareVillageRecipe: ShardRecipe;
|
|
28
|
+
//# sourceMappingURL=si-bare-village.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"si-bare-village.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/si-bare-village.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAE1G,eAAO,MAAM,mBAAmB,EAAE,WAwDjC,CAAA"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `si-bare-village` — the Slovenian no-street counter-shard (#901 run-2). Slovenia's rural
|
|
7
|
+
* addressing has NO street line: the village name is the street-level token and repeats as the
|
|
8
|
+
* locality ("Zabiče 8, 6250 Zabiče"). The v1.9.8 gate FALSIFIED the fr-bare-street shard alone on
|
|
9
|
+
* exactly this class (SI resolve −3.4pp; "Apače 108" split into street "Apače 10" + house "8") —
|
|
10
|
+
* the bare-street boundary lesson generalizes onto a form where the leading name must keep its
|
|
11
|
+
* number whole and the trailing mention must stay locality-bound. This shard is the paired
|
|
12
|
+
* counter-distribution: same lesson ("name before number, comma, then admin"), opposite polarity
|
|
13
|
+
* on the trailing mention.
|
|
14
|
+
*
|
|
15
|
+
* Three real-order templates cycle per tuple (mirrors the coord-golden orders so the eval and the
|
|
16
|
+
* training distribution agree):
|
|
17
|
+
*
|
|
18
|
+
* 1. canonical "«V» «n», «pc» «V»"
|
|
19
|
+
* 2. bare "«V» «n», «V»" (no postcode — the anchor-free form, the fr-bare lesson)
|
|
20
|
+
* 3. pc-first "«pc» «V», «V» «n»"
|
|
21
|
+
*
|
|
22
|
+
* Gold spans: leading «V» = street (matches OA ground truth — the village IS the address line),
|
|
23
|
+
* «n» = house_number (NEVER split mid-digits), «pc» = postcode (never swallowing the neighbor),
|
|
24
|
+
* trailing «V» = locality (the binding the resolver needs).
|
|
25
|
+
*/
|
|
26
|
+
import { alignAndWrite, makeMulberry32, readTuples, shardSourceID } from "./scaffold.js";
|
|
27
|
+
export const siBareVillageRecipe = {
|
|
28
|
+
name: "si-bare-village",
|
|
29
|
+
description: "SI no-street village form (#901 run-2): '«V» «n», «pc» «V»' — the fr-bare-street counter-distribution",
|
|
30
|
+
mode: "tuples",
|
|
31
|
+
async run(opts, write) {
|
|
32
|
+
makeMulberry32(opts.seed);
|
|
33
|
+
let read = 0;
|
|
34
|
+
let emitted = 0;
|
|
35
|
+
let skipped = 0;
|
|
36
|
+
for await (const t of readTuples(opts.input)) {
|
|
37
|
+
read++;
|
|
38
|
+
const village = String(t.locality ?? "").trim();
|
|
39
|
+
const number = String(t.number ?? "").trim();
|
|
40
|
+
const postcode = String(t.postcode ?? "").trim();
|
|
41
|
+
if (!village || !number || !postcode) {
|
|
42
|
+
skipped++;
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const order = read % 3;
|
|
46
|
+
let raw;
|
|
47
|
+
const components = {
|
|
48
|
+
street: village,
|
|
49
|
+
house_number: number,
|
|
50
|
+
locality: village,
|
|
51
|
+
};
|
|
52
|
+
if (order === 0) {
|
|
53
|
+
components.postcode = postcode;
|
|
54
|
+
raw = `${village} ${number}, ${postcode} ${village}`;
|
|
55
|
+
}
|
|
56
|
+
else if (order === 1) {
|
|
57
|
+
raw = `${village} ${number}, ${village}`;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
components.postcode = postcode;
|
|
61
|
+
raw = `${postcode} ${village}, ${village} ${number}`;
|
|
62
|
+
}
|
|
63
|
+
const source_id = shardSourceID("synth-si-bare-village", { ...components, o: String(order), v: String(read) });
|
|
64
|
+
const canonical = {
|
|
65
|
+
raw,
|
|
66
|
+
components,
|
|
67
|
+
country: "SI",
|
|
68
|
+
locale: "sl-SI",
|
|
69
|
+
source: "synth-si-bare-village",
|
|
70
|
+
source_id,
|
|
71
|
+
corpus_version: "0.9.9",
|
|
72
|
+
license: "Synthetic — si-bare-village; (village, number, postcode) from OpenAddresses SI (per-source attribution in the model card)",
|
|
73
|
+
};
|
|
74
|
+
if (alignAndWrite(write, canonical, "si-bare-village"))
|
|
75
|
+
emitted++;
|
|
76
|
+
else
|
|
77
|
+
skipped++;
|
|
78
|
+
}
|
|
79
|
+
return { read, emitted, skipped };
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
//# sourceMappingURL=si-bare-village.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"si-bare-village.js","sourceRoot":"","sources":["../../../src/shard-recipes/si-bare-village.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,MAAM,CAAC,MAAM,mBAAmB,GAAgB;IAC/C,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,uGAAuG;IACpH,IAAI,EAAE,QAAQ;IACd,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YAC/C,IAAI,EAAE,CAAA;YACN,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEhD,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtC,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAA;YACtB,IAAI,GAAW,CAAA;YACf,MAAM,UAAU,GAA2B;gBAC1C,MAAM,EAAE,OAAO;gBACf,YAAY,EAAE,MAAM;gBACpB,QAAQ,EAAE,OAAO;aACjB,CAAA;YAED,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACjB,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;gBAC9B,GAAG,GAAG,GAAG,OAAO,IAAI,MAAM,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAA;YACrD,CAAC;iBAAM,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBACxB,GAAG,GAAG,GAAG,OAAO,IAAI,MAAM,KAAK,OAAO,EAAE,CAAA;YACzC,CAAC;iBAAM,CAAC;gBACP,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;gBAC9B,GAAG,GAAG,GAAG,QAAQ,IAAI,OAAO,KAAK,OAAO,IAAI,MAAM,EAAE,CAAA;YACrD,CAAC;YACD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC9G,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EACN,2HAA2H;aAC5H,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,iBAAiB,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBAC5D,OAAO,EAAE,CAAA;QACf,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -50,6 +50,24 @@ export interface LocaleSynthesisOpts {
|
|
|
50
50
|
* correction).
|
|
51
51
|
*/
|
|
52
52
|
order?: "native" | "international";
|
|
53
|
+
/**
|
|
54
|
+
* Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
|
|
55
|
+
* `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
|
|
56
|
+
* evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
|
|
57
|
+
* either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
|
|
58
|
+
* currently glues the suffix onto the city).
|
|
59
|
+
*/
|
|
60
|
+
postcodeShape?: "conventional" | "as-source";
|
|
61
|
+
/**
|
|
62
|
+
* How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
|
|
63
|
+
* — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
|
|
64
|
+
* form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
|
|
65
|
+
* <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
|
|
66
|
+
* (DE/IT/NL) render identically under both. Mixing both stops an ES shard from teaching the comma as THE street→house
|
|
67
|
+
* boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
|
|
68
|
+
* house-first space-joined).
|
|
69
|
+
*/
|
|
70
|
+
nativeHouseJoin?: "template" | "space";
|
|
53
71
|
}
|
|
54
72
|
/** @deprecated Alias — use LocaleSynthesisOpts. */
|
|
55
73
|
export type GermanSynthesisOpts = LocaleSynthesisOpts;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;IAClC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,WAAW,CAAA;IAC5C;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,UAAU,GAAG,OAAO,CAAA;CACtC;AACD,mDAAmD;AACnD,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAA;AA2CrD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CA4D7B;AAED,uGAAuG;AACvG,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CAE7B"}
|
|
@@ -82,7 +82,9 @@ export function synthesizeLocaleRow(base, country, opts = {}) {
|
|
|
82
82
|
// ~80% keep the house number (the rest are street-only forms, also idiomatic).
|
|
83
83
|
if (base.house_number && random() < 0.8)
|
|
84
84
|
components.house_number = base.house_number;
|
|
85
|
-
// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it).
|
|
85
|
+
// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
|
|
86
|
+
// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
|
|
87
|
+
// normalizes the postcode itself, so a glued input can't survive rendering directly.
|
|
86
88
|
if (base.postcode && random() < 0.85)
|
|
87
89
|
components.postcode = normalizePostcode(base.postcode, country);
|
|
88
90
|
// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
|
|
@@ -96,9 +98,25 @@ export function synthesizeLocaleRow(base, country, opts = {}) {
|
|
|
96
98
|
// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
|
|
97
99
|
// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
|
|
98
100
|
const renderCountry = order === "international" ? "US" : country;
|
|
99
|
-
|
|
101
|
+
let raw = formatAddress(components, renderCountry, { separator: ", " });
|
|
100
102
|
if (!raw)
|
|
101
103
|
return null;
|
|
104
|
+
// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
|
|
105
|
+
// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
|
|
106
|
+
// space-join (the substring isn't present), and skipped when the house number was dropped above.
|
|
107
|
+
if (order === "native" && opts.nativeHouseJoin === "space" && components.house_number) {
|
|
108
|
+
raw = raw.replace(`${components.street}, ${components.house_number}`, `${components.street} ${components.house_number}`);
|
|
109
|
+
}
|
|
110
|
+
// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
|
|
111
|
+
// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
|
|
112
|
+
// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
|
|
113
|
+
if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
|
|
114
|
+
const sourceForm = base.postcode.trim();
|
|
115
|
+
if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
|
|
116
|
+
raw = raw.replace(components.postcode, sourceForm);
|
|
117
|
+
components.postcode = sourceForm;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
102
120
|
// Every component must align — drop the row if the template didn't surface one verbatim, or a
|
|
103
121
|
// numeric component collides with a neighbouring digit run.
|
|
104
122
|
for (const value of Object.values(components)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAuD3C,yFAAyF;AACzF,MAAM,UAAU,GAA2B;IAC1C,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,OAAe;IAC3D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAEpD,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,EAAE,CAAA;IAC/C,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,qGAAqG;AACrG,SAAS,YAAY,CAAC,GAAW,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACtC,2FAA2F;IAC3F,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAA;IAElF,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE5E,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAAe,EACf,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAA;IAEpC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAE/C,MAAM,UAAU,GAA+B,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAA;IAE/F,+EAA+E;IAC/E,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,EAAE,GAAG,GAAG;QAAE,UAAU,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAA;IAEpF,4FAA4F;IAC5F,0FAA0F;IAC1F,qFAAqF;IACrF,IAAI,IAAI,CAAC,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI;QAAE,UAAU,CAAC,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAErG,iGAAiG;IACjG,oGAAoG;IACpG,kGAAkG;IAClG,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,KAAK,KAAK,eAAe,IAAI,IAAI,CAAC,MAAM;QAAE,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAE7E,mGAAmG;IACnG,+FAA+F;IAC/F,oGAAoG;IACpG,MAAM,aAAa,GAAG,KAAK,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAA;IAChE,IAAI,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,qGAAqG;IACrG,6FAA6F;IAC7F,iGAAiG;IACjG,IAAI,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,KAAK,OAAO,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QACvF,GAAG,GAAG,GAAG,CAAC,OAAO,CAChB,GAAG,UAAU,CAAC,MAAM,KAAK,UAAU,CAAC,YAAY,EAAE,EAClD,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,YAAY,EAAE,CACjD,CAAA;IACF,CAAC;IAED,qGAAqG;IACrG,8FAA8F;IAC9F,6FAA6F;IAC7F,IAAI,IAAI,CAAC,aAAa,KAAK,WAAW,IAAI,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEvC,IAAI,UAAU,IAAI,UAAU,KAAK,UAAU,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3F,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAClD,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAA;QACjC,CAAC;IACF,CAAC;IAED,8FAA8F;IAC9F,4DAA4D;IAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;IACrD,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAA;AACjF,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAA4B,EAAE;IAE9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC7C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.3.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
|
-
"license": "AGPL-3.0-only",
|
|
5
|
+
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/sister-software/mailwoman.git",
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@dsnp/parquetjs": "1.8.7",
|
|
29
|
-
"@mailwoman/codex": "5.
|
|
30
|
-
"@mailwoman/core": "5.
|
|
31
|
-
"@mailwoman/formatter": "5.
|
|
29
|
+
"@mailwoman/codex": "5.3.0",
|
|
30
|
+
"@mailwoman/core": "5.3.0",
|
|
31
|
+
"@mailwoman/formatter": "5.3.0",
|
|
32
32
|
"csv-parse": "^5.6.0",
|
|
33
33
|
"fast-glob": "^3.3.3",
|
|
34
34
|
"fastest-levenshtein": "^1.0.16",
|