@mailwoman/corpus 5.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +2 -1
- package/out/src/build.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +39 -5
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +142 -68
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/synthesize-german.d.ts +18 -0
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +20 -2
- package/out/src/synthesize-german.js.map +1 -1
- package/package.json +5 -5
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAYH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAkMxF"}
|
package/out/src/build.js
CHANGED
|
@@ -50,6 +50,7 @@ import { createReadStream, createWriteStream, existsSync, readFileSync } from "n
|
|
|
50
50
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
51
51
|
import { join } from "node:path";
|
|
52
52
|
import { createInterface } from "node:readline";
|
|
53
|
+
import { $public } from "@mailwoman/core/env";
|
|
53
54
|
import { defaultAdapterRegistry } from "./adapter.js";
|
|
54
55
|
import { alignRow } from "./align.js";
|
|
55
56
|
import { licenseExcluded } from "./license.js";
|
|
@@ -89,7 +90,7 @@ export async function buildCorpus(opts) {
|
|
|
89
90
|
// redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
|
|
90
91
|
const adapterDir = join(intermediateDir, adapter.id);
|
|
91
92
|
const cachedManifest = join(adapterDir, "MANIFEST.json");
|
|
92
|
-
if (
|
|
93
|
+
if ($public.MAILWOMAN_RESUME === "1" &&
|
|
93
94
|
existsSync(cachedManifest) &&
|
|
94
95
|
existsSync(join(adapterDir, "canonical.jsonl"))) {
|
|
95
96
|
const cached = JSON.parse(readFileSync(cachedManifest, "utf8"));
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA8D/E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAExD,IACC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA8D/E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAExD,IACC,OAAO,CAAC,gBAAgB,KAAK,GAAG;YAChC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBACnB,SAAQ;YACT,CAAC;YACD,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBAEvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAExG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAElE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -7,13 +7,47 @@
|
|
|
7
7
|
* OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
|
|
8
8
|
* {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
|
|
9
9
|
* rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
|
|
10
|
-
* source CSV (`unzip -p
|
|
11
|
-
* countrywide work in bounded memory), then
|
|
12
|
-
* `random`. Ported from
|
|
10
|
+
* source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
|
|
11
|
+
* reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
|
|
12
|
+
* draws `--count` rows from the pool with the passed `random`. Ported from
|
|
13
|
+
* scripts/build-locale-shard.mjs.
|
|
13
14
|
*
|
|
14
|
-
* The reservoir uses its OWN seeded PRNG ({@link
|
|
15
|
-
* `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
15
|
+
* The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the
|
|
16
|
+
* emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
17
|
+
*
|
|
18
|
+
* SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the
|
|
19
|
+
* 2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed
|
|
20
|
+
* forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000
|
|
21
|
+
* eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse
|
|
22
|
+
* the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national
|
|
23
|
+
* convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued
|
|
24
|
+
* source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
|
|
25
|
+
* country, so DE/FR emit streams are unchanged for a given seed.
|
|
16
26
|
*/
|
|
17
27
|
import { type ShardRecipe } from "./scaffold.js";
|
|
28
|
+
/**
|
|
29
|
+
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
30
|
+
* the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop
|
|
31
|
+
* the tuple.
|
|
32
|
+
*
|
|
33
|
+
* Cleaned classes:
|
|
34
|
+
*
|
|
35
|
+
* 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of
|
|
36
|
+
* ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city.
|
|
37
|
+
* Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits.
|
|
38
|
+
* 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk
|
|
39
|
+
* (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names
|
|
40
|
+
* `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality
|
|
41
|
+
* labels.
|
|
42
|
+
*
|
|
43
|
+
* Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names):
|
|
44
|
+
*
|
|
45
|
+
* - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking
|
|
46
|
+
* `de`/`di` makes them full names, unlike the German glued-abbreviation class.
|
|
47
|
+
* - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim.
|
|
48
|
+
* - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation
|
|
49
|
+
* lever, not this shard's.
|
|
50
|
+
*/
|
|
51
|
+
export declare function cleanCityNoise(city: string): string | null;
|
|
18
52
|
export declare const localeRecipe: ShardRecipe;
|
|
19
53
|
//# sourceMappingURL=locale.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAWH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAqFhE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAwHD,eAAO,MAAM,YAAY,EAAE,WAsH1B,CAAA"}
|
|
@@ -7,54 +7,67 @@
|
|
|
7
7
|
* OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
|
|
8
8
|
* {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
|
|
9
9
|
* rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
|
|
10
|
-
* source CSV (`unzip -p
|
|
11
|
-
* countrywide work in bounded memory), then
|
|
12
|
-
* `random`. Ported from
|
|
10
|
+
* source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
|
|
11
|
+
* reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
|
|
12
|
+
* draws `--count` rows from the pool with the passed `random`. Ported from
|
|
13
|
+
* scripts/build-locale-shard.mjs.
|
|
13
14
|
*
|
|
14
|
-
* The reservoir uses its OWN seeded PRNG ({@link
|
|
15
|
-
* `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
15
|
+
* The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the
|
|
16
|
+
* emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws.
|
|
17
|
+
*
|
|
18
|
+
* SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the
|
|
19
|
+
* 2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed
|
|
20
|
+
* forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000
|
|
21
|
+
* eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse
|
|
22
|
+
* the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national
|
|
23
|
+
* convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued
|
|
24
|
+
* source shape, the rest the spaced conventional one. These draws are consumed ONLY for their
|
|
25
|
+
* country, so DE/FR emit streams are unchanged for a given seed.
|
|
16
26
|
*/
|
|
17
27
|
import { spawn } from "node:child_process";
|
|
28
|
+
import { createReadStream } from "node:fs";
|
|
18
29
|
import { createInterface } from "node:readline";
|
|
30
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
19
31
|
import { stableSourceID } from "../adapter.js";
|
|
20
32
|
import { alignRow } from "../align.js";
|
|
21
33
|
import { synthesizeLocaleRow } from "../synthesize-german.js";
|
|
22
34
|
import { makeMulberry32 } from "./scaffold.js";
|
|
23
35
|
/**
|
|
24
|
-
* Per-country OA sources
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* carries a `
|
|
36
|
+
* Per-country OA sources + the source name used in the corpus. DE/FR still point at the legacy `/tmp/oa-cache` zips
|
|
37
|
+
* (their historical build inputs — materialize them there to regenerate). ES/NL read the extracted countrywide CSVs and
|
|
38
|
+
* IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is OA-conformed, so the old
|
|
39
|
+
* raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is empty; the
|
|
40
|
+
* international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad autónoma, IT =
|
|
41
|
+
* regione, NL = province).
|
|
28
42
|
*/
|
|
29
43
|
const COUNTRY_SOURCES = {
|
|
30
44
|
DE: {
|
|
31
45
|
source: "synth-german",
|
|
46
|
+
corpusVersion: "0.4.0",
|
|
32
47
|
parts: [
|
|
33
48
|
{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
|
|
34
49
|
{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
|
|
35
50
|
],
|
|
36
51
|
},
|
|
37
|
-
FR: {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
52
|
+
FR: {
|
|
53
|
+
source: "synth-fr",
|
|
54
|
+
corpusVersion: "0.4.0",
|
|
55
|
+
parts: [{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }],
|
|
56
|
+
},
|
|
57
|
+
NL: {
|
|
58
|
+
source: "synth-nl",
|
|
59
|
+
corpusVersion: "0.9.9",
|
|
60
|
+
parts: [{ path: dataRootPath("openaddresses", "extracted", "nl", "countrywide.csv") }],
|
|
61
|
+
},
|
|
62
|
+
IT: {
|
|
63
|
+
source: "synth-it",
|
|
64
|
+
corpusVersion: "0.9.9",
|
|
65
|
+
parts: [{ zip: dataRootPath("oa-cache", "it__countrywide.zip"), csv: "it/countrywide.csv" }],
|
|
66
|
+
},
|
|
43
67
|
ES: {
|
|
44
68
|
source: "synth-es",
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
zip: "/tmp/oa-cache/es__countrywide.zip",
|
|
48
|
-
csv: "es_addresses.csv",
|
|
49
|
-
conform: {
|
|
50
|
-
number: "numero",
|
|
51
|
-
street: ["tipo_vial", "nombre_via"],
|
|
52
|
-
city: "municipio",
|
|
53
|
-
region: "comunidad_autonoma",
|
|
54
|
-
postcode: "cod_postal",
|
|
55
|
-
},
|
|
56
|
-
},
|
|
57
|
-
],
|
|
69
|
+
corpusVersion: "0.9.9",
|
|
70
|
+
parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
|
|
58
71
|
},
|
|
59
72
|
};
|
|
60
73
|
/**
|
|
@@ -63,6 +76,47 @@ const COUNTRY_SOURCES = {
|
|
|
63
76
|
* sources (≤ ~1.2M) fit entirely, so they're sampled losslessly.
|
|
64
77
|
*/
|
|
65
78
|
const RESERVOIR_CAP = 1_200_000;
|
|
79
|
+
/**
|
|
80
|
+
* Fraction of NATIVE-order ES rows whose street→house join is space-collapsed (`CALLE MAYOR 12`) instead of the
|
|
81
|
+
* template's comma (`CALLE MAYOR, 12`). Both are real Spanish surfaces — the comma is the official convention, the
|
|
82
|
+
* space is what OA-derived feeds (and all 3,000 `openaddresses-es-sample.jsonl` rows) carry. 0.5 teaches both.
|
|
83
|
+
*/
|
|
84
|
+
const ES_SPACE_JOIN_FRACTION = 0.5;
|
|
85
|
+
/**
|
|
86
|
+
* Fraction of NL rows whose postcode keeps OA's glued shape (`1187LM`) instead of the spaced national convention (`1187
|
|
87
|
+
* LM`). The eval sample is 100% glued; the conventional spaced form is the `1012 LM` two-letter-suffix shape the model
|
|
88
|
+
* currently glues onto the city (#241). 0.5 teaches both.
|
|
89
|
+
*/
|
|
90
|
+
const NL_GLUED_POSTCODE_FRACTION = 0.5;
|
|
91
|
+
/**
|
|
92
|
+
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
93
|
+
* the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop
|
|
94
|
+
* the tuple.
|
|
95
|
+
*
|
|
96
|
+
* Cleaned classes:
|
|
97
|
+
*
|
|
98
|
+
* 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of
|
|
99
|
+
* ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city.
|
|
100
|
+
* Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits.
|
|
101
|
+
* 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk
|
|
102
|
+
* (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names
|
|
103
|
+
* `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality
|
|
104
|
+
* labels.
|
|
105
|
+
*
|
|
106
|
+
* Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names):
|
|
107
|
+
*
|
|
108
|
+
* - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking
|
|
109
|
+
* `de`/`di` makes them full names, unlike the German glued-abbreviation class.
|
|
110
|
+
* - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim.
|
|
111
|
+
* - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation
|
|
112
|
+
* lever, not this shard's.
|
|
113
|
+
*/
|
|
114
|
+
export function cleanCityNoise(city) {
|
|
115
|
+
if (/,|\d{4}/.test(city))
|
|
116
|
+
return null;
|
|
117
|
+
const stripped = city.replace(/\s*\(\p{L}{1,3}\)\s*$/u, "").trim();
|
|
118
|
+
return stripped || null;
|
|
119
|
+
}
|
|
66
120
|
/** Minimal RFC-4180-ish splitter (handles quoted fields). */
|
|
67
121
|
function splitCSV(line) {
|
|
68
122
|
const out = [];
|
|
@@ -95,57 +149,58 @@ function splitCSV(line) {
|
|
|
95
149
|
return out;
|
|
96
150
|
}
|
|
97
151
|
/**
|
|
98
|
-
* Stream real tuples out of
|
|
99
|
-
*
|
|
100
|
-
* from the emit loop's PRNG. NO global dedup (a
|
|
101
|
-
*
|
|
152
|
+
* Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV line-by-line
|
|
153
|
+
* — `unzip -p | readline` for zip parts, `createReadStream | readline` for extracted parts (both bounded memory) — and
|
|
154
|
+
* keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO global dedup (a
|
|
155
|
+
* 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the region falls
|
|
156
|
+
* back to `part.region` when the row's REGION cell is empty (DE).
|
|
102
157
|
*/
|
|
103
158
|
function readTuples(part, rng) {
|
|
104
159
|
return new Promise((resolve) => {
|
|
105
|
-
|
|
106
|
-
|
|
160
|
+
let input;
|
|
161
|
+
if (part.path) {
|
|
162
|
+
input = createReadStream(part.path, { encoding: "utf8" });
|
|
163
|
+
input.on("error", (err) => {
|
|
164
|
+
console.error(` WARN: read failed for ${part.path}: ${err.message}`);
|
|
165
|
+
resolve([]);
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
const child = spawn("unzip", ["-p", part.zip, part.csv]);
|
|
170
|
+
child.on("error", (err) => {
|
|
171
|
+
console.error(` WARN: unzip failed for ${part.zip}: ${err.message}`);
|
|
172
|
+
resolve([]);
|
|
173
|
+
});
|
|
174
|
+
input = child.stdout;
|
|
175
|
+
}
|
|
176
|
+
const rl = createInterface({ input, crlfDelay: Infinity });
|
|
107
177
|
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
108
178
|
const reservoir = [];
|
|
109
179
|
let cols = null;
|
|
110
180
|
let header = null;
|
|
111
181
|
let seen = 0;
|
|
182
|
+
let dropped = 0;
|
|
112
183
|
rl.on("line", (line) => {
|
|
113
184
|
if (!line)
|
|
114
185
|
return;
|
|
115
186
|
if (header === null) {
|
|
116
187
|
header = splitCSV(line).map((h) => h.trim().toLowerCase());
|
|
117
|
-
const ix = (name) => header.indexOf(
|
|
118
|
-
|
|
119
|
-
// upstream schema (ES — CNIG columns, street split across `tipo_vial` + `nombre_via`).
|
|
120
|
-
const c = part.conform;
|
|
121
|
-
cols = c
|
|
122
|
-
? {
|
|
123
|
-
num: ix(c.number),
|
|
124
|
-
streetParts: (Array.isArray(c.street) ? c.street : [c.street]).map(ix),
|
|
125
|
-
city: ix(c.city),
|
|
126
|
-
region: c.region ? ix(c.region) : -1,
|
|
127
|
-
post: ix(c.postcode),
|
|
128
|
-
}
|
|
129
|
-
: {
|
|
130
|
-
num: ix("number"),
|
|
131
|
-
streetParts: [ix("street")],
|
|
132
|
-
city: ix("city"),
|
|
133
|
-
region: ix("region"),
|
|
134
|
-
post: ix("postcode"),
|
|
135
|
-
};
|
|
188
|
+
const ix = (name) => header.indexOf(name);
|
|
189
|
+
cols = { num: ix("number"), street: ix("street"), city: ix("city"), region: ix("region"), post: ix("postcode") };
|
|
136
190
|
return;
|
|
137
191
|
}
|
|
138
192
|
if (cols === null)
|
|
139
193
|
return;
|
|
140
194
|
const cells = splitCSV(line);
|
|
141
|
-
const street = cols.
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
195
|
+
const street = get(cells, cols.street);
|
|
196
|
+
const rawCity = get(cells, cols.city);
|
|
197
|
+
if (!street || !rawCity)
|
|
198
|
+
return;
|
|
199
|
+
const locality = cleanCityNoise(rawCity);
|
|
200
|
+
if (!locality) {
|
|
201
|
+
dropped++;
|
|
148
202
|
return;
|
|
203
|
+
}
|
|
149
204
|
const tuple = {
|
|
150
205
|
house_number: get(cells, cols.num),
|
|
151
206
|
street,
|
|
@@ -165,13 +220,9 @@ function readTuples(part, rng) {
|
|
|
165
220
|
}
|
|
166
221
|
});
|
|
167
222
|
rl.on("close", () => {
|
|
168
|
-
console.error(` ${part.csv}: ${reservoir.length} sampled of ${seen} rows`);
|
|
223
|
+
console.error(` ${part.path ?? part.csv}: ${reservoir.length} sampled of ${seen} rows (${dropped} city-noise drops)`);
|
|
169
224
|
resolve(reservoir);
|
|
170
225
|
});
|
|
171
|
-
child.on("error", (err) => {
|
|
172
|
-
console.error(` WARN: unzip failed for ${part.zip}: ${err.message}`);
|
|
173
|
-
resolve([]);
|
|
174
|
-
});
|
|
175
226
|
});
|
|
176
227
|
}
|
|
177
228
|
export const localeRecipe = {
|
|
@@ -208,7 +259,7 @@ export const localeRecipe = {
|
|
|
208
259
|
pool.push(x); // NOT pool.push(...t) — spreading huge arrays overflows the stack
|
|
209
260
|
}
|
|
210
261
|
if (pool.length === 0) {
|
|
211
|
-
throw new Error(`No ${country} tuples found — are the
|
|
262
|
+
throw new Error(`No ${country} tuples found — are the source CSVs/zips present? (see COUNTRY_SOURCES)`);
|
|
212
263
|
}
|
|
213
264
|
let emitted = 0;
|
|
214
265
|
let skipped = 0;
|
|
@@ -217,12 +268,35 @@ export const localeRecipe = {
|
|
|
217
268
|
while (emitted < count && guard++ < count * 6) {
|
|
218
269
|
const base = pool[Math.floor(random() * N)];
|
|
219
270
|
const order = random() < intlFraction ? "international" : "native";
|
|
220
|
-
|
|
271
|
+
// Per-country surface-shape draws (#241) — consumed ONLY for that country, so the DE/FR emit
|
|
272
|
+
// streams for a given seed are unchanged by their existence.
|
|
273
|
+
const nativeHouseJoin = country === "ES" ? (random() < ES_SPACE_JOIN_FRACTION ? "space" : "template") : undefined;
|
|
274
|
+
const postcodeShape = country === "NL"
|
|
275
|
+
? random() < NL_GLUED_POSTCODE_FRACTION
|
|
276
|
+
? "as-source"
|
|
277
|
+
: "conventional"
|
|
278
|
+
: undefined;
|
|
279
|
+
const synth = synthesizeLocaleRow(base, country, { random, order, nativeHouseJoin, postcodeShape });
|
|
221
280
|
if (!synth) {
|
|
222
281
|
skipped++;
|
|
223
282
|
continue;
|
|
224
283
|
}
|
|
225
284
|
if (opts.golden) {
|
|
285
|
+
// Golden rows must round-trip through alignRow exactly like training rows (#241 done-when): a
|
|
286
|
+
// render that can't be BIO-labeled can't serve as a parser golden either. Consumes no RNG draw.
|
|
287
|
+
const goldenCanonical = {
|
|
288
|
+
raw: synth.raw,
|
|
289
|
+
components: synth.components,
|
|
290
|
+
country,
|
|
291
|
+
locale: synth.locale,
|
|
292
|
+
source,
|
|
293
|
+
source_id: "golden:align-check",
|
|
294
|
+
};
|
|
295
|
+
const goldenAligned = alignRow(goldenCanonical);
|
|
296
|
+
if (goldenAligned.kind !== "labeled" || !goldenAligned.row) {
|
|
297
|
+
skipped++;
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
226
300
|
write(JSON.stringify({ raw: synth.raw, components: synth.components, country, order }) + "\n");
|
|
227
301
|
emitted++;
|
|
228
302
|
continue;
|
|
@@ -240,7 +314,7 @@ export const localeRecipe = {
|
|
|
240
314
|
locale: synth.locale,
|
|
241
315
|
source,
|
|
242
316
|
source_id: sourceID,
|
|
243
|
-
corpus_version:
|
|
317
|
+
corpus_version: countrySource.corpusVersion,
|
|
244
318
|
license: `OpenAddresses ${country} tuples, rendered ${order}-order — see ingest SOURCES`,
|
|
245
319
|
};
|
|
246
320
|
const aligned = alignRow(canonical);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAwB,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAyBhE;;;;;;;GAOG;AACH,MAAM,eAAe,GAAwC;IAC5D,EAAE,EAAE;QACH,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE;YACN,EAAE,GAAG,EAAE,8BAA8B,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;YAC/E,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;SAC7F;KACD;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAChF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAC5F;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;CACD,CAAA;AAED;;;;GAIG;AACH,MAAM,aAAa,GAAG,SAAS,CAAA;AAE/B;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAElE,OAAO,QAAQ,IAAI,IAAI,CAAA;AACxB,CAAC;AAED,6DAA6D;AAC7D,SAAS,QAAQ,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,GAAG,GAAG,KAAK,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QAElB,IAAI,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACzB,GAAG,IAAI,GAAG,CAAA;oBACV,CAAC,EAAE,CAAA;gBACJ,CAAC;;oBAAM,GAAG,GAAG,KAAK,CAAA;YACnB,CAAC;;gBAAM,GAAG,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,CAAC,KAAK,GAAG;YAAE,GAAG,GAAG,IAAI,CAAA;aAC3B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,GAAG,EAAE,CAAA;QACT,CAAC;;YAAM,GAAG,IAAI,CAAC,CAAA;IAChB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEb,OAAO,GAAG,CAAA;AACX,CAAC;AAUD;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,IAAgB,EAAE,GAAiB;IACtD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,IAAI,KAA4B,CAAA;QAEhC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YACzD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAChC,OAAO,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrE,OAAO,CAAC,EAAE,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;QACH,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,GAAI,EAAE,IAAI,CAAC,GAAI,CAAC,CAAC,CAAA;YAC1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;gBACrE,OAAO,CAAC,EAAE,CAAC,CAAA;YACZ,CAAC,CAAC,CAAA;YACF,KAAK,GAAG,KAAK,CAAC,MAAO,CAAA;QACtB,CAAC;QACD,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC1D,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAC/G,MAAM,SAAS,GAAsB,EAAE,CAAA;QACvC,IAAI,IAAI,GAAuB,IAAI,CAAA;QACnC,IAAI,MAAM,GAAoB,IAAI,CAAA;QAClC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,CAAC,IAAI;gBAAE,OAAM;YAEjB,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;gBAC1D,MAAM,EAAE,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAC1D,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,CAAA;gBAEhH,OAAM;YACP,CAAC;YAED,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAM;YACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;YAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YACtC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;gBAAE,OAAM;YAC/B,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YAExC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO,EAAE,CAAA;gBAET,OAAM;YACP,CAAC;YACD,MAAM,KAAK,GAAoB;gBAC9B,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBAClC,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE;gBACpD,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;aAC/B,CAAA;YACD,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBACtC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;gBAElC,cAAc;gBACd,IAAI,CAAC,GAAG,aAAa;oBAAE,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;YAC5C,CAAC;QACF,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,OAAO,CAAC,KAAK,CACZ,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,MAAM,eAAe,IAAI,UAAU,OAAO,oBAAoB,CACvG,CAAA;YACD,OAAO,CAAC,SAAS,CAAC,CAAA;QACnB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAgB;IACxC,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,kGAAkG;IAC/G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAW,EAAE,6CAA6C,EAAE;QACtF,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,oDAAoD,EAAE;KAClG;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,kGAAkG;QAClG,0FAA0F;QAC1F,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QAE9C,IAAI,CAAC,aAAa,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,0CAA0C,OAAO,YAAY,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvG,CAAA;QACF,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;QAE7C,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;QAC1E,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,MAAM,CAAA;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAA;QAChC,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAA;QAE/B,MAAM,IAAI,GAAsB,EAAE,CAAA;QAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1C,kGAAkG;YAClG,yDAAyD;YACzD,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YAChF,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,EAAE,CAAE,EAAE,YAAY,CAAC,CAAA;YAEpD,KAAK,MAAM,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,kEAAkE;QACnG,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,MAAM,OAAO,yEAAyE,CAAC,CAAA;QACxG,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAA;YAClE,6FAA6F;YAC7F,6DAA6D;YAC7D,MAAM,eAAe,GACpB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAE,UAAoB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAChH,MAAM,aAAa,GAClB,OAAO,KAAK,IAAI;gBACf,CAAC,CAAC,MAAM,EAAE,GAAG,0BAA0B;oBACtC,CAAC,CAAE,WAAqB;oBACxB,CAAC,CAAE,cAAwB;gBAC5B,CAAC,CAAC,SAAS,CAAA;YACb,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAA;YAEnG,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,8FAA8F;gBAC9F,gGAAgG;gBAChG,MAAM,eAAe,GAAG;oBACvB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO;oBACP,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,oBAAoB;iBAC/B,CAAA;gBACD,MAAM,aAAa,GAAG,QAAQ,CAAC,eAAiD,CAAC,CAAA;gBAEjF,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC5D,OAAO,EAAE,CAAA;oBACT,SAAQ;gBACT,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAC9F,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC/B,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY;gBAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;aACnC,CAAC,CAAA;YACF,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,aAAa,CAAC,aAAa;gBAC3C,OAAO,EAAE,iBAAiB,OAAO,qBAAqB,KAAK,6BAA6B;aACxF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC/G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -50,6 +50,24 @@ export interface LocaleSynthesisOpts {
|
|
|
50
50
|
* correction).
|
|
51
51
|
*/
|
|
52
52
|
order?: "native" | "international";
|
|
53
|
+
/**
|
|
54
|
+
* Postcode surface shape. `"conventional"` (default) canonicalizes to the country's rendered form (NL: OA's glued
|
|
55
|
+
* `1011AB` → the spaced `1011 AB`); `"as-source"` keeps the source's own surface — the form OA (and the OA-derived
|
|
56
|
+
* evals) feed, which for NL is 100% glued. Only NL differs today; every other country passes through identically
|
|
57
|
+
* either way. Mixing both teaches the two-letter-suffix `1012 LM` shape AND the glued feed shape (#241 — the model
|
|
58
|
+
* currently glues the suffix onto the city).
|
|
59
|
+
*/
|
|
60
|
+
postcodeShape?: "conventional" | "as-source";
|
|
61
|
+
/**
|
|
62
|
+
* How the NATIVE-order render joins street and house number. The OpenCage ES template comma-joins (`Calle Mayor, 12`
|
|
63
|
+
* — the official Spanish convention); OA-derived feeds and our ES eval space-join (`CALLE MAYOR 12`, the observed
|
|
64
|
+
* form on all 3,000 eval rows). `"template"` (default) keeps the template's own join; `"space"` collapses `<street>,
|
|
65
|
+
* <house_number>` → `<street> <house_number>` after rendering. Countries whose template already space-joins
|
|
66
|
+
* (DE/IT/NL) render identically under both. Mixing both stops an ES shard from teaching the comma as THE street→house
|
|
67
|
+
* boundary signal (#241 format-diversity audit). International order ignores this (the US template is already
|
|
68
|
+
* house-first space-joined).
|
|
69
|
+
*/
|
|
70
|
+
nativeHouseJoin?: "template" | "space";
|
|
53
71
|
}
|
|
54
72
|
/** @deprecated Alias — use LocaleSynthesisOpts. */
|
|
55
73
|
export type GermanSynthesisOpts = LocaleSynthesisOpts;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;IAClC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,WAAW,CAAA;IAC5C;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,UAAU,GAAG,OAAO,CAAA;CACtC;AACD,mDAAmD;AACnD,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAA;AA2CrD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CA4D7B;AAED,uGAAuG;AACvG,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CAE7B"}
|
|
@@ -82,7 +82,9 @@ export function synthesizeLocaleRow(base, country, opts = {}) {
|
|
|
82
82
|
// ~80% keep the house number (the rest are street-only forms, also idiomatic).
|
|
83
83
|
if (base.house_number && random() < 0.8)
|
|
84
84
|
components.house_number = base.house_number;
|
|
85
|
-
// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it).
|
|
85
|
+
// ~85% keep the postcode (canonicalized to the country's rendered form — NL spaces it). The
|
|
86
|
+
// `postcodeShape: "as-source"` rewrite happens AFTER the render: the OpenCage NL template
|
|
87
|
+
// normalizes the postcode itself, so a glued input can't survive rendering directly.
|
|
86
88
|
if (base.postcode && random() < 0.85)
|
|
87
89
|
components.postcode = normalizePostcode(base.postcode, country);
|
|
88
90
|
// International order carries the REGION in the tail ("City, Region Postcode") — the layout real
|
|
@@ -96,9 +98,25 @@ export function synthesizeLocaleRow(base, country, opts = {}) {
|
|
|
96
98
|
// house-first, postcode-after-city, with a region slot for the tail. Neither branch consumes a
|
|
97
99
|
// `random()` draw for the template, so the RNG sequence existing callers/tests depend on is stable.
|
|
98
100
|
const renderCountry = order === "international" ? "US" : country;
|
|
99
|
-
|
|
101
|
+
let raw = formatAddress(components, renderCountry, { separator: ", " });
|
|
100
102
|
if (!raw)
|
|
101
103
|
return null;
|
|
104
|
+
// Native-order space-join (see {@link LocaleSynthesisOpts.nativeHouseJoin}): collapse the template's
|
|
105
|
+
// `<street>, <hn>` comma to a space — the OA/feed layout. A no-op for templates that already
|
|
106
|
+
// space-join (the substring isn't present), and skipped when the house number was dropped above.
|
|
107
|
+
if (order === "native" && opts.nativeHouseJoin === "space" && components.house_number) {
|
|
108
|
+
raw = raw.replace(`${components.street}, ${components.house_number}`, `${components.street} ${components.house_number}`);
|
|
109
|
+
}
|
|
110
|
+
// `postcodeShape: "as-source"` (see {@link LocaleSynthesisOpts.postcodeShape}): rewrite BOTH raw and
|
|
111
|
+
// the component back to the source's own surface (NL glued `1011AB`). Post-render because the
|
|
112
|
+
// OpenCage NL template normalizes the postcode to the spaced form no matter what it's given.
|
|
113
|
+
if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
|
|
114
|
+
const sourceForm = base.postcode.trim();
|
|
115
|
+
if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
|
|
116
|
+
raw = raw.replace(components.postcode, sourceForm);
|
|
117
|
+
components.postcode = sourceForm;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
102
120
|
// Every component must align — drop the row if the template didn't surface one verbatim, or a
|
|
103
121
|
// numeric component collides with a neighbouring digit run.
|
|
104
122
|
for (const value of Object.values(components)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAuD3C,yFAAyF;AACzF,MAAM,UAAU,GAA2B;IAC1C,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,OAAe;IAC3D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAEpD,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,EAAE,CAAA;IAC/C,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,qGAAqG;AACrG,SAAS,YAAY,CAAC,GAAW,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACtC,2FAA2F;IAC3F,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAA;IAElF,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE5E,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAAe,EACf,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAA;IAEpC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAE/C,MAAM,UAAU,GAA+B,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAA;IAE/F,+EAA+E;IAC/E,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,EAAE,GAAG,GAAG;QAAE,UAAU,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAA;IAEpF,4FAA4F;IAC5F,0FAA0F;IAC1F,qFAAqF;IACrF,IAAI,IAAI,CAAC,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI;QAAE,UAAU,CAAC,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAErG,iGAAiG;IACjG,oGAAoG;IACpG,kGAAkG;IAClG,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,KAAK,KAAK,eAAe,IAAI,IAAI,CAAC,MAAM;QAAE,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAE7E,mGAAmG;IACnG,+FAA+F;IAC/F,oGAAoG;IACpG,MAAM,aAAa,GAAG,KAAK,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAA;IAChE,IAAI,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,qGAAqG;IACrG,6FAA6F;IAC7F,iGAAiG;IACjG,IAAI,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,KAAK,OAAO,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QACvF,GAAG,GAAG,GAAG,CAAC,OAAO,CAChB,GAAG,UAAU,CAAC,MAAM,KAAK,UAAU,CAAC,YAAY,EAAE,EAClD,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,YAAY,EAAE,CACjD,CAAA;IACF,CAAC;IAED,qGAAqG;IACrG,8FAA8F;IAC9F,6FAA6F;IAC7F,IAAI,IAAI,CAAC,aAAa,KAAK,WAAW,IAAI,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEvC,IAAI,UAAU,IAAI,UAAU,KAAK,UAAU,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3F,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAClD,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAA;QACjC,CAAC;IACF,CAAC;IAED,8FAA8F;IAC9F,4DAA4D;IAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;IACrD,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAA;AACjF,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAA4B,EAAE;IAE9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC7C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.2.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
|
-
"license": "AGPL-3.0-only",
|
|
5
|
+
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/sister-software/mailwoman.git",
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
28
|
"@dsnp/parquetjs": "1.8.7",
|
|
29
|
-
"@mailwoman/codex": "5.
|
|
30
|
-
"@mailwoman/core": "5.
|
|
31
|
-
"@mailwoman/formatter": "5.
|
|
29
|
+
"@mailwoman/codex": "5.2.0",
|
|
30
|
+
"@mailwoman/core": "5.2.0",
|
|
31
|
+
"@mailwoman/formatter": "5.2.0",
|
|
32
32
|
"csv-parse": "^5.6.0",
|
|
33
33
|
"fast-glob": "^3.3.3",
|
|
34
34
|
"fastest-levenshtein": "^1.0.16",
|