@mailwoman/corpus 6.5.0 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/build.js +5 -5
- package/out/src/codex/us-fips-state.d.ts +6 -6
- package/out/src/codex/us-fips-state.js +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +2 -1
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +6 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +38 -3
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -4
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/synthesize-german.d.ts +6 -0
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +7 -0
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +3 -3
- package/package.json +4 -4
package/out/src/build.js
CHANGED
|
@@ -139,7 +139,7 @@ export async function buildCorpus(opts) {
|
|
|
139
139
|
};
|
|
140
140
|
for (const adapterRun of adapterRuns) {
|
|
141
141
|
opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
|
|
142
|
-
for await (const row of
|
|
142
|
+
for await (const row of streamJSONL(adapterRun.jsonl_path)) {
|
|
143
143
|
licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
|
|
144
144
|
// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
|
|
145
145
|
// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
|
|
@@ -203,9 +203,9 @@ export async function buildCorpus(opts) {
|
|
|
203
203
|
// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
|
|
204
204
|
opts.onProgress?.("shard", "writing parquet shards");
|
|
205
205
|
const shardManifest = await writeShards({
|
|
206
|
-
train:
|
|
207
|
-
val:
|
|
208
|
-
test:
|
|
206
|
+
train: streamJSONL(labeledPaths.train),
|
|
207
|
+
val: streamJSONL(labeledPaths.val),
|
|
208
|
+
test: streamJSONL(labeledPaths.test),
|
|
209
209
|
}, {
|
|
210
210
|
outputDir: opts.outputDir,
|
|
211
211
|
corpusVersion: opts.corpusVersion,
|
|
@@ -235,7 +235,7 @@ export async function buildCorpus(opts) {
|
|
|
235
235
|
await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
236
236
|
return manifest;
|
|
237
237
|
}
|
|
238
|
-
async function*
|
|
238
|
+
async function* streamJSONL(path) {
|
|
239
239
|
// JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
|
|
240
240
|
// lines are dropped at the row level) and throws SyntaxError on a malformed row —
|
|
241
241
|
// same fail-loud behavior as the prior readline + bare `JSON.parse`.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* join.
|
|
14
14
|
*/
|
|
15
15
|
/** Per-state record: two-letter postal abbreviation + full canonical display name. */
|
|
16
|
-
export interface
|
|
16
|
+
export interface USStateInfo {
|
|
17
17
|
abbreviation: string;
|
|
18
18
|
name: string;
|
|
19
19
|
}
|
|
@@ -21,17 +21,17 @@ export interface UsStateInfo {
|
|
|
21
21
|
* FIPS state-or-territory code → `{ abbreviation, name }`. Includes all 50 states, DC, and the five primary territories
|
|
22
22
|
* (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER column `statefp`.
|
|
23
23
|
*/
|
|
24
|
-
export declare const US_FIPS_STATE: Readonly<Record<string,
|
|
24
|
+
export declare const US_FIPS_STATE: Readonly<Record<string, USStateInfo>>;
|
|
25
25
|
/** Lookup helper. Returns null when the FIPS code isn't recognized. */
|
|
26
|
-
export declare function lookupFipsState(statefp: string | null | undefined):
|
|
26
|
+
export declare function lookupFipsState(statefp: string | null | undefined): USStateInfo | null;
|
|
27
27
|
/**
|
|
28
|
-
* Inverted view: two-letter postal abbreviation → `
|
|
28
|
+
* Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
|
|
29
29
|
* source data ships the abbreviation rather than the FIPS code (FCC BDC, most federal CSVs).
|
|
30
30
|
*/
|
|
31
|
-
export declare const US_STATE_BY_ABBREVIATION: Readonly<Record<string,
|
|
31
|
+
export declare const US_STATE_BY_ABBREVIATION: Readonly<Record<string, USStateInfo>>;
|
|
32
32
|
/**
|
|
33
33
|
* Lookup helper for adapters carrying 2-char USPS abbreviations (`"CA"`, `"VT"`). Case-folded; null for any value
|
|
34
34
|
* outside the 50 states + DC + the five primary territories.
|
|
35
35
|
*/
|
|
36
|
-
export declare function lookupStateAbbreviation(abbreviation: string | null | undefined):
|
|
36
|
+
export declare function lookupStateAbbreviation(abbreviation: string | null | undefined): USStateInfo | null;
|
|
37
37
|
//# sourceMappingURL=us-fips-state.d.ts.map
|
|
@@ -82,7 +82,7 @@ export function lookupFipsState(statefp) {
|
|
|
82
82
|
return US_FIPS_STATE[statefp] ?? null;
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
|
-
* Inverted view: two-letter postal abbreviation → `
|
|
85
|
+
* Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
|
|
86
86
|
* source data ships the abbreviation rather than the FIPS code (FCC BDC, most federal CSVs).
|
|
87
87
|
*/
|
|
88
88
|
export const US_STATE_BY_ABBREVIATION = Object.freeze(Object.fromEntries(Object.values(US_FIPS_STATE).map((info) => [info.abbreviation, info])));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fr-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAKH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA4C1G,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAajD;AAQD,eAAO,MAAM,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"fr-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAKH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA4C1G,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAajD;AAQD,eAAO,MAAM,gBAAgB,EAAE,WAqK9B,CAAA"}
|
|
@@ -167,8 +167,9 @@ export const frFragmentRecipe = {
|
|
|
167
167
|
read++;
|
|
168
168
|
const fullStreet = String(tuple.street ?? "").trim();
|
|
169
169
|
const locality = String(tuple.locality ?? "").trim();
|
|
170
|
-
if (locality)
|
|
170
|
+
if (locality) {
|
|
171
171
|
localities.add(locality);
|
|
172
|
+
}
|
|
172
173
|
if (!fullStreet) {
|
|
173
174
|
skipped++;
|
|
174
175
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fr-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,mFAAmF;AACnF,MAAM,aAAa,GAAG;IACrB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CACrH,CAAA;AACD,gGAAgG;AAChG,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/C,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CACtC,KAAK;KACH,SAAS,CAAC,KAAK,CAAC;KAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KACrB,WAAW,EAAE;KACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAA;AAET;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;IACxB,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,KAAc,EAAU,EAAE,CACrD,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CACxB,IAAI;SACF,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;SACnE,IAAI,CAAC,GAAG,CAAC,CACX;SACA,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAC5D,kFAAkF;AAClF,MAAM,OAAO,GACZ,qIAAqI,CAAA;AAEtI,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,kJAAkJ;IACnJ,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EAAE,uFAAuF;SACpG;QACD;YACC,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,kFAAkF;SAC/F;QACD;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,qFAAqF;SAClG;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,4GAA4G;gBAC5G,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAA;QAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAA;QAE9C,wFAAwF;QACxF,+DAA+D;QAC/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QAEpC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ;
|
|
1
|
+
{"version":3,"file":"fr-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,mFAAmF;AACnF,MAAM,aAAa,GAAG;IACrB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CACrH,CAAA;AACD,gGAAgG;AAChG,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/C,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CACtC,KAAK;KACH,SAAS,CAAC,KAAK,CAAC;KAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KACrB,WAAW,EAAE;KACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAA;AAET;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;IACxB,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,KAAc,EAAU,EAAE,CACrD,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CACxB,IAAI;SACF,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;SACnE,IAAI,CAAC,GAAG,CAAC,CACX;SACA,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAC5D,kFAAkF;AAClF,MAAM,OAAO,GACZ,qIAAqI,CAAA;AAEtI,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,kJAAkJ;IACnJ,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EAAE,uFAAuF;SACpG;QACD;YACC,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,kFAAkF;SAC/F;QACD;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,qFAAqF;SAClG;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,4GAA4G;gBAC5G,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAA;QAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAA;QAE9C,wFAAwF;QACxF,+DAA+D;QAC/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QAEpC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ,EAAE,CAAC;gBACd,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,oEAAoE;YACpE,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBACpC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAA;YAExD,yFAAyF;YACzF,qDAAqD;YACrD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,EAAE,GAAG,MAAM,CAAA;YACvC,MAAM,UAAU,GAA2B,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;YAC5E,IAAI,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;YAC/B,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAA;YAE1G,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,CAAE,CAAA;gBAC1E,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,IAAI,CAAA;gBAC7B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAE,CAAA;gBAC5E,MAAM,WAAW,GAAG,KAAK;oBACxB,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,KAAK;wBACrC,CAAC,CAAC,GAAG,MAAM,IAAI,MAAM,EAAE;wBACvB,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,EAAE;oBACvB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;gBACjB,UAAU,CAAC,YAAY,GAAG,WAAW,CAAA;gBACrC,GAAG,GAAG,GAAG,WAAW,IAAI,MAAM,IAAI,MAAM,EAAE,CAAA;gBAC1C,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,oBAAoB,CAAA;YAC3D,CAAC;YAED,MAAM,QAAQ,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAEvF,IACC,aAAa,CACZ,KAAK,EACL;gBACC,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,+FAA+F;aACxG,EACD,eAAe,KAAK,EAAE,CACtB,EACA,CAAC;gBACF,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,8FAA8F;QAC9F,8FAA8F;QAC9F,8FAA8F;QAC9F,uFAAuF;QACvF,MAAM,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAA;QAE3F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,mFAAmF;YACnF,2EAA2E;YAC3E,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAE,CAAC,CAAA;YACnE,MAAM,QAAQ,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;YAEtF,IACC,aAAa,CACZ,KAAK,EACL;gBACC,GAAG,EAAE,IAAI;gBACT,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC9B,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,kFAAkF;aAC3F,EACD,2BAA2B,CAC3B,EACA,CAAC;gBACF,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,wBAAwB,YAAY,8DAA8D,CAAC,CAAA;QAClH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -37,6 +37,12 @@ export interface LocalePart {
|
|
|
37
37
|
csv?: string;
|
|
38
38
|
path?: string;
|
|
39
39
|
region?: string;
|
|
40
|
+
/**
|
|
41
|
+
* NZ — the OA DISTRICT column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`). When set, map
|
|
42
|
+
* DISTRICT→locality and CITY→dependent_locality (falling back to CITY→locality when DISTRICT is empty, ~18% of NZ
|
|
43
|
+
* rows). Without this, the default CITY→locality mapping wrongly trains the suburb as the locality.
|
|
44
|
+
*/
|
|
45
|
+
districtAsLocality?: boolean;
|
|
40
46
|
}
|
|
41
47
|
/**
|
|
42
48
|
* OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAUH,OAAO,EAAE,KAAK,eAAe,EAAuB,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"locale.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAUH,OAAO,EAAE,KAAK,eAAe,EAAuB,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAA;CAC5B;AAiFD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM1D;AAWD;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA2GhG;AAED,eAAO,MAAM,YAAY,EAAE,WAwH1B,CAAA"}
|
|
@@ -69,6 +69,15 @@ const COUNTRY_SOURCES = {
|
|
|
69
69
|
corpusVersion: "0.9.9",
|
|
70
70
|
parts: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
|
|
71
71
|
},
|
|
72
|
+
NZ: {
|
|
73
|
+
// LINZ-derived OA countrywide extract (2.12M rows). `districtAsLocality` inverts the CITY/DISTRICT
|
|
74
|
+
// mapping: DISTRICT holds the city (Auckland), CITY the suburb (Birkenhead). NZ OA carries no postcode.
|
|
75
|
+
source: "synth-nz",
|
|
76
|
+
corpusVersion: "0.9.9",
|
|
77
|
+
// Eval holdout: a probe build excludes ~12% of NZ localities (a locality-bucket split) for a source-disjoint
|
|
78
|
+
// coord board; that split is a BUILD-TIME concern (scratchpad), so the committed recipe reads the full CSV.
|
|
79
|
+
parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
|
|
80
|
+
},
|
|
72
81
|
};
|
|
73
82
|
/**
|
|
74
83
|
* Per-part reservoir cap. Streaming + Algorithm-R reservoir sampling to this size keeps memory bounded regardless of
|
|
@@ -159,7 +168,14 @@ export async function readTuples(part, rng) {
|
|
|
159
168
|
if (header === null) {
|
|
160
169
|
header = cells.map((h) => h.trim().toLowerCase());
|
|
161
170
|
const ix = (name) => header.indexOf(name);
|
|
162
|
-
cols = {
|
|
171
|
+
cols = {
|
|
172
|
+
num: ix("number"),
|
|
173
|
+
street: ix("street"),
|
|
174
|
+
city: ix("city"),
|
|
175
|
+
district: ix("district"),
|
|
176
|
+
region: ix("region"),
|
|
177
|
+
post: ix("postcode"),
|
|
178
|
+
};
|
|
163
179
|
continue;
|
|
164
180
|
}
|
|
165
181
|
if (cols === null)
|
|
@@ -168,7 +184,25 @@ export async function readTuples(part, rng) {
|
|
|
168
184
|
const rawCity = get(cells, cols.city);
|
|
169
185
|
if (!street || !rawCity)
|
|
170
186
|
continue;
|
|
171
|
-
|
|
187
|
+
// Default: CITY → locality. NZ (`districtAsLocality`) inverts it — the OA DISTRICT holds the city
|
|
188
|
+
// (`Auckland`) and CITY holds the suburb (`Birkenhead`), so DISTRICT → locality and CITY →
|
|
189
|
+
// dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
|
|
190
|
+
// sub-locality. See {@link LocalePart.districtAsLocality}.
|
|
191
|
+
let locality;
|
|
192
|
+
let dependent_locality;
|
|
193
|
+
if (part.districtAsLocality) {
|
|
194
|
+
const cleanedDistrict = cleanCityNoise(get(cells, cols.district));
|
|
195
|
+
if (cleanedDistrict) {
|
|
196
|
+
locality = cleanedDistrict;
|
|
197
|
+
dependent_locality = cleanCityNoise(rawCity) ?? undefined;
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
locality = cleanCityNoise(rawCity);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
locality = cleanCityNoise(rawCity);
|
|
205
|
+
}
|
|
172
206
|
if (!locality) {
|
|
173
207
|
dropped++;
|
|
174
208
|
continue;
|
|
@@ -179,6 +213,7 @@ export async function readTuples(part, rng) {
|
|
|
179
213
|
locality,
|
|
180
214
|
region: get(cells, cols.region) || part.region || "",
|
|
181
215
|
postcode: get(cells, cols.post),
|
|
216
|
+
...(dependent_locality ? { dependent_locality } : {}),
|
|
182
217
|
};
|
|
183
218
|
seen++;
|
|
184
219
|
if (reservoir.length < RESERVOIR_CAP) {
|
|
@@ -205,7 +240,7 @@ export const localeRecipe = {
|
|
|
205
240
|
description: "Per-locale coverage rows (DE/FR/NL/IT/ES) from real OA tuples, both orders → synthesizeLocaleRow",
|
|
206
241
|
mode: "generate",
|
|
207
242
|
options: [
|
|
208
|
-
{ flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES). Default DE" },
|
|
243
|
+
{ flag: "--country <cc>", description: "Target country (DE|FR|NL|IT|ES|NZ). Default DE" },
|
|
209
244
|
{ flag: "--intl-fraction <f>", description: "Fraction rendered international order. Default 0.4" },
|
|
210
245
|
],
|
|
211
246
|
async run(opts, write) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAwB,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"locale.js","sourceRoot":"","sources":["../../../src/shard-recipes/locale.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAE1C,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAwB,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AACnF,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AA+BhE;;;;;;;GAOG;AACH,MAAM,eAAe,GAAwC;IAC5D,EAAE,EAAE;QACH,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE;YACN,EAAE,GAAG,EAAE,8BAA8B,EAAE,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;YAC/E,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE;SAC7F;KACD;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAChF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,qBAAqB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC;KAC5F;IACD,EAAE,EAAE;QACH,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,CAAC;KACtF;IACD,EAAE,EAAE;QACH,mGAAmG;QACnG,wGAAwG;QACxG,MAAM,EAAE,UAAU;QAClB,aAAa,EAAE,OAAO;QACtB,6GAA6G;QAC7G,4GAA4G;QAC5G,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,eAAe,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,CAAC,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC;KAChH;CACD,CAAA;AAED;;;;GAIG;AACH,MAAM,aAAa,GAAG,SAAS,CAAA;AAE/B;;;;GAIG;AACH,MAAM,sBAAsB,GAAG,GAAG,CAAA;AAElC;;;;GAIG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAElE,OAAO,QAAQ,IAAI,IAAI,CAAA;AACxB,CAAC;AAWD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAgB,EAAE,GAAiB;IACnE,IAAI,KAA4B,CAAA;IAEhC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACf,8FAA8F;QAC9F,qEAAqE;QACrE,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;SAAM,CAAC;QACP,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,GAAI,EAAE,IAAI,CAAC,GAAI,CAAC,CAAC,CAAA;QAC1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACzB,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;QACtE,CAAC,CAAC,CAAA;QACF,KAAK,GAAG,KAAK,CAAC,MAAO,CAAA;IACtB,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/G,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,IAAI,IAAI,GAAuB,IAAI,CAAA;IACnC,IAAI,MAAM,GAAoB,IAAI,CAAA;IAClC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,CAAC;QACJ,gGAAgG;QAChG,iGAAiG;QACjG,mGAAmG;QACnG,sGAAsG;QACtG,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,cAAc,CAAC,SAAS,CAAW,KAAK,EAAE;YACnE,IAAI,EAAE,OAAO;YACb,MAAM,EAAE,KAAK;YACb,mBAAmB,EAAE,IAAI;SACzB,CAAC,EAAE,CAAC;YACJ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;gBACjD,MAAM,EAAE,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;gBAC1D,IAAI,GAAG;oBACN,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC;oBACjB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC;oBACpB,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC;oBAChB,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC;oBACxB,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC;oBACpB,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC;iBACpB,CAAA;gBAED,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,KAAK,IAAI;gBAAE,SAAQ;YAC3B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YACtC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEjC,kGAAkG;YAClG,2FAA2F;YAC3F,qGAAqG;YACrG,2DAA2D;YAC3D,IAAI,QAAuB,CAAA;YAC3B,IAAI,kBAAsC,CAAA;YAE1C,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC7B,MAAM,eAAe,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAA;gBAEjE,IAAI,eAAe,EAAE,CAAC;oBACrB,QAAQ,GAAG,eAAe,CAAA;oBAC1B,kBAAkB,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,SAAS,CAAA;gBAC1D,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;gBACnC,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;YACnC,CAAC;YAED,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACf,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YACD,MAAM,KAAK,GAAoB;gBAC9B,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;gBAClC,MAAM;gBACN,QAAQ;gBACR,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE;gBACpD,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC/B,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAA;YACD,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,aAAa,EAAE,CAAC;gBACtC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAA;gBAElC,cAAc;gBACd,IAAI,CAAC,GAAG,aAAa,EAAE,CAAC;oBACvB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAA;QAE5F,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,MAAM,eAAe,IAAI,UAAU,OAAO,oBAAoB,CAAC,CAAA;IAEtH,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAgB;IACxC,IAAI,EAAE,QAAQ;IACd,WAAW,EAAE,kGAAkG;IAC/G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE;QACR,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAW,EAAE,gDAAgD,EAAE;QACzF,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,oDAAoD,EAAE;KAClG;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,kGAAkG;QAClG,0FAA0F;QAC1F,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QAE9C,IAAI,CAAC,aAAa,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,0CAA0C,OAAO,YAAY,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvG,CAAA;QACF,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,GAAG,CAAA;QAE7C,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,IAAI,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAA;QAC1E,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,MAAM,CAAA;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAA;QAChC,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAA;QAE/B,MAAM,IAAI,GAAsB,EAAE,CAAA;QAElC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC1C,kGAAkG;YAClG,yDAAyD;YACzD,MAAM,YAAY,GAAG,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YAChF,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,EAAE,CAAE,EAAE,YAAY,CAAC,CAAA;YAEpD,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACb,CAAC,CAAC,kEAAkE;QACrE,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,MAAM,OAAO,yEAAyE,CAAC,CAAA;QACxG,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAA;YAClE,6FAA6F;YAC7F,6DAA6D;YAC7D,MAAM,eAAe,GACpB,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,sBAAsB,CAAC,CAAC,CAAE,OAAiB,CAAC,CAAC,CAAE,UAAoB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;YAChH,MAAM,aAAa,GAClB,OAAO,KAAK,IAAI;gBACf,CAAC,CAAC,MAAM,EAAE,GAAG,0BAA0B;oBACtC,CAAC,CAAE,WAAqB;oBACxB,CAAC,CAAE,cAAwB;gBAC5B,CAAC,CAAC,SAAS,CAAA;YACb,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,CAAC,CAAA;YAEnG,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,8FAA8F;gBAC9F,gGAAgG;gBAChG,MAAM,eAAe,GAAG;oBACvB,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO;oBACP,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM;oBACN,SAAS,EAAE,oBAAoB;iBAC/B,CAAA;gBACD,MAAM,aAAa,GAAG,QAAQ,CAAC,eAAiD,CAAC,CAAA;gBAEjF,IAAI,aAAa,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;oBAC5D,OAAO,EAAE,CAAA;oBACT,SAAQ;gBACT,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAC9F,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,EAAE;gBACvC,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM;gBAC/B,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,YAAY;gBAC3C,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;gBACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;aACnC,CAAC,CAAA;YACF,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,aAAa,CAAC,aAAa;gBAC3C,OAAO,EAAE,iBAAiB,OAAO,qBAAqB,KAAK,6BAA6B;aACxF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC/G,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"no-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAgB1G,eAAO,MAAM,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"no-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAgB1G,eAAO,MAAM,gBAAgB,EAAE,WAsJ9B,CAAA"}
|
|
@@ -114,10 +114,12 @@ export const noFragmentRecipe = {
|
|
|
114
114
|
corpus_version: "0.11.0",
|
|
115
115
|
license: "Synthetic — no-fragment; (street, number, postcode, city) from OpenAddresses NO / Kartverket",
|
|
116
116
|
};
|
|
117
|
-
if (alignAndWrite(write, canonical, "no-fragment"))
|
|
117
|
+
if (alignAndWrite(write, canonical, "no-fragment")) {
|
|
118
118
|
emitted++;
|
|
119
|
-
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
120
121
|
skipped++;
|
|
122
|
+
}
|
|
121
123
|
};
|
|
122
124
|
for await (const tuple of readTuples(opts.input)) {
|
|
123
125
|
read++;
|
|
@@ -125,10 +127,12 @@ export const noFragmentRecipe = {
|
|
|
125
127
|
const locality = titleNO(String(tuple.locality ?? "").trim());
|
|
126
128
|
const number = String(tuple.number ?? "").trim();
|
|
127
129
|
const postcode = String(tuple.postcode ?? "").trim();
|
|
128
|
-
if (locality)
|
|
130
|
+
if (locality) {
|
|
129
131
|
localities.add(locality);
|
|
130
|
-
|
|
132
|
+
}
|
|
133
|
+
if (postcode) {
|
|
131
134
|
postcodes.add(postcode);
|
|
135
|
+
}
|
|
132
136
|
if (!street) {
|
|
133
137
|
skipped++;
|
|
134
138
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"no-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G;;;;GAIG;AACH,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAExG,6FAA6F;AAC7F,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CACzC,KAAK;KACH,KAAK,CAAC,KAAK,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpE,IAAI,CAAC,GAAG,CAAC,CAAA;AAEZ,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,2JAA2J;IAC5J,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EACV,gHAAgH;SACjH;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;QACD;YACC,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,iGAAiG;SAC9G;QACD;YACC,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EACV,kMAAkM;SACnM;QACD;YACC,IAAI,EAAE,8BAA8B;YACpC,WAAW,EAAE,+EAA+E;SAC5F;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,qGAAqG;gBACrG,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAA;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1E,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAA;QAEzD,yFAAyF;QACzF,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QAEnC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,UAAkC,EAAE,KAAa,EAAQ,EAAE;YACrF,wFAAwF;YACxF,kGAAkG;YAClG,MAAM,SAAS,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAA;YAC5G,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EAAE,8FAA8F;aACvG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"no-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G;;;;GAIG;AACH,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAExG,6FAA6F;AAC7F,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CACzC,KAAK;KACH,KAAK,CAAC,KAAK,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpE,IAAI,CAAC,GAAG,CAAC,CAAA;AAEZ,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,2JAA2J;IAC5J,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EACV,gHAAgH;SACjH;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;QACD;YACC,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,iGAAiG;SAC9G;QACD;YACC,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EACV,kMAAkM;SACnM;QACD;YACC,IAAI,EAAE,8BAA8B;YACpC,WAAW,EAAE,+EAA+E;SAC5F;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,qGAAqG;gBACrG,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAA;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1E,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAA;QAEzD,yFAAyF;QACzF,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QAEnC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,UAAkC,EAAE,KAAa,EAAQ,EAAE;YACrF,wFAAwF;YACxF,kGAAkG;YAClG,MAAM,SAAS,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAA;YAC5G,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EAAE,8FAA8F;aACvG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ,EAAE,CAAC;gBACd,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACd,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACxB,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,iEAAiE;YACjE,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAChC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,0FAA0F;YAC1F,2FAA2F;YAC3F,0EAA0E;YAC1E,IAAI,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;gBAC5B,IAAI,MAAM,EAAE,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;oBACvC,MAAM,GAAG,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,CAAE,CAAA;oBAEpE,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAA;gBACtD,CAAC;qBAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;oBAC3B,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAE,CAAA;oBAEjE,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAA;gBACpD,CAAC;gBACD,SAAQ;YACT,CAAC;YAED,0FAA0F;YAC1F,8FAA8F;YAC9F,IAAI,CAAC,MAAM,IAAI,MAAM,EAAE,GAAG,cAAc,EAAE,CAAC;gBAC1C,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,CAAA;YACxC,CAAC;iBAAM,CAAC;gBACP,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAA;gBAC7D,0FAA0F;gBAC1F,wFAAwF;gBACxF,2FAA2F;gBAC3F,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;gBACjD,MAAM,MAAM,GAAG,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAA;gBAElE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,CAAA;gBACrE,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAA;IAChD,CAAC;CACD,CAAA"}
|
|
@@ -27,6 +27,12 @@ export interface LocaleBaseTuple {
|
|
|
27
27
|
house_number?: string;
|
|
28
28
|
street: string;
|
|
29
29
|
locality: string;
|
|
30
|
+
/**
|
|
31
|
+
* A sub-locality that sits BELOW the locality (a suburb / district). NZ is the case that needs it: the OA DISTRICT
|
|
32
|
+
* column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`), so the real envelope carries both (`31
|
|
33
|
+
* Rawene Road, Birkenhead, Auckland`). Rendered between street and locality in both orders when present.
|
|
34
|
+
*/
|
|
35
|
+
dependent_locality?: string;
|
|
30
36
|
region?: string;
|
|
31
37
|
postcode?: string;
|
|
32
38
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;IAClC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,WAAW,CAAA;IAC5C;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,UAAU,GAAG,OAAO,CAAA;CACtC;AACD,mDAAmD;AACnD,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.d.ts","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AACD,+CAA+C;AAC/C,MAAM,MAAM,eAAe,GAAG,eAAe,CAAA;AAE7C,MAAM,WAAW,oBAAoB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;CACd;AACD,oDAAoD;AACpD,MAAM,MAAM,oBAAoB,GAAG,oBAAoB,CAAA;AAEvD,MAAM,WAAW,mBAAmB;IACnC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAA;IAClC;;;;;;OAMG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,WAAW,CAAA;IAC5C;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,UAAU,GAAG,OAAO,CAAA;CACtC;AACD,mDAAmD;AACnD,MAAM,MAAM,mBAAmB,GAAG,mBAAmB,CAAA;AA4CrD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CAyE7B;AAED,uGAAuG;AACvG,wBAAgB,mBAAmB,CAClC,IAAI,EAAE,eAAe,EACrB,IAAI,GAAE,mBAAwB,GAC5B,oBAAoB,GAAG,IAAI,CAE7B"}
|
|
@@ -31,6 +31,7 @@ const LOCALE_TAG = {
|
|
|
31
31
|
GB: "en-GB",
|
|
32
32
|
FR: "fr-FR",
|
|
33
33
|
US: "en-US",
|
|
34
|
+
NZ: "en-NZ",
|
|
34
35
|
};
|
|
35
36
|
/**
|
|
36
37
|
* Canonicalize a postcode to the form the country's template renders, so the stored component aligns verbatim against
|
|
@@ -79,6 +80,12 @@ export function synthesizeLocaleRow(base, country, opts = {}) {
|
|
|
79
80
|
if (!base.street || !base.locality)
|
|
80
81
|
return null;
|
|
81
82
|
const components = { street: base.street, locality: base.locality };
|
|
83
|
+
// Sub-locality (suburb / district) sits between street and locality and renders in BOTH orders — it's part
|
|
84
|
+
// of the address body, not the admin-region tail that native order drops. NZ needs it (suburb + city both on
|
|
85
|
+
// the envelope). The tokenPresent gate below drops the row if the template didn't surface it verbatim.
|
|
86
|
+
if (base.dependent_locality) {
|
|
87
|
+
components.dependent_locality = base.dependent_locality;
|
|
88
|
+
}
|
|
82
89
|
// ~80% keep the house number (the rest are street-only forms, also idiomatic).
|
|
83
90
|
if (base.house_number && random() < 0.8) {
|
|
84
91
|
components.house_number = base.house_number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"synthesize-german.js","sourceRoot":"","sources":["../../src/synthesize-german.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AA6D3C,yFAAyF;AACzF,MAAM,UAAU,GAA2B;IAC1C,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,OAAe;IAC3D,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,2BAA2B,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAEpD,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,EAAE,CAAA;IAC/C,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,qGAAqG;AACrG,SAAS,YAAY,CAAC,GAAW,EAAE,KAAa;IAC/C,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IACtC,2FAA2F;IAC3F,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,OAAO,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAA;IAElF,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAE5E,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAAe,EACf,OAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAA;IAEpC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAE/C,MAAM,UAAU,GAA+B,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAA;IAE/F,2GAA2G;IAC3G,6GAA6G;IAC7G,uGAAuG;IACvG,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC7B,UAAU,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB,CAAA;IACxD,CAAC;IAED,+EAA+E;IAC/E,IAAI,IAAI,CAAC,YAAY,IAAI,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QACzC,UAAU,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAA;IAC5C,CAAC;IAED,4FAA4F;IAC5F,0FAA0F;IAC1F,qFAAqF;IACrF,IAAI,IAAI,CAAC,QAAQ,IAAI,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC;QACtC,UAAU,CAAC,QAAQ,GAAG,iBAAiB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAChE,CAAC;IAED,iGAAiG;IACjG,oGAAoG;IACpG,kGAAkG;IAClG,mGAAmG;IACnG,6FAA6F;IAC7F,IAAI,KAAK,KAAK,eAAe,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC9C,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAChC,CAAC;IAED,mGAAmG;IACnG,+FAA+F;IAC/F,oGAAoG;IACpG,MAAM,aAAa,GAAG,KAAK,KAAK,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAA;IAChE,IAAI,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,qGAAqG;IACrG,6FAA6F;IAC7F,iGAAiG;IACjG,IAAI,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,KAAK,OAAO,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;QACvF,GAAG,GAAG,GAAG,CAAC,OAAO,CAChB,GAAG,UAAU,CAAC,MAAM,KAAK,UAAU,CAAC,YAAY,EAAE,EAClD,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,YAAY,EAAE,CACjD,CAAA;IACF,CAAC;IAED,qGAAqG;IACrG,8FAA8F;IAC9F,6FAA6F;IAC7F,IAAI,IAAI,CAAC,aAAa,KAAK,WAAW,IAAI,UAAU,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChF,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;QAEvC,IAAI,UAAU,IAAI,UAAU,KAAK,UAAU,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3F,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;YAClD,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAA;QACjC,CAAC;IACF,CAAC;IAED,8FAA8F;IAC9F,4DAA4D;IAC5D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;IACrD,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,EAAE,CAAA;AACjF,CAAC;AAED,uGAAuG;AACvG,MAAM,UAAU,mBAAmB,CAClC,IAAqB,EACrB,OAA4B,EAAE;IAE9B,OAAO,mBAAmB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC7C,CAAC"}
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
* /tmp/part-po-box.parquet
|
|
31
31
|
*/
|
|
32
32
|
/** Options for {@linkcode jsonlToParquet}. */
|
|
33
|
-
export interface
|
|
33
|
+
export interface JSONLToParquetOptions {
|
|
34
34
|
/** The labeled-row JSONL to convert. */
|
|
35
35
|
input: string;
|
|
36
36
|
/** The parquet shard to write. */
|
|
@@ -39,11 +39,11 @@ export interface JsonlToParquetOptions {
|
|
|
39
39
|
rowGroupSize?: number;
|
|
40
40
|
}
|
|
41
41
|
/** Summary returned by {@linkcode jsonlToParquet}. */
|
|
42
|
-
export interface
|
|
42
|
+
export interface JSONLToParquetSummary {
|
|
43
43
|
read: number;
|
|
44
44
|
written: number;
|
|
45
45
|
outPath: string;
|
|
46
46
|
}
|
|
47
47
|
/** Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */
|
|
48
|
-
export declare function jsonlToParquet(options:
|
|
48
|
+
export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
|
|
49
49
|
//# sourceMappingURL=jsonl-to-parquet.d.ts.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.1.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -82,9 +82,9 @@
|
|
|
82
82
|
},
|
|
83
83
|
"dependencies": {
|
|
84
84
|
"@dsnp/parquetjs": "1.8.8",
|
|
85
|
-
"@mailwoman/codex": "
|
|
86
|
-
"@mailwoman/core": "
|
|
87
|
-
"@mailwoman/formatter": "
|
|
85
|
+
"@mailwoman/codex": "7.1.0",
|
|
86
|
+
"@mailwoman/core": "7.1.0",
|
|
87
|
+
"@mailwoman/formatter": "7.1.0",
|
|
88
88
|
"csv-parse": "^7.0.1",
|
|
89
89
|
"fast-glob": "^3.3.3",
|
|
90
90
|
"fastest-levenshtein": "^1.0.16",
|