@mailwoman/corpus 6.4.0 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/build.js +5 -5
- package/out/src/codex/us-fips-state.d.ts +6 -6
- package/out/src/codex/us-fips-state.js +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +2 -1
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +30 -6
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +7 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +3 -3
- package/package.json +4 -4
package/out/src/build.js
CHANGED
|
@@ -139,7 +139,7 @@ export async function buildCorpus(opts) {
|
|
|
139
139
|
};
|
|
140
140
|
for (const adapterRun of adapterRuns) {
|
|
141
141
|
opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
|
|
142
|
-
for await (const row of
|
|
142
|
+
for await (const row of streamJSONL(adapterRun.jsonl_path)) {
|
|
143
143
|
licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
|
|
144
144
|
// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
|
|
145
145
|
// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
|
|
@@ -203,9 +203,9 @@ export async function buildCorpus(opts) {
|
|
|
203
203
|
// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
|
|
204
204
|
opts.onProgress?.("shard", "writing parquet shards");
|
|
205
205
|
const shardManifest = await writeShards({
|
|
206
|
-
train:
|
|
207
|
-
val:
|
|
208
|
-
test:
|
|
206
|
+
train: streamJSONL(labeledPaths.train),
|
|
207
|
+
val: streamJSONL(labeledPaths.val),
|
|
208
|
+
test: streamJSONL(labeledPaths.test),
|
|
209
209
|
}, {
|
|
210
210
|
outputDir: opts.outputDir,
|
|
211
211
|
corpusVersion: opts.corpusVersion,
|
|
@@ -235,7 +235,7 @@ export async function buildCorpus(opts) {
|
|
|
235
235
|
await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
236
236
|
return manifest;
|
|
237
237
|
}
|
|
238
|
-
async function*
|
|
238
|
+
async function* streamJSONL(path) {
|
|
239
239
|
// JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
|
|
240
240
|
// lines are dropped at the row level) and throws SyntaxError on a malformed row —
|
|
241
241
|
// same fail-loud behavior as the prior readline + bare `JSON.parse`.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* join.
|
|
14
14
|
*/
|
|
15
15
|
/** Per-state record: two-letter postal abbreviation + full canonical display name. */
|
|
16
|
-
export interface
|
|
16
|
+
export interface USStateInfo {
|
|
17
17
|
abbreviation: string;
|
|
18
18
|
name: string;
|
|
19
19
|
}
|
|
@@ -21,17 +21,17 @@ export interface UsStateInfo {
|
|
|
21
21
|
* FIPS state-or-territory code → `{ abbreviation, name }`. Includes all 50 states, DC, and the five primary territories
|
|
22
22
|
* (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER column `statefp`.
|
|
23
23
|
*/
|
|
24
|
-
export declare const US_FIPS_STATE: Readonly<Record<string,
|
|
24
|
+
export declare const US_FIPS_STATE: Readonly<Record<string, USStateInfo>>;
|
|
25
25
|
/** Lookup helper. Returns null when the FIPS code isn't recognized. */
|
|
26
|
-
export declare function lookupFipsState(statefp: string | null | undefined):
|
|
26
|
+
export declare function lookupFipsState(statefp: string | null | undefined): USStateInfo | null;
|
|
27
27
|
/**
|
|
28
|
-
* Inverted view: two-letter postal abbreviation → `
|
|
28
|
+
* Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
|
|
29
29
|
* source data ships the abbreviation rather than the FIPS code (FCC BDC, most federal CSVs).
|
|
30
30
|
*/
|
|
31
|
-
export declare const US_STATE_BY_ABBREVIATION: Readonly<Record<string,
|
|
31
|
+
export declare const US_STATE_BY_ABBREVIATION: Readonly<Record<string, USStateInfo>>;
|
|
32
32
|
/**
|
|
33
33
|
* Lookup helper for adapters carrying 2-char USPS abbreviations (`"CA"`, `"VT"`). Case-folded; null for any value
|
|
34
34
|
* outside the 50 states + DC + the five primary territories.
|
|
35
35
|
*/
|
|
36
|
-
export declare function lookupStateAbbreviation(abbreviation: string | null | undefined):
|
|
36
|
+
export declare function lookupStateAbbreviation(abbreviation: string | null | undefined): USStateInfo | null;
|
|
37
37
|
//# sourceMappingURL=us-fips-state.d.ts.map
|
|
@@ -82,7 +82,7 @@ export function lookupFipsState(statefp) {
|
|
|
82
82
|
return US_FIPS_STATE[statefp] ?? null;
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
|
-
* Inverted view: two-letter postal abbreviation → `
|
|
85
|
+
* Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
|
|
86
86
|
* source data ships the abbreviation rather than the FIPS code (FCC BDC, most federal CSVs).
|
|
87
87
|
*/
|
|
88
88
|
export const US_STATE_BY_ABBREVIATION = Object.freeze(Object.fromEntries(Object.values(US_FIPS_STATE).map((info) => [info.abbreviation, info])));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fr-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAKH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA4C1G,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAajD;AAQD,eAAO,MAAM,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"fr-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAKH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA4C1G,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAajD;AAQD,eAAO,MAAM,gBAAgB,EAAE,WAqK9B,CAAA"}
|
|
@@ -167,8 +167,9 @@ export const frFragmentRecipe = {
|
|
|
167
167
|
read++;
|
|
168
168
|
const fullStreet = String(tuple.street ?? "").trim();
|
|
169
169
|
const locality = String(tuple.locality ?? "").trim();
|
|
170
|
-
if (locality)
|
|
170
|
+
if (locality) {
|
|
171
171
|
localities.add(locality);
|
|
172
|
+
}
|
|
172
173
|
if (!fullStreet) {
|
|
173
174
|
skipped++;
|
|
174
175
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fr-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,mFAAmF;AACnF,MAAM,aAAa,GAAG;IACrB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CACrH,CAAA;AACD,gGAAgG;AAChG,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/C,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CACtC,KAAK;KACH,SAAS,CAAC,KAAK,CAAC;KAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KACrB,WAAW,EAAE;KACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAA;AAET;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;IACxB,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,KAAc,EAAU,EAAE,CACrD,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CACxB,IAAI;SACF,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;SACnE,IAAI,CAAC,GAAG,CAAC,CACX;SACA,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAC5D,kFAAkF;AAClF,MAAM,OAAO,GACZ,qIAAqI,CAAA;AAEtI,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,kJAAkJ;IACnJ,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EAAE,uFAAuF;SACpG;QACD;YACC,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,kFAAkF;SAC/F;QACD;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,qFAAqF;SAClG;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,4GAA4G;gBAC5G,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAA;QAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAA;QAE9C,wFAAwF;QACxF,+DAA+D;QAC/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QAEpC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ;
|
|
1
|
+
{"version":3,"file":"fr-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/fr-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAA;AACvE,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G,mFAAmF;AACnF,MAAM,aAAa,GAAG;IACrB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;CACrH,CAAA;AACD,gGAAgG;AAChG,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;AAE/C,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CACtC,KAAK;KACH,SAAS,CAAC,KAAK,CAAC;KAChB,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;KACrB,WAAW,EAAE;KACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;KACpB,IAAI,EAAE,CAAA;AAET;;;;;;;;GAQG;AACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC;IACxB,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,GAAG;IACH,GAAG;IACH,KAAK;IACL,MAAM;IACN,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;CACL,CAAC,CAAA;AAEF,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,MAAM,GAAG,GAAG,CAAC,KAAa,EAAE,KAAc,EAAU,EAAE,CACrD,CAAC,KAAK,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CACxB,IAAI;SACF,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;SACnE,IAAI,CAAC,GAAG,CAAC,CACX;SACA,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,2CAA2C,CAAA;AAC5D,kFAAkF;AAClF,MAAM,OAAO,GACZ,qIAAqI,CAAA;AAEtI,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,kJAAkJ;IACnJ,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EAAE,uFAAuF;SACpG;QACD;YACC,IAAI,EAAE,eAAe;YACrB,WAAW,EAAE,kFAAkF;SAC/F;QACD;YACC,IAAI,EAAE,iBAAiB;YACvB,WAAW,EAAE,qFAAqF;SAClG;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,4GAA4G;gBAC5G,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAA;QAClC,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAA;QAE9C,wFAAwF;QACxF,+DAA+D;QAC/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QAEpC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ,EAAE,CAAC;gBACd,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YAED,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,oEAAoE;YACpE,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;gBACpC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAA;YAExD,yFAAyF;YACzF,qDAAqD;YACrD,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACxB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,EAAE,GAAG,MAAM,CAAA;YACvC,MAAM,UAAU,GAA2B,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;YAC5E,IAAI,GAAG,GAAG,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;YAC/B,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAA;YAE1G,IAAI,aAAa,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,CAAE,CAAA;gBAC1E,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,IAAI,CAAA;gBAC7B,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAE,CAAA;gBAC5E,MAAM,WAAW,GAAG,KAAK;oBACxB,CAAC,CAAC,MAAM,KAAK,KAAK,IAAI,MAAM,KAAK,KAAK;wBACrC,CAAC,CAAC,GAAG,MAAM,IAAI,MAAM,EAAE;wBACvB,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,EAAE;oBACvB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;gBACjB,UAAU,CAAC,YAAY,GAAG,WAAW,CAAA;gBACrC,GAAG,GAAG,GAAG,WAAW,IAAI,MAAM,IAAI,MAAM,EAAE,CAAA;gBAC1C,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,oBAAoB,CAAA;YAC3D,CAAC;YAED,MAAM,QAAQ,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAEvF,IACC,aAAa,CACZ,KAAK,EACL;gBACC,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,+FAA+F;aACxG,EACD,eAAe,KAAK,EAAE,CACtB,EACA,CAAC;gBACF,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,8FAA8F;QAC9F,8FAA8F;QAC9F,8FAA8F;QAC9F,uFAAuF;QACvF,MAAM,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAA;QAE3F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,mFAAmF;YACnF,2EAA2E;YAC3E,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAE,CAAC,CAAA;YACnE,MAAM,QAAQ,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;YAEtF,IACC,aAAa,CACZ,KAAK,EACL;gBACC,GAAG,EAAE,IAAI;gBACT,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC9B,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS,EAAE,QAAQ;gBACnB,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,kFAAkF;aAC3F,EACD,2BAA2B,CAC3B,EACA,CAAC;gBACF,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,wBAAwB,YAAY,8DAA8D,CAAC,CAAA;QAClH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"no-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAgB1G,eAAO,MAAM,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"no-fragment.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAIH,OAAO,EAA6C,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAgB1G,eAAO,MAAM,gBAAgB,EAAE,WAsJ9B,CAAA"}
|
|
@@ -64,6 +64,14 @@ export const noFragmentRecipe = {
|
|
|
64
64
|
flag: "--counter-prob <n>",
|
|
65
65
|
description: "Share of ALL rows that are counter-distribution (bare locality OR bare postcode) (default 0.30)",
|
|
66
66
|
},
|
|
67
|
+
{
|
|
68
|
+
flag: "--long-number-boost <n>",
|
|
69
|
+
description: "knob 3: emit N copies of each street+number row whose number has >= --long-number-min-digits digits — oversample the failing long-number class the model calls a postcode (default 1 = no boost)",
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
flag: "--long-number-min-digits <n>",
|
|
73
|
+
description: "knob 3: minimum digit count for a number to be 'long' and boosted (default 3)",
|
|
74
|
+
},
|
|
67
75
|
],
|
|
68
76
|
async run(opts, write) {
|
|
69
77
|
const random = makeMulberry32(opts.seed);
|
|
@@ -81,6 +89,8 @@ export const noFragmentRecipe = {
|
|
|
81
89
|
throw new Error(`no-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`);
|
|
82
90
|
const bareStreetProb = opts.bareProb ?? 0.3;
|
|
83
91
|
const counterProb = opts.counterProb ?? 0.3;
|
|
92
|
+
const longNumberBoost = Math.max(1, Math.floor(opts.longNumberBoost ?? 1));
|
|
93
|
+
const longNumberMinDigits = opts.longNumberMinDigits ?? 3;
|
|
84
94
|
// Harvested from the tuples — every NO row carries its locality and postcode, so the two
|
|
85
95
|
// counter-classes need no second source.
|
|
86
96
|
const localities = new Set();
|
|
@@ -89,8 +99,11 @@ export const noFragmentRecipe = {
|
|
|
89
99
|
let emitted = 0;
|
|
90
100
|
let skipped = 0;
|
|
91
101
|
let contaminated = 0;
|
|
102
|
+
let emitSeq = 0;
|
|
92
103
|
const emit = (raw, components, klass) => {
|
|
93
|
-
|
|
104
|
+
// emitSeq keeps every emit distinct — knob 3 emits N copies of one long-number row, and
|
|
105
|
+
// (components, read) alone would collide their source_id and let downstream dedup drop the boost.
|
|
106
|
+
const source_id = shardSourceID("synth-no-fragment", { ...components, k: klass, v: `${read}:${emitSeq++}` });
|
|
94
107
|
const canonical = {
|
|
95
108
|
raw,
|
|
96
109
|
components,
|
|
@@ -101,10 +114,12 @@ export const noFragmentRecipe = {
|
|
|
101
114
|
corpus_version: "0.11.0",
|
|
102
115
|
license: "Synthetic — no-fragment; (street, number, postcode, city) from OpenAddresses NO / Kartverket",
|
|
103
116
|
};
|
|
104
|
-
if (alignAndWrite(write, canonical, "no-fragment"))
|
|
117
|
+
if (alignAndWrite(write, canonical, "no-fragment")) {
|
|
105
118
|
emitted++;
|
|
106
|
-
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
107
121
|
skipped++;
|
|
122
|
+
}
|
|
108
123
|
};
|
|
109
124
|
for await (const tuple of readTuples(opts.input)) {
|
|
110
125
|
read++;
|
|
@@ -112,10 +127,12 @@ export const noFragmentRecipe = {
|
|
|
112
127
|
const locality = titleNO(String(tuple.locality ?? "").trim());
|
|
113
128
|
const number = String(tuple.number ?? "").trim();
|
|
114
129
|
const postcode = String(tuple.postcode ?? "").trim();
|
|
115
|
-
if (locality)
|
|
130
|
+
if (locality) {
|
|
116
131
|
localities.add(locality);
|
|
117
|
-
|
|
132
|
+
}
|
|
133
|
+
if (postcode) {
|
|
118
134
|
postcodes.add(postcode);
|
|
135
|
+
}
|
|
119
136
|
if (!street) {
|
|
120
137
|
skipped++;
|
|
121
138
|
continue;
|
|
@@ -146,7 +163,14 @@ export const noFragmentRecipe = {
|
|
|
146
163
|
}
|
|
147
164
|
else {
|
|
148
165
|
const klass = number.includes("/") ? "slash-hn" : "street-hn";
|
|
149
|
-
|
|
166
|
+
// knob 3: the failing class is street + LONG number (Leppdalsvegen 1285 -> postcode). The
|
|
167
|
+
// digit count, not the slash, is what tips the length prior toward postcode. Oversample
|
|
168
|
+
// those rows to fight the prior with volume and teach the street/number boundary directly.
|
|
169
|
+
const digits = (number.match(/\d/g) ?? []).length;
|
|
170
|
+
const copies = digits >= longNumberMinDigits ? longNumberBoost : 1;
|
|
171
|
+
for (let c = 0; c < copies; c++) {
|
|
172
|
+
emit(`${street} ${number}`, { street, house_number: number }, klass);
|
|
173
|
+
}
|
|
150
174
|
}
|
|
151
175
|
}
|
|
152
176
|
return { read, emitted, skipped, contaminated };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"no-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G;;;;GAIG;AACH,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAExG,6FAA6F;AAC7F,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CACzC,KAAK;KACH,KAAK,CAAC,KAAK,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpE,IAAI,CAAC,GAAG,CAAC,CAAA;AAEZ,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,2JAA2J;IAC5J,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EACV,gHAAgH;SACjH;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;QACD;YACC,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,iGAAiG;SAC9G;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,qGAAqG;gBACrG,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAA;
|
|
1
|
+
{"version":3,"file":"no-fragment.js","sourceRoot":"","sources":["../../../src/shard-recipes/no-fragment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,UAAU,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE1G;;;;GAIG;AACH,MAAM,IAAI,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAExG,6FAA6F;AAC7F,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CACzC,KAAK;KACH,KAAK,CAAC,KAAK,CAAC;KACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KACpE,IAAI,CAAC,GAAG,CAAC,CAAA;AAEZ,MAAM,CAAC,MAAM,gBAAgB,GAAgB;IAC5C,IAAI,EAAE,aAAa;IACnB,WAAW,EACV,2JAA2J;IAC5J,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;QACR;YACC,IAAI,EAAE,2BAA2B;YACjC,WAAW,EACV,gHAAgH;SACjH;QACD;YACC,IAAI,EAAE,wBAAwB;YAC9B,WAAW,EAAE,mGAAmG;SAChH;QACD;YACC,IAAI,EAAE,oBAAoB;YAC1B,WAAW,EAAE,iGAAiG;SAC9G;QACD;YACC,IAAI,EAAE,yBAAyB;YAC/B,WAAW,EACV,kMAAkM;SACnM;QACD;YACC,IAAI,EAAE,8BAA8B;YACpC,WAAW,EAAE,+EAA+E;SAC5F;KACD;IACD,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,WAAW,GAAG,IAAI,CAAC,eAAe,CAAA;QAExC,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACd,uFAAuF;gBACtF,qGAAqG;gBACrG,sFAAsF,CACvF,CAAA;QACF,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,GAAG,CACvB,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CACjD,CAAA;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,WAAW,sBAAsB,CAAC,CAAA;QAE1G,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;QAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAA;QAC3C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1E,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAA;QAEzD,yFAAyF;QACzF,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAA;QAEnC,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,UAAkC,EAAE,KAAa,EAAQ,EAAE;YACrF,wFAAwF;YACxF,kGAAkG;YAClG,MAAM,SAAS,GAAG,aAAa,CAAC,mBAAmB,EAAE,EAAE,GAAG,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,IAAI,OAAO,EAAE,EAAE,EAAE,CAAC,CAAA;YAC5G,MAAM,SAAS,GAAG;gBACjB,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,mBAAmB;gBAC3B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EAAE,8FAA8F;aACvG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,UAAU,CAAC,IAAI,CAAC,KAAM,CAAC,EAAE,CAAC;YACnD,IAAI,EAAE,CAAA;YACN,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC7D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;YAEpD,IAAI,QAAQ,EAAE,CAAC;gBACd,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACzB,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACd,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;YACxB,CAAC;YAED,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,iEAAiE;YACjE,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;gBAChC,YAAY,EAAE,CAAA;gBACd,SAAQ;YACT,CAAC;YAED,0FAA0F;YAC1F,2FAA2F;YAC3F,0EAA0E;YAC1E,IAAI,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;gBAC5B,IAAI,MAAM,EAAE,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,EAAE,CAAC;oBACvC,MAAM,GAAG,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,CAAE,CAAA;oBAEpE,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,EAAE,uBAAuB,CAAC,CAAA;gBACtD,CAAC;qBAAM,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC;oBAC3B,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAE,CAAA;oBAEjE,IAAI,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAA;gBACpD,CAAC;gBACD,SAAQ;YACT,CAAC;YAED,0FAA0F;YAC1F,8FAA8F;YAC9F,IAAI,CAAC,MAAM,IAAI,MAAM,EAAE,GAAG,cAAc,EAAE,CAAC;gBAC1C,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,aAAa,CAAC,CAAA;YACxC,CAAC;iBAAM,CAAC;gBACP,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAA;gBAC7D,0FAA0F;gBAC1F,wFAAwF;gBACxF,2FAA2F;gBAC3F,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;gBACjD,MAAM,MAAM,GAAG,MAAM,IAAI,mBAAmB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAA;gBAElE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjC,IAAI,CAAC,GAAG,MAAM,IAAI,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,CAAA;gBACrE,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,CAAA;IAChD,CAAC;CACD,CAAA"}
|
|
@@ -80,6 +80,13 @@ export interface ShardRecipeOpts {
|
|
|
80
80
|
excludeSurfaces?: string;
|
|
81
81
|
/** `no-fragment`: share of rows that are counter-distribution (bare locality OR bare postcode). */
|
|
82
82
|
counterProb?: number;
|
|
83
|
+
/**
|
|
84
|
+
* `no-fragment` knob 3: emit N copies of each street+number row whose number has >= longNumberMinDigits digits
|
|
85
|
+
* (oversample the failing long-number class). Default 1 = no boost.
|
|
86
|
+
*/
|
|
87
|
+
longNumberBoost?: number;
|
|
88
|
+
/** `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3. */
|
|
89
|
+
longNumberMinDigits?: number;
|
|
83
90
|
}
|
|
84
91
|
/** Tally a recipe returns. */
|
|
85
92
|
export interface ShardStats {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED,+FAA+F;AAC/F,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAQlD;AAED,6CAA6C;AAC7C,eAAO,MAAM,UAAU,gBAAU,CAAA;AAEjC;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAUzD;AAED,mGAAmG;AACnG,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED,kGAAkG;AAClG,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mGAAmG;IACnG,WAAW,CAAC,EAAE,MAAM,CAAA;
|
|
1
|
+
{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GAAG,MAAM,CAElG;AAED,+FAA+F;AAC/F,MAAM,WAAW,UAAU;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAA;CACpB;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAQlD;AAED,6CAA6C;AAC7C,eAAO,MAAM,UAAU,gBAAU,CAAA;AAEjC;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CAUzD;AAED,mGAAmG;AACnG,wBAAuB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,CAAC,UAAU,CAAC,CAgB3E;AAED,kGAAkG;AAClG,MAAM,WAAW,iBAAiB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,EAC7B,SAAS,EAAE,iBAAiB,EAC5B,WAAW,EAAE,MAAM,EACnB,WAAW,GAAE,MAAM,GAAG,IAAW,GAC/B,OAAO,CAOT;AAED,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC/B,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,mGAAmG;IACnG,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,2GAA2G;IAC3G,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,8BAA8B;AAC9B,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,gFAAgF;AAChF,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACnB;AAED,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC3B,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;IACZ,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAA;IACnB,gFAAgF;IAChF,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAA;IAC3B,qEAAqE;IACrE,OAAO,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC7B;;;OAGG;IACH,GAAG,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC9E"}
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
* /tmp/part-po-box.parquet
|
|
31
31
|
*/
|
|
32
32
|
/** Options for {@linkcode jsonlToParquet}. */
|
|
33
|
-
export interface
|
|
33
|
+
export interface JSONLToParquetOptions {
|
|
34
34
|
/** The labeled-row JSONL to convert. */
|
|
35
35
|
input: string;
|
|
36
36
|
/** The parquet shard to write. */
|
|
@@ -39,11 +39,11 @@ export interface JsonlToParquetOptions {
|
|
|
39
39
|
rowGroupSize?: number;
|
|
40
40
|
}
|
|
41
41
|
/** Summary returned by {@linkcode jsonlToParquet}. */
|
|
42
|
-
export interface
|
|
42
|
+
export interface JSONLToParquetSummary {
|
|
43
43
|
read: number;
|
|
44
44
|
written: number;
|
|
45
45
|
outPath: string;
|
|
46
46
|
}
|
|
47
47
|
/** Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */
|
|
48
|
-
export declare function jsonlToParquet(options:
|
|
48
|
+
export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
|
|
49
49
|
//# sourceMappingURL=jsonl-to-parquet.d.ts.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "7.0.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -82,9 +82,9 @@
|
|
|
82
82
|
},
|
|
83
83
|
"dependencies": {
|
|
84
84
|
"@dsnp/parquetjs": "1.8.8",
|
|
85
|
-
"@mailwoman/codex": "
|
|
86
|
-
"@mailwoman/core": "
|
|
87
|
-
"@mailwoman/formatter": "
|
|
85
|
+
"@mailwoman/codex": "7.0.0",
|
|
86
|
+
"@mailwoman/core": "7.0.0",
|
|
87
|
+
"@mailwoman/formatter": "7.0.0",
|
|
88
88
|
"csv-parse": "^7.0.1",
|
|
89
89
|
"fast-glob": "^3.3.3",
|
|
90
90
|
"fastest-levenshtein": "^1.0.16",
|