@mailwoman/corpus 4.10.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/out/src/adapters/ban/adapter.d.ts +6 -2
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +8 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +5 -2
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -7
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/build.d.ts +14 -0
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +25 -0
- package/out/src/build.js.map +1 -1
- package/out/src/license.d.ts +37 -0
- package/out/src/license.d.ts.map +1 -0
- package/out/src/license.js +46 -0
- package/out/src/license.js.map +1 -0
- package/out/src/synthesize-boundary-stress.d.ts +77 -0
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -0
- package/out/src/synthesize-boundary-stress.js +449 -0
- package/out/src/synthesize-boundary-stress.js.map +1 -0
- package/out/src/synthesize-po-box.d.ts +3 -1
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +56 -3
- package/out/src/synthesize-po-box.js.map +1 -1
- package/package.json +4 -4
package/README.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# @mailwoman/corpus
|
|
2
|
+
|
|
3
|
+
**BIO-labeled training-corpus pipeline** for the Mailwoman address parser.
|
|
4
|
+
|
|
5
|
+
Generates sequence-labeling training data from reference sources
|
|
6
|
+
(OpenAddresses, libpostal dictionaries, synthetic shards) and assembles them into
|
|
7
|
+
the TSV format consumed by the Modal training pipeline. This package produces
|
|
8
|
+
the data that trains `@mailwoman/neural-weights-*`.
|
|
9
|
+
|
|
10
|
+
```ts
|
|
11
|
+
// The corpus pipeline is primarily build-time CLI tooling.
|
|
12
|
+
// Key entry points:
|
|
13
|
+
import { expandGolden } from "@mailwoman/corpus" // Expand reference addresses
|
|
14
|
+
import { synthesizeShard } from "@mailwoman/corpus" // Generate synthetic training shards
|
|
15
|
+
import { alignRow } from "@mailwoman/corpus" // Align raw address → BIO tokens
|
|
16
|
+
import { validateCorpus } from "@mailwoman/corpus" // Validate corpus integrity
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## What it produces
|
|
20
|
+
|
|
21
|
+
The corpus pipeline assembles training data from multiple sources:
|
|
22
|
+
|
|
23
|
+
| Source | Description |
|
|
24
|
+
| -------------------- | ------------------------------------------------------------------------ |
|
|
25
|
+
| **OpenAddresses** | Real government address point data (US, FR, DE, …) |
|
|
26
|
+
| **NAD** | National Address Database (US-specific) |
|
|
27
|
+
| **libpostal** | Multilingual street/place name dictionaries |
|
|
28
|
+
| **Synthetic shards** | Generated address variations (boundary stress, order variants, all-caps) |
|
|
29
|
+
| **Overture Maps** | Address theme ingestion (alpha) |
|
|
30
|
+
|
|
31
|
+
Output format: TSV rows with `raw<TAB>BIO_labels` consumed by the Python
|
|
32
|
+
training pipeline (`corpus-python/`).
|
|
33
|
+
|
|
34
|
+
## Key modules
|
|
35
|
+
|
|
36
|
+
| Module | Purpose |
|
|
37
|
+
| ----------------------- | ------------------------------------------------------------------ |
|
|
38
|
+
| **`expand-golden.ts`** | Expand reference addresses into training rows with alignment |
|
|
39
|
+
| **`align.ts`** | Tokenize raw address → BIO label sequence |
|
|
40
|
+
| **`validate.ts`** | Validate corpus integrity, label coverage, shard balance |
|
|
41
|
+
| **`synthesize-*.ts`** | Synthetic shard generators (boundary stress, order variants, etc.) |
|
|
42
|
+
| **`ingest/`** | Overture Maps + NAD ingestion |
|
|
43
|
+
| **`shard-registry.ts`** | Shard metadata and composition |
|
|
44
|
+
| **`stats.ts`** | Per-shard and per-tag statistics |
|
|
45
|
+
|
|
46
|
+
## Build-time tooling
|
|
47
|
+
|
|
48
|
+
The corpus is assembled via scripts in `scripts/`:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Validate the corpus
|
|
52
|
+
node scripts/validate-corpus.mjs
|
|
53
|
+
|
|
54
|
+
# Rebuild shards
|
|
55
|
+
node scripts/build-boundary-stress-shard.mjs
|
|
56
|
+
|
|
57
|
+
# Corpus statistics
|
|
58
|
+
node scripts/corpus-stats.mjs
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Design
|
|
62
|
+
|
|
63
|
+
- **BIO (Begin/Inside/Outside) labeling** over SentencePiece tokens.
|
|
64
|
+
- **Character-offset aligned** — labels track the raw string, not the
|
|
65
|
+
normalized form, so the model learns real input distributions.
|
|
66
|
+
- **Source-homogeneous shards** — each shard comes from one source, ordered by
|
|
67
|
+
type, so eval splits are honest (no bleed between train and held-out).
|
|
68
|
+
|
|
69
|
+
## Related
|
|
70
|
+
|
|
71
|
+
- [`@mailwoman/neural`](../neural) — the runtime that loads and runs the trained model
|
|
72
|
+
- [`@mailwoman/neural-weights-en-us`](../neural-weights-en-us) — the trained model itself
|
|
73
|
+
- [Corpus Construction concepts](https://mailwoman.sister.software/articles/concepts/corpus-construction/)
|
|
74
|
+
- [Training Pipeline concepts](https://mailwoman.sister.software/articles/concepts/training-pipeline/)
|
|
75
|
+
- [CONTRIBUTING_MODEL_WORK](https://mailwoman.sister.software/articles/plan/CONTRIBUTING_MODEL_WORK/)
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
[AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html)
|
|
@@ -18,8 +18,12 @@
|
|
|
18
18
|
* left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
|
|
19
19
|
* for Phase 1 the row's region is simply absent).
|
|
20
20
|
*
|
|
21
|
-
* License:
|
|
22
|
-
*
|
|
21
|
+
* License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
|
|
22
|
+
* attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
|
|
23
|
+
* allowed for training with attribution; the ODbL option's share-alike obligation would defeat
|
|
24
|
+
* the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
|
|
25
|
+
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
|
+
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
23
27
|
*
|
|
24
28
|
* The adapter is streaming-aware: it uses `csv-parse` in streaming mode so a 25M-row dump never
|
|
25
29
|
* sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAMH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAGjF,eAAO,MAAM,cAAc,QAAQ,CAAA;AA4CnC,wBAAgB,gBAAgB,IAAI,aAAa,CAwEhD;AAED,eAAO,MAAM,UAAU,eAAqB,CAAA"}
|
|
@@ -18,8 +18,12 @@
|
|
|
18
18
|
* left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
|
|
19
19
|
* for Phase 1 the row's region is simply absent).
|
|
20
20
|
*
|
|
21
|
-
* License:
|
|
22
|
-
*
|
|
21
|
+
* License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
|
|
22
|
+
* attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
|
|
23
|
+
* allowed for training with attribution; the ODbL option's share-alike obligation would defeat
|
|
24
|
+
* the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
|
|
25
|
+
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
|
+
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
23
27
|
*
|
|
24
28
|
* The adapter is streaming-aware: it uses `csv-parse` in streaming mode so a 25M-row dump never
|
|
25
29
|
* sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
|
|
@@ -64,7 +68,7 @@ function composeRaw(house, street, postcode, locality) {
|
|
|
64
68
|
export function createBanAdapter() {
|
|
65
69
|
return {
|
|
66
70
|
id: BAN_ADAPTER_ID,
|
|
67
|
-
defaultLicense: "
|
|
71
|
+
defaultLicense: "Licence Ouverte 2.0",
|
|
68
72
|
description: "Base Adresse Nationale (FR): house-number-level street addresses (~25M rows).",
|
|
69
73
|
async *rows(opts) {
|
|
70
74
|
if (opts.country && opts.country !== "FR") {
|
|
@@ -122,7 +126,7 @@ export function createBanAdapter() {
|
|
|
122
126
|
source: BAN_ADAPTER_ID,
|
|
123
127
|
source_id: sourceId,
|
|
124
128
|
corpus_version: "",
|
|
125
|
-
license: "
|
|
129
|
+
license: "Licence Ouverte 2.0",
|
|
126
130
|
};
|
|
127
131
|
emitted++;
|
|
128
132
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAErD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAEzD,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAA;AAenC;;;GAGG;AACH,SAAS,kBAAkB,CAAC,MAAc,EAAE,GAAW;IACtD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACpB,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IACjB,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc,EAAE,QAAgB,EAAE,QAAgB;IACpF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACnE,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACtC,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,IAAI,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACpD,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,EAAE,EAAE,cAAc;QAClB,cAAc,EAAE,qBAAqB;QACrC,WAAW,EAAE,+EAA+E;QAE5F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YACrE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACzB,QAAQ,CAAC;gBACR,SAAS,EAAE,GAAG;gBACd,OAAO,EAAE,IAAI;gBACb,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,IAAI;gBAClB,kBAAkB,EAAE,IAAI;aACxB,CAAC,CACF,CAAA;YAED,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAA+B,EAAE,CAAC;oBAC5D,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBAC/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,MAAK;oBAE5D,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAA;oBACvE,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAC7C,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAClD,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAElD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAClC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEjC,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;oBAE5C,MAAM,UAAU,GAA+B,EAAE,CAAA;oBACjD,IAAI,KAAK;wBAAE,UAAU,CAAC,YAAY,GAAG,KAAK,CAAA;oBAC1C,IAAI,UAAU,CAAC,MAAM;wBAAE,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;oBACnE,IAAI,UAAU,CAAC,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;oBAC5D,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAC5C,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAE5C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;oBACzD,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAElB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBACpD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAQ;oBAE/C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE;wBACjC,CAAC,CAAC,GAAG,cAAc,IAAI,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE;wBACzC,CAAC,CAAC,cAAc,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;oBAE1C,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,qBAAqB;qBAC9B,CAAA;oBACD,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA"}
|
|
@@ -46,8 +46,11 @@ export interface OpenaddressesAdapterOptions {
|
|
|
46
46
|
*/
|
|
47
47
|
defaultLicense?: string;
|
|
48
48
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
49
|
+
* Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a
|
|
50
|
+
* deliberate BUILD-level act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT
|
|
51
|
+
* a silent adapter default (#26 — "purposely exclude, don't opt in to include"). Set false only
|
|
52
|
+
* for an explicit adapter-scoped drop; the build-level `--exclude-share-alike` is the normal
|
|
53
|
+
* path.
|
|
51
54
|
*/
|
|
52
55
|
allowShareAlike?: boolean;
|
|
53
56
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAOH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEjF,eAAO,MAAM,wBAAwB,kBAAkB,CAAA;AACvD,eAAO,MAAM,6BAA6B,cAAc,CAAA;AA8CxD,MAAM,WAAW,2BAA2B;IAC3C;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CACzB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,GAAE,2BAAgC,GAAG,aAAa,CAwFhG;AAED,eAAO,MAAM,oBAAoB,eAA+B,CAAA"}
|
|
@@ -39,6 +39,7 @@ import { createReadStream } from "node:fs";
|
|
|
39
39
|
import { createInterface } from "node:readline";
|
|
40
40
|
import { stableSourceId } from "../../adapter.js";
|
|
41
41
|
import { formatAddress, reconcileComponents } from "../../format.js";
|
|
42
|
+
import { SHARE_ALIKE_PATTERN } from "../../license.js";
|
|
42
43
|
export const OPENADDRESSES_ADAPTER_ID = "openaddresses";
|
|
43
44
|
export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
44
45
|
/** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
|
|
@@ -73,19 +74,13 @@ function parseFeatureLine(line) {
|
|
|
73
74
|
return null;
|
|
74
75
|
return normalizeProperties(obj.properties);
|
|
75
76
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Licenses that require share-alike or create a copyleft obligation on derived works. Rows carrying
|
|
78
|
-
* any of these licenses are dropped by default because they would contaminate proprietary-weights
|
|
79
|
-
* training. Per the licensing strategy (#26).
|
|
80
|
-
*/
|
|
81
|
-
const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
|
|
82
77
|
/**
|
|
83
78
|
* Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default
|
|
84
79
|
* fallback for dumps known to carry a single license throughout (e.g. a PDDL-only state slice).
|
|
85
80
|
*/
|
|
86
81
|
export function createOpenaddressesAdapter(opts = {}) {
|
|
87
82
|
const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE;
|
|
88
|
-
const allowShareAlike = opts.allowShareAlike ??
|
|
83
|
+
const allowShareAlike = opts.allowShareAlike ?? true;
|
|
89
84
|
return {
|
|
90
85
|
id: OPENADDRESSES_ADAPTER_ID,
|
|
91
86
|
defaultLicense,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAGtD,MAAM,CAAC,MAAM,wBAAwB,GAAG,eAAe,CAAA;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,WAAW,CAAA;AAmBxD,0FAA0F;AAC1F,SAAS,mBAAmB,CAAC,GAAY;IACxC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAA;IAC9C,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;QACrE,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAA;aAC9C,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;IACjE,CAAC;IACD,OAAO,GAAmB,CAAA;AAC3B,CAAC;AAED,+FAA+F;AAC/F,SAAS,gBAAgB,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACpD,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACtD,MAAM,GAAG,GAAG,MAAiD,CAAA;IAC7D,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IACvC,OAAO,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;AAC3C,CAAC;AAoBD;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAoC,EAAE;IAChF,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,6BAA6B,CAAA;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,OAAO;QACN,EAAE,EAAE,wBAAwB;QAC5B,cAAc;QACd,WAAW,EAAE,6EAA6E;QAE1F,KAAK,CAAC,CAAC,IAAI,CAAC,WAA2B;YACtC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACd,0HAA0H,CAC1H,CAAA;YACF,CAAC;YACD,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAA;YAEnC,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YAC5E,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;YAErE,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,iBAAiB,GAAG,CAAC,CAAA;YACzB,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBAChC,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBACtC,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,WAAW,CAAC,KAAK;wBAAE,MAAK;oBAE1E,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;oBACpC,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBAEpB,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACrC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACrC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBAE7C,iFAAiF;oBACjF,wDAAwD;oBACxD,IAAI,CAAC,MAAM;wBAAE,SAAQ;oBACrB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEhC,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,cAAc,CAAC,CAAC,IAAI,EAAE,CAAA;oBAEhE,IAAI,CAAC,eAAe,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC3D,iBAAiB,EAAE,CAAA;wBACnB,SAAQ;oBACT,CAAC;oBAED,MAAM,UAAU,GAA+B,EAAE,CAAA;oBACjD,IAAI,WAAW;wBAAE,UAAU,CAAC,YAAY,GAAG,WAAW,CAAA;oBACtD,IAAI,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;oBACtC,IAAI,IAAI;wBAAE,UAAU,CAAC,IAAI,GAAG,IAAI,CAAA;oBAChC,IAAI,IAAI;wBAAE,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAA;oBACpC,IAAI,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;oBACtC,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBACnE,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAElB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBACpD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAQ;oBAE/C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAA;oBAC3D,MAAM,QAAQ,GAAG,YAAY;wBAC5B,CAAC,CAAC,GAAG,wBAAwB,IAAI,YAAY,EAAE;wBAC/C,CAAC,CAAC,cAAc,CAAC,wBAAwB,EAAE,OAAO,CAAC,CAAA;oBAEpD,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO;wBACP,MAAM,EAAE,wBAAwB;wBAChC,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO;qBACP,CAAA;oBACD,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,MAAM,CAAC,OAAO,EAAE,CAAA;gBAChB,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,iBAAiB,8BAA8B,OAAO,SAAS,CAAC,CAAA;gBAC1G,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAA"}
|
package/out/src/build.d.ts
CHANGED
|
@@ -74,6 +74,14 @@ export interface BuildCorpusOptions {
|
|
|
74
74
|
rowsPerShard?: number;
|
|
75
75
|
/** Progress hook. Errors thrown abort the build. */
|
|
76
76
|
onProgress?: (stage: BuildStage, message: string) => void;
|
|
77
|
+
/**
|
|
78
|
+
* License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see
|
|
79
|
+
* `compileLicenseExcludes` / `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license`
|
|
80
|
+
* matches any is dropped at ingest. Default (omitted) includes EVERYTHING — exclusion is a
|
|
81
|
+
* deliberate act, not a silent default. A proprietary-weights build passes the share-alike set
|
|
82
|
+
* (`--exclude-share-alike`).
|
|
83
|
+
*/
|
|
84
|
+
excludeLicenses?: readonly RegExp[];
|
|
77
85
|
}
|
|
78
86
|
/** Top-level manifest tying every stage together. */
|
|
79
87
|
export interface BuildCorpusManifest {
|
|
@@ -91,6 +99,12 @@ export interface BuildCorpusManifest {
|
|
|
91
99
|
};
|
|
92
100
|
quarantine_count: number;
|
|
93
101
|
total_aligned_rows: number;
|
|
102
|
+
/**
|
|
103
|
+
* Resolved license set across all INCLUDED rows (license string → row count), + the count dropped
|
|
104
|
+
* by `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
|
|
105
|
+
*/
|
|
106
|
+
licenses: Record<string, number>;
|
|
107
|
+
excluded_by_license: number;
|
|
94
108
|
}
|
|
95
109
|
/**
|
|
96
110
|
* Drive the full corpus build to completion.
|
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AASH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwLxF"}
|
package/out/src/build.js
CHANGED
|
@@ -52,6 +52,7 @@ import { join } from "node:path";
|
|
|
52
52
|
import { createInterface } from "node:readline";
|
|
53
53
|
import { defaultAdapterRegistry } from "./adapter.js";
|
|
54
54
|
import { alignRow } from "./align.js";
|
|
55
|
+
import { licenseExcluded } from "./license.js";
|
|
55
56
|
import { writeShards } from "./parquet.js";
|
|
56
57
|
import { runAdapter } from "./runner.js";
|
|
57
58
|
import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "./split.js";
|
|
@@ -127,12 +128,27 @@ export async function buildCorpus(opts) {
|
|
|
127
128
|
let quarantined = 0;
|
|
128
129
|
const counts = { train: 0, val: 0, test: 0 };
|
|
129
130
|
const holdouts = defaultHoldouts();
|
|
131
|
+
// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
|
|
132
|
+
// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
|
|
133
|
+
// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
|
|
134
|
+
const excludeLicenses = opts.excludeLicenses ?? [];
|
|
135
|
+
const licenseCounts = new Map();
|
|
136
|
+
let excludedByLicense = 0;
|
|
130
137
|
const writeQuarantine = (row, reason) => {
|
|
131
138
|
quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
|
|
132
139
|
};
|
|
133
140
|
for (const adapterRun of adapterRuns) {
|
|
134
141
|
opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
|
|
135
142
|
for await (const row of streamJsonl(adapterRun.jsonl_path)) {
|
|
143
|
+
licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
|
|
144
|
+
// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
|
|
145
|
+
// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
|
|
146
|
+
// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
|
|
147
|
+
// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
|
|
148
|
+
if (licenseExcluded(row.license, excludeLicenses)) {
|
|
149
|
+
excludedByLicense++;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
136
152
|
const fanned = [row];
|
|
137
153
|
if (synthesize) {
|
|
138
154
|
for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
|
|
@@ -194,6 +210,13 @@ export async function buildCorpus(opts) {
|
|
|
194
210
|
corpusVersion: opts.corpusVersion,
|
|
195
211
|
rowsPerShard,
|
|
196
212
|
});
|
|
213
|
+
// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
|
|
214
|
+
// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
|
|
215
|
+
const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1]);
|
|
216
|
+
opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
|
|
217
|
+
(excludedByLicense > 0
|
|
218
|
+
? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
|
|
219
|
+
: " | NO license exclusion applied (all rows kept)"));
|
|
197
220
|
// 6. Top-level manifest.
|
|
198
221
|
opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
|
|
199
222
|
const manifest = {
|
|
@@ -205,6 +228,8 @@ export async function buildCorpus(opts) {
|
|
|
205
228
|
shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
|
|
206
229
|
quarantine_count: quarantined,
|
|
207
230
|
total_aligned_rows: aligned,
|
|
231
|
+
licenses: Object.fromEntries(licenseSummary),
|
|
232
|
+
excluded_by_license: excludedByLicense,
|
|
208
233
|
};
|
|
209
234
|
await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
210
235
|
return manifest;
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA+D/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACxD,IACC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;YACpC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBACnB,SAAQ;YACT,CAAC;YACD,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBACvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Corpus licensing — the single source of truth for the training-data license policy (#26).
|
|
7
|
+
*
|
|
8
|
+
* Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
|
|
9
|
+
* build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
|
|
10
|
+
* license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
|
|
11
|
+
* inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
|
|
12
|
+
* })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
|
|
13
|
+
* string unless the operator named it. This avoids the trap of silently dropping allowed data
|
|
14
|
+
* mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
|
|
15
|
+
* ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
|
|
16
|
+
* dropped 48M allowed rows).
|
|
17
|
+
*
|
|
18
|
+
* Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
|
|
19
|
+
* — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
|
|
20
|
+
* proprietary-weights build via `--exclude-share-alike`.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
|
|
24
|
+
* `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
|
|
25
|
+
* it.
|
|
26
|
+
*/
|
|
27
|
+
export declare const SHARE_ALIKE_PATTERN: RegExp;
|
|
28
|
+
/**
|
|
29
|
+
* Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
|
|
30
|
+
* case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
|
|
31
|
+
* `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
|
|
32
|
+
* spec is a literal license prefix, not a user-supplied regex.
|
|
33
|
+
*/
|
|
34
|
+
export declare function compileLicenseExcludes(spec: string): RegExp[];
|
|
35
|
+
/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
|
|
36
|
+
export declare function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean;
|
|
37
|
+
//# sourceMappingURL=license.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED,kGAAkG;AAClG,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAGjG"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Corpus licensing — the single source of truth for the training-data license policy (#26).
|
|
7
|
+
*
|
|
8
|
+
* Posture (operator, 2026-06-19): **exclusion is a deliberate act, not a silent default.** The
|
|
9
|
+
* build INCLUDES every row an adapter yields (stamping its `license`); a build that needs a clean
|
|
10
|
+
* license set — e.g. the proprietary `@mailwoman/neural-weights-*` weights, which must not
|
|
11
|
+
* inherit a share-alike obligation — PURPOSELY excludes kinds via `buildCorpus({ excludeLicenses
|
|
12
|
+
* })` (CLI `--exclude-licenses` / `--exclude-share-alike`). Nothing is dropped on a license
|
|
13
|
+
* string unless the operator named it. This avoids the trap of silently dropping allowed data
|
|
14
|
+
* mis-stamped with a conservative license (e.g. BAN, which is dual-licensed Licence Ouverte OR
|
|
15
|
+
* ODbL — we elect Licence Ouverte; a default-deny on the old `ODbL` stamp would have wrongly
|
|
16
|
+
* dropped 48M allowed rows).
|
|
17
|
+
*
|
|
18
|
+
* Tier reference (#26): A = PD/CC0 (allowed); B = CC-BY / Licence Ouverte (allowed WITH attribution
|
|
19
|
+
* — the model card must carry it); C = share-alike (ODbL, CC-BY-SA, CC-SA) — exclude for a
|
|
20
|
+
* proprietary-weights build via `--exclude-share-alike`.
|
|
21
|
+
*/
|
|
22
|
+
/**
|
|
23
|
+
* Licenses that require share-alike / create a copyleft obligation on derived works (Tier C). The
|
|
24
|
+
* `--exclude-share-alike` convenience expands to this; `allowShareAlike: false` adapters also use
|
|
25
|
+
* it.
|
|
26
|
+
*/
|
|
27
|
+
export const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
|
|
28
|
+
/**
|
|
29
|
+
* Compile a `--exclude-licenses` spec (comma-separated, e.g. `"ODbL,CC-BY-SA"`) into anchored,
|
|
30
|
+
* case-insensitive prefix patterns. Each entry matches a license string that STARTS with it, so
|
|
31
|
+
* `CC-BY-SA` catches `CC-BY-SA-3.0`, `CC-BY-SA-4.0`, etc. Regex metacharacters are escaped — the
|
|
32
|
+
* spec is a literal license prefix, not a user-supplied regex.
|
|
33
|
+
*/
|
|
34
|
+
export function compileLicenseExcludes(spec) {
|
|
35
|
+
return spec
|
|
36
|
+
.split(",")
|
|
37
|
+
.map((s) => s.trim())
|
|
38
|
+
.filter(Boolean)
|
|
39
|
+
.map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"));
|
|
40
|
+
}
|
|
41
|
+
/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
|
|
42
|
+
export function licenseExcluded(license, patterns) {
|
|
43
|
+
const l = license ?? "";
|
|
44
|
+
return patterns.some((p) => p.test(l));
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=license.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AAC9E,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IACvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Boundary-instability synthesizer (#375 — the highest-leverage parser lever). The failure taxonomy
|
|
7
|
+
*
|
|
8
|
+
* The within-token-punctuation decomposition (#702) found one failure FAMILY surfacing under many
|
|
9
|
+
* names: the model mis-places token boundaries between adjacent components when the boundary is
|
|
10
|
+
* ambiguous or unmarked. This generator emits diverse BIO-labeled rows that put the gold boundary
|
|
11
|
+
* exactly where the model wobbles, so a retrain learns the boundary from context, not the
|
|
12
|
+
* lexeme.
|
|
13
|
+
*
|
|
14
|
+
* The four token-aligned stress shapes, all in BASE LOCALES (US/FR/DE) so the shard never
|
|
15
|
+
* introduces tokens the base corpus lacks (the #511 base-consistency lint flagged an earlier
|
|
16
|
+
* AU-bearing draft: AU 4-digit postcodes collide with US house numbers, and AU localities are
|
|
17
|
+
* absent from the US/FR/DE base — a real contradiction). Each component is a whitespace-separated
|
|
18
|
+
* token run, so `alignRow` labels it cleanly:
|
|
19
|
+
*
|
|
20
|
+
* 1. `street-eats-affix` — multi-word street + suffix (`Country Club Rd` → street + street_suffix),
|
|
21
|
+
* the #1 wobble: the model keeps the suffix in the street.
|
|
22
|
+
* 2. `comma-less-city-state` — no comma between street / locality / region (`100 Main St Springfield
|
|
23
|
+
* IL 62701`), the #694 family: concatenated input loses the segmentation cue. US-only (US
|
|
24
|
+
* zips are base-consistent; the boundary is locale-agnostic).
|
|
25
|
+
* 3. `fr-prefix` — FR street-type prefix split from the name (`Rue Jean-Baptiste Lebas` →
|
|
26
|
+
* street_prefix
|
|
27
|
+
*
|
|
28
|
+
* - Street), postcode-first order.
|
|
29
|
+
* 4. `house-number-after-street` — FR/DE number-follows-street (`Neuve-des-Capucines 5` → street +
|
|
30
|
+
* house_number), the model absorbs the number into the street.
|
|
31
|
+
*
|
|
32
|
+
* Two BALANCING shapes (added 2026-06-18 after the v1.6.0 probes). The first pass at weight 1.0
|
|
33
|
+
* lifted the boundaries but over-fit a NARROW distribution — every row was a clean, full,
|
|
34
|
+
* structured address — so the model regressed on out-of-distribution real rows (held-out US
|
|
35
|
+
* locality 66.3→58.2%). These two widen the distribution the shard teaches over, per the
|
|
36
|
+
* diagnosis (scripts/eval/locality-regression-probe): 5. `bare-locality` — locality with NO
|
|
37
|
+
* street (`Public Library, Lisbon ND`, `75003 Paris`), the ship-blocker: 84% of the v1.6.0
|
|
38
|
+
* locality regression was DROPPED locality on bare "City, STATE" rows, because every other shape
|
|
39
|
+
* placed a street before the city. Bare / comma-less / postcode'd / venue-prefixed forms, US +
|
|
40
|
+
* FR. 6. `house-number-before-street` — the confounding mirror of #4 (same FR vocab, number
|
|
41
|
+
* BEFORE the street). A balanced before:after mix breaks the positional shortcut behind the #4
|
|
42
|
+
* order-bias.
|
|
43
|
+
*
|
|
44
|
+
* EXCLUDED: the region+postcode glue (`NY14201` — sub-token, no punctuation to split) and the
|
|
45
|
+
* AU/NZ/UK slash unit-convention (`4/2A` → unit+house_number). The slash labels cleanly (the
|
|
46
|
+
* tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
|
|
47
|
+
* AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
|
|
48
|
+
* separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
|
|
49
|
+
* base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
|
|
50
|
+
*/
|
|
51
|
+
import type { CanonicalRow } from "./types.js";
|
|
52
|
+
export type BoundaryStressTemplate = "street-eats-affix" | "comma-less-city-state" | "fr-prefix" | "house-number-after-street" | "bare-locality" | "house-number-before-street";
|
|
53
|
+
export interface BoundaryStressBaseTuple {
|
|
54
|
+
locality: string;
|
|
55
|
+
region: string;
|
|
56
|
+
postcode: string;
|
|
57
|
+
country: string;
|
|
58
|
+
}
|
|
59
|
+
export interface BoundaryStressSynthesisOpts {
|
|
60
|
+
random?: () => number;
|
|
61
|
+
/** Force a specific shape (tests + balanced shard composition). */
|
|
62
|
+
forceTemplate?: BoundaryStressTemplate;
|
|
63
|
+
}
|
|
64
|
+
export interface SynthesizedBoundaryStressRow {
|
|
65
|
+
raw: string;
|
|
66
|
+
components: CanonicalRow["components"];
|
|
67
|
+
locale: string;
|
|
68
|
+
template: BoundaryStressTemplate;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple
|
|
72
|
+
* is drawn from the internal pools (so the generator is self-contained; a build script can pass
|
|
73
|
+
* real tuples for scale + diversity). Every component value is a verbatim substring of `raw`, so
|
|
74
|
+
* `alignRow` locates + BIO-labels it.
|
|
75
|
+
*/
|
|
76
|
+
export declare function synthesizeBoundaryStressRow(base: BoundaryStressBaseTuple | undefined, opts?: BoundaryStressSynthesisOpts): SynthesizedBoundaryStressRow;
|
|
77
|
+
//# sourceMappingURL=synthesize-boundary-stress.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"synthesize-boundary-stress.d.ts","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC/B,mBAAmB,GACnB,uBAAuB,GACvB,WAAW,GACX,2BAA2B,GAG3B,eAAe,GACf,4BAA4B,CAAA;AAE/B,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,mEAAmE;IACnE,aAAa,CAAC,EAAE,sBAAsB,CAAA;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,sBAAsB,CAAA;CAChC;AAyRD;;;;;GAKG;AACH,wBAAgB,2BAA2B,CAC1C,IAAI,EAAE,uBAAuB,GAAG,SAAS,EACzC,IAAI,GAAE,2BAAgC,GACpC,4BAA4B,CA4H9B"}
|
|
@@ -0,0 +1,449 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Boundary-instability synthesizer (#375 — the highest-leverage parser lever). The failure taxonomy
|
|
7
|
+
*
|
|
8
|
+
* The within-token-punctuation decomposition (#702) found one failure FAMILY surfacing under many
|
|
9
|
+
* names: the model mis-places token boundaries between adjacent components when the boundary is
|
|
10
|
+
* ambiguous or unmarked. This generator emits diverse BIO-labeled rows that put the gold boundary
|
|
11
|
+
* exactly where the model wobbles, so a retrain learns the boundary from context, not the
|
|
12
|
+
* lexeme.
|
|
13
|
+
*
|
|
14
|
+
* The four token-aligned stress shapes, all in BASE LOCALES (US/FR/DE) so the shard never
|
|
15
|
+
* introduces tokens the base corpus lacks (the #511 base-consistency lint flagged an earlier
|
|
16
|
+
* AU-bearing draft: AU 4-digit postcodes collide with US house numbers, and AU localities are
|
|
17
|
+
* absent from the US/FR/DE base — a real contradiction). Each component is a whitespace-separated
|
|
18
|
+
* token run, so `alignRow` labels it cleanly:
|
|
19
|
+
*
|
|
20
|
+
* 1. `street-eats-affix` — multi-word street + suffix (`Country Club Rd` → street + street_suffix),
|
|
21
|
+
* the #1 wobble: the model keeps the suffix in the street.
|
|
22
|
+
* 2. `comma-less-city-state` — no comma between street / locality / region (`100 Main St Springfield
|
|
23
|
+
* IL 62701`), the #694 family: concatenated input loses the segmentation cue. US-only (US
|
|
24
|
+
* zips are base-consistent; the boundary is locale-agnostic).
|
|
25
|
+
* 3. `fr-prefix` — FR street-type prefix split from the name (`Rue Jean-Baptiste Lebas` →
|
|
26
|
+
* street_prefix
|
|
27
|
+
*
|
|
28
|
+
* - Street), postcode-first order.
|
|
29
|
+
* 4. `house-number-after-street` — FR/DE number-follows-street (`Neuve-des-Capucines 5` → street +
|
|
30
|
+
* house_number), the model absorbs the number into the street.
|
|
31
|
+
*
|
|
32
|
+
* Two BALANCING shapes (added 2026-06-18 after the v1.6.0 probes). The first pass at weight 1.0
|
|
33
|
+
* lifted the boundaries but over-fit a NARROW distribution — every row was a clean, full,
|
|
34
|
+
* structured address — so the model regressed on out-of-distribution real rows (held-out US
|
|
35
|
+
* locality 66.3→58.2%). These two widen the distribution the shard teaches over, per the
|
|
36
|
+
* diagnosis (scripts/eval/locality-regression-probe): 5. `bare-locality` — locality with NO
|
|
37
|
+
* street (`Public Library, Lisbon ND`, `75003 Paris`), the ship-blocker: 84% of the v1.6.0
|
|
38
|
+
* locality regression was DROPPED locality on bare "City, STATE" rows, because every other shape
|
|
39
|
+
* placed a street before the city. Bare / comma-less / postcode'd / venue-prefixed forms, US +
|
|
40
|
+
* FR. 6. `house-number-before-street` — the confounding mirror of #4 (same FR vocab, number
|
|
41
|
+
* BEFORE the street). A balanced before:after mix breaks the positional shortcut behind the #4
|
|
42
|
+
* order-bias.
|
|
43
|
+
*
|
|
44
|
+
* EXCLUDED: the region+postcode glue (`NY14201` — sub-token, no punctuation to split) and the
|
|
45
|
+
* AU/NZ/UK slash unit-convention (`4/2A` → unit+house_number). The slash labels cleanly (the
|
|
46
|
+
* tokenizer splits `/`) and is the worst within-token class — but it inherently requires non-base
|
|
47
|
+
* AU/NZ/UK locales, which contradict the US/FR/DE base (the lint catch). It belongs in a
|
|
48
|
+
* separately-scoped AU/NZ/UK boundary-coverage shard that ALSO adds AU base coverage, not in this
|
|
49
|
+
* base-locale shard. `synthesize-boundary-stress.test.ts` proves the alignments.
|
|
50
|
+
*/
|
|
51
|
+
function pick(arr, random) {
|
|
52
|
+
return arr[Math.floor(random() * arr.length)];
|
|
53
|
+
}
|
|
54
|
+
// Multi-word street names — the suffix boundary only bites when "Club" could be read as part of the
|
|
55
|
+
// name. Single-word names alone teach nothing about the suffix edge.
|
|
56
|
+
// Multi-word names are what make the suffix boundary BITE (the model must not read the trailing
|
|
57
|
+
// suffix word as part of the name). Kept diverse so the shard teaches the boundary, not the lexeme.
|
|
58
|
+
const MULTIWORD_STREETS = [
|
|
59
|
+
"Country Club",
|
|
60
|
+
"Martin Luther King",
|
|
61
|
+
"Forest Hill",
|
|
62
|
+
"Lake View",
|
|
63
|
+
"Spring Valley",
|
|
64
|
+
"Cedar Ridge",
|
|
65
|
+
"Old Mill",
|
|
66
|
+
"Sunset Park",
|
|
67
|
+
"Maple Grove",
|
|
68
|
+
"Stone Creek",
|
|
69
|
+
"Glen Cove",
|
|
70
|
+
"Pine Bluff",
|
|
71
|
+
"Fox Hollow",
|
|
72
|
+
"Briar Patch",
|
|
73
|
+
"West End",
|
|
74
|
+
"College Station",
|
|
75
|
+
"Quail Hollow",
|
|
76
|
+
"Eagle Ridge",
|
|
77
|
+
"Deer Run",
|
|
78
|
+
"Bear Creek",
|
|
79
|
+
"Willow Bend",
|
|
80
|
+
"Cypress Point",
|
|
81
|
+
"Laurel Oak",
|
|
82
|
+
"Magnolia Park",
|
|
83
|
+
"Cherry Hill",
|
|
84
|
+
"Walnut Grove",
|
|
85
|
+
"Birch Hollow",
|
|
86
|
+
"Aspen Grove",
|
|
87
|
+
"Juniper Ridge",
|
|
88
|
+
"Hidden Valley",
|
|
89
|
+
"Rolling Hills",
|
|
90
|
+
"Tanglewood",
|
|
91
|
+
"Meadow Brook",
|
|
92
|
+
"Clover Field",
|
|
93
|
+
"Sunrise Point",
|
|
94
|
+
"Harbor View",
|
|
95
|
+
"Bay Shore",
|
|
96
|
+
"Ocean Breeze",
|
|
97
|
+
"Mountain View",
|
|
98
|
+
"Valley Forge",
|
|
99
|
+
"Liberty Square",
|
|
100
|
+
"Washington Crossing",
|
|
101
|
+
"Kings Highway",
|
|
102
|
+
"Queens Gate",
|
|
103
|
+
"Princeton Junction",
|
|
104
|
+
];
|
|
105
|
+
const SINGLE_STREETS = [
|
|
106
|
+
"Main",
|
|
107
|
+
"Oak",
|
|
108
|
+
"Maple",
|
|
109
|
+
"Park",
|
|
110
|
+
"Washington",
|
|
111
|
+
"Lincoln",
|
|
112
|
+
"Church",
|
|
113
|
+
"River",
|
|
114
|
+
"Pine",
|
|
115
|
+
"Cedar",
|
|
116
|
+
"Elm",
|
|
117
|
+
"Jefferson",
|
|
118
|
+
"Madison",
|
|
119
|
+
"Adams",
|
|
120
|
+
"Jackson",
|
|
121
|
+
"Franklin",
|
|
122
|
+
"Highland",
|
|
123
|
+
"Sunset",
|
|
124
|
+
"Lakeview",
|
|
125
|
+
"Hillcrest",
|
|
126
|
+
"Cambridge",
|
|
127
|
+
"Devonshire",
|
|
128
|
+
"Sherwood",
|
|
129
|
+
"Kingston",
|
|
130
|
+
"Berkshire",
|
|
131
|
+
"Aberdeen",
|
|
132
|
+
"Belmont",
|
|
133
|
+
"Carlisle",
|
|
134
|
+
"Dover",
|
|
135
|
+
"Easton",
|
|
136
|
+
"Fairfax",
|
|
137
|
+
"Greenwood",
|
|
138
|
+
"1st",
|
|
139
|
+
"2nd",
|
|
140
|
+
"3rd",
|
|
141
|
+
"4th",
|
|
142
|
+
"5th",
|
|
143
|
+
"12th",
|
|
144
|
+
"42nd",
|
|
145
|
+
];
|
|
146
|
+
const SUFFIXES = [
|
|
147
|
+
"St",
|
|
148
|
+
"Street",
|
|
149
|
+
"Ave",
|
|
150
|
+
"Avenue",
|
|
151
|
+
"Rd",
|
|
152
|
+
"Road",
|
|
153
|
+
"Blvd",
|
|
154
|
+
"Boulevard",
|
|
155
|
+
"Ln",
|
|
156
|
+
"Lane",
|
|
157
|
+
"Dr",
|
|
158
|
+
"Drive",
|
|
159
|
+
"Pkwy",
|
|
160
|
+
"Parkway",
|
|
161
|
+
"Way",
|
|
162
|
+
"Ct",
|
|
163
|
+
"Court",
|
|
164
|
+
"Pl",
|
|
165
|
+
"Place",
|
|
166
|
+
"Cir",
|
|
167
|
+
"Circle",
|
|
168
|
+
"Ter",
|
|
169
|
+
"Terrace",
|
|
170
|
+
"Hwy",
|
|
171
|
+
"Trail",
|
|
172
|
+
"Loop",
|
|
173
|
+
"Cres",
|
|
174
|
+
"Crescent",
|
|
175
|
+
"Row",
|
|
176
|
+
"Walk",
|
|
177
|
+
];
|
|
178
|
+
const DIRECTIONALS = ["N", "S", "E", "W", "NE", "NW", "SE", "SW"];
|
|
179
|
+
// FR street-type prefixes + hyphenated honorific street names (the hyphen is incidental; the boundary
|
|
180
|
+
// stress is the prefix↔name split + the number-after-street order).
|
|
181
|
+
const FR_PREFIXES = [
|
|
182
|
+
"Rue",
|
|
183
|
+
"Avenue",
|
|
184
|
+
"Boulevard",
|
|
185
|
+
"Place",
|
|
186
|
+
"Impasse",
|
|
187
|
+
"Chemin",
|
|
188
|
+
"Quai",
|
|
189
|
+
"Cours",
|
|
190
|
+
"Allée",
|
|
191
|
+
"Passage",
|
|
192
|
+
"Square",
|
|
193
|
+
"Villa",
|
|
194
|
+
"Sentier",
|
|
195
|
+
"Promenade",
|
|
196
|
+
];
|
|
197
|
+
const FR_NAMES = [
|
|
198
|
+
"Jean-Baptiste Lebas",
|
|
199
|
+
"Neuve-des-Capucines",
|
|
200
|
+
"Charles-de-Gaulle",
|
|
201
|
+
"du Général-Leclerc",
|
|
202
|
+
"de la République",
|
|
203
|
+
"des Trois-Frères",
|
|
204
|
+
"Victor-Hugo",
|
|
205
|
+
"Jean-Jaurès",
|
|
206
|
+
"de l'Abreuvoir",
|
|
207
|
+
"Émile-Zola",
|
|
208
|
+
"Gambetta",
|
|
209
|
+
"Jean-Moulin",
|
|
210
|
+
"des Martyrs-de-la-Résistance",
|
|
211
|
+
"du Maréchal-Foch",
|
|
212
|
+
"Pierre-et-Marie-Curie",
|
|
213
|
+
"Antoine-de-Saint-Exupéry",
|
|
214
|
+
"de la Liberté",
|
|
215
|
+
"des Quatre-Vents",
|
|
216
|
+
"du Faubourg-Saint-Antoine",
|
|
217
|
+
"Saint-Honoré",
|
|
218
|
+
"de la Pompe",
|
|
219
|
+
"des Petits-Champs",
|
|
220
|
+
"Léon-Blum",
|
|
221
|
+
"Aristide-Briand",
|
|
222
|
+
];
|
|
223
|
+
// Org/venue prefixes for the bare-locality shape — the v1.6.0 locality drop hit org-PREFIXED real rows
|
|
224
|
+
// hardest ("LISBON PUBLIC LIBRARY, …, Lisbon ND"; "Alburg Health Center"). Teaching the locality WITH a
|
|
225
|
+
// leading venue keeps the model emitting it on facility-style addresses (NPPES/HRSA shapes). `venue` is a
|
|
226
|
+
// base ComponentTag.
|
|
227
|
+
//
|
|
228
|
+
// #511-LINTED 2026-06-18 (scripts/lint-venue-vocab — scan of nppes/hrsa/tiger/nad/wof-admin): every token
|
|
229
|
+
// here is venue-DOMINANT in the base, so the shard agrees with it. The first draft was naive — 9 terms
|
|
230
|
+
// were dropped because their tokens are dominantly street/locality and would CONTRADICT the base the way
|
|
231
|
+
// Madison-as-street did (#511): "Fire" 93% street, "Veterans" 94% street, "City" 68% locality, "Hall" 63%
|
|
232
|
+
// street, "Memorial"/"Hospital" 62-63% street, "Recreation" 79% street, "Town" 48% locality, "Library" 51%
|
|
233
|
+
// street, "County" 68% street, "Arts"/"Courthouse"/"Municipal" dependent_locality. Kept tokens: Clinic 98%,
|
|
234
|
+
// Practice 98%, Dental 100%, Health 99%, Medical 88%, Community 92%, Department 90%, Group 87%, Center 65%,
|
|
235
|
+
// School 70%, Public 89%, Elementary/Family 97% (all venue).
|
|
236
|
+
const VENUES = [
|
|
237
|
+
"Community Center",
|
|
238
|
+
"Health Center",
|
|
239
|
+
"Medical Center",
|
|
240
|
+
"Medical Clinic",
|
|
241
|
+
"Family Clinic",
|
|
242
|
+
"Community Clinic",
|
|
243
|
+
"Dental Clinic",
|
|
244
|
+
"Family Practice",
|
|
245
|
+
"Medical Practice",
|
|
246
|
+
"Dental Group",
|
|
247
|
+
"Medical Group",
|
|
248
|
+
"Health Department",
|
|
249
|
+
"Elementary School",
|
|
250
|
+
"Public School",
|
|
251
|
+
];
|
|
252
|
+
// Localities DERIVED from the base corpus (#511): every name here is verified locality-DOMINANT in the
|
|
253
|
+
// training data (B-locality ≫ I-street), so the shard agrees with the base instead of fighting it. The
|
|
254
|
+
// night's targeted scan caught the prior vocab (Madison, Portland, Springfield IL…) at 92–100% STREET
|
|
255
|
+
// in the base ("Madison Ave"), the "5th Avenue Theatre" #511 trap. See 2026-06-17-locality-vocab-fix.
|
|
256
|
+
const US_TUPLES = [
|
|
257
|
+
{ locality: "Albuquerque", region: "NM", postcode: "87102", country: "US" },
|
|
258
|
+
{ locality: "Indianapolis", region: "IN", postcode: "46203", country: "US" },
|
|
259
|
+
{ locality: "Sacramento", region: "CA", postcode: "95823", country: "US" },
|
|
260
|
+
{ locality: "Rochester", region: "NY", postcode: "14606", country: "US" },
|
|
261
|
+
{ locality: "Jacksonville", region: "FL", postcode: "32209", country: "US" },
|
|
262
|
+
{ locality: "Portsmouth", region: "VA", postcode: "23704", country: "US" },
|
|
263
|
+
{ locality: "Merced", region: "CA", postcode: "95340", country: "US" },
|
|
264
|
+
{ locality: "Miami", region: "FL", postcode: "33125", country: "US" },
|
|
265
|
+
{ locality: "Tampa", region: "FL", postcode: "33624", country: "US" },
|
|
266
|
+
{ locality: "Orlando", region: "FL", postcode: "32827", country: "US" },
|
|
267
|
+
{ locality: "Tulsa", region: "OK", postcode: "74133", country: "US" },
|
|
268
|
+
{ locality: "Louisville", region: "KY", postcode: "40203", country: "US" },
|
|
269
|
+
{ locality: "Nashville", region: "TN", postcode: "37207", country: "US" },
|
|
270
|
+
{ locality: "Spokane", region: "WA", postcode: "99202", country: "US" },
|
|
271
|
+
{ locality: "Akron", region: "OH", postcode: "44313", country: "US" },
|
|
272
|
+
{ locality: "Fairbanks", region: "AK", postcode: "99701", country: "US" },
|
|
273
|
+
{ locality: "Plano", region: "TX", postcode: "75024", country: "US" },
|
|
274
|
+
{ locality: "Shreveport", region: "LA", postcode: "71103", country: "US" },
|
|
275
|
+
{ locality: "Southfield", region: "MI", postcode: "48034", country: "US" },
|
|
276
|
+
{ locality: "Glendale", region: "CA", postcode: "91203", country: "US" },
|
|
277
|
+
{ locality: "Philadelphia", region: "PA", postcode: "19104", country: "US" },
|
|
278
|
+
{ locality: "Brooklyn", region: "NY", postcode: "11230", country: "US" },
|
|
279
|
+
{ locality: "Bronx", region: "NY", postcode: "10461", country: "US" },
|
|
280
|
+
{ locality: "Fairport", region: "NY", postcode: "14450", country: "US" },
|
|
281
|
+
{ locality: "Syracuse", region: "NE", postcode: "68446", country: "US" },
|
|
282
|
+
{ locality: "Marion", region: "AR", postcode: "72364", country: "US" },
|
|
283
|
+
{ locality: "Chicago", region: "IL", postcode: "60625", country: "US" },
|
|
284
|
+
{ locality: "Springfield", region: "MA", postcode: "01108", country: "US" },
|
|
285
|
+
];
|
|
286
|
+
// FR localities DERIVED from the FR (ban) shards specifically — where these famous cities are 95–99%
|
|
287
|
+
// locality-DOMINANT (Paris 515605/24789, Marseille 247014/1752, Lyon 106239/3114). NB: the all-shard
|
|
288
|
+
// scan falsely flagged them street-dominant by undersampling the FR block (parts 180–209) and mixing in
|
|
289
|
+
// US street-contexts; the FR-block scan is the honest distribution. Dept-diverse (28 depts), region
|
|
290
|
+
// empty (French addresses carry no region token; the generator's region-optional path handles it).
|
|
291
|
+
const FR_TUPLES = [
|
|
292
|
+
{ locality: "Paris", region: "", postcode: "75003", country: "FR" },
|
|
293
|
+
{ locality: "Marseille", region: "", postcode: "13016", country: "FR" },
|
|
294
|
+
{ locality: "Lyon", region: "", postcode: "69009", country: "FR" },
|
|
295
|
+
{ locality: "Perpignan", region: "", postcode: "66000", country: "FR" },
|
|
296
|
+
{ locality: "Toulon", region: "", postcode: "83100", country: "FR" },
|
|
297
|
+
{ locality: "Avignon", region: "", postcode: "84140", country: "FR" },
|
|
298
|
+
{ locality: "Poitiers", region: "", postcode: "86000", country: "FR" },
|
|
299
|
+
{ locality: "Arles", region: "", postcode: "13280", country: "FR" },
|
|
300
|
+
{ locality: "Annecy", region: "", postcode: "74940", country: "FR" },
|
|
301
|
+
{ locality: "Mulhouse", region: "", postcode: "68200", country: "FR" },
|
|
302
|
+
{ locality: "Carpentras", region: "", postcode: "84200", country: "FR" },
|
|
303
|
+
{ locality: "Antony", region: "", postcode: "92160", country: "FR" },
|
|
304
|
+
{ locality: "Sartrouville", region: "", postcode: "78500", country: "FR" },
|
|
305
|
+
{ locality: "Épinal", region: "", postcode: "88000", country: "FR" },
|
|
306
|
+
{ locality: "Meyzieu", region: "", postcode: "69330", country: "FR" },
|
|
307
|
+
{ locality: "Sens", region: "", postcode: "89100", country: "FR" },
|
|
308
|
+
{ locality: "Brunoy", region: "", postcode: "91800", country: "FR" },
|
|
309
|
+
{ locality: "Rambouillet", region: "", postcode: "78120", country: "FR" },
|
|
310
|
+
];
|
|
311
|
+
// NB: no DE_TUPLES — German cities are street-dominated too ("Berliner Straße"), and the base yielded
|
|
312
|
+
// zero locality-dominant DE towns in the scan, so house-number-after-street is FR-only here. DE's
|
|
313
|
+
// native-order number-after-street is covered by the dedicated synth-german shard.
|
|
314
|
+
const houseNumber = (random) => String(1 + Math.floor(random() * 4999));
|
|
315
|
+
const localeFor = { US: "en-US", FR: "fr-FR", DE: "de-DE" };
|
|
316
|
+
const ALL_TEMPLATES = [
|
|
317
|
+
"street-eats-affix",
|
|
318
|
+
"comma-less-city-state",
|
|
319
|
+
"fr-prefix",
|
|
320
|
+
"house-number-after-street",
|
|
321
|
+
"bare-locality",
|
|
322
|
+
"house-number-before-street",
|
|
323
|
+
];
|
|
324
|
+
/**
|
|
325
|
+
* Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple
|
|
326
|
+
* is drawn from the internal pools (so the generator is self-contained; a build script can pass
|
|
327
|
+
* real tuples for scale + diversity). Every component value is a verbatim substring of `raw`, so
|
|
328
|
+
* `alignRow` locates + BIO-labels it.
|
|
329
|
+
*/
|
|
330
|
+
export function synthesizeBoundaryStressRow(base, opts = {}) {
|
|
331
|
+
const random = opts.random ?? Math.random;
|
|
332
|
+
const template = opts.forceTemplate ?? pick(ALL_TEMPLATES, random);
|
|
333
|
+
if (template === "bare-locality") {
|
|
334
|
+
// The v1.6.0 ship-blocker fix: locality was DROPPED on bare/short "City, STATE" rows (84% of the
|
|
335
|
+
// regression) because every prior shape placed a street before the city, so the model learned "the
|
|
336
|
+
// city follows a street" and stopped emitting locality without one. Teach the locality with NO street,
|
|
337
|
+
// across the forms real data carries it — bare, comma-LESS, postcode'd, and venue/org-prefixed.
|
|
338
|
+
const b = base ?? (random() < 0.3 ? pick(FR_TUPLES, random) : pick(US_TUPLES, random));
|
|
339
|
+
const venue = random() < 0.45 ? pick(VENUES, random) : "";
|
|
340
|
+
// ~12% carry a trailing country token — the v1.7.1 country patch (DeepSeek 2026-06-18). The pure
|
|
341
|
+
// "City, STATE" bare rows carry NO country token, which cost ~4pp on us.country_homograph in v1.7.0;
|
|
342
|
+
// teaching "…, USA"/"…, France" recovers it as a single-variable additive without diluting locality.
|
|
343
|
+
const withCountry = random() < 0.12;
|
|
344
|
+
if (b.country === "FR") {
|
|
345
|
+
// FR carries no region token; "{postcode} {locality}" is the bare FR form.
|
|
346
|
+
const core = `${b.postcode} ${b.locality}${withCountry ? ", France" : ""}`;
|
|
347
|
+
return {
|
|
348
|
+
raw: venue ? `${venue}, ${core}` : core,
|
|
349
|
+
components: {
|
|
350
|
+
...(venue ? { venue } : {}),
|
|
351
|
+
postcode: b.postcode,
|
|
352
|
+
locality: b.locality,
|
|
353
|
+
...(withCountry ? { country: "France" } : {}),
|
|
354
|
+
},
|
|
355
|
+
locale: "fr-FR",
|
|
356
|
+
template,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
const withZip = random() < 0.5;
|
|
360
|
+
const comma = random() < 0.6 ? "," : ""; // include the comma-LESS "City STATE" form too
|
|
361
|
+
// "United States" (United 98% / States 98% country in the base), NOT "USA" — the #511 lint found
|
|
362
|
+
// "USA" is locality-DOMINANT (75%, only 6% country) in the base; labeling it country would contradict.
|
|
363
|
+
const countryName = "United States";
|
|
364
|
+
const core = `${b.locality}${comma} ${b.region}${withZip ? ` ${b.postcode}` : ""}${withCountry ? `, ${countryName}` : ""}`;
|
|
365
|
+
return {
|
|
366
|
+
raw: venue ? `${venue}, ${core}` : core,
|
|
367
|
+
components: {
|
|
368
|
+
...(venue ? { venue } : {}),
|
|
369
|
+
locality: b.locality,
|
|
370
|
+
region: b.region,
|
|
371
|
+
...(withZip ? { postcode: b.postcode } : {}),
|
|
372
|
+
...(withCountry ? { country: countryName } : {}),
|
|
373
|
+
},
|
|
374
|
+
locale: "en-US",
|
|
375
|
+
template,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
if (template === "fr-prefix" ||
|
|
379
|
+
template === "house-number-after-street" ||
|
|
380
|
+
template === "house-number-before-street") {
|
|
381
|
+
// FR-only (no base-consistent DE locality vocab; see the DE_TUPLES note above).
|
|
382
|
+
const b = base ?? pick(FR_TUPLES, random);
|
|
383
|
+
const name = pick(FR_NAMES, random);
|
|
384
|
+
const hn = houseNumber(random);
|
|
385
|
+
if (template === "house-number-before-street") {
|
|
386
|
+
// The confounding MIRROR of house-number-after-street: the SAME FR street vocab with the number
|
|
387
|
+
// BEFORE the name. A balanced before:after mix (the build/recipe sets the ratio, ~7:3 to keep US
|
|
388
|
+
// house_number 99.8% safe) teaches the model a street-adjacent number is a house_number by FORM,
|
|
389
|
+
// not position — the probe found v1.6.0 confidently absorbs the TRAILING number into street (I-street
|
|
390
|
+
// P=0.96), the order-bias.
|
|
391
|
+
const raw = `${hn} ${name}, ${b.postcode} ${b.locality}`;
|
|
392
|
+
return {
|
|
393
|
+
raw,
|
|
394
|
+
components: { house_number: hn, street: name, postcode: b.postcode, locality: b.locality },
|
|
395
|
+
locale: localeFor[b.country] ?? "fr-FR",
|
|
396
|
+
template,
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
if (template === "fr-prefix") {
|
|
400
|
+
const prefix = pick(FR_PREFIXES, random);
|
|
401
|
+
// "{hn} {prefix} {name}, {postcode} {locality}" — postcode-first, prefix split from the name.
|
|
402
|
+
const raw = `${hn} ${prefix} ${name}, ${b.postcode} ${b.locality}`;
|
|
403
|
+
return {
|
|
404
|
+
raw,
|
|
405
|
+
components: {
|
|
406
|
+
house_number: hn,
|
|
407
|
+
street_prefix: prefix,
|
|
408
|
+
street: name,
|
|
409
|
+
postcode: b.postcode,
|
|
410
|
+
locality: b.locality,
|
|
411
|
+
},
|
|
412
|
+
locale: localeFor[b.country] ?? "fr-FR",
|
|
413
|
+
template,
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
// house-number-after-street: "{name} {hn}, {postcode} {locality}" — number FOLLOWS the street.
|
|
417
|
+
const raw = `${name} ${hn}, ${b.postcode} ${b.locality}`;
|
|
418
|
+
return {
|
|
419
|
+
raw,
|
|
420
|
+
components: { street: name, house_number: hn, postcode: b.postcode, locality: b.locality },
|
|
421
|
+
locale: localeFor[b.country] ?? "fr-FR",
|
|
422
|
+
template,
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
// en-US street shapes (street-eats-affix + comma-less). US-only — US zips are base-consistent and
|
|
426
|
+
// the boundary these teach is locale-agnostic; no need to introduce a non-base locale.
|
|
427
|
+
const b = base ?? pick(US_TUPLES, random);
|
|
428
|
+
const hn = houseNumber(random);
|
|
429
|
+
const dir = random() < 0.4 ? pick(DIRECTIONALS, random) : "";
|
|
430
|
+
const name = random() < 0.7 ? pick(MULTIWORD_STREETS, random) : pick(SINGLE_STREETS, random);
|
|
431
|
+
const suffix = pick(SUFFIXES, random);
|
|
432
|
+
const streetCore = `${dir ? `${dir} ` : ""}${name} ${suffix}`;
|
|
433
|
+
const components = {
|
|
434
|
+
house_number: hn,
|
|
435
|
+
...(dir ? { street_prefix: dir } : {}),
|
|
436
|
+
street: name,
|
|
437
|
+
street_suffix: suffix,
|
|
438
|
+
locality: b.locality,
|
|
439
|
+
region: b.region,
|
|
440
|
+
postcode: b.postcode,
|
|
441
|
+
};
|
|
442
|
+
const raw = template === "comma-less-city-state"
|
|
443
|
+
? // no commas anywhere — the segmentation cue is gone
|
|
444
|
+
`${hn} ${streetCore} ${b.locality} ${b.region} ${b.postcode}`
|
|
445
|
+
: // standard delimited, multi-word street stresses the suffix boundary
|
|
446
|
+
`${hn} ${streetCore}, ${b.locality}, ${b.region} ${b.postcode}`;
|
|
447
|
+
return { raw, components, locale: localeFor[b.country] ?? "en-US", template };
|
|
448
|
+
}
|
|
449
|
+
//# sourceMappingURL=synthesize-boundary-stress.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"synthesize-boundary-stress.js","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAkCH,SAAS,IAAI,CAAI,GAAqB,EAAE,MAAoB;IAC3D,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAE,CAAA;AAC/C,CAAC;AAED,oGAAoG;AACpG,qEAAqE;AACrE,gGAAgG;AAChG,oGAAoG;AACpG,MAAM,iBAAiB,GAAG;IACzB,cAAc;IACd,oBAAoB;IACpB,aAAa;IACb,WAAW;IACX,eAAe;IACf,aAAa;IACb,UAAU;IACV,aAAa;IACb,aAAa;IACb,aAAa;IACb,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,UAAU;IACV,iBAAiB;IACjB,cAAc;IACd,aAAa;IACb,UAAU;IACV,YAAY;IACZ,aAAa;IACb,eAAe;IACf,YAAY;IACZ,eAAe;IACf,aAAa;IACb,cAAc;IACd,cAAc;IACd,aAAa;IACb,eAAe;IACf,eAAe;IACf,eAAe;IACf,YAAY;IACZ,cAAc;IACd,cAAc;IACd,eAAe;IACf,aAAa;IACb,WAAW;IACX,cAAc;IACd,eAAe;IACf,cAAc;IACd,gBAAgB;IAChB,qBAAqB;IACrB,eAAe;IACf,aAAa;IACb,oBAAoB;CACX,CAAA;AACV,MAAM,cAAc,GAAG;IACtB,MAAM;IACN,KAAK;IACL,OAAO;IACP,MAAM;IACN,YAAY;IACZ,SAAS;IACT,QAAQ;IACR,OAAO;IACP,MAAM;IACN,OAAO;IACP,KAAK;IACL,WAAW;IACX,SAAS;IACT,OAAO;IACP,SAAS;IACT,UAAU;IACV,UAAU;IACV,QAAQ;IACR,UAAU;IACV,WAAW;IACX,WAAW;IACX,YAAY;IACZ,UAAU;IACV,UAAU;IACV,WAAW;IACX,UAAU;IACV,SAAS;IACT,UAAU;IACV,OAAO;IACP,QAAQ;IACR,SAAS;IACT,WAAW;IACX,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;CACG,CAAA;AACV,MAAM,QAAQ,GAAG;IAChB,IAAI;IACJ,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,IAAI;IACJ,MAAM;IACN,MAAM;IACN,WAAW;IACX,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,OAAO;IACP,MAAM;IACN,SAAS;IACT,KAAK;IACL,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,OAAO;IACP,KAAK;IACL,QAAQ;IACR,KAAK;IACL,SAAS;IACT,KAAK;IACL,OAAO;IACP,MAAM;IACN,MAAM;IACN,UAAU;IACV,KAAK;IACL,MAAM;CACG,CAAA;AACV,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU,CAAA;AAE1E,sGAAsG;AACtG,oEAAoE;AACpE,MAAM,WAAW,GAAG;IACnB,KAAK;IACL,QAAQ;IACR,WAAW;IACX,OAAO;IACP,SAAS;IACT,QAAQ;IACR,MAAM;IACN,OAAO;IACP,OAAO;IACP,SAAS;IACT,QAAQ;IACR,OAAO;IACP,SAAS;IACT,WAAW;CACF,CAAA;AACV,MAAM,QAAQ,GAAG;IAChB,qBAAqB;IACrB,qBAAqB;IACrB,mBAAmB;IACnB,oBAAoB;IACpB,kBAAkB;IAClB,kBAAkB;IAClB,aAAa;IACb,aAAa;IACb,gBAAgB;IAChB,YAAY;IACZ,UAAU;IACV,aAAa;IACb,8BAA8B;IAC9B,kBAAkB;IAClB,uBAAuB;IACvB,0BAA0B;IAC1B,eAAe;IACf,kBAAkB;IAClB,2BAA2B;IAC3B,cAAc;IACd,aAAa;IACb,mBAAmB;IACnB,WAAW;IACX,iBAAiB;CACR,CAAA;AAEV,uGAAuG;AACvG,wGAAwG;AACxG,0GAA0G;AAC1G,qBAAqB;AACrB,EAAE;AACF,0GAA0G;AAC1G,uGAAuG;AACvG,yGAAyG;AACzG,0GAA0G;AAC1G,2GAA2G;AAC3G,4GAA4G;AAC5G,4GAA4G;AAC5G,6DAA6D;AAC7D,MAAM,MAAM,GAAG;IACd,kBAAkB;IAClB,eAAe;IACf,gBAAgB;IAChB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;IAClB,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,cAAc;IACd,eAAe;IACf,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;CACN,CAAA;AAEV,uGAAuG;AACvG,uGAAuG;AACvG,sGAAsG;AACtG,sGAAsG;AACtG,MAAM,SAAS,GAA2C;IACzD,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC3E,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC5E,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACzE,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC5E,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACtE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACvE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACzE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACvE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACzE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACxE,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC5E,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACxE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACxE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACtE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACvE,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;CAC3E,CAAA;AACD,qGAAqG;AACrG,qGAAqG;AACrG,wGAAwG;AACxG,oGAAoG;AACpG,mGAAmG;AACnG,MAAM,SAAS,GAA2C;IACzD,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACnE,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACvE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAClE,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACvE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACpE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACtE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACnE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACpE,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACtE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACxE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACpE,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAC1E,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACpE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACrE,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IAClE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;IACpE,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE;CACzE,CAAA;AACD,sGAAsG;AACtG,kGAAkG;AAClG,mFAAmF;AACnF,MAAM,WAAW,GAAG,CAAC,MAAoB,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;AAC7F,MAAM,SAAS,GAA2B,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,CAAA;AAEnF,MAAM,aAAa,GAAsC;IACxD,mBAAmB;IACnB,uBAAuB;IACvB,WAAW;IACX,2BAA2B;IAC3B,eAAe;IACf,4BAA4B;CAC5B,CAAA;AAED;;;;;GAKG;AACH,MAAM,UAAU,2BAA2B,CAC1C,IAAyC,EACzC,OAAoC,EAAE;IAEtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IAElE,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;QAClC,iGAAiG;QACjG,mGAAmG;QACnG,uGAAuG;QACvG,gGAAgG;QAChG,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAA;QACtF,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QACzD,iGAAiG;QACjG,qGAAqG;QACrG,qGAAqG;QACrG,MAAM,WAAW,GAAG,MAAM,EAAE,GAAG,IAAI,CAAA;QACnC,IAAI,CAAC,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YACxB,2EAA2E;YAC3E,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;YAC1E,OAAO;gBACN,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;gBACvC,UAAU,EAAE;oBACX,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC7C;gBACD,MAAM,EAAE,OAAO;gBACf,QAAQ;aACR,CAAA;QACF,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,EAAE,GAAG,GAAG,CAAA;QAC9B,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA,CAAC,+CAA+C;QACvF,iGAAiG;QACjG,uGAAuG;QACvG,MAAM,WAAW,GAAG,eAAe,CAAA;QACnC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAC1H,OAAO;YACN,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;YACvC,UAAU,EAAE;gBACX,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5C,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChD;YACD,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IACC,QAAQ,KAAK,WAAW;QACxB,QAAQ,KAAK,2BAA2B;QACxC,QAAQ,KAAK,4BAA4B,EACxC,CAAC;QACF,gFAAgF;QAChF,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACzC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACnC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;QAC9B,IAAI,QAAQ,KAAK,4BAA4B,EAAE,CAAC;YAC/C,gGAAgG;YAChG,iGAAiG;YACjG,iGAAiG;YACjG,sGAAsG;YACtG,2BAA2B;YAC3B,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,IAAI,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;YACxD,OAAO;gBACN,GAAG;gBACH,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;gBAC1F,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO;gBACvC,QAAQ;aACR,CAAA;QACF,CAAC;QACD,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;YACxC,8FAA8F;YAC9F,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;YAClE,OAAO;gBACN,GAAG;gBACH,UAAU,EAAE;oBACX,YAAY,EAAE,EAAE;oBAChB,aAAa,EAAE,MAAM;oBACrB,MAAM,EAAE,IAAI;oBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;iBACpB;gBACD,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO;gBACvC,QAAQ;aACR,CAAA;QACF,CAAC;QACD,+FAA+F;QAC/F,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;QACxD,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC1F,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO;YACvC,QAAQ;SACR,CAAA;IACF,CAAC;IAED,kGAAkG;IAClG,uFAAuF;IACvF,MAAM,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAA;IAC9B,MAAM,GAAG,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC5D,MAAM,IAAI,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAA;IAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACrC,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,MAAM,EAAE,CAAA;IAC7D,MAAM,UAAU,GAA+B;QAC9C,YAAY,EAAE,EAAE;QAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtC,MAAM,EAAE,IAAI;QACZ,aAAa,EAAE,MAAM;QACrB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACpB,CAAA;IACD,MAAM,GAAG,GACR,QAAQ,KAAK,uBAAuB;QACnC,CAAC,CAAC,oDAAoD;YACrD,GAAG,EAAE,IAAI,UAAU,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE;QAC9D,CAAC,CAAC,qEAAqE;YACtE,GAAG,EAAE,IAAI,UAAU,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;IAClE,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAA;AAC9E,CAAC"}
|
|
@@ -58,7 +58,7 @@ export interface SynthesizedPoBoxRow {
|
|
|
58
58
|
raw: string;
|
|
59
59
|
components: CanonicalRow["components"];
|
|
60
60
|
locale: string;
|
|
61
|
-
template: "po-box" | "pmb-with-street";
|
|
61
|
+
template: "po-box" | "pmb-with-street" | "military-po-box";
|
|
62
62
|
}
|
|
63
63
|
export interface PoBoxSynthesisOpts {
|
|
64
64
|
/** Random function — pass deterministic seed for tests. Default Math.random. */
|
|
@@ -77,6 +77,8 @@ export declare function synthesizePoBoxRow(base: PoBoxBaseTuple & {
|
|
|
77
77
|
street?: string;
|
|
78
78
|
houseNumber?: string;
|
|
79
79
|
}, opts?: PoBoxSynthesisOpts): SynthesizedPoBoxRow | null;
|
|
80
|
+
/** Generate one US military/diplomatic PO-box row (#517). Self-contained — draws no base tuple. */
|
|
81
|
+
export declare function synthesizeMilitaryPoBoxRow(opts?: PoBoxSynthesisOpts): SynthesizedPoBoxRow;
|
|
80
82
|
/**
|
|
81
83
|
* Map a country code (ISO-3166-1 alpha-2 or alpha-3, or country display name) to the locale code we
|
|
82
84
|
* have a PO box template for.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-po-box.d.ts","sourceRoot":"","sources":["../../src/synthesize-po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IAE9B,GAAG,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;CAC3B;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,EAAE,aAAa,CAAC,cAAc,
|
|
1
|
+
{"version":3,"file":"synthesize-po-box.d.ts","sourceRoot":"","sources":["../../src/synthesize-po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;IAE9B,GAAG,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;CAC3B;AAED;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB,EAAE,aAAa,CAAC,cAAc,CA2CjE,CAAA;AAID;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,MAAM,GAAG,MAAM,CAY/E;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzE;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,QAAQ,GAAG,iBAAiB,GAAG,iBAAiB,CAAA;CAC1D;AAED,MAAM,WAAW,kBAAkB;IAClC,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,uDAAuD;IACvD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,MAAM,KAAK,MAAM,CAAA;IAC7C,4FAA4F;IAC5F,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAWD;;;;GAIG;AACH,wBAAgB,kBAAkB,CACjC,IAAI,EAAE,cAAc,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAChE,IAAI,GAAE,kBAAuB,GAC3B,mBAAmB,GAAG,IAAI,CAqD5B;AAuBD,mGAAmG;AACnG,wBAAgB,0BAA0B,CAAC,IAAI,GAAE,kBAAuB,GAAG,mBAAmB,CAsB7F;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAYvD;AAED,qFAAqF;AACrF,wBAAgB,gBAAgB,IAAI,aAAa,CAAC,MAAM,CAAC,CAExD"}
|
|
@@ -48,6 +48,10 @@ export const PO_BOX_LOCALE_TEMPLATES = [
|
|
|
48
48
|
locale: "en-AU",
|
|
49
49
|
leaders: ["PO Box", "P.O. Box", "Post Office Box", "GPO Box", "Locked Bag"],
|
|
50
50
|
},
|
|
51
|
+
{
|
|
52
|
+
locale: "en-NZ",
|
|
53
|
+
leaders: ["PO Box", "P.O. Box", "Post Office Box", "Private Bag", "Private Box"],
|
|
54
|
+
},
|
|
51
55
|
{
|
|
52
56
|
locale: "fr-FR",
|
|
53
57
|
leaders: ["BP", "B.P.", "Boîte Postale", "BP."],
|
|
@@ -146,14 +150,17 @@ export function synthesizePoBoxRow(base, opts = {}) {
|
|
|
146
150
|
template: "pmb-with-street",
|
|
147
151
|
};
|
|
148
152
|
}
|
|
149
|
-
// Standard PO box: replaces the street line entirely.
|
|
150
|
-
|
|
153
|
+
// Standard PO box: replaces the street line entirely. Region-optional — NZ (and other region-less
|
|
154
|
+
// locales) read "Private Bag 12, Auckland 1010" with no region token between locality and postcode.
|
|
155
|
+
const hasRegion = Boolean(base.region && base.region.trim());
|
|
156
|
+
const tail = hasRegion ? `${base.locality}, ${base.region} ${base.postcode}` : `${base.locality} ${base.postcode}`;
|
|
157
|
+
const raw = `${poBoxPhrase}, ${tail}`;
|
|
151
158
|
return {
|
|
152
159
|
raw,
|
|
153
160
|
components: {
|
|
154
161
|
po_box: poBoxPhrase,
|
|
155
162
|
locality: base.locality,
|
|
156
|
-
region: base.region,
|
|
163
|
+
...(hasRegion ? { region: base.region } : {}),
|
|
157
164
|
postcode: base.postcode,
|
|
158
165
|
country: base.country,
|
|
159
166
|
},
|
|
@@ -161,6 +168,50 @@ export function synthesizePoBoxRow(base, opts = {}) {
|
|
|
161
168
|
template: "po-box",
|
|
162
169
|
};
|
|
163
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* The US military/diplomatic PO-box class (#517). A distinct shape the leader-based locale
|
|
173
|
+
* templates can't express: a unit line (`PSC <id> Box <box>`, `CMR <id> Box <box>`, `Unit <id> [Box
|
|
174
|
+
* <box>]`) tagged `po_box`, then the post-office code (APO/FPO/DPO) as the locality and the
|
|
175
|
+
* armed-forces region (AA/AE/AP) as the region, with a theatre-specific ZIP. Authoritative
|
|
176
|
+
* reference + citations: `@mailwoman/codex` `codex/us/military-address.ts`; the small constants are
|
|
177
|
+
* inlined here so the generator is self-contained.
|
|
178
|
+
*/
|
|
179
|
+
const MIL_UNITS = [
|
|
180
|
+
{ code: "PSC", boxRequired: true },
|
|
181
|
+
{ code: "CMR", boxRequired: true },
|
|
182
|
+
{ code: "Unit", boxRequired: false },
|
|
183
|
+
];
|
|
184
|
+
const MIL_PO_CODES = ["APO", "FPO", "DPO"];
|
|
185
|
+
// region → plausible ZIP prefix (AE Europe 09xxx, AP Pacific 962-966xx, AA Americas 340xx).
|
|
186
|
+
const MIL_REGION_ZIP = [
|
|
187
|
+
{ region: "AE", zip: (r) => `09${String(Math.floor(r() * 1000)).padStart(3, "0")}` },
|
|
188
|
+
{ region: "AP", zip: (r) => `96${String(200 + Math.floor(r() * 100)).padStart(3, "0")}` },
|
|
189
|
+
{ region: "AA", zip: (r) => `340${String(Math.floor(r() * 100)).padStart(2, "0")}` },
|
|
190
|
+
];
|
|
191
|
+
/** Generate one US military/diplomatic PO-box row (#517). Self-contained — draws no base tuple. */
|
|
192
|
+
export function synthesizeMilitaryPoBoxRow(opts = {}) {
|
|
193
|
+
const random = opts.random ?? Math.random;
|
|
194
|
+
const unit = MIL_UNITS[Math.floor(random() * MIL_UNITS.length)];
|
|
195
|
+
const unitId = String(1 + Math.floor(random() * 9999));
|
|
196
|
+
const { region, zip } = MIL_REGION_ZIP[Math.floor(random() * MIL_REGION_ZIP.length)];
|
|
197
|
+
const zipStr = zip(random);
|
|
198
|
+
const po = MIL_PO_CODES[Math.floor(random() * MIL_PO_CODES.length)];
|
|
199
|
+
const hasBox = unit.boxRequired || random() < 0.5;
|
|
200
|
+
const unitLine = hasBox ? `${unit.code} ${unitId} Box ${1 + Math.floor(random() * 9999)}` : `${unit.code} ${unitId}`;
|
|
201
|
+
const raw = `${unitLine}, ${po} ${region} ${zipStr}`;
|
|
202
|
+
return {
|
|
203
|
+
raw,
|
|
204
|
+
components: {
|
|
205
|
+
po_box: unitLine,
|
|
206
|
+
locality: po,
|
|
207
|
+
region,
|
|
208
|
+
postcode: zipStr,
|
|
209
|
+
country: "US",
|
|
210
|
+
},
|
|
211
|
+
locale: "en-US",
|
|
212
|
+
template: "military-po-box",
|
|
213
|
+
};
|
|
214
|
+
}
|
|
164
215
|
/**
|
|
165
216
|
* Map a country code (ISO-3166-1 alpha-2 or alpha-3, or country display name) to the locale code we
|
|
166
217
|
* have a PO box template for.
|
|
@@ -175,6 +226,8 @@ export function countryToLocale(country) {
|
|
|
175
226
|
return "en-GB";
|
|
176
227
|
if (c === "AU" || c === "AUS" || c === "AUSTRALIA")
|
|
177
228
|
return "en-AU";
|
|
229
|
+
if (c === "NZ" || c === "NZL" || c === "NEW ZEALAND")
|
|
230
|
+
return "en-NZ";
|
|
178
231
|
if (c === "FR" || c === "FRA" || c === "FRANCE")
|
|
179
232
|
return "fr-FR";
|
|
180
233
|
if (c === "ES" || c === "ESP" || c === "SPAIN")
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-po-box.js","sourceRoot":"","sources":["../../src/synthesize-po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAkC;IACrE;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,KAAK,CAAC;QACrF,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,iBAAiB,CAAC;QACzD,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,CAAC;KAClD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,CAAC;KAC3E;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC;KAC/C;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC;KACrD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,qBAAqB,CAAC;KAC7D;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,CAAC;KACvD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,SAAS,EAAE,mBAAmB,EAAE,IAAI,CAAC;KAC/C;CACD,CAAA;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAyB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5G;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAW,EAAE,MAAoB;IACtE,IAAI,MAAM,EAAE,GAAG,GAAG;QAAE,OAAO,GAAG,CAAA;IAC9B,MAAM,QAAQ,GAAiC;QAC9C,qDAAqD;QACrD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,wCAAwC;QACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,mDAAmD;QACnD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5B,CAAA;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAE,CAAA;IAC3D,OAAO,CAAC,CAAC,GAAG,CAAC,CAAA;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,MAAc;IAChE,OAAO,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;AAC7B,CAAC;AAkBD,SAAS,iBAAiB,CAAC,MAAoB;IAC9C,sFAAsF;IACtF,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAClB,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA,CAAC,OAAO;IACjE,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA,CAAC,UAAU;IACvE,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA,CAAC,YAAY;IAC5E,OAAO,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAA,CAAC,cAAc;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,IAAgE,EAChE,OAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAA;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;IAErC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAA;IAChE,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAE,CAAA;IACtE,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEtD,mEAAmE;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAA;IAC7D,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAG,CAAC,GAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAI,CAAC,MAAM,CAAC,CAAE,CAAA;QACnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAO,CAAA;QACzF,MAAM,GAAG,GAAG,GAAG,UAAU,KAAK,SAAS,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC5F,OAAO;YACN,GAAG;YACH,UAAU,EAAE;gBACX,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/D,MAAM,EAAE,IAAI,CAAC,MAAO;gBACpB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACrB;YACD,MAAM;YACN,QAAQ,EAAE,iBAAiB;SAC3B,CAAA;IACF,CAAC;IAED,
|
|
1
|
+
{"version":3,"file":"synthesize-po-box.js","sourceRoot":"","sources":["../../src/synthesize-po-box.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAkBH;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAkC;IACrE;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,KAAK,CAAC;QACrF,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,iBAAiB,CAAC;QACzD,GAAG,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC;KACjB;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,CAAC;KAClD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,SAAS,EAAE,YAAY,CAAC;KAC3E;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,iBAAiB,EAAE,aAAa,EAAE,aAAa,CAAC;KAChF;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,CAAC;KAC/C;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC;KACrD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,qBAAqB,CAAC;KAC7D;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,IAAI,CAAC;KACvD;IACD;QACC,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC,SAAS,EAAE,mBAAmB,EAAE,IAAI,CAAC;KAC/C;CACD,CAAA;AAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAyB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAE5G;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAW,EAAE,MAAoB;IACtE,IAAI,MAAM,EAAE,GAAG,GAAG;QAAE,OAAO,GAAG,CAAA;IAC9B,MAAM,QAAQ,GAAiC;QAC9C,qDAAqD;QACrD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,wCAAwC;QACxC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,mDAAmD;QACnD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KAC5B,CAAA;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAE,CAAA;IAC3D,OAAO,CAAC,CAAC,GAAG,CAAC,CAAA;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,MAAc;IAChE,OAAO,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;AAC7B,CAAC;AAkBD,SAAS,iBAAiB,CAAC,MAAoB;IAC9C,sFAAsF;IACtF,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAClB,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA,CAAC,OAAO;IACjE,IAAI,CAAC,GAAG,GAAG;QAAE,OAAO,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAA,CAAC,UAAU;IACvE,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA,CAAC,YAAY;IAC5E,OAAO,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAA,CAAC,cAAc;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CACjC,IAAgE,EAChE,OAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAA;IACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAA;IAErC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC5C,MAAM,GAAG,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IACzC,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAErB,MAAM,MAAM,GAAG,qBAAqB,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAA;IAChE,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAE,CAAA;IACtE,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAEtD,mEAAmE;IACnE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,MAAM,EAAE,GAAG,QAAQ,CAAA;IAC7D,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,GAAG,CAAC,GAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,GAAI,CAAC,MAAM,CAAC,CAAE,CAAA;QACnE,MAAM,SAAS,GAAG,kBAAkB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAO,CAAA;QACzF,MAAM,GAAG,GAAG,GAAG,UAAU,KAAK,SAAS,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;QAC5F,OAAO;YACN,GAAG;YACH,UAAU,EAAE;gBACX,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/D,MAAM,EAAE,IAAI,CAAC,MAAO;gBACpB,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,OAAO,EAAE,IAAI,CAAC,OAAO;aACrB;YACD,MAAM;YACN,QAAQ,EAAE,iBAAiB;SAC3B,CAAA;IACF,CAAC;IAED,kGAAkG;IAClG,oGAAoG;IACpG,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IAC5D,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAA;IAClH,MAAM,GAAG,GAAG,GAAG,WAAW,KAAK,IAAI,EAAE,CAAA;IACrC,OAAO;QACN,GAAG;QACH,UAAU,EAAE;YACX,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;SACrB;QACD,MAAM;QACN,QAAQ,EAAE,QAAQ;KAClB,CAAA;AACF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,SAAS,GAA0D;IACxE,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;IAClC,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE;IAClC,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE;CACpC,CAAA;AACD,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAU,CAAA;AACnD,4FAA4F;AAC5F,MAAM,cAAc,GAAwE;IAC3F,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;IACpF,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;IACzF,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;CACpF,CAAA;AAED,mGAAmG;AACnG,MAAM,UAAU,0BAA0B,CAAC,OAA2B,EAAE;IACvE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAE,CAAA;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,cAAc,CAAC,MAAM,CAAC,CAAE,CAAA;IACrF,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1B,MAAM,EAAE,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAE,CAAA;IACpE,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,EAAE,GAAG,GAAG,CAAA;IACjD,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,EAAE,CAAA;IACpH,MAAM,GAAG,GAAG,GAAG,QAAQ,KAAK,EAAE,IAAI,MAAM,IAAI,MAAM,EAAE,CAAA;IACpD,OAAO;QACN,GAAG;QACH,UAAU,EAAE;YACX,MAAM,EAAE,QAAQ;YAChB,QAAQ,EAAE,EAAE;YACZ,MAAM;YACN,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,IAAI;SACb;QACD,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,iBAAiB;KAC3B,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC9C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IACtC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,eAAe;QAAE,OAAO,OAAO,CAAA;IACtE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAC/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,gBAAgB;QAAE,OAAO,OAAO,CAAA;IACrF,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,WAAW;QAAE,OAAO,OAAO,CAAA;IAClE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,aAAa;QAAE,OAAO,OAAO,CAAA;IACpE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAC/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,OAAO,CAAA;IAC9D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAA;IAC/D,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,WAAW;QAAE,OAAO,OAAO,CAAA;IAClE,OAAO,OAAO,CAAA;AACf,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,gBAAgB;IAC/B,OAAO,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;AACpD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.11.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
19
|
"@dsnp/parquetjs": "1.8.7",
|
|
20
|
-
"@mailwoman/codex": "4.
|
|
21
|
-
"@mailwoman/core": "4.
|
|
22
|
-
"@mailwoman/formatter": "4.
|
|
20
|
+
"@mailwoman/codex": "4.11.0",
|
|
21
|
+
"@mailwoman/core": "4.11.0",
|
|
22
|
+
"@mailwoman/formatter": "4.11.0",
|
|
23
23
|
"csv-parse": "^5.6.0",
|
|
24
24
|
"fast-glob": "^3.3.3",
|
|
25
25
|
"fastest-levenshtein": "^1.0.16",
|