@mailwoman/corpus 4.9.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -0
- package/out/src/adapters/ban/adapter.d.ts +6 -2
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +8 -4
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +5 -2
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -7
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/build.d.ts +14 -0
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +25 -0
- package/out/src/build.js.map +1 -1
- package/out/src/format.d.ts +6 -71
- package/out/src/format.d.ts.map +1 -1
- package/out/src/format.js +6 -143
- package/out/src/format.js.map +1 -1
- package/out/src/license.d.ts +37 -0
- package/out/src/license.d.ts.map +1 -0
- package/out/src/license.js +46 -0
- package/out/src/license.js.map +1 -0
- package/out/src/synthesize-boundary-stress.d.ts +77 -0
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -0
- package/out/src/synthesize-boundary-stress.js +449 -0
- package/out/src/synthesize-boundary-stress.js.map +1 -0
- package/out/src/synthesize-po-box.d.ts +3 -1
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +56 -3
- package/out/src/synthesize-po-box.js.map +1 -1
- package/package.json +4 -4
package/README.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# @mailwoman/corpus
|
|
2
|
+
|
|
3
|
+
**BIO-labeled training-corpus pipeline** for the Mailwoman address parser.
|
|
4
|
+
|
|
5
|
+
Generates sequence-labeling training data from reference sources
|
|
6
|
+
(OpenAddresses, libpostal dictionaries, synthetic shards) and assembles them into
|
|
7
|
+
the TSV format consumed by the Modal training pipeline. This package produces
|
|
8
|
+
the data that trains `@mailwoman/neural-weights-*`.
|
|
9
|
+
|
|
10
|
+
```ts
|
|
11
|
+
// The corpus pipeline is primarily build-time CLI tooling.
|
|
12
|
+
// Key entry points:
|
|
13
|
+
import { expandGolden } from "@mailwoman/corpus" // Expand reference addresses
|
|
14
|
+
import { synthesizeShard } from "@mailwoman/corpus" // Generate synthetic training shards
|
|
15
|
+
import { alignRow } from "@mailwoman/corpus" // Align raw address → BIO tokens
|
|
16
|
+
import { validateCorpus } from "@mailwoman/corpus" // Validate corpus integrity
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## What it produces
|
|
20
|
+
|
|
21
|
+
The corpus pipeline assembles training data from multiple sources:
|
|
22
|
+
|
|
23
|
+
| Source | Description |
|
|
24
|
+
| -------------------- | ------------------------------------------------------------------------ |
|
|
25
|
+
| **OpenAddresses** | Real government address point data (US, FR, DE, …) |
|
|
26
|
+
| **NAD** | National Address Database (US-specific) |
|
|
27
|
+
| **libpostal** | Multilingual street/place name dictionaries |
|
|
28
|
+
| **Synthetic shards** | Generated address variations (boundary stress, order variants, all-caps) |
|
|
29
|
+
| **Overture Maps** | Address theme ingestion (alpha) |
|
|
30
|
+
|
|
31
|
+
Output format: TSV rows with `raw<TAB>BIO_labels` consumed by the Python
|
|
32
|
+
training pipeline (`corpus-python/`).
|
|
33
|
+
|
|
34
|
+
## Key modules
|
|
35
|
+
|
|
36
|
+
| Module | Purpose |
|
|
37
|
+
| ----------------------- | ------------------------------------------------------------------ |
|
|
38
|
+
| **`expand-golden.ts`** | Expand reference addresses into training rows with alignment |
|
|
39
|
+
| **`align.ts`** | Tokenize raw address → BIO label sequence |
|
|
40
|
+
| **`validate.ts`** | Validate corpus integrity, label coverage, shard balance |
|
|
41
|
+
| **`synthesize-*.ts`** | Synthetic shard generators (boundary stress, order variants, etc.) |
|
|
42
|
+
| **`ingest/`** | Overture Maps + NAD ingestion |
|
|
43
|
+
| **`shard-registry.ts`** | Shard metadata and composition |
|
|
44
|
+
| **`stats.ts`** | Per-shard and per-tag statistics |
|
|
45
|
+
|
|
46
|
+
## Build-time tooling
|
|
47
|
+
|
|
48
|
+
The corpus is assembled via scripts in `scripts/`:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Validate the corpus
|
|
52
|
+
node scripts/validate-corpus.mjs
|
|
53
|
+
|
|
54
|
+
# Rebuild shards
|
|
55
|
+
node scripts/build-boundary-stress-shard.mjs
|
|
56
|
+
|
|
57
|
+
# Corpus statistics
|
|
58
|
+
node scripts/corpus-stats.mjs
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Design
|
|
62
|
+
|
|
63
|
+
- **BIO (Begin/Inside/Outside) labeling** over SentencePiece tokens.
|
|
64
|
+
- **Character-offset aligned** — labels track the raw string, not the
|
|
65
|
+
normalized form, so the model learns real input distributions.
|
|
66
|
+
- **Source-homogeneous shards** — each shard comes from one source, ordered by
|
|
67
|
+
type, so eval splits are honest (no bleed between train and held-out).
|
|
68
|
+
|
|
69
|
+
## Related
|
|
70
|
+
|
|
71
|
+
- [`@mailwoman/neural`](../neural) — the runtime that loads and runs the trained model
|
|
72
|
+
- [`@mailwoman/neural-weights-en-us`](../neural-weights-en-us) — the trained model itself
|
|
73
|
+
- [Corpus Construction concepts](https://mailwoman.sister.software/articles/concepts/corpus-construction/)
|
|
74
|
+
- [Training Pipeline concepts](https://mailwoman.sister.software/articles/concepts/training-pipeline/)
|
|
75
|
+
- [CONTRIBUTING_MODEL_WORK](https://mailwoman.sister.software/articles/plan/CONTRIBUTING_MODEL_WORK/)
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
[AGPL-3.0-only](https://www.gnu.org/licenses/agpl-3.0.html)
|
|
@@ -18,8 +18,12 @@
|
|
|
18
18
|
* left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
|
|
19
19
|
* for Phase 1 the row's region is simply absent).
|
|
20
20
|
*
|
|
21
|
-
* License:
|
|
22
|
-
*
|
|
21
|
+
* License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
|
|
22
|
+
* attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
|
|
23
|
+
* allowed for training with attribution; the ODbL option's share-alike obligation would defeat
|
|
24
|
+
* the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
|
|
25
|
+
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
|
+
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
23
27
|
*
|
|
24
28
|
* The adapter is streaming-aware: it uses `csv-parse` in streaming mode so a 25M-row dump never
|
|
25
29
|
* sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAMH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAGjF,eAAO,MAAM,cAAc,QAAQ,CAAA;AA4CnC,wBAAgB,gBAAgB,IAAI,aAAa,CAwEhD;AAED,eAAO,MAAM,UAAU,eAAqB,CAAA"}
|
|
@@ -18,8 +18,12 @@
|
|
|
18
18
|
* left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
|
|
19
19
|
* for Phase 1 the row's region is simply absent).
|
|
20
20
|
*
|
|
21
|
-
* License:
|
|
22
|
-
*
|
|
21
|
+
* License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
|
|
22
|
+
* attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
|
|
23
|
+
* allowed for training with attribution; the ODbL option's share-alike obligation would defeat
|
|
24
|
+
* the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
|
|
25
|
+
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
|
+
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
23
27
|
*
|
|
24
28
|
* The adapter is streaming-aware: it uses `csv-parse` in streaming mode so a 25M-row dump never
|
|
25
29
|
* sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
|
|
@@ -64,7 +68,7 @@ function composeRaw(house, street, postcode, locality) {
|
|
|
64
68
|
export function createBanAdapter() {
|
|
65
69
|
return {
|
|
66
70
|
id: BAN_ADAPTER_ID,
|
|
67
|
-
defaultLicense: "
|
|
71
|
+
defaultLicense: "Licence Ouverte 2.0",
|
|
68
72
|
description: "Base Adresse Nationale (FR): house-number-level street addresses (~25M rows).",
|
|
69
73
|
async *rows(opts) {
|
|
70
74
|
if (opts.country && opts.country !== "FR") {
|
|
@@ -122,7 +126,7 @@ export function createBanAdapter() {
|
|
|
122
126
|
source: BAN_ADAPTER_ID,
|
|
123
127
|
source_id: sourceId,
|
|
124
128
|
corpus_version: "",
|
|
125
|
-
license: "
|
|
129
|
+
license: "Licence Ouverte 2.0",
|
|
126
130
|
};
|
|
127
131
|
emitted++;
|
|
128
132
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,KAAK,IAAI,QAAQ,EAAE,MAAM,WAAW,CAAA;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAErD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAEzD,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAA;AAenC;;;GAGG;AACH,SAAS,kBAAkB,CAAC,MAAc,EAAE,GAAW;IACtD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IACpB,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IACjB,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc,EAAE,QAAgB,EAAE,QAAgB;IACpF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACnE,IAAI,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACtC,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACtE,IAAI,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACpD,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,EAAE,EAAE,cAAc;QAClB,cAAc,EAAE,qBAAqB;QACrC,WAAW,EAAE,+EAA+E;QAE5F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YACrE,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACzB,QAAQ,CAAC;gBACR,SAAS,EAAE,GAAG;gBACd,OAAO,EAAE,IAAI;gBACb,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,IAAI;gBAClB,kBAAkB,EAAE,IAAI;aACxB,CAAC,CACF,CAAA;YAED,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAA+B,EAAE,CAAC;oBAC5D,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBAC/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,MAAK;oBAE5D,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAA;oBACvE,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAC7C,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAClD,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBAElD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAClC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEjC,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;oBAE5C,MAAM,UAAU,GAA+B,EAAE,CAAA;oBACjD,IAAI,KAAK;wBAAE,UAAU,CAAC,YAAY,GAAG,KAAK,CAAA;oBAC1C,IAAI,UAAU,CAAC,MAAM;wBAAE,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;oBACnE,IAAI,UAAU,CAAC,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;oBAC5D,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAC5C,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAE5C,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;oBACzD,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAElB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBACpD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAQ;oBAE/C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE;wBACjC,CAAC,CAAC,GAAG,cAAc,IAAI,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE;wBACzC,CAAC,CAAC,cAAc,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;oBAE1C,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,cAAc;wBACtB,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,qBAAqB;qBAC9B,CAAA;oBACD,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,MAAM,CAAC,OAAO,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA"}
|
|
@@ -46,8 +46,11 @@ export interface OpenaddressesAdapterOptions {
|
|
|
46
46
|
*/
|
|
47
47
|
defaultLicense?: string;
|
|
48
48
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
49
|
+
* Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a
|
|
50
|
+
* deliberate BUILD-level act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT
|
|
51
|
+
* a silent adapter default (#26 — "purposely exclude, don't opt in to include"). Set false only
|
|
52
|
+
* for an explicit adapter-scoped drop; the build-level `--exclude-share-alike` is the normal
|
|
53
|
+
* path.
|
|
51
54
|
*/
|
|
52
55
|
allowShareAlike?: boolean;
|
|
53
56
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAOH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEjF,eAAO,MAAM,wBAAwB,kBAAkB,CAAA;AACvD,eAAO,MAAM,6BAA6B,cAAc,CAAA;AA8CxD,MAAM,WAAW,2BAA2B;IAC3C;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CACzB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,IAAI,GAAE,2BAAgC,GAAG,aAAa,CAwFhG;AAED,eAAO,MAAM,oBAAoB,eAA+B,CAAA"}
|
|
@@ -39,6 +39,7 @@ import { createReadStream } from "node:fs";
|
|
|
39
39
|
import { createInterface } from "node:readline";
|
|
40
40
|
import { stableSourceId } from "../../adapter.js";
|
|
41
41
|
import { formatAddress, reconcileComponents } from "../../format.js";
|
|
42
|
+
import { SHARE_ALIKE_PATTERN } from "../../license.js";
|
|
42
43
|
export const OPENADDRESSES_ADAPTER_ID = "openaddresses";
|
|
43
44
|
export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
44
45
|
/** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
|
|
@@ -73,19 +74,13 @@ function parseFeatureLine(line) {
|
|
|
73
74
|
return null;
|
|
74
75
|
return normalizeProperties(obj.properties);
|
|
75
76
|
}
|
|
76
|
-
/**
|
|
77
|
-
* Licenses that require share-alike or create a copyleft obligation on derived works. Rows carrying
|
|
78
|
-
* any of these licenses are dropped by default because they would contaminate proprietary-weights
|
|
79
|
-
* training. Per the licensing strategy (#26).
|
|
80
|
-
*/
|
|
81
|
-
const SHARE_ALIKE_PATTERN = /^ODbL|^Open Database License|^CC-BY-SA|^CC-SA/i;
|
|
82
77
|
/**
|
|
83
78
|
* Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default
|
|
84
79
|
* fallback for dumps known to carry a single license throughout (e.g. a PDDL-only state slice).
|
|
85
80
|
*/
|
|
86
81
|
export function createOpenaddressesAdapter(opts = {}) {
|
|
87
82
|
const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE;
|
|
88
|
-
const allowShareAlike = opts.allowShareAlike ??
|
|
83
|
+
const allowShareAlike = opts.allowShareAlike ?? true;
|
|
89
84
|
return {
|
|
90
85
|
id: OPENADDRESSES_ADAPTER_ID,
|
|
91
86
|
defaultLicense,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/openaddresses/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AAGtD,MAAM,CAAC,MAAM,wBAAwB,GAAG,eAAe,CAAA;AACvD,MAAM,CAAC,MAAM,6BAA6B,GAAG,WAAW,CAAA;AAmBxD,0FAA0F;AAC1F,SAAS,mBAAmB,CAAC,GAAY;IACxC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAA;IAC9C,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;QACrE,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAA;aAC9C,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;IACjE,CAAC;IACD,OAAO,GAAmB,CAAA;AAC3B,CAAC;AAED,+FAA+F;AAC/F,SAAS,gBAAgB,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACpD,IAAI,MAAe,CAAA;IACnB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC7B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACtD,MAAM,GAAG,GAAG,MAAiD,CAAA;IAC7D,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAA;IACvC,OAAO,mBAAmB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;AAC3C,CAAC;AAoBD;;;GAGG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAAoC,EAAE;IAChF,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,6BAA6B,CAAA;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,OAAO;QACN,EAAE,EAAE,wBAAwB;QAC5B,cAAc;QACd,WAAW,EAAE,6EAA6E;QAE1F,KAAK,CAAC,CAAC,IAAI,CAAC,WAA2B;YACtC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CACd,0HAA0H,CAC1H,CAAA;YACF,CAAC;YACD,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAA;YAEnC,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;YAC5E,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;YAErE,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,iBAAiB,GAAG,CAAC,CAAA;YACzB,IAAI,CAAC;gBACJ,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBAChC,IAAI,WAAW,CAAC,MAAM,EAAE,OAAO;wBAAE,MAAK;oBACtC,IAAI,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,WAAW,CAAC,KAAK;wBAAE,MAAK;oBAE1E,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;oBACpC,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBAEpB,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACrC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACrC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;oBAE7C,iFAAiF;oBACjF,wDAAwD;oBACxD,IAAI,CAAC,MAAM;wBAAE,SAAQ;oBACrB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEhC,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,cAAc,CAAC,CAAC,IAAI,EAAE,CAAA;oBAEhE,IAAI,CAAC,eAAe,IAAI,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;wBAC3D,iBAAiB,EAAE,CAAA;wBACnB,SAAQ;oBACT,CAAC;oBAED,MAAM,UAAU,GAA+B,EAAE,CAAA;oBACjD,IAAI,WAAW;wBAAE,UAAU,CAAC,YAAY,GAAG,WAAW,CAAA;oBACtD,IAAI,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;oBACtC,IAAI,IAAI;wBAAE,UAAU,CAAC,IAAI,GAAG,IAAI,CAAA;oBAChC,IAAI,IAAI;wBAAE,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAA;oBACpC,IAAI,MAAM;wBAAE,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;oBACtC,IAAI,QAAQ;wBAAE,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;oBAE5C,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBACnE,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAElB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBACpD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAQ;oBAE/C,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,CAAA;oBAC3D,MAAM,QAAQ,GAAG,YAAY;wBAC5B,CAAC,CAAC,GAAG,wBAAwB,IAAI,YAAY,EAAE;wBAC/C,CAAC,CAAC,cAAc,CAAC,wBAAwB,EAAE,OAAO,CAAC,CAAA;oBAEpD,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO;wBACP,MAAM,EAAE,wBAAwB;wBAChC,SAAS,EAAE,QAAQ;wBACnB,cAAc,EAAE,EAAE;wBAClB,OAAO;qBACP,CAAA;oBACD,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,MAAM,CAAC,OAAO,EAAE,CAAA;gBAChB,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;oBAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,iBAAiB,8BAA8B,OAAO,SAAS,CAAC,CAAA;gBAC1G,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAA"}
|
package/out/src/build.d.ts
CHANGED
|
@@ -74,6 +74,14 @@ export interface BuildCorpusOptions {
|
|
|
74
74
|
rowsPerShard?: number;
|
|
75
75
|
/** Progress hook. Errors thrown abort the build. */
|
|
76
76
|
onProgress?: (stage: BuildStage, message: string) => void;
|
|
77
|
+
/**
|
|
78
|
+
* License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see
|
|
79
|
+
* `compileLicenseExcludes` / `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license`
|
|
80
|
+
* matches any is dropped at ingest. Default (omitted) includes EVERYTHING — exclusion is a
|
|
81
|
+
* deliberate act, not a silent default. A proprietary-weights build passes the share-alike set
|
|
82
|
+
* (`--exclude-share-alike`).
|
|
83
|
+
*/
|
|
84
|
+
excludeLicenses?: readonly RegExp[];
|
|
77
85
|
}
|
|
78
86
|
/** Top-level manifest tying every stage together. */
|
|
79
87
|
export interface BuildCorpusManifest {
|
|
@@ -91,6 +99,12 @@ export interface BuildCorpusManifest {
|
|
|
91
99
|
};
|
|
92
100
|
quarantine_count: number;
|
|
93
101
|
total_aligned_rows: number;
|
|
102
|
+
/**
|
|
103
|
+
* Resolved license set across all INCLUDED rows (license string → row count), + the count dropped
|
|
104
|
+
* by `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
|
|
105
|
+
*/
|
|
106
|
+
licenses: Record<string, number>;
|
|
107
|
+
excluded_by_license: number;
|
|
94
108
|
}
|
|
95
109
|
/**
|
|
96
110
|
* Drive the full corpus build to completion.
|
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AASH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF,2CAA2C;AAC3C,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF,gDAAgD;AAChD,MAAM,WAAW,kBAAkB;IAClC,kEAAkE;IAClE,SAAS,EAAE,MAAM,CAAA;IAEjB,0FAA0F;IAC1F,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C,yFAAyF;IACzF,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwLxF"}
|
package/out/src/build.js
CHANGED
|
@@ -52,6 +52,7 @@ import { join } from "node:path";
|
|
|
52
52
|
import { createInterface } from "node:readline";
|
|
53
53
|
import { defaultAdapterRegistry } from "./adapter.js";
|
|
54
54
|
import { alignRow } from "./align.js";
|
|
55
|
+
import { licenseExcluded } from "./license.js";
|
|
55
56
|
import { writeShards } from "./parquet.js";
|
|
56
57
|
import { runAdapter } from "./runner.js";
|
|
57
58
|
import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "./split.js";
|
|
@@ -127,12 +128,27 @@ export async function buildCorpus(opts) {
|
|
|
127
128
|
let quarantined = 0;
|
|
128
129
|
const counts = { train: 0, val: 0, test: 0 };
|
|
129
130
|
const holdouts = defaultHoldouts();
|
|
131
|
+
// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
|
|
132
|
+
// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
|
|
133
|
+
// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
|
|
134
|
+
const excludeLicenses = opts.excludeLicenses ?? [];
|
|
135
|
+
const licenseCounts = new Map();
|
|
136
|
+
let excludedByLicense = 0;
|
|
130
137
|
const writeQuarantine = (row, reason) => {
|
|
131
138
|
quarantineStream.write(`${JSON.stringify({ row, reason })}\n`);
|
|
132
139
|
};
|
|
133
140
|
for (const adapterRun of adapterRuns) {
|
|
134
141
|
opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`);
|
|
135
142
|
for await (const row of streamJsonl(adapterRun.jsonl_path)) {
|
|
143
|
+
licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1);
|
|
144
|
+
// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
|
|
145
|
+
// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
|
|
146
|
+
// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
|
|
147
|
+
// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
|
|
148
|
+
if (licenseExcluded(row.license, excludeLicenses)) {
|
|
149
|
+
excludedByLicense++;
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
136
152
|
const fanned = [row];
|
|
137
153
|
if (synthesize) {
|
|
138
154
|
for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
|
|
@@ -194,6 +210,13 @@ export async function buildCorpus(opts) {
|
|
|
194
210
|
corpusVersion: opts.corpusVersion,
|
|
195
211
|
rowsPerShard,
|
|
196
212
|
});
|
|
213
|
+
// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
|
|
214
|
+
// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
|
|
215
|
+
const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1]);
|
|
216
|
+
opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
|
|
217
|
+
(excludedByLicense > 0
|
|
218
|
+
? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
|
|
219
|
+
: " | NO license exclusion applied (all rows kept)"));
|
|
197
220
|
// 6. Top-level manifest.
|
|
198
221
|
opts.onProgress?.("manifest", "writing top-level MANIFEST.json");
|
|
199
222
|
const manifest = {
|
|
@@ -205,6 +228,8 @@ export async function buildCorpus(opts) {
|
|
|
205
228
|
shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
|
|
206
229
|
quarantine_count: quarantined,
|
|
207
230
|
total_aligned_rows: aligned,
|
|
231
|
+
licenses: Object.fromEntries(licenseSummary),
|
|
232
|
+
excluded_by_license: excludedByLicense,
|
|
208
233
|
};
|
|
209
234
|
await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
|
210
235
|
return manifest;
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACzG,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA+D/E;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QACrD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAC/E,SAAQ;QACT,CAAC;QACD,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QACxD,IACC,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG;YACpC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YACxB,SAAQ;QACT,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QACF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IACD,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IACD,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAC/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBACnB,SAAQ;YACT,CAAC;YACD,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBACvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,GAAa,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBACzE,WAAW,EAAE,CAAA;oBACb,SAAQ;gBACT,CAAC;gBACD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAC9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBACf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBACrC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC;QAAE,CAAC,CAAC,GAAG,EAAE,CAAA;IACtD,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAChD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IACpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAChE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACxG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC3D,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAClE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAM,CAAA;IAC/B,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
package/out/src/format.d.ts
CHANGED
|
@@ -3,77 +3,12 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Thin re-export of `@mailwoman/formatter`.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* `@fragaria/address-formatter` returns multi-line strings (newline-separated). This module honors
|
|
14
|
-
* that by default and exposes a `separator` option to coerce to single-line when an adapter wants
|
|
15
|
-
* `", "` joined output for the corpus.
|
|
16
|
-
*
|
|
17
|
-
* Limitations (documented, not blockers for Phase 1):
|
|
18
|
-
*
|
|
19
|
-
* - `unit`: OpenCage's vocabulary doesn't have a slot, so units are appended to the road line
|
|
20
|
-
* (`"Pennsylvania Ave NW Apt 4B"`). Phase 2 or beyond can change this if needed.
|
|
21
|
-
* - `intersection_a` / `intersection_b`: joined as `"<a> & <b>"` into the road field.
|
|
22
|
-
* - `cedex` (FR): folded into `postcode` (e.g. `"75008 CEDEX 08"`) so the FR template renders it in
|
|
23
|
-
* the right slot.
|
|
24
|
-
* - JP-specific tags (`prefecture`, `municipality`, ...): no mapping yet — left for Phase 6 when JP
|
|
25
|
-
* becomes a live locale.
|
|
26
|
-
*/
|
|
27
|
-
import type { ComponentTag } from "@mailwoman/core/types";
|
|
28
|
-
/** Options accepted by `formatAddress`. */
|
|
29
|
-
export interface FormatAddressOptions {
|
|
30
|
-
/**
|
|
31
|
-
* Append the country name as a final line (`"USA"`, `"France"`). Default `false`: most corpus
|
|
32
|
-
* rows are intra-country and the country prefix is redundant noise.
|
|
33
|
-
*/
|
|
34
|
-
appendCountry?: boolean;
|
|
35
|
-
/**
|
|
36
|
-
* Apply OpenCage's per-country abbreviation rules (e.g. `"Avenue"` → `"Ave"`, `"Boulevard"` →
|
|
37
|
-
* `"Blvd"`). Default `false`. Phase 1 prefers unabbreviated output because synthesis
|
|
38
|
-
* (`synthesize.ts`) handles abbreviation swaps as its own augmentation pass.
|
|
39
|
-
*/
|
|
40
|
-
abbreviate?: boolean;
|
|
41
|
-
/**
|
|
42
|
-
* Replace the template's newlines with this separator. Default `undefined` (keep newlines). Use
|
|
43
|
-
* `", "` to get a single-line output, or `" "` to remove all internal punctuation.
|
|
44
|
-
*/
|
|
45
|
-
separator?: string;
|
|
46
|
-
}
|
|
47
|
-
type ComponentDict = Partial<Record<ComponentTag, string>>;
|
|
48
|
-
/**
|
|
49
|
-
* After rendering, drop any component whose value isn't actually present in the formatted `raw`.
|
|
50
|
-
* OpenCage's per-country templates legitimately omit some inputs:
|
|
51
|
-
*
|
|
52
|
-
* - **FR**: regions are absorbed by the postcode and rarely render verbatim (`"75008 Paris, France"`
|
|
53
|
-
* — `"Île-de-France"` is dropped).
|
|
54
|
-
* - **US**: state names are abbreviated (`"Oregon"` → `"OR"`); the verbatim form may not survive.
|
|
55
|
-
*
|
|
56
|
-
* Alignment downstream requires `components[tag]` to occur in `raw` (within fuzzy-match tolerance).
|
|
57
|
-
* If a component was dropped or transformed beyond Levenshtein reach, keeping it in the dict
|
|
58
|
-
* guarantees a quarantine reject. Reconciliation prunes the dropouts up front so the row arrives at
|
|
59
|
-
* alignment with a self-consistent (`raw`, `components`) pair.
|
|
60
|
-
*
|
|
61
|
-
* Comparison is case- and whitespace-insensitive. The retained value in `components` is the
|
|
62
|
-
* **original** input — alignment will fuzzy-match it against the raw character span.
|
|
63
|
-
*/
|
|
64
|
-
export declare function reconcileComponents(components: ComponentDict, raw: string): ComponentDict;
|
|
65
|
-
/**
|
|
66
|
-
* Render a component dict into an idiomatic per-country address string.
|
|
67
|
-
*
|
|
68
|
-
* Returns an empty string if `components` is empty after translation. Throws nothing — bad inputs
|
|
69
|
-
* (empty dict, unsupported tag) silently degrade to the longest meaningful prefix.
|
|
70
|
-
*/
|
|
71
|
-
export declare function formatAddress(components: ComponentDict, country: string, opts?: FormatAddressOptions): string;
|
|
72
|
-
/**
|
|
73
|
-
* Translate a `ComponentTag` dict to the OpenCage vocabulary that `@fragaria/address-formatter`
|
|
74
|
-
* expects. Exported for testing and for adapters that want to pre-build the dict for batch
|
|
75
|
-
* formatting.
|
|
8
|
+
* The formatter implementation moved to its own workspace (`@mailwoman/formatter`) so the parser,
|
|
9
|
+
* the corpus pipeline, and the record matcher can share one locale-aware renderer plus the
|
|
10
|
+
* canonical match key. This module stays put as the stable `@mailwoman/corpus/format` entry point
|
|
11
|
+
* the synthesis adapters import `formatAddress` / `reconcileComponents` from.
|
|
76
12
|
*/
|
|
77
|
-
export
|
|
78
|
-
export {};
|
|
13
|
+
export * from "@mailwoman/formatter";
|
|
79
14
|
//# sourceMappingURL=format.d.ts.map
|
package/out/src/format.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/format.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/format.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,cAAc,sBAAsB,CAAA"}
|
package/out/src/format.js
CHANGED
|
@@ -3,149 +3,12 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
6
|
+
* Thin re-export of `@mailwoman/formatter`.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* `@fragaria/address-formatter` returns multi-line strings (newline-separated). This module honors
|
|
14
|
-
* that by default and exposes a `separator` option to coerce to single-line when an adapter wants
|
|
15
|
-
* `", "` joined output for the corpus.
|
|
16
|
-
*
|
|
17
|
-
* Limitations (documented, not blockers for Phase 1):
|
|
18
|
-
*
|
|
19
|
-
* - `unit`: OpenCage's vocabulary doesn't have a slot, so units are appended to the road line
|
|
20
|
-
* (`"Pennsylvania Ave NW Apt 4B"`). Phase 2 or beyond can change this if needed.
|
|
21
|
-
* - `intersection_a` / `intersection_b`: joined as `"<a> & <b>"` into the road field.
|
|
22
|
-
* - `cedex` (FR): folded into `postcode` (e.g. `"75008 CEDEX 08"`) so the FR template renders it in
|
|
23
|
-
* the right slot.
|
|
24
|
-
* - JP-specific tags (`prefecture`, `municipality`, ...): no mapping yet — left for Phase 6 when JP
|
|
25
|
-
* becomes a live locale.
|
|
26
|
-
*/
|
|
27
|
-
import addressFormatter from "@fragaria/address-formatter";
|
|
28
|
-
/**
|
|
29
|
-
* After rendering, drop any component whose value isn't actually present in the formatted `raw`.
|
|
30
|
-
* OpenCage's per-country templates legitimately omit some inputs:
|
|
31
|
-
*
|
|
32
|
-
* - **FR**: regions are absorbed by the postcode and rarely render verbatim (`"75008 Paris, France"`
|
|
33
|
-
* — `"Île-de-France"` is dropped).
|
|
34
|
-
* - **US**: state names are abbreviated (`"Oregon"` → `"OR"`); the verbatim form may not survive.
|
|
35
|
-
*
|
|
36
|
-
* Alignment downstream requires `components[tag]` to occur in `raw` (within fuzzy-match tolerance).
|
|
37
|
-
* If a component was dropped or transformed beyond Levenshtein reach, keeping it in the dict
|
|
38
|
-
* guarantees a quarantine reject. Reconciliation prunes the dropouts up front so the row arrives at
|
|
39
|
-
* alignment with a self-consistent (`raw`, `components`) pair.
|
|
40
|
-
*
|
|
41
|
-
* Comparison is case- and whitespace-insensitive. The retained value in `components` is the
|
|
42
|
-
* **original** input — alignment will fuzzy-match it against the raw character span.
|
|
43
|
-
*/
|
|
44
|
-
export function reconcileComponents(components, raw) {
|
|
45
|
-
const haystack = raw.toLowerCase().replace(/\s+/g, " ");
|
|
46
|
-
const out = {};
|
|
47
|
-
for (const [k, v] of Object.entries(components)) {
|
|
48
|
-
if (!v)
|
|
49
|
-
continue;
|
|
50
|
-
const needle = v.toLowerCase().replace(/\s+/g, " ");
|
|
51
|
-
if (haystack.includes(needle)) {
|
|
52
|
-
out[k] = v;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return out;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Render a component dict into an idiomatic per-country address string.
|
|
59
|
-
*
|
|
60
|
-
* Returns an empty string if `components` is empty after translation. Throws nothing — bad inputs
|
|
61
|
-
* (empty dict, unsupported tag) silently degrade to the longest meaningful prefix.
|
|
62
|
-
*/
|
|
63
|
-
export function formatAddress(components, country, opts = {}) {
|
|
64
|
-
const ocComponents = toOpenCageComponents(components, country);
|
|
65
|
-
if (Object.keys(ocComponents).length === 0)
|
|
66
|
-
return "";
|
|
67
|
-
const raw = addressFormatter.format(ocComponents, {
|
|
68
|
-
abbreviate: opts.abbreviate ?? false,
|
|
69
|
-
appendCountry: opts.appendCountry ?? false,
|
|
70
|
-
});
|
|
71
|
-
const trimmed = raw.replace(/\s+$/g, "");
|
|
72
|
-
return opts.separator !== undefined ? trimmed.replace(/\n+/g, opts.separator) : trimmed;
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Translate a `ComponentTag` dict to the OpenCage vocabulary that `@fragaria/address-formatter`
|
|
76
|
-
* expects. Exported for testing and for adapters that want to pre-build the dict for batch
|
|
77
|
-
* formatting.
|
|
78
|
-
*/
|
|
79
|
-
export function toOpenCageComponents(components, country) {
|
|
80
|
-
const out = {};
|
|
81
|
-
const road = composeRoad(components);
|
|
82
|
-
if (road)
|
|
83
|
-
out.road = road;
|
|
84
|
-
if (components.house_number)
|
|
85
|
-
out.house_number = components.house_number;
|
|
86
|
-
if (components.venue)
|
|
87
|
-
out.house = components.venue;
|
|
88
|
-
if (components.locality)
|
|
89
|
-
out.city = components.locality;
|
|
90
|
-
if (components.dependent_locality)
|
|
91
|
-
out.suburb = components.dependent_locality;
|
|
92
|
-
if (components.subregion)
|
|
93
|
-
out.county = components.subregion;
|
|
94
|
-
if (components.region)
|
|
95
|
-
out.state = components.region;
|
|
96
|
-
const postcode = composePostcode(components);
|
|
97
|
-
if (postcode)
|
|
98
|
-
out.postcode = postcode;
|
|
99
|
-
if (components.po_box)
|
|
100
|
-
out.po_box = components.po_box;
|
|
101
|
-
if (components.attention)
|
|
102
|
-
out.attention = components.attention;
|
|
103
|
-
if (components.country)
|
|
104
|
-
out.country = components.country;
|
|
105
|
-
// country_code drives template selection, not output. Only emit it if at least one other
|
|
106
|
-
// component is present — otherwise the FR/US templates render the bare code as a fallback
|
|
107
|
-
// line ("US"), which is never what a corpus consumer wants.
|
|
108
|
-
const cc = country.trim().toLowerCase();
|
|
109
|
-
if (cc && Object.keys(out).length > 0)
|
|
110
|
-
out.country_code = cc;
|
|
111
|
-
return out;
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Build the `road` line from prefix / particle / street / suffix / unit / intersection components.
|
|
115
|
-
* Order:
|
|
116
|
-
*
|
|
117
|
-
* ```
|
|
118
|
-
* [intersection_a & intersection_b]
|
|
119
|
-
* OR
|
|
120
|
-
* [street_prefix] [street_prefix_particle] [street] [street_suffix] [unit]
|
|
121
|
-
* ```
|
|
122
|
-
*/
|
|
123
|
-
function composeRoad(components) {
|
|
124
|
-
if (components.intersection_a && components.intersection_b) {
|
|
125
|
-
return `${components.intersection_a} & ${components.intersection_b}`;
|
|
126
|
-
}
|
|
127
|
-
const parts = [];
|
|
128
|
-
if (components.street_prefix)
|
|
129
|
-
parts.push(components.street_prefix);
|
|
130
|
-
if (components.street_prefix_particle)
|
|
131
|
-
parts.push(components.street_prefix_particle);
|
|
132
|
-
if (components.street)
|
|
133
|
-
parts.push(components.street);
|
|
134
|
-
if (components.street_suffix)
|
|
135
|
-
parts.push(components.street_suffix);
|
|
136
|
-
if (components.unit)
|
|
137
|
-
parts.push(components.unit);
|
|
138
|
-
return parts.join(" ").replace(/\s+/g, " ").trim();
|
|
139
|
-
}
|
|
140
|
-
/**
|
|
141
|
-
* Fold CEDEX into postcode for FR-style output: `"75008"` + cedex `"CEDEX 08"` → `"75008 CEDEX
|
|
142
|
-
* 08"`. If only one is present, return it. If neither, return empty.
|
|
8
|
+
* The formatter implementation moved to its own workspace (`@mailwoman/formatter`) so the parser,
|
|
9
|
+
* the corpus pipeline, and the record matcher can share one locale-aware renderer plus the
|
|
10
|
+
* canonical match key. This module stays put as the stable `@mailwoman/corpus/format` entry point
|
|
11
|
+
* the synthesis adapters import `formatAddress` / `reconcileComponents` from.
|
|
143
12
|
*/
|
|
144
|
-
|
|
145
|
-
const base = components.postcode?.trim() ?? "";
|
|
146
|
-
const cedex = components.cedex?.trim() ?? "";
|
|
147
|
-
if (base && cedex)
|
|
148
|
-
return `${base} ${cedex}`.replace(/\s+/g, " ");
|
|
149
|
-
return base || cedex;
|
|
150
|
-
}
|
|
13
|
+
export * from "@mailwoman/formatter";
|
|
151
14
|
//# sourceMappingURL=format.js.map
|
package/out/src/format.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/format.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/format.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,cAAc,sBAAsB,CAAA"}
|