@mailwoman/corpus 9.3.0 → 9.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/recipes/anchor-absorption.ts +2 -2
- package/lib/recipes/bare-country.ts +2 -2
- package/lib/recipes/boundary-stress.ts +3 -1
- package/lib/recipes/country-balanced.ts +2 -1
- package/lib/recipes/cz-pcfirst-preposition.ts +2 -1
- package/lib/recipes/fr-admin-split.ts +2 -1
- package/lib/recipes/fr-bare-street.ts +2 -1
- package/lib/recipes/fr-fragment.ts +2 -1
- package/lib/recipes/fr-lieudit.ts +2 -2
- package/lib/recipes/fr-order.ts +2 -1
- package/lib/recipes/german.ts +2 -1
- package/lib/recipes/house-venue.ts +2 -1
- package/lib/recipes/intersection.ts +2 -1
- package/lib/recipes/locale.ts +2 -1
- package/lib/recipes/nl-postcode.ts +3 -1
- package/lib/recipes/no-fragment.ts +2 -8
- package/lib/recipes/no-street-led.ts +2 -8
- package/lib/recipes/no-street.ts +2 -1
- package/lib/recipes/po-box-cedex.ts +1 -1
- package/lib/recipes/po-box.ts +2 -1
- package/lib/recipes/scaffold.ts +0 -1
- package/lib/recipes/si-bare-village.ts +3 -1
- package/lib/recipes/street-affix.ts +3 -2
- package/lib/recipes/street-bare.ts +3 -1
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub-venue-sources.ts +2 -1
- package/lib/recipes/sub-venue.ts +2 -1
- package/lib/recipes/trailing-region.ts +3 -8
- package/lib/recipes/unit.ts +2 -2
- package/lib/tools/fetch/acra-sg.ts +174 -0
- package/lib/tools/fetch/download.ts +217 -2
- package/lib/tools/fetch/gcis-tw.ts +175 -0
- package/lib/tools/fetch/houjin-jp.ts +135 -0
- package/lib/tools/fetch/index.ts +27 -0
- package/lib/tools/fetch/juso-kr.ts +165 -0
- package/lib/tools/fetch/localdata-kr.ts +194 -0
- package/lib/tools/golden-relabel-street.ts +6 -3
- package/out/recipes/anchor-absorption.d.ts.map +1 -1
- package/out/recipes/anchor-absorption.js +2 -1
- package/out/recipes/anchor-absorption.js.map +1 -1
- package/out/recipes/bare-country.d.ts.map +1 -1
- package/out/recipes/bare-country.js +2 -1
- package/out/recipes/bare-country.js.map +1 -1
- package/out/recipes/boundary-stress.d.ts.map +1 -1
- package/out/recipes/boundary-stress.js +2 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/country-balanced.d.ts.map +1 -1
- package/out/recipes/country-balanced.js +2 -1
- package/out/recipes/country-balanced.js.map +1 -1
- package/out/recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/recipes/cz-pcfirst-preposition.js +2 -1
- package/out/recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/recipes/fr-admin-split.d.ts +1 -1
- package/out/recipes/fr-admin-split.d.ts.map +1 -1
- package/out/recipes/fr-admin-split.js +1 -1
- package/out/recipes/fr-admin-split.js.map +1 -1
- package/out/recipes/fr-bare-street.d.ts.map +1 -1
- package/out/recipes/fr-bare-street.js +2 -1
- package/out/recipes/fr-bare-street.js.map +1 -1
- package/out/recipes/fr-fragment.d.ts.map +1 -1
- package/out/recipes/fr-fragment.js +2 -1
- package/out/recipes/fr-fragment.js.map +1 -1
- package/out/recipes/fr-lieudit.d.ts +1 -1
- package/out/recipes/fr-lieudit.d.ts.map +1 -1
- package/out/recipes/fr-lieudit.js +1 -1
- package/out/recipes/fr-lieudit.js.map +1 -1
- package/out/recipes/fr-order.d.ts.map +1 -1
- package/out/recipes/fr-order.js +2 -1
- package/out/recipes/fr-order.js.map +1 -1
- package/out/recipes/german.d.ts.map +1 -1
- package/out/recipes/german.js +2 -1
- package/out/recipes/german.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +2 -1
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/intersection.d.ts.map +1 -1
- package/out/recipes/intersection.js +2 -1
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/locale.d.ts +1 -1
- package/out/recipes/locale.d.ts.map +1 -1
- package/out/recipes/locale.js +1 -1
- package/out/recipes/locale.js.map +1 -1
- package/out/recipes/nl-postcode.d.ts.map +1 -1
- package/out/recipes/nl-postcode.js +2 -1
- package/out/recipes/nl-postcode.js.map +1 -1
- package/out/recipes/no-fragment.d.ts.map +1 -1
- package/out/recipes/no-fragment.js +2 -1
- package/out/recipes/no-fragment.js.map +1 -1
- package/out/recipes/no-street-led.d.ts.map +1 -1
- package/out/recipes/no-street-led.js +2 -1
- package/out/recipes/no-street-led.js.map +1 -1
- package/out/recipes/no-street.d.ts.map +1 -1
- package/out/recipes/no-street.js +2 -1
- package/out/recipes/no-street.js.map +1 -1
- package/out/recipes/po-box-cedex.d.ts.map +1 -1
- package/out/recipes/po-box-cedex.js +2 -1
- package/out/recipes/po-box-cedex.js.map +1 -1
- package/out/recipes/po-box.d.ts.map +1 -1
- package/out/recipes/po-box.js +2 -1
- package/out/recipes/po-box.js.map +1 -1
- package/out/recipes/scaffold.d.ts +0 -1
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +0 -12
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/si-bare-village.d.ts.map +1 -1
- package/out/recipes/si-bare-village.js +2 -1
- package/out/recipes/si-bare-village.js.map +1 -1
- package/out/recipes/street-affix.d.ts.map +1 -1
- package/out/recipes/street-affix.js +3 -3
- package/out/recipes/street-affix.js.map +1 -1
- package/out/recipes/street-bare.d.ts.map +1 -1
- package/out/recipes/street-bare.js +2 -1
- package/out/recipes/street-bare.js.map +1 -1
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +2 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/recipes/sub-venue-sources.js +2 -1
- package/out/recipes/sub-venue-sources.js.map +1 -1
- package/out/recipes/sub-venue.d.ts.map +1 -1
- package/out/recipes/sub-venue.js +2 -1
- package/out/recipes/sub-venue.js.map +1 -1
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +2 -1
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.d.ts.map +1 -1
- package/out/recipes/unit.js +2 -1
- package/out/recipes/unit.js.map +1 -1
- package/out/tools/fetch/acra-sg.d.ts +23 -0
- package/out/tools/fetch/acra-sg.d.ts.map +1 -0
- package/out/tools/fetch/acra-sg.js +126 -0
- package/out/tools/fetch/acra-sg.js.map +1 -0
- package/out/tools/fetch/download.d.ts +65 -0
- package/out/tools/fetch/download.d.ts.map +1 -1
- package/out/tools/fetch/download.js +160 -2
- package/out/tools/fetch/download.js.map +1 -1
- package/out/tools/fetch/gcis-tw.d.ts +23 -0
- package/out/tools/fetch/gcis-tw.d.ts.map +1 -0
- package/out/tools/fetch/gcis-tw.js +135 -0
- package/out/tools/fetch/gcis-tw.js.map +1 -0
- package/out/tools/fetch/houjin-jp.d.ts +24 -0
- package/out/tools/fetch/houjin-jp.d.ts.map +1 -0
- package/out/tools/fetch/houjin-jp.js +102 -0
- package/out/tools/fetch/houjin-jp.js.map +1 -0
- package/out/tools/fetch/index.d.ts +27 -0
- package/out/tools/fetch/index.d.ts.map +1 -1
- package/out/tools/fetch/index.js +27 -0
- package/out/tools/fetch/index.js.map +1 -1
- package/out/tools/fetch/juso-kr.d.ts +30 -0
- package/out/tools/fetch/juso-kr.d.ts.map +1 -0
- package/out/tools/fetch/juso-kr.js +120 -0
- package/out/tools/fetch/juso-kr.js.map +1 -0
- package/out/tools/fetch/localdata-kr.d.ts +30 -0
- package/out/tools/fetch/localdata-kr.d.ts.map +1 -0
- package/out/tools/fetch/localdata-kr.js +139 -0
- package/out/tools/fetch/localdata-kr.js.map +1 -0
- package/out/tools/golden-relabel-street.d.ts.map +1 -1
- package/out/tools/golden-relabel-street.js +1 -2
- package/out/tools/golden-relabel-street.js.map +1 -1
- package/package.json +10 -10
- package/lib/name-prone-us-suffixes.ts +0 -12
- package/out/name-prone-us-suffixes.d.ts +0 -12
- package/out/name-prone-us-suffixes.d.ts.map +0 -1
- package/out/name-prone-us-suffixes.js +0 -12
- package/out/name-prone-us-suffixes.js.map +0 -1
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
|
|
15
15
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
16
16
|
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
17
|
+
import { makeLcg } from "@mailwoman/core/utils"
|
|
17
18
|
import type { PathBuilderLike } from "path-ts"
|
|
18
19
|
|
|
19
|
-
import {
|
|
20
|
+
import { sliceSourceID, type CorpusRecipe } from "#recipes/scaffold"
|
|
20
21
|
import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption"
|
|
21
22
|
import { alignRow } from "#utils"
|
|
22
|
-
|
|
23
23
|
/**
|
|
24
24
|
* The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
|
|
25
25
|
* helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
|
|
@@ -18,9 +18,9 @@
|
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
20
|
import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country"
|
|
21
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
21
22
|
|
|
22
|
-
import { alignAndWrite,
|
|
23
|
-
|
|
23
|
+
import { alignAndWrite, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
24
24
|
/**
|
|
25
25
|
* Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
|
|
26
26
|
*/
|
|
@@ -12,7 +12,9 @@
|
|
|
12
12
|
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import {
|
|
15
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
16
|
+
|
|
17
|
+
import { type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
16
18
|
import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
|
|
17
19
|
import { alignRow } from "#utils"
|
|
18
20
|
|
|
@@ -28,10 +28,11 @@ import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
|
|
|
28
28
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
29
29
|
import { isPresent } from "@mailwoman/core/objects"
|
|
30
30
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
31
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
31
32
|
import type { PathBuilderLike } from "path-ts"
|
|
32
33
|
|
|
33
34
|
import { stableSourceID } from "#adapters/utils"
|
|
34
|
-
import {
|
|
35
|
+
import { readOATuples, type CorpusRecipe } from "#recipes/scaffold"
|
|
35
36
|
import { pick } from "#synthesizers/utils"
|
|
36
37
|
import type { CanonicalRow } from "#types"
|
|
37
38
|
import { alignRow } from "#utils"
|
|
@@ -14,8 +14,9 @@
|
|
|
14
14
|
* cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
-
import {
|
|
17
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
18
18
|
|
|
19
|
+
import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
19
20
|
/**
|
|
20
21
|
* The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
|
|
21
22
|
* `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
|
|
@@ -33,10 +33,11 @@
|
|
|
33
33
|
import { departementForCodePostal } from "@mailwoman/codex/fr"
|
|
34
34
|
import { tempRootPath } from "@mailwoman/core/data-root"
|
|
35
35
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
36
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
36
37
|
import { CSVSpliterator, Delimiters } from "spliterator"
|
|
37
38
|
|
|
38
39
|
import { stableSourceID } from "#adapters/utils"
|
|
39
|
-
import {
|
|
40
|
+
import type { CorpusRecipe } from "#recipes/scaffold"
|
|
40
41
|
import type { CanonicalRow } from "#types"
|
|
41
42
|
import { alignRow } from "#utils"
|
|
42
43
|
|
|
@@ -21,9 +21,10 @@
|
|
|
21
21
|
*/
|
|
22
22
|
|
|
23
23
|
import { FR_VOIE_TYPES } from "@mailwoman/codex/fr"
|
|
24
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
24
25
|
|
|
25
26
|
import { decomposeFrStreet } from "#adapters/ban/street-decompose"
|
|
26
|
-
import { alignAndWrite,
|
|
27
|
+
import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
27
28
|
|
|
28
29
|
/**
|
|
29
30
|
* Canonical voie type (lowercase, accent-kept) → its most common written abbreviation, from the codex table's first
|
|
@@ -68,10 +68,11 @@
|
|
|
68
68
|
* blow-up). Disable that mask for any run including this slice.
|
|
69
69
|
*/
|
|
70
70
|
|
|
71
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
71
72
|
import { TextSpliterator } from "spliterator"
|
|
72
73
|
|
|
73
74
|
import { decomposeFrStreet } from "#adapters/ban/street-decompose"
|
|
74
|
-
import { alignAndWrite,
|
|
75
|
+
import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
75
76
|
import { pick } from "#synthesizers/utils"
|
|
76
77
|
|
|
77
78
|
/**
|
|
@@ -30,15 +30,15 @@ import { dataRootPath } from "@mailwoman/core/data-root"
|
|
|
30
30
|
import { readDirectory } from "@mailwoman/core/fs/readers"
|
|
31
31
|
import { isPresent } from "@mailwoman/core/objects"
|
|
32
32
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
33
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
33
34
|
import { join, type PathBuilderLike } from "path-ts"
|
|
34
35
|
|
|
35
36
|
import { decomposeFrStreet } from "#adapters/ban/street-decompose"
|
|
36
37
|
import { stableSourceID } from "#adapters/utils"
|
|
37
|
-
import {
|
|
38
|
+
import type { CorpusRecipe } from "#recipes/scaffold"
|
|
38
39
|
import { pick } from "#synthesizers/utils"
|
|
39
40
|
import type { CanonicalRow } from "#types"
|
|
40
41
|
import { alignRow } from "#utils"
|
|
41
|
-
|
|
42
42
|
/**
|
|
43
43
|
* Matches the `ban` adapter's Tier-B election for BAN data.
|
|
44
44
|
*/
|
package/lib/recipes/fr-order.ts
CHANGED
|
@@ -33,9 +33,10 @@
|
|
|
33
33
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
34
34
|
import { isPresent } from "@mailwoman/core/objects"
|
|
35
35
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
36
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
36
37
|
|
|
37
38
|
import { stableSourceID } from "#adapters/utils"
|
|
38
|
-
import {
|
|
39
|
+
import { readOATuples, type CorpusRecipe } from "#recipes/scaffold"
|
|
39
40
|
import { pick } from "#synthesizers/utils"
|
|
40
41
|
import type { CanonicalRow } from "#types"
|
|
41
42
|
import { alignRow } from "#utils"
|
package/lib/recipes/german.ts
CHANGED
|
@@ -27,12 +27,13 @@
|
|
|
27
27
|
|
|
28
28
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
29
29
|
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
30
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
30
31
|
import type { WOFDatabase } from "@mailwoman/resolver-wof-sqlite/schema"
|
|
31
32
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
32
33
|
import type { PathBuilderLike } from "path-ts"
|
|
33
34
|
|
|
34
35
|
import { stableSourceID } from "#adapters/utils"
|
|
35
|
-
import {
|
|
36
|
+
import { readZippedCSVRecords, type CorpusRecipe } from "#recipes/scaffold"
|
|
36
37
|
import { synthesizeGermanRow, type LocaleBaseTuple } from "#synthesizers/german"
|
|
37
38
|
import { alignRow } from "#utils"
|
|
38
39
|
|
|
@@ -40,11 +40,12 @@ import { pathExists } from "@mailwoman/core/fs/readers"
|
|
|
40
40
|
import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
|
|
41
41
|
import { repoRootPath } from "@mailwoman/core/paths"
|
|
42
42
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
43
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
43
44
|
import type { PathBuilderLike } from "path-ts"
|
|
44
45
|
import { JSONSpliterator } from "spliterator"
|
|
45
46
|
|
|
46
47
|
import { stableSourceID } from "#adapters/utils"
|
|
47
|
-
import {
|
|
48
|
+
import { readZippedCSVRecords, type CorpusRecipe } from "#recipes/scaffold"
|
|
48
49
|
import { pick, weightedPick } from "#synthesizers/utils"
|
|
49
50
|
import type { CanonicalRow, LabeledRow } from "#types"
|
|
50
51
|
import { alignRow } from "#utils"
|
package/lib/recipes/locale.ts
CHANGED
|
@@ -30,11 +30,12 @@ import { dataRootPath } from "@mailwoman/core/data-root"
|
|
|
30
30
|
import { openReadStream } from "@mailwoman/core/fs/streams"
|
|
31
31
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
32
32
|
import { isPresent } from "@mailwoman/core/objects"
|
|
33
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
33
34
|
import type { PathBuilderLike } from "path-ts"
|
|
34
35
|
import { CSVSpliterator } from "spliterator"
|
|
35
36
|
|
|
36
37
|
import { stableSourceID } from "#adapters/utils"
|
|
37
|
-
import {
|
|
38
|
+
import type { CorpusRecipe } from "#recipes/scaffold"
|
|
38
39
|
import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "#synthesizers/german"
|
|
39
40
|
import { pick } from "#synthesizers/utils"
|
|
40
41
|
import { alignRow } from "#utils"
|
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
* keep polarity balanced (the v1.9.9 lesson).
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import {
|
|
21
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
22
|
+
|
|
23
|
+
import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
22
24
|
|
|
23
25
|
/**
|
|
24
26
|
* "1012LG" → "1012 LG". The tuples carry the unspaced OA form; the spaced form is the failing case.
|
|
@@ -35,16 +35,10 @@
|
|
|
35
35
|
* list. Diacritic-KEEPING normalizer, matching the board (see the norm docstring).
|
|
36
36
|
*/
|
|
37
37
|
|
|
38
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
38
39
|
import { TextSpliterator } from "spliterator"
|
|
39
40
|
|
|
40
|
-
import {
|
|
41
|
-
alignAndWrite,
|
|
42
|
-
foldNOSurface,
|
|
43
|
-
makeMulberry32,
|
|
44
|
-
readTuples,
|
|
45
|
-
type CorpusRecipe,
|
|
46
|
-
sliceSourceID,
|
|
47
|
-
} from "#recipes/scaffold"
|
|
41
|
+
import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
48
42
|
|
|
49
43
|
/**
|
|
50
44
|
* Title-case a Kartverket ALL-CAPS locality (HELLVIK -> Hellvik); #690, all-caps is OOD.
|
|
@@ -16,16 +16,10 @@
|
|
|
16
16
|
* 3. pc-first "«pc» «city», «st» «n»" (the 7% floor — the anchor the others converge to)
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
19
20
|
import { TextSpliterator } from "spliterator"
|
|
20
21
|
|
|
21
|
-
import {
|
|
22
|
-
alignAndWrite,
|
|
23
|
-
foldNOSurface,
|
|
24
|
-
makeMulberry32,
|
|
25
|
-
readTuples,
|
|
26
|
-
type CorpusRecipe,
|
|
27
|
-
sliceSourceID,
|
|
28
|
-
} from "#recipes/scaffold"
|
|
22
|
+
import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
29
23
|
|
|
30
24
|
/**
|
|
31
25
|
* Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
package/lib/recipes/no-street.ts
CHANGED
|
@@ -38,11 +38,11 @@ import { isPOBox } from "@mailwoman/codex/us"
|
|
|
38
38
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
39
39
|
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
40
40
|
import { readZipEntry } from "@mailwoman/core/fs/zip"
|
|
41
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
41
42
|
import type { PathBuilderLike } from "path-ts"
|
|
42
43
|
import { TextSpliterator, TSVSpliterator } from "spliterator"
|
|
43
44
|
|
|
44
45
|
import {
|
|
45
|
-
makeMulberry32,
|
|
46
46
|
readCSVRecords,
|
|
47
47
|
readZippedCSVRecords,
|
|
48
48
|
sliceSourceID,
|
package/lib/recipes/po-box.ts
CHANGED
package/lib/recipes/scaffold.ts
CHANGED
|
@@ -79,7 +79,6 @@ export interface SliceTuple {
|
|
|
79
79
|
* A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
|
|
80
80
|
* seed), or `--seed N` stops being byte-reproducible.
|
|
81
81
|
*/
|
|
82
|
-
export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils"
|
|
83
82
|
|
|
84
83
|
/**
|
|
85
84
|
* One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
|
|
@@ -24,7 +24,9 @@
|
|
|
24
24
|
* trailing «V» = locality (the binding the resolver needs).
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
|
-
import {
|
|
27
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
28
|
+
|
|
29
|
+
import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
28
30
|
|
|
29
31
|
/**
|
|
30
32
|
* Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
@@ -35,14 +35,15 @@ import {
|
|
|
35
35
|
matchTrailingSuffix,
|
|
36
36
|
renderDirectional,
|
|
37
37
|
US_STREET_SUFFIX_PREFERRED_ABBR,
|
|
38
|
+
NAME_PRONE_US_SUFFIXES,
|
|
38
39
|
} from "@mailwoman/codex/us"
|
|
39
40
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
40
41
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
42
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
41
43
|
import type { PathBuilderLike } from "path-ts"
|
|
42
44
|
|
|
43
45
|
import { stableSourceID } from "#adapters/utils"
|
|
44
|
-
import {
|
|
45
|
-
import { makeMulberry32, readCSVRecords, readOATuples, sliceSourceID, type CorpusRecipe } from "#recipes/scaffold"
|
|
46
|
+
import { readCSVRecords, readOATuples, sliceSourceID, type CorpusRecipe } from "#recipes/scaffold"
|
|
46
47
|
import { pick } from "#synthesizers/utils"
|
|
47
48
|
import type { CanonicalRow } from "#types"
|
|
48
49
|
import { alignRow } from "#utils"
|
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
* reproduces the legacy run byte-for-byte.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import {
|
|
20
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
21
|
+
|
|
22
|
+
import { sliceSourceID, type CanonicalSliceRow, type CorpusRecipe } from "#recipes/scaffold"
|
|
21
23
|
import { DEFAULT_US_BASES } from "#synthesizers/intersection"
|
|
22
24
|
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
23
25
|
import { alignRow } from "#utils"
|
package/lib/recipes/street.ts
CHANGED
|
@@ -7,8 +7,10 @@
|
|
|
7
7
|
* {@link synthesizeStreetRow} → aligned LabeledRow. Ported from scripts/build-street-slice.mjs.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import { makeLcg } from "@mailwoman/core/utils"
|
|
11
|
+
|
|
10
12
|
import { stableSourceID } from "#adapters/utils"
|
|
11
|
-
import { alignAndWrite,
|
|
13
|
+
import { alignAndWrite, readTuples, type CorpusRecipe } from "#recipes/scaffold"
|
|
12
14
|
import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
|
|
13
15
|
|
|
14
16
|
/**
|
|
@@ -28,12 +28,13 @@
|
|
|
28
28
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
29
29
|
import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
|
|
30
30
|
import { resolveModulePath } from "@mailwoman/core/module/resolvers"
|
|
31
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
31
32
|
import type { POIDatabase } from "@mailwoman/resolver-wof-sqlite/poi"
|
|
32
33
|
import { DatabaseClient } from "@mailwoman/sqlite/client"
|
|
33
34
|
import type { PathBuilderLike } from "path-ts"
|
|
34
35
|
|
|
35
36
|
import { readTuples as readLocaleTuples, type LocalePart } from "#recipes/locale"
|
|
36
|
-
import {
|
|
37
|
+
import { readTuples as readSliceTuples } from "#recipes/scaffold"
|
|
37
38
|
import type { LocaleBaseTuple } from "#synthesizers/german"
|
|
38
39
|
import { pick, weightedPick } from "#synthesizers/utils"
|
|
39
40
|
import {
|
package/lib/recipes/sub-venue.ts
CHANGED
|
@@ -63,9 +63,10 @@
|
|
|
63
63
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
64
64
|
import { isPresent } from "@mailwoman/core/objects"
|
|
65
65
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
66
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
66
67
|
import type { PathBuilderLike } from "path-ts"
|
|
67
68
|
|
|
68
|
-
import {
|
|
69
|
+
import { sliceSourceID, type CorpusRecipe } from "#recipes/scaffold"
|
|
69
70
|
import {
|
|
70
71
|
buildIdentifierModel,
|
|
71
72
|
buildStreetNegatives,
|
|
@@ -66,14 +66,9 @@
|
|
|
66
66
|
* fact about a place; the postcode is not, because it does.
|
|
67
67
|
*/
|
|
68
68
|
|
|
69
|
-
import {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
type PostcodePlacement,
|
|
73
|
-
readTuples,
|
|
74
|
-
type CorpusRecipe,
|
|
75
|
-
sliceSourceID,
|
|
76
|
-
} from "#recipes/scaffold"
|
|
69
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
70
|
+
|
|
71
|
+
import { alignAndWrite, type PostcodePlacement, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
|
|
77
72
|
|
|
78
73
|
/**
|
|
79
74
|
* Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
package/lib/recipes/unit.ts
CHANGED
|
@@ -24,14 +24,14 @@
|
|
|
24
24
|
import { US_UNIT_DESIGNATOR_PREFERRED_ABBR, type USUnitDesignator } from "@mailwoman/codex/us"
|
|
25
25
|
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
26
26
|
import type { ComponentTag } from "@mailwoman/core/types"
|
|
27
|
+
import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
|
|
27
28
|
import type { PathBuilderLike } from "path-ts"
|
|
28
29
|
|
|
29
30
|
import { stableSourceID } from "#adapters/utils"
|
|
30
|
-
import {
|
|
31
|
+
import { readOATuples, type CorpusRecipe } from "#recipes/scaffold"
|
|
31
32
|
import { pick } from "#synthesizers/utils"
|
|
32
33
|
import type { CanonicalRow } from "#types"
|
|
33
34
|
import { alignRow } from "#utils"
|
|
34
|
-
|
|
35
35
|
/**
|
|
36
36
|
* A cached OpenAddresses extract: the zip, the CSV member, and the implied (file-level) region.
|
|
37
37
|
*/
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
|
|
7
|
+
* letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
|
|
8
|
+
* `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
|
|
9
|
+
* so the corpus adapter renders the string and the spans fall out by construction. A Singapore
|
|
10
|
+
* postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
|
|
11
|
+
*
|
|
12
|
+
* License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
|
|
13
|
+
* adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
|
|
14
|
+
*
|
|
15
|
+
* data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
|
|
16
|
+
* `poll-download` answers it once the export is ready.
|
|
17
|
+
*
|
|
18
|
+
* Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
|
|
22
|
+
import { pathExists } from "@mailwoman/core/fs/readers"
|
|
23
|
+
import { makeDirectories } from "@mailwoman/core/fs/writers"
|
|
24
|
+
import { sha256File } from "@mailwoman/core/hash"
|
|
25
|
+
import { sleep } from "@mailwoman/core/utils/sleep"
|
|
26
|
+
import { join } from "path-ts"
|
|
27
|
+
|
|
28
|
+
import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
|
|
29
|
+
import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
|
|
30
|
+
|
|
31
|
+
const SLUG = "acra-sg"
|
|
32
|
+
const COLLECTION_URL = "https://api-production.data.gov.sg/v2/public/api/collections/2/metadata"
|
|
33
|
+
const DATASET_API = "https://api-production.data.gov.sg/v2/public/api/datasets"
|
|
34
|
+
const DOWNLOAD_API = "https://api-open.data.gov.sg/v1/public/api/datasets"
|
|
35
|
+
const LICENSE = "Singapore Open Data Licence version 1.0 — https://data.gov.sg/open-data-licence"
|
|
36
|
+
|
|
37
|
+
const ATTRIBUTION =
|
|
38
|
+
"Contains information from ACRA Information on Corporate Entities accessed on <date> from data.gov.sg which is made available under the terms of the Singapore Open Data Licence version 1.0 https://data.gov.sg/open-data-licence"
|
|
39
|
+
|
|
40
|
+
const POLL_INTERVAL_MS = 3000
|
|
41
|
+
const POLL_ATTEMPTS = 40
|
|
42
|
+
|
|
43
|
+
export type FetchACRASGOptions = BaseFetchOptions
|
|
44
|
+
|
|
45
|
+
interface CollectionMetadata {
|
|
46
|
+
data?: { collectionMetadata?: { childDatasets?: string[] } }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface DatasetMetadata {
|
|
50
|
+
data?: { name?: string; lastUpdatedAt?: string }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
interface PollResponse {
|
|
54
|
+
data?: { status?: string; url?: string }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function readJSON<T>(url: string): Promise<T> {
|
|
58
|
+
const res = await fetch(url, { headers: { accept: "application/json" } })
|
|
59
|
+
|
|
60
|
+
if (!res.ok) throw new Error(`acra-sg: ${url} answered HTTP ${res.status}`)
|
|
61
|
+
|
|
62
|
+
return (await res.json()) as T
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Ask the portal to prepare the CSV export, then poll until it names the signed URL.
|
|
67
|
+
*/
|
|
68
|
+
async function signedURLFor(datasetID: string): Promise<string | undefined> {
|
|
69
|
+
await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`)
|
|
70
|
+
|
|
71
|
+
for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) {
|
|
72
|
+
await sleep(POLL_INTERVAL_MS)
|
|
73
|
+
const poll = await readJSON<PollResponse>(`${DOWNLOAD_API}/${datasetID}/poll-download`)
|
|
74
|
+
|
|
75
|
+
if (poll.data?.status === "DOWNLOAD_SUCCESS" && poll.data.url) return poll.data.url
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return undefined
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function filenameFor(name: string, datasetID: string): string {
|
|
82
|
+
const letter = /\('([^']+)'\)/.exec(name)?.[1]
|
|
83
|
+
|
|
84
|
+
return `acra-entities-${(letter ?? datasetID).toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")}.csv`
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export async function fetchACRASG(options: FetchACRASGOptions, report?: (line: string) => void): Promise<FetchSummary> {
|
|
88
|
+
const destDir = join(options.outRoot, SLUG)
|
|
89
|
+
await makeDirectories(destDir)
|
|
90
|
+
const manifestPath = join(destDir, "MANIFEST.json")
|
|
91
|
+
|
|
92
|
+
const collection = await readJSON<CollectionMetadata>(COLLECTION_URL)
|
|
93
|
+
const datasetIDs = collection.data?.collectionMetadata?.childDatasets ?? []
|
|
94
|
+
report?.(`=== ${SLUG}: ${datasetIDs.length} datasets in collection 2`)
|
|
95
|
+
|
|
96
|
+
const previous = await loadCollectionFiles(manifestPath)
|
|
97
|
+
const files = new Map<string, SourceManifest>()
|
|
98
|
+
let fetched = 0
|
|
99
|
+
let skipped = 0
|
|
100
|
+
const failedCodes: string[] = []
|
|
101
|
+
|
|
102
|
+
for (const datasetID of datasetIDs) {
|
|
103
|
+
const metadata = await readJSON<DatasetMetadata>(`${DATASET_API}/${datasetID}/metadata`)
|
|
104
|
+
const name = metadata.data?.name ?? datasetID
|
|
105
|
+
const filename = filenameFor(name, datasetID)
|
|
106
|
+
const dest = join(destDir, filename)
|
|
107
|
+
const before = previous.get(filename)
|
|
108
|
+
|
|
109
|
+
if (before && (await pathExists(dest))) {
|
|
110
|
+
files.set(filename, before)
|
|
111
|
+
|
|
112
|
+
skipped++
|
|
113
|
+
|
|
114
|
+
continue
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
report?.(`--- ${name} (${datasetID})`)
|
|
118
|
+
let bytes: number
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
bytes = await withRetries(
|
|
122
|
+
async () => {
|
|
123
|
+
const url = await signedURLFor(datasetID)
|
|
124
|
+
|
|
125
|
+
if (!url) throw new Error(`the export never reached DOWNLOAD_SUCCESS for ${datasetID}`)
|
|
126
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(1_800_000) })
|
|
127
|
+
|
|
128
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} on the signed URL for ${datasetID}`)
|
|
129
|
+
|
|
130
|
+
return streamBodyToFile(res, dest)
|
|
131
|
+
},
|
|
132
|
+
{ report, label: name }
|
|
133
|
+
)
|
|
134
|
+
} catch (error) {
|
|
135
|
+
report?.(` ✗ ${(error as Error).message}`)
|
|
136
|
+
failedCodes.push(datasetID)
|
|
137
|
+
|
|
138
|
+
continue
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (bytes < BYTES_PER_KIB) {
|
|
142
|
+
report?.(` ✗ ${bytes} bytes — an error page, not the register`)
|
|
143
|
+
failedCodes.push(datasetID)
|
|
144
|
+
|
|
145
|
+
continue
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const sha = await sha256File(dest)
|
|
149
|
+
|
|
150
|
+
files.set(filename, {
|
|
151
|
+
source_url: `https://data.gov.sg/datasets/${datasetID}/view`,
|
|
152
|
+
downloaded_at: new Date().toISOString(),
|
|
153
|
+
filename,
|
|
154
|
+
sha256: sha,
|
|
155
|
+
bytes,
|
|
156
|
+
})
|
|
157
|
+
|
|
158
|
+
fetched++
|
|
159
|
+
report?.(` ✓ ${(bytes / 1024 / 1024).toFixed(1)} MB sha256=${sha}`)
|
|
160
|
+
|
|
161
|
+
const manifest: SourceCollectionManifest = {
|
|
162
|
+
source: SLUG,
|
|
163
|
+
source_url: "https://data.gov.sg/collections/2/view",
|
|
164
|
+
license: LICENSE,
|
|
165
|
+
attribution: ATTRIBUTION,
|
|
166
|
+
downloaded_at: new Date().toISOString(),
|
|
167
|
+
files: [...files.values()],
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
await writeManifest(manifestPath, manifest)
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return { fetched, skipped, failed: failedCodes.length, failedCodes }
|
|
174
|
+
}
|