@mailwoman/corpus 10.6.0 → 10.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/PROVENANCE.md +24 -0
- package/data/address-source-register.json +18948 -8757
- package/data/corpus-source-inputs.json +4756 -0
- package/data/license-decisions.json +7260 -0
- package/data/licenses/whosonfirst-licenses.md +3 -2
- package/data/reviewed-ve-postcode-tuples.json +5 -72
- package/data/source-additions.json +2450 -0
- package/data/source-resolutions.json +87 -0
- package/data/training-manifests/v0.7.10-gb-county.effective.json +2335 -0
- package/data/training-manifests/v0.7.10-gb-county.json +1132 -0
- package/data/training-manifests/v0.7.2-address-systems.effective.json +3 -2
- package/data/training-manifests/v0.7.7-no-ppd.json +550 -0
- package/data/training-manifests/v0.7.8-rights-clean.json +538 -0
- package/data/training-manifests/v0.7.9-country-gap.json +1132 -0
- package/data/us-situs-sources.json +128 -0
- package/lib/adapters/cordis/adapter.ts +1 -1
- package/lib/adapters/libpostal-dictionary.ts +23 -3
- package/lib/adapters/osm/adapter.ts +42 -11
- package/lib/adapters.ts +14 -0
- package/lib/az/adapters/opendata/adapter.ts +389 -0
- package/lib/bs/adapters/gbpa/adapter.ts +248 -0
- package/lib/build/types.ts +16 -0
- package/lib/build.ts +21 -0
- package/lib/ee/adapters/ads/adapter.ts +18 -0
- package/lib/fo/adapters/adressur/adapter.ts +242 -0
- package/lib/gb/adapters/os-open/adapter.ts +12 -0
- package/lib/gb/county.ts +59 -0
- package/lib/gb/os-open/open/names.ts +41 -0
- package/lib/gu/adapters/bsp/adapter.ts +434 -0
- package/lib/im/adapters/land-registry/adapter.ts +340 -0
- package/lib/international/recipes/locale.ts +164 -19
- package/lib/li/adapters/llv/adapter.ts +246 -0
- package/lib/recipes/bare/postcode.ts +10 -34
- package/lib/recipes/country-balanced.ts +95 -78
- package/lib/recipes/intersection.ts +22 -30
- package/lib/recipes/po/box/cedex/recipe.ts +8 -5
- package/lib/recipes/po/box/cedex/sources.ts +2 -23
- package/lib/recipes/scaffold.ts +28 -3
- package/lib/recipes/sources.ts +112 -1
- package/lib/recipes/street/affix.ts +17 -29
- package/lib/recipes/sub/venue/context.ts +33 -9
- package/lib/recipes/sub/venue/sources.ts +54 -13
- package/lib/recipes/sub/venue.ts +68 -23
- package/lib/recipes/unit.ts +20 -7
- package/lib/registers.ts +33 -0
- package/lib/source-register/corpus-inputs.ts +528 -0
- package/lib/source-register/effective-manifest.ts +37 -5
- package/lib/source-register/types.ts +20 -0
- package/lib/south-asia/recipes/register.ts +1 -1
- package/lib/tools/overlay/manifest.ts +109 -1
- package/lib/tools/source-register/build.ts +47 -1
- package/lib/tools/variety-sample.ts +633 -0
- package/lib/us/adapters/usgov/nad/adapter.ts +39 -10
- package/lib/us/recipes/skeleton-sources.ts +156 -0
- package/lib/us/tools/nad-skeletons.ts +395 -0
- package/lib/uy/adapters/ide/adapter.ts +275 -0
- package/lib/ve/recipes/reviewed-postcode-tail.ts +29 -6
- package/out/adapters/cordis/adapter.d.ts +1 -1
- package/out/adapters/cordis/adapter.js +1 -1
- package/out/adapters/libpostal-dictionary.d.ts +9 -0
- package/out/adapters/libpostal-dictionary.d.ts.map +1 -1
- package/out/adapters/libpostal-dictionary.js +19 -3
- package/out/adapters/libpostal-dictionary.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +12 -1
- package/out/adapters/osm/adapter.d.ts.map +1 -1
- package/out/adapters/osm/adapter.js +30 -13
- package/out/adapters/osm/adapter.js.map +1 -1
- package/out/adapters.d.ts.map +1 -1
- package/out/adapters.js +14 -0
- package/out/adapters.js.map +1 -1
- package/out/az/adapters/opendata/adapter.d.ts +88 -0
- package/out/az/adapters/opendata/adapter.d.ts.map +1 -0
- package/out/az/adapters/opendata/adapter.js +282 -0
- package/out/az/adapters/opendata/adapter.js.map +1 -0
- package/out/bs/adapters/gbpa/adapter.d.ts +64 -0
- package/out/bs/adapters/gbpa/adapter.d.ts.map +1 -0
- package/out/bs/adapters/gbpa/adapter.js +187 -0
- package/out/bs/adapters/gbpa/adapter.js.map +1 -0
- package/out/build/types.d.ts +15 -0
- package/out/build/types.d.ts.map +1 -1
- package/out/build/types.js.map +1 -1
- package/out/build.d.ts.map +1 -1
- package/out/build.js +17 -0
- package/out/build.js.map +1 -1
- package/out/ee/adapters/ads/adapter.d.ts +6 -0
- package/out/ee/adapters/ads/adapter.d.ts.map +1 -1
- package/out/ee/adapters/ads/adapter.js +16 -0
- package/out/ee/adapters/ads/adapter.js.map +1 -1
- package/out/fo/adapters/adressur/adapter.d.ts +63 -0
- package/out/fo/adapters/adressur/adapter.d.ts.map +1 -0
- package/out/fo/adapters/adressur/adapter.js +177 -0
- package/out/fo/adapters/adressur/adapter.js.map +1 -0
- package/out/gb/adapters/os-open/adapter.d.ts +8 -0
- package/out/gb/adapters/os-open/adapter.d.ts.map +1 -1
- package/out/gb/adapters/os-open/adapter.js +11 -0
- package/out/gb/adapters/os-open/adapter.js.map +1 -1
- package/out/gb/county.d.ts +30 -0
- package/out/gb/county.d.ts.map +1 -0
- package/out/gb/county.js +51 -0
- package/out/gb/county.js.map +1 -0
- package/out/gb/os-open/open/names.d.ts +17 -0
- package/out/gb/os-open/open/names.d.ts.map +1 -1
- package/out/gb/os-open/open/names.js +19 -0
- package/out/gb/os-open/open/names.js.map +1 -1
- package/out/gu/adapters/bsp/adapter.d.ts +83 -0
- package/out/gu/adapters/bsp/adapter.d.ts.map +1 -0
- package/out/gu/adapters/bsp/adapter.js +346 -0
- package/out/gu/adapters/bsp/adapter.js.map +1 -0
- package/out/im/adapters/land-registry/adapter.d.ts +68 -0
- package/out/im/adapters/land-registry/adapter.d.ts.map +1 -0
- package/out/im/adapters/land-registry/adapter.js +261 -0
- package/out/im/adapters/land-registry/adapter.js.map +1 -0
- package/out/international/recipes/locale.d.ts +24 -4
- package/out/international/recipes/locale.d.ts.map +1 -1
- package/out/international/recipes/locale.js +107 -15
- package/out/international/recipes/locale.js.map +1 -1
- package/out/li/adapters/llv/adapter.d.ts +69 -0
- package/out/li/adapters/llv/adapter.d.ts.map +1 -0
- package/out/li/adapters/llv/adapter.js +181 -0
- package/out/li/adapters/llv/adapter.js.map +1 -0
- package/out/recipes/bare/postcode.d.ts.map +1 -1
- package/out/recipes/bare/postcode.js +9 -30
- package/out/recipes/bare/postcode.js.map +1 -1
- package/out/recipes/country-balanced.d.ts.map +1 -1
- package/out/recipes/country-balanced.js +82 -73
- package/out/recipes/country-balanced.js.map +1 -1
- package/out/recipes/intersection.d.ts +3 -2
- package/out/recipes/intersection.d.ts.map +1 -1
- package/out/recipes/intersection.js +20 -26
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/recipe.js +5 -4
- package/out/recipes/po/box/cedex/recipe.js.map +1 -1
- package/out/recipes/po/box/cedex/sources.d.ts +2 -45
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -1
- package/out/recipes/po/box/cedex/sources.js +0 -20
- package/out/recipes/po/box/cedex/sources.js.map +1 -1
- package/out/recipes/scaffold.d.ts +15 -2
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +21 -1
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/sources.d.ts.map +1 -1
- package/out/recipes/sources.js +112 -1
- package/out/recipes/sources.js.map +1 -1
- package/out/recipes/street/affix.d.ts.map +1 -1
- package/out/recipes/street/affix.js +8 -20
- package/out/recipes/street/affix.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts +14 -3
- package/out/recipes/sub/venue/context.d.ts.map +1 -1
- package/out/recipes/sub/venue/context.js +29 -9
- package/out/recipes/sub/venue/context.js.map +1 -1
- package/out/recipes/sub/venue/sources.d.ts +41 -8
- package/out/recipes/sub/venue/sources.d.ts.map +1 -1
- package/out/recipes/sub/venue/sources.js +26 -7
- package/out/recipes/sub/venue/sources.js.map +1 -1
- package/out/recipes/sub/venue.d.ts.map +1 -1
- package/out/recipes/sub/venue.js +35 -14
- package/out/recipes/sub/venue.js.map +1 -1
- package/out/recipes/unit.d.ts +4 -0
- package/out/recipes/unit.d.ts.map +1 -1
- package/out/recipes/unit.js +7 -6
- package/out/recipes/unit.js.map +1 -1
- package/out/registers.d.ts +33 -0
- package/out/registers.d.ts.map +1 -1
- package/out/registers.js +33 -0
- package/out/registers.js.map +1 -1
- package/out/source-register/corpus-inputs.d.ts +240 -0
- package/out/source-register/corpus-inputs.d.ts.map +1 -0
- package/out/source-register/corpus-inputs.js +275 -0
- package/out/source-register/corpus-inputs.js.map +1 -0
- package/out/source-register/effective-manifest.d.ts +2 -0
- package/out/source-register/effective-manifest.d.ts.map +1 -1
- package/out/source-register/effective-manifest.js +33 -4
- package/out/source-register/effective-manifest.js.map +1 -1
- package/out/source-register/types.d.ts +21 -1
- package/out/source-register/types.d.ts.map +1 -1
- package/out/source-register/types.js +20 -0
- package/out/source-register/types.js.map +1 -1
- package/out/south-asia/recipes/register.js +1 -1
- package/out/south-asia/recipes/register.js.map +1 -1
- package/out/tools/overlay/manifest.d.ts +7 -0
- package/out/tools/overlay/manifest.d.ts.map +1 -1
- package/out/tools/overlay/manifest.js +72 -1
- package/out/tools/overlay/manifest.js.map +1 -1
- package/out/tools/source-register/build.d.ts +9 -0
- package/out/tools/source-register/build.d.ts.map +1 -1
- package/out/tools/source-register/build.js +25 -3
- package/out/tools/source-register/build.js.map +1 -1
- package/out/tools/variety-sample.d.ts +196 -0
- package/out/tools/variety-sample.d.ts.map +1 -0
- package/out/tools/variety-sample.js +417 -0
- package/out/tools/variety-sample.js.map +1 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts +80 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -1
- package/out/us/adapters/usgov/nad/adapter.js +27 -8
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -1
- package/out/us/recipes/skeleton-sources.d.ts +92 -0
- package/out/us/recipes/skeleton-sources.d.ts.map +1 -0
- package/out/us/recipes/skeleton-sources.js +121 -0
- package/out/us/recipes/skeleton-sources.js.map +1 -0
- package/out/us/tools/nad-skeletons.d.ts +120 -0
- package/out/us/tools/nad-skeletons.d.ts.map +1 -0
- package/out/us/tools/nad-skeletons.js +262 -0
- package/out/us/tools/nad-skeletons.js.map +1 -0
- package/out/uy/adapters/ide/adapter.d.ts +64 -0
- package/out/uy/adapters/ide/adapter.d.ts.map +1 -0
- package/out/uy/adapters/ide/adapter.js +206 -0
- package/out/uy/adapters/ide/adapter.js.map +1 -0
- package/out/ve/recipes/reviewed-postcode-tail.d.ts +6 -2
- package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -1
- package/out/ve/recipes/reviewed-postcode-tail.js +16 -5
- package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -1
- package/package.json +14 -14
- package/lib/recipes/unit/sources.ts +0 -44
- package/lib/tools/fetch/ppd.ts +0 -121
- package/out/recipes/unit/sources.d.ts +0 -28
- package/out/recipes/unit/sources.d.ts.map +0 -1
- package/out/recipes/unit/sources.js +0 -30
- package/out/recipes/unit/sources.js.map +0 -1
- package/out/tools/fetch/ppd.d.ts +0 -15
- package/out/tools/fetch/ppd.d.ts.map +0 -1
- package/out/tools/fetch/ppd.js +0 -86
- package/out/tools/fetch/ppd.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reviewed-postcode-tail.js","sourceRoot":"","sources":["../../../lib/ve/recipes/reviewed-postcode-tail.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"reviewed-postcode-tail.js","sourceRoot":"","sources":["../../../lib/ve/recipes/reviewed-postcode-tail.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAA;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AAG/D,OAAO,EAAE,aAAa,EAAqB,cAAc,EAAE,MAAM,mBAAmB,CAAA;AACpF,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEtC;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,mBAAmB,CAAC,8BAA8B,CAAC,CAAA;AAkDhG,MAAM,QAAQ,GAAG,qBAAqB,CAAA;AAEtC;;GAEG;AACH,MAAM,UAAU,gCAAgC;IAC/C,OAAO,iBAAiB,CAAC,yDAAyD,CAAC,CAAA;AACpF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC/C,IAAI,GAAoB,gCAAgC,EAAE;IAE1D,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAA4B,IAAI,CAAC,CAAA;IAEzE,IACC,CAAC,QAAQ,CAAC,OAAO;QACjB,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QACnC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QACtC,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAC7C,CAAC;QACF,MAAM,IAAI,KAAK,CACd,oGAAoG;YACnG,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,SAAS,CACzC,CAAA;IACF,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAA;QACnC,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,QAAQ,SAAS,KAAK,CAAC,QAAQ,SAAS,KAAK,CAAC,MAAM,EAAE,CAAA;QAE5E,IACC,CAAC,KAAK,CAAC,EAAE;YACT,CAAC,KAAK,CAAC,QAAQ;YACf,CAAC,KAAK,CAAC,QAAQ;YACf,CAAC,KAAK,CAAC,MAAM;YACb,KAAK,CAAC,OAAO,KAAK,WAAW;YAC7B,KAAK,CAAC,EAAE,KAAK,IAAI;YACjB,KAAK,CAAC,MAAM,KAAK,OAAO;YACxB,KAAK,CAAC,iBAAiB,KAAK,gBAAgB;YAC5C,CAAC,UAAU,EAAE,SAAS;YACtB,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;YAC7B,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YACtC,UAAU,CAAC,YAAY,KAAK,UAAU;YACtC,CAAC,UAAU,CAAC,iBAAiB,EAC5B,CAAC;YACF,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,CAAC,EAAE,IAAI,cAAc,EAAE,CAAC,CAAA;QAClF,CAAC;QAED,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAA;QAC3G,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACjB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChB,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,CAAA;AACvB,CAAC;AAED,MAAM,WAAW,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AAE1F;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,KAA4B;IACxE,MAAM,IAAI,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,CAAA;IACnE,MAAM,WAAW,GAAG,GAAG,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAA;IACjH,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;IAExG,MAAM,QAAQ,GAAc;QAC3B,EAAE,EAAE,EAAE,mBAAmB,EAAE,GAAG,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,EAAE;QAC/D;YACC,EAAE,EAAE,sBAAsB;YAC1B,GAAG,EAAE,IAAI;YACT,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;SACxF;QACD,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;QACnH,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW,CAAC,WAAW,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE;QACtE;YACC,EAAE,EAAE,cAAc;YAClB,GAAG,EAAE,sCAAsC,WAAW,EAAE;YACxD,UAAU,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,iBAAiB,EAAE,GAAG,IAAI,EAAE;SAC7E;KACD,CAAA;IAED,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAClD,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAE9C,IAAI,cAAc,KAAK,KAAK,CAAC,QAAQ,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACxE,QAAQ,CAAC,IAAI,CAAC;YACb,EAAE,EAAE,eAAe;YACnB,GAAG,EAAE,GAAG,cAAc,IAAI,KAAK,CAAC,QAAQ,KAAK,YAAY,KAAK,KAAK,CAAC,OAAO,EAAE;YAC7E,UAAU,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE;SAChH,CAAC,CAAA;IACH,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAiB;IACvD,IAAI,EAAE,wBAAwB;IAC9B,WAAW,EAAE,iFAAiF;IAC9F,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAE3D,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,oGAAoG;gBACnG,gCAAgC,CACjC,CAAA;QACF,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,KAAK,MAAM,OAAO,IAAI,4BAA4B,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3D,MAAM,QAAQ,GAAG,cAAc,CAAC,6BAA6B,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;gBAExG,MAAM,SAAS,GAAG;oBACjB,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,OAAO,EAAE,KAAK,CAAC,EAAE;oBACjB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,6BAA6B;oBACrC,SAAS,EAAE,QAAQ;oBACnB,cAAc,EAAE,QAAQ;oBACxB,OAAO,EAAE,kDAAkD,KAAK,CAAC,UAAU,CAAC,SAAS,6DAA6D;iBAClJ,CAAA;gBAED,IACC,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,wBAAwB,EAAE;oBACzD,mDAAmD;oBACnD,qDAAqD;oBACrD,gCAAgC;oBAChC,QAAQ,EAAE,cAAc,CAAC,cAAc;oBACvC,OAAO,EAAE,aAAa,CAAC,QAAQ;oBAC/B,YAAY,EAAE,KAAK,CAAC,EAAE;iBACtB,CAAC,EACD,CAAC;oBACF,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IACjD,CAAC;CACD,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "10.
|
|
3
|
+
"version": "10.7.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only OR LicenseRef-Commercial",
|
|
6
6
|
"repository": {
|
|
@@ -116,19 +116,19 @@
|
|
|
116
116
|
}
|
|
117
117
|
},
|
|
118
118
|
"dependencies": {
|
|
119
|
-
"@mailwoman/ban": "10.
|
|
120
|
-
"@mailwoman/bdc": "10.
|
|
121
|
-
"@mailwoman/codex": "10.
|
|
122
|
-
"@mailwoman/core": "10.
|
|
123
|
-
"@mailwoman/evidence": "10.
|
|
124
|
-
"@mailwoman/normalize": "10.
|
|
125
|
-
"@mailwoman/poi-taxonomy": "10.
|
|
126
|
-
"@mailwoman/query-shape": "10.
|
|
127
|
-
"@mailwoman/record": "10.
|
|
128
|
-
"@mailwoman/resolver-wof-sqlite": "10.
|
|
129
|
-
"@mailwoman/spatial": "10.
|
|
130
|
-
"@mailwoman/sqlite": "10.
|
|
131
|
-
"@mailwoman/tiger": "10.
|
|
119
|
+
"@mailwoman/ban": "10.7.0",
|
|
120
|
+
"@mailwoman/bdc": "10.7.0",
|
|
121
|
+
"@mailwoman/codex": "10.7.0",
|
|
122
|
+
"@mailwoman/core": "10.7.0",
|
|
123
|
+
"@mailwoman/evidence": "10.7.0",
|
|
124
|
+
"@mailwoman/normalize": "10.7.0",
|
|
125
|
+
"@mailwoman/poi-taxonomy": "10.7.0",
|
|
126
|
+
"@mailwoman/query-shape": "10.7.0",
|
|
127
|
+
"@mailwoman/record": "10.7.0",
|
|
128
|
+
"@mailwoman/resolver-wof-sqlite": "10.7.0",
|
|
129
|
+
"@mailwoman/spatial": "10.7.0",
|
|
130
|
+
"@mailwoman/sqlite": "10.7.0",
|
|
131
|
+
"@mailwoman/tiger": "10.7.0",
|
|
132
132
|
"apache-arrow": "^21.2.0",
|
|
133
133
|
"fastest-levenshtein": "^1.0.16",
|
|
134
134
|
"parquet-wasm": "0.9.0",
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Lists the cached OpenAddresses extracts read by the `unit` recipe.
|
|
6
|
-
* It also identifies the held-out extract excluded from training.
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
11
|
-
import type { PathBuilderLike } from "path-ts"
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* A cached OpenAddresses extract.
|
|
15
|
-
*
|
|
16
|
-
* Its fields identify the ZIP, CSV member and implied file-level region.
|
|
17
|
-
*/
|
|
18
|
-
export interface UnitSource {
|
|
19
|
-
zip: PathBuilderLike
|
|
20
|
-
csv: string
|
|
21
|
-
region: string
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Every non-Vermont state cached; `region` is implied by the file.
|
|
26
|
-
*/
|
|
27
|
-
export const TRAIN_SOURCES: readonly UnitSource[] = [
|
|
28
|
-
{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
|
|
29
|
-
{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
|
|
30
|
-
{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
|
|
31
|
-
{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
|
|
32
|
-
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
|
|
33
|
-
{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
|
|
34
|
-
{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
|
|
35
|
-
]
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Vermont, the corpus holdout — `--golden` reads this and no other source.
|
|
39
|
-
*/
|
|
40
|
-
export const EVAL_SOURCE: UnitSource = {
|
|
41
|
-
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
42
|
-
csv: "us/vt/statewide.csv",
|
|
43
|
-
region: "VT",
|
|
44
|
-
}
|
package/lib/tools/fetch/ppd.ts
DELETED
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
import { titleCaseGB } from "@mailwoman/codex/gb"
|
|
2
|
-
import { dataRootPath } from "@mailwoman/core/data-root"
|
|
3
|
-
import { openReadStream, openWriteStream } from "@mailwoman/core/fs/streams"
|
|
4
|
-
import { stringifyJSON } from "@mailwoman/core/json"
|
|
5
|
-
import { runIfScript } from "@mailwoman/core/scripting"
|
|
6
|
-
import { parseArguments } from "@mailwoman/core/scripting/arguments"
|
|
7
|
-
import { resolvePath } from "path-ts"
|
|
8
|
-
import { CSVSpliterator } from "spliterator"
|
|
9
|
-
|
|
10
|
-
const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/
|
|
11
|
-
|
|
12
|
-
export interface PPDExtractStats {
|
|
13
|
-
kept: number
|
|
14
|
-
skippedSAON: number
|
|
15
|
-
skippedPAON: number
|
|
16
|
-
skippedNoStreet: number
|
|
17
|
-
skippedNoPostcode: number
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
const quote = (value: string): string => (value ? `"${value.replaceAll('"', '""')}"` : "")
|
|
21
|
-
|
|
22
|
-
/**
|
|
23
|
-
* Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
|
|
24
|
-
* — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
|
|
25
|
-
*
|
|
26
|
-
* Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
|
|
27
|
-
*/
|
|
28
|
-
export async function extractPPDTuples(
|
|
29
|
-
input: AsyncIterable<string[]> | Iterable<string[]>,
|
|
30
|
-
write: (line: string) => void
|
|
31
|
-
): Promise<PPDExtractStats> {
|
|
32
|
-
const stats: PPDExtractStats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 }
|
|
33
|
-
write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE")
|
|
34
|
-
|
|
35
|
-
for await (const cells of input) {
|
|
36
|
-
// HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
|
|
37
|
-
const postcode = cells[3]
|
|
38
|
-
const paon = cells[7]
|
|
39
|
-
const saon = cells[8]
|
|
40
|
-
const street = cells[9]
|
|
41
|
-
const locality = cells[10]
|
|
42
|
-
const town = cells[11]
|
|
43
|
-
const county = cells[13]
|
|
44
|
-
|
|
45
|
-
if (saon) {
|
|
46
|
-
stats.skippedSAON++
|
|
47
|
-
|
|
48
|
-
continue
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
if (!paon || !HOUSE_NUMBER_PATTERN.test(paon)) {
|
|
52
|
-
stats.skippedPAON++
|
|
53
|
-
|
|
54
|
-
continue
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
if (!street) {
|
|
58
|
-
stats.skippedNoStreet++
|
|
59
|
-
|
|
60
|
-
continue
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
if (!postcode) {
|
|
64
|
-
stats.skippedNoPostcode++
|
|
65
|
-
|
|
66
|
-
continue
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
const number = paon.replace(/\s*-\s*/, "-")
|
|
70
|
-
const city = locality && locality !== town ? titleCaseGB(locality) : ""
|
|
71
|
-
|
|
72
|
-
write(
|
|
73
|
-
[
|
|
74
|
-
number,
|
|
75
|
-
quote(titleCaseGB(street)),
|
|
76
|
-
quote(city),
|
|
77
|
-
quote(titleCaseGB(town ?? "")),
|
|
78
|
-
quote(titleCaseGB(county ?? "")),
|
|
79
|
-
postcode,
|
|
80
|
-
].join(",")
|
|
81
|
-
)
|
|
82
|
-
|
|
83
|
-
stats.kept++
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
return stats
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
|
|
91
|
-
*/
|
|
92
|
-
async function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats> {
|
|
93
|
-
// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself
|
|
94
|
-
// (see readTuples in recipes/international/locale.ts).
|
|
95
|
-
// `header: false` yields every row as data — PPD ships no header row.
|
|
96
|
-
const rows = CSVSpliterator.fromAsync<string[]>(openReadStream(inputPath), {
|
|
97
|
-
header: false,
|
|
98
|
-
})
|
|
99
|
-
|
|
100
|
-
const out = openWriteStream(outputPath, { encoding: "utf8" })
|
|
101
|
-
const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"))
|
|
102
|
-
|
|
103
|
-
await new Promise<void>((res) => {
|
|
104
|
-
out.end(res)
|
|
105
|
-
})
|
|
106
|
-
|
|
107
|
-
return stats
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
runIfScript(import.meta, async () => {
|
|
111
|
-
const { values } = parseArguments({
|
|
112
|
-
options: {
|
|
113
|
-
input: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "pp-complete.csv")) },
|
|
114
|
-
output: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")) },
|
|
115
|
-
},
|
|
116
|
-
})
|
|
117
|
-
|
|
118
|
-
const stats = await runPPDExtract(values.input!, values.output!)
|
|
119
|
-
|
|
120
|
-
console.log(`[ppd] ${stringifyJSON(stats)}`)
|
|
121
|
-
})
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Lists the cached OpenAddresses extracts read by the `unit` recipe.
|
|
6
|
-
* It also identifies the held-out extract excluded from training.
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
import type { PathBuilderLike } from "path-ts";
|
|
10
|
-
/**
|
|
11
|
-
* A cached OpenAddresses extract.
|
|
12
|
-
*
|
|
13
|
-
* Its fields identify the ZIP, CSV member and implied file-level region.
|
|
14
|
-
*/
|
|
15
|
-
export interface UnitSource {
|
|
16
|
-
zip: PathBuilderLike;
|
|
17
|
-
csv: string;
|
|
18
|
-
region: string;
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Every non-Vermont state cached; `region` is implied by the file.
|
|
22
|
-
*/
|
|
23
|
-
export declare const TRAIN_SOURCES: readonly UnitSource[];
|
|
24
|
-
/**
|
|
25
|
-
* Vermont, the corpus holdout — `--golden` reads this and no other source.
|
|
26
|
-
*/
|
|
27
|
-
export declare const EVAL_SOURCE: UnitSource;
|
|
28
|
-
//# sourceMappingURL=sources.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sources.d.ts","sourceRoot":"","sources":["../../../lib/recipes/unit/sources.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C;;;;GAIG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,eAAe,CAAA;IACpB,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,eAAO,MAAM,aAAa,EAAE,SAAS,UAAU,EAQ9C,CAAA;AAED;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,UAIzB,CAAA"}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
* @file Lists the cached OpenAddresses extracts read by the `unit` recipe.
|
|
6
|
-
* It also identifies the held-out extract excluded from training.
|
|
7
|
-
*
|
|
8
|
-
*/
|
|
9
|
-
import { dataRootPath } from "@mailwoman/core/data-root";
|
|
10
|
-
/**
|
|
11
|
-
* Every non-Vermont state cached; `region` is implied by the file.
|
|
12
|
-
*/
|
|
13
|
-
export const TRAIN_SOURCES = [
|
|
14
|
-
{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
|
|
15
|
-
{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
|
|
16
|
-
{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
|
|
17
|
-
{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
|
|
18
|
-
{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
|
|
19
|
-
{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
|
|
20
|
-
{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
|
|
21
|
-
];
|
|
22
|
-
/**
|
|
23
|
-
* Vermont, the corpus holdout — `--golden` reads this and no other source.
|
|
24
|
-
*/
|
|
25
|
-
export const EVAL_SOURCE = {
|
|
26
|
-
zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
|
|
27
|
-
csv: "us/vt/statewide.csv",
|
|
28
|
-
region: "VT",
|
|
29
|
-
};
|
|
30
|
-
//# sourceMappingURL=sources.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sources.js","sourceRoot":"","sources":["../../../lib/recipes/unit/sources.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AAcxD;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAA0B;IACnD,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,sBAAsB,CAAC,EAAE,GAAG,EAAE,oBAAoB,EAAE,MAAM,EAAE,IAAI,EAAE;IAClG,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,mBAAmB,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE;IAC5F,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACpG,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACpG,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,kBAAkB,CAAC,EAAE,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE;IAC1F,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACpG,EAAE,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;CACpG,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAe;IACtC,GAAG,EAAE,YAAY,CAAC,UAAU,EAAE,uBAAuB,CAAC;IACtD,GAAG,EAAE,qBAAqB;IAC1B,MAAM,EAAE,IAAI;CACZ,CAAA"}
|
package/out/tools/fetch/ppd.d.ts
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
export interface PPDExtractStats {
|
|
2
|
-
kept: number;
|
|
3
|
-
skippedSAON: number;
|
|
4
|
-
skippedPAON: number;
|
|
5
|
-
skippedNoStreet: number;
|
|
6
|
-
skippedNoPostcode: number;
|
|
7
|
-
}
|
|
8
|
-
/**
|
|
9
|
-
* Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
|
|
10
|
-
* — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
|
|
11
|
-
*
|
|
12
|
-
* Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
|
|
13
|
-
*/
|
|
14
|
-
export declare function extractPPDTuples(input: AsyncIterable<string[]> | Iterable<string[]>, write: (line: string) => void): Promise<PPDExtractStats>;
|
|
15
|
-
//# sourceMappingURL=ppd.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAWA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B"}
|
package/out/tools/fetch/ppd.js
DELETED
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
import { titleCaseGB } from "@mailwoman/codex/gb";
|
|
2
|
-
import { dataRootPath } from "@mailwoman/core/data-root";
|
|
3
|
-
import { openReadStream, openWriteStream } from "@mailwoman/core/fs/streams";
|
|
4
|
-
import { stringifyJSON } from "@mailwoman/core/json";
|
|
5
|
-
import { runIfScript } from "@mailwoman/core/scripting";
|
|
6
|
-
import { parseArguments } from "@mailwoman/core/scripting/arguments";
|
|
7
|
-
import { resolvePath } from "path-ts";
|
|
8
|
-
import { CSVSpliterator } from "spliterator";
|
|
9
|
-
const HOUSE_NUMBER_PATTERN = /^\d+[A-Za-z]?(\s*-\s*\d+[A-Za-z]?)?$/;
|
|
10
|
-
const quote = (value) => (value ? `"${value.replaceAll('"', '""')}"` : "");
|
|
11
|
-
/**
|
|
12
|
-
* Convert PPD rows (`id,price,date,postcode,type,new,tenure,paon,saon,street,locality,town,district,county,cat,status`
|
|
13
|
-
* — headerless) into OA-shaped tuple lines via `write`, applying the skip rules + title-casing.
|
|
14
|
-
*
|
|
15
|
-
* Accepts a plain array of rows (tests) or a streamed async source (the real 31M-row extraction).
|
|
16
|
-
*/
|
|
17
|
-
export async function extractPPDTuples(input, write) {
|
|
18
|
-
const stats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 };
|
|
19
|
-
write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE");
|
|
20
|
-
for await (const cells of input) {
|
|
21
|
-
// HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
|
|
22
|
-
const postcode = cells[3];
|
|
23
|
-
const paon = cells[7];
|
|
24
|
-
const saon = cells[8];
|
|
25
|
-
const street = cells[9];
|
|
26
|
-
const locality = cells[10];
|
|
27
|
-
const town = cells[11];
|
|
28
|
-
const county = cells[13];
|
|
29
|
-
if (saon) {
|
|
30
|
-
stats.skippedSAON++;
|
|
31
|
-
continue;
|
|
32
|
-
}
|
|
33
|
-
if (!paon || !HOUSE_NUMBER_PATTERN.test(paon)) {
|
|
34
|
-
stats.skippedPAON++;
|
|
35
|
-
continue;
|
|
36
|
-
}
|
|
37
|
-
if (!street) {
|
|
38
|
-
stats.skippedNoStreet++;
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
if (!postcode) {
|
|
42
|
-
stats.skippedNoPostcode++;
|
|
43
|
-
continue;
|
|
44
|
-
}
|
|
45
|
-
const number = paon.replace(/\s*-\s*/, "-");
|
|
46
|
-
const city = locality && locality !== town ? titleCaseGB(locality) : "";
|
|
47
|
-
write([
|
|
48
|
-
number,
|
|
49
|
-
quote(titleCaseGB(street)),
|
|
50
|
-
quote(city),
|
|
51
|
-
quote(titleCaseGB(town ?? "")),
|
|
52
|
-
quote(titleCaseGB(county ?? "")),
|
|
53
|
-
postcode,
|
|
54
|
-
].join(","));
|
|
55
|
-
stats.kept++;
|
|
56
|
-
}
|
|
57
|
-
return stats;
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
|
|
61
|
-
*/
|
|
62
|
-
async function runPPDExtract(inputPath, outputPath) {
|
|
63
|
-
// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself
|
|
64
|
-
// (see readTuples in recipes/international/locale.ts).
|
|
65
|
-
// `header: false` yields every row as data — PPD ships no header row.
|
|
66
|
-
const rows = CSVSpliterator.fromAsync(openReadStream(inputPath), {
|
|
67
|
-
header: false,
|
|
68
|
-
});
|
|
69
|
-
const out = openWriteStream(outputPath, { encoding: "utf8" });
|
|
70
|
-
const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"));
|
|
71
|
-
await new Promise((res) => {
|
|
72
|
-
out.end(res);
|
|
73
|
-
});
|
|
74
|
-
return stats;
|
|
75
|
-
}
|
|
76
|
-
runIfScript(import.meta, async () => {
|
|
77
|
-
const { values } = parseArguments({
|
|
78
|
-
options: {
|
|
79
|
-
input: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "pp-complete.csv")) },
|
|
80
|
-
output: { type: "string", default: resolvePath(dataRootPath("ppd", "2026-07-22", "gb-tuples.csv")) },
|
|
81
|
-
},
|
|
82
|
-
});
|
|
83
|
-
const stats = await runPPDExtract(values.input, values.output);
|
|
84
|
-
console.log(`[ppd] ${stringifyJSON(stats)}`);
|
|
85
|
-
});
|
|
86
|
-
//# sourceMappingURL=ppd.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../lib/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACjE,6EAA6E;IAC7E,uDAAuD;IACvD,sEAAsE;IACtE,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,cAAc,CAAC,SAAS,CAAC,EAAE;QAC1E,MAAM,EAAE,KAAK;KACb,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,eAAe,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC7D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,OAAO,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC;QACjC,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,EAAE;YACrG,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,CAAC,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC,EAAE;SACpG;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC7C,CAAC,CAAC,CAAA"}
|