@mailwoman/corpus 5.8.0 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/index.d.ts +19 -19
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.js +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/align.d.ts +2 -2
- package/out/src/build.d.ts +4 -4
- package/out/src/index.d.ts +22 -21
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/index.d.ts +3 -3
- package/out/src/parquet-wrapper/reader.d.ts +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +4 -2
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +3 -1
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +3 -3
- package/out/src/runner.d.ts +2 -2
- package/out/src/shard-recipes/anchor-absorption.d.ts +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +1 -1
- package/out/src/shard-recipes/german.d.ts +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +1 -1
- package/out/src/shard-recipes/index.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts +2 -2
- package/out/src/shard-recipes/nl-postcode.d.ts +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +1 -1
- package/out/src/shard-recipes/no-street.d.ts +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +1 -1
- package/out/src/shard-recipes/po-box.d.ts +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +1 -1
- package/out/src/shard-recipes/street.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/split.d.ts +1 -1
- package/out/src/split.js +1 -1
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +1 -1
- package/out/src/synthesize-german.d.ts +1 -1
- package/out/src/synthesize-house-venue.d.ts +1 -1
- package/out/src/synthesize-intersection.d.ts +1 -1
- package/out/src/synthesize-no-street.d.ts +1 -1
- package/out/src/synthesize-po-box.d.ts +1 -1
- package/out/src/synthesize-street.d.ts +1 -1
- package/out/src/synthesize.d.ts +2 -2
- package/out/src/tools/align-shard.d.ts +8 -0
- package/out/src/tools/align-shard.d.ts.map +1 -0
- package/out/src/tools/align-shard.js +58 -0
- package/out/src/tools/align-shard.js.map +1 -0
- package/out/src/tools/audit.d.ts +28 -0
- package/out/src/tools/audit.d.ts.map +1 -0
- package/out/src/tools/audit.js +273 -0
- package/out/src/tools/audit.js.map +1 -0
- package/out/src/tools/corpus-stats.d.ts +44 -0
- package/out/src/tools/corpus-stats.d.ts.map +1 -0
- package/out/src/tools/corpus-stats.js +151 -0
- package/out/src/tools/corpus-stats.js.map +1 -0
- package/out/src/tools/fetch/ban.d.ts +31 -0
- package/out/src/tools/fetch/ban.d.ts.map +1 -0
- package/out/src/tools/fetch/ban.js +248 -0
- package/out/src/tools/fetch/ban.js.map +1 -0
- package/out/src/tools/fetch/download.d.ts +49 -0
- package/out/src/tools/fetch/download.d.ts.map +1 -0
- package/out/src/tools/fetch/download.js +81 -0
- package/out/src/tools/fetch/download.js.map +1 -0
- package/out/src/tools/fetch/hrsa.d.ts +18 -0
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -0
- package/out/src/tools/fetch/hrsa.js +46 -0
- package/out/src/tools/fetch/hrsa.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts +25 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -0
- package/out/src/tools/fetch/imls-pls.js +120 -0
- package/out/src/tools/fetch/imls-pls.js.map +1 -0
- package/out/src/tools/fetch/index.d.ts +118 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -0
- package/out/src/tools/fetch/index.js +117 -0
- package/out/src/tools/fetch/index.js.map +1 -0
- package/out/src/tools/fetch/nad.d.ts +56 -0
- package/out/src/tools/fetch/nad.d.ts.map +1 -0
- package/out/src/tools/fetch/nad.js +225 -0
- package/out/src/tools/fetch/nad.js.map +1 -0
- package/out/src/tools/fetch/nppes.d.ts +26 -0
- package/out/src/tools/fetch/nppes.d.ts.map +1 -0
- package/out/src/tools/fetch/nppes.js +136 -0
- package/out/src/tools/fetch/nppes.js.map +1 -0
- package/out/src/tools/fetch/openaddresses.d.ts +59 -0
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -0
- package/out/src/tools/fetch/openaddresses.js +307 -0
- package/out/src/tools/fetch/openaddresses.js.map +1 -0
- package/out/src/tools/fetch/state-hi-schools.d.ts +25 -0
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -0
- package/out/src/tools/fetch/state-hi-schools.js +171 -0
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -0
- package/out/src/tools/fetch/state-sources.d.ts +22 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -0
- package/out/src/tools/fetch/state-sources.js +110 -0
- package/out/src/tools/fetch/state-sources.js.map +1 -0
- package/out/src/tools/fetch/tiger-full.d.ts +36 -0
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -0
- package/out/src/tools/fetch/tiger-full.js +239 -0
- package/out/src/tools/fetch/tiger-full.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts +74 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -0
- package/out/src/tools/golden-expand.js +394 -0
- package/out/src/tools/golden-expand.js.map +1 -0
- package/out/src/tools/golden-promote.d.ts +60 -0
- package/out/src/tools/golden-promote.d.ts.map +1 -0
- package/out/src/tools/golden-promote.js +223 -0
- package/out/src/tools/golden-promote.js.map +1 -0
- package/out/src/tools/index.d.ts +24 -0
- package/out/src/tools/index.d.ts.map +1 -0
- package/out/src/tools/index.js +24 -0
- package/out/src/tools/index.js.map +1 -0
- package/out/src/tools/ingest-csv.d.ts +45 -0
- package/out/src/tools/ingest-csv.d.ts.map +1 -0
- package/out/src/tools/ingest-csv.js +309 -0
- package/out/src/tools/ingest-csv.js.map +1 -0
- package/out/src/tools/jsonl-to-parquet.d.ts +49 -0
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -0
- package/out/src/tools/jsonl-to-parquet.js +157 -0
- package/out/src/tools/jsonl-to-parquet.js.map +1 -0
- package/out/src/tools/lint-shard-vocab.d.ts +66 -0
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -0
- package/out/src/tools/lint-shard-vocab.js +266 -0
- package/out/src/tools/lint-shard-vocab.js.map +1 -0
- package/out/src/tools/lint-shard.d.ts +73 -0
- package/out/src/tools/lint-shard.d.ts.map +1 -0
- package/out/src/tools/lint-shard.js +350 -0
- package/out/src/tools/lint-shard.js.map +1 -0
- package/out/src/tools/overlay-manifest.d.ts +35 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -0
- package/out/src/tools/overlay-manifest.js +85 -0
- package/out/src/tools/overlay-manifest.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +28 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -0
- package/out/src/tools/shard-kryptonite.js +101 -0
- package/out/src/tools/shard-kryptonite.js.map +1 -0
- package/out/src/tools/shard-translit.d.ts +42 -0
- package/out/src/tools/shard-translit.d.ts.map +1 -0
- package/out/src/tools/shard-translit.js +196 -0
- package/out/src/tools/shard-translit.js.map +1 -0
- package/package.json +63 -7
package/out/src/adapter.d.ts
CHANGED
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
* Everything here is pure (no I/O); side-effecting code goes in `./runner.ts`.
|
|
21
21
|
*/
|
|
22
22
|
import type { ComponentTag } from "@mailwoman/core/types";
|
|
23
|
-
import type { CanonicalRow, CorpusAdapter } from "./types.
|
|
23
|
+
import type { CanonicalRow, CorpusAdapter } from "./types.ts";
|
|
24
24
|
/**
|
|
25
25
|
* Lookup table for corpus adapters.
|
|
26
26
|
*
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
|
|
30
30
|
* and `opts.country` for a self-consistency check (errors if country !== FR).
|
|
31
31
|
*/
|
|
32
|
-
import type { CorpusAdapter } from "../../types.
|
|
32
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
33
33
|
export declare const BAN_ADAPTER_ID = "ban";
|
|
34
34
|
export declare function createBanAdapter(): CorpusAdapter;
|
|
35
35
|
export declare const banAdapter: CorpusAdapter;
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
* The CostQuest Fabric source data has its own license; consumers who substitute that path should
|
|
34
34
|
* re-stamp accordingly.
|
|
35
35
|
*/
|
|
36
|
-
import type { CorpusAdapter } from "../../types.
|
|
36
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
37
37
|
export declare const FCC_BDC_ADAPTER_ID = "fcc-bdc";
|
|
38
38
|
export declare const FCC_BDC_DEFAULT_LICENSE = "Public Domain";
|
|
39
39
|
interface SplitAddress {
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
*
|
|
28
28
|
* License: stamped `"CC-BY-4.0"` per row (GeoNames' terms); provenance is the `geonames-<id>` key.
|
|
29
29
|
*/
|
|
30
|
-
import type { CorpusAdapter } from "../../types.
|
|
30
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
31
31
|
export declare const GEONAMES_ADAPTER_ID = "geonames";
|
|
32
32
|
export declare const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
33
33
|
export declare function createGeonamesAdapter(): CorpusAdapter;
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* non-US countries; for US, the postcode-last sources are the right order. License:
|
|
23
23
|
* `"CC-BY-4.0"` per row (attribute "GeoNames").
|
|
24
24
|
*/
|
|
25
|
-
import type { CorpusAdapter } from "../../types.
|
|
25
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
26
26
|
export declare const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal";
|
|
27
27
|
export declare const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
28
28
|
export declare function createGeonamesPostalAdapter(): CorpusAdapter;
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
|
|
28
28
|
* line). Open G-NAF licence — attribute "Geoscape Australia".
|
|
29
29
|
*/
|
|
30
|
-
import type { CorpusAdapter } from "../../types.
|
|
30
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
31
31
|
export declare const GNAF_ADAPTER_ID = "gnaf";
|
|
32
32
|
/** Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style). */
|
|
33
33
|
export declare const GNAF_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
|
|
19
19
|
* implementations; the registered ids are unchanged).
|
|
20
20
|
*/
|
|
21
|
-
import type { CorpusAdapter } from "../types.
|
|
21
|
+
import type { CorpusAdapter } from "../types.ts";
|
|
22
22
|
/**
|
|
23
23
|
* Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
|
|
24
24
|
* Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
|
|
@@ -30,22 +30,22 @@ import type { CorpusAdapter } from "../types.js";
|
|
|
30
30
|
* CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
|
|
31
31
|
*/
|
|
32
32
|
export declare const BUILTIN_ADAPTERS: readonly CorpusAdapter[];
|
|
33
|
-
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.
|
|
34
|
-
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.
|
|
35
|
-
export { GEONAMES_POSTAL_ADAPTER_ID, GEONAMES_POSTAL_DEFAULT_LICENSE, geonamesPostalAdapter, } from "./geonames-postal/adapter.
|
|
36
|
-
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.
|
|
37
|
-
export { OPENADDRESSES_ADAPTER_ID, OPENADDRESSES_DEFAULT_LICENSE, openaddressesAdapter, } from "./openaddresses/adapter.
|
|
38
|
-
export { STATE_HI_SCHOOLS_ADAPTER_ID, STATE_HI_SCHOOLS_DEFAULT_LICENSE, stateHiSchoolsAdapter, } from "./state-hi-schools/adapter.
|
|
39
|
-
export { STATE_IA_CONTRACTORS_ADAPTER_ID, STATE_IA_CONTRACTORS_DEFAULT_LICENSE, stateIaContractorsAdapter, } from "./state-ia-contractors/adapter.
|
|
40
|
-
export { STATE_NY_NOTARIES_ADAPTER_ID, STATE_NY_NOTARIES_DEFAULT_LICENSE, stateNyNotariesAdapter, } from "./state-ny-notaries/adapter.
|
|
41
|
-
export { STATE_TX_NOTARIES_ADAPTER_ID, STATE_TX_NOTARIES_DEFAULT_LICENSE, stateTxNotariesAdapter, } from "./state-tx-notaries/adapter.
|
|
42
|
-
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.
|
|
43
|
-
export { USGOV_HRSA_FQHC_ADAPTER_ID, USGOV_HRSA_FQHC_DEFAULT_LICENSE, usgovHrsaFqhcAdapter, } from "./usgov-hrsa-fqhc/adapter.
|
|
44
|
-
export { USGOV_IMLS_PLS_ADAPTER_ID, USGOV_IMLS_PLS_DEFAULT_LICENSE, usgovImlsPlsAdapter, } from "./usgov-imls-pls/adapter.
|
|
45
|
-
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.
|
|
46
|
-
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.
|
|
47
|
-
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.
|
|
48
|
-
export { USGOV_SAMHSA_ADAPTER_ID, USGOV_SAMHSA_DEFAULT_LICENSE, usgovSamhsaTreatmentLocatorAdapter, } from "./usgov-samhsa-treatment-locator/adapter.
|
|
49
|
-
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.
|
|
50
|
-
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.
|
|
33
|
+
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts";
|
|
34
|
+
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts";
|
|
35
|
+
export { GEONAMES_POSTAL_ADAPTER_ID, GEONAMES_POSTAL_DEFAULT_LICENSE, geonamesPostalAdapter, } from "./geonames-postal/adapter.ts";
|
|
36
|
+
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts";
|
|
37
|
+
export { OPENADDRESSES_ADAPTER_ID, OPENADDRESSES_DEFAULT_LICENSE, openaddressesAdapter, } from "./openaddresses/adapter.ts";
|
|
38
|
+
export { STATE_HI_SCHOOLS_ADAPTER_ID, STATE_HI_SCHOOLS_DEFAULT_LICENSE, stateHiSchoolsAdapter, } from "./state-hi-schools/adapter.ts";
|
|
39
|
+
export { STATE_IA_CONTRACTORS_ADAPTER_ID, STATE_IA_CONTRACTORS_DEFAULT_LICENSE, stateIaContractorsAdapter, } from "./state-ia-contractors/adapter.ts";
|
|
40
|
+
export { STATE_NY_NOTARIES_ADAPTER_ID, STATE_NY_NOTARIES_DEFAULT_LICENSE, stateNyNotariesAdapter, } from "./state-ny-notaries/adapter.ts";
|
|
41
|
+
export { STATE_TX_NOTARIES_ADAPTER_ID, STATE_TX_NOTARIES_DEFAULT_LICENSE, stateTxNotariesAdapter, } from "./state-tx-notaries/adapter.ts";
|
|
42
|
+
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts";
|
|
43
|
+
export { USGOV_HRSA_FQHC_ADAPTER_ID, USGOV_HRSA_FQHC_DEFAULT_LICENSE, usgovHrsaFqhcAdapter, } from "./usgov-hrsa-fqhc/adapter.ts";
|
|
44
|
+
export { USGOV_IMLS_PLS_ADAPTER_ID, USGOV_IMLS_PLS_DEFAULT_LICENSE, usgovImlsPlsAdapter, } from "./usgov-imls-pls/adapter.ts";
|
|
45
|
+
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts";
|
|
46
|
+
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts";
|
|
47
|
+
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts";
|
|
48
|
+
export { USGOV_SAMHSA_ADAPTER_ID, USGOV_SAMHSA_DEFAULT_LICENSE, usgovSamhsaTreatmentLocatorAdapter, } from "./usgov-samhsa-treatment-locator/adapter.ts";
|
|
49
|
+
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts";
|
|
50
|
+
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts";
|
|
51
51
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -35,7 +35,7 @@
|
|
|
35
35
|
* inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
|
|
36
36
|
* for non-US locales where district names DO appear on the envelope.
|
|
37
37
|
*/
|
|
38
|
-
import type { CorpusAdapter } from "../../types.
|
|
38
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
39
39
|
export declare const OPENADDRESSES_ADAPTER_ID = "openaddresses";
|
|
40
40
|
export declare const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0";
|
|
41
41
|
export interface OpenaddressesAdapterOptions {
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
* (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
|
|
30
30
|
* `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
|
|
31
31
|
*/
|
|
32
|
-
import type { CorpusAdapter } from "../../types.
|
|
32
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
33
33
|
export declare const OVERTURE_ADAPTER_ID = "overture";
|
|
34
34
|
export declare const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0";
|
|
35
35
|
export declare function createOvertureAdapter(): CorpusAdapter;
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
*
|
|
29
29
|
* License: stamped `"Public Domain"` per Hawaii state government open-data terms.
|
|
30
30
|
*/
|
|
31
|
-
import type { CorpusAdapter } from "../../types.
|
|
31
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
32
32
|
export declare const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools";
|
|
33
33
|
export declare const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain";
|
|
34
34
|
export declare function createStateHiSchoolsAdapter(): CorpusAdapter;
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*
|
|
16
16
|
* License: stamped `"Public Domain"` per Iowa state government open-data terms.
|
|
17
17
|
*/
|
|
18
|
-
import type { CorpusAdapter } from "../../types.
|
|
18
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
19
19
|
export declare const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors";
|
|
20
20
|
export declare const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain";
|
|
21
21
|
export declare function createStateIaContractorsAdapter(): CorpusAdapter;
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
*
|
|
14
14
|
* License: stamped `"Public Domain"` per New York state government open-data terms.
|
|
15
15
|
*/
|
|
16
|
-
import type { CorpusAdapter } from "../../types.
|
|
16
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
17
17
|
export declare const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries";
|
|
18
18
|
export declare const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain";
|
|
19
19
|
export declare function createStateNyNotariesAdapter(): CorpusAdapter;
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*
|
|
15
15
|
* License: stamped `"Public Domain"` per Texas state government open-data terms.
|
|
16
16
|
*/
|
|
17
|
-
import type { CorpusAdapter } from "../../types.
|
|
17
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
18
18
|
export declare const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries";
|
|
19
19
|
export declare const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain";
|
|
20
20
|
export declare function createStateTxNotariesAdapter(): CorpusAdapter;
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
* ARE valid (CMRA addresses). Those are produced when `pmbRatio > 0` AND the input tuple
|
|
20
20
|
* carries a `street` field.
|
|
21
21
|
*/
|
|
22
|
-
import { type PoBoxBaseTuple } from "../../synthesize-po-box.
|
|
23
|
-
import type { CorpusAdapter } from "../../types.
|
|
22
|
+
import { type PoBoxBaseTuple } from "../../synthesize-po-box.ts";
|
|
23
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
24
24
|
export declare const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box";
|
|
25
25
|
export declare const SYNTH_PO_BOX_LICENSE = "Synthetic \u2014 derived from CC-BY / public-domain input tuples";
|
|
26
26
|
export interface PoBoxInputRow extends PoBoxBaseTuple {
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
* License: stamped `"Public Domain"` per Census Bureau guidance on TIGER/Line. No per-row override
|
|
34
34
|
* needed — every row in TIGER is the same license.
|
|
35
35
|
*/
|
|
36
|
-
import type { CorpusAdapter } from "../../types.
|
|
36
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
37
37
|
export declare const TIGER_ADAPTER_ID = "tiger";
|
|
38
38
|
export declare const TIGER_DEFAULT_LICENSE = "Public Domain";
|
|
39
39
|
/** Build a TIGER adapter. Pure factory so multiple instances can be created in tests. */
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
* License: stamped `"Public Domain"` per the HRSA Data Warehouse's federal government distribution
|
|
29
29
|
* terms.
|
|
30
30
|
*/
|
|
31
|
-
import type { CorpusAdapter } from "../../types.
|
|
31
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
32
32
|
export declare const USGOV_HRSA_FQHC_ADAPTER_ID = "usgov-hrsa-fqhc";
|
|
33
33
|
export declare const USGOV_HRSA_FQHC_DEFAULT_LICENSE = "Public Domain";
|
|
34
34
|
export declare function createUsgovHrsaFqhcAdapter(): CorpusAdapter;
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*
|
|
18
18
|
* License: stamped `"Public Domain"` per IMLS federal government distribution terms.
|
|
19
19
|
*/
|
|
20
|
-
import type { CorpusAdapter } from "../../types.
|
|
20
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
21
21
|
export declare const USGOV_IMLS_PLS_ADAPTER_ID = "usgov-imls-pls";
|
|
22
22
|
export declare const USGOV_IMLS_PLS_DEFAULT_LICENSE = "Public Domain";
|
|
23
23
|
export declare function createUsgovImlsPlsAdapter(): CorpusAdapter;
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* line. STATE is already a USPS abbreviation in the source. License: `"Public Domain"` (US
|
|
19
19
|
* federal).
|
|
20
20
|
*/
|
|
21
|
-
import type { CorpusAdapter } from "../../types.
|
|
21
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
22
22
|
export declare const USGOV_IRS_BMF_ADAPTER_ID = "usgov-irs-bmf";
|
|
23
23
|
export declare const USGOV_IRS_BMF_DEFAULT_LICENSE = "Public Domain";
|
|
24
24
|
export declare function createUsgovIrsBmfAdapter(): CorpusAdapter;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* commercial address space with full structured components.
|
|
12
12
|
*
|
|
13
13
|
* The adapter consumes NDJSON shards produced by `fetch-nad.ts`'s featureserver mode (operator
|
|
14
|
-
* pre-downloads via `
|
|
14
|
+
* pre-downloads via `mailwoman corpus fetch nad`). Each shard is per-OID-range
|
|
15
15
|
* `oids_<start>-<end>.ndjson` with a sibling `.manifest.json`. Adapter iterates every `.ndjson`
|
|
16
16
|
* in the input directory, skipping the `quarantined-bash-bug/` subdir (legacy of the bash-
|
|
17
17
|
* fetcher's silent-page-failure bug).
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
*
|
|
30
30
|
* License: stamped `"Public Domain"` per 17 U.S.C. § 105 (US federal works).
|
|
31
31
|
*/
|
|
32
|
-
import type { CorpusAdapter } from "../../types.
|
|
32
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
33
33
|
export declare const USGOV_NAD_ADAPTER_ID = "usgov-nad";
|
|
34
34
|
export declare const USGOV_NAD_DEFAULT_LICENSE = "Public Domain";
|
|
35
35
|
export declare function createUsgovNADAdapter(): CorpusAdapter;
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* commercial address space with full structured components.
|
|
12
12
|
*
|
|
13
13
|
* The adapter consumes NDJSON shards produced by `fetch-nad.ts`'s featureserver mode (operator
|
|
14
|
-
* pre-downloads via `
|
|
14
|
+
* pre-downloads via `mailwoman corpus fetch nad`). Each shard is per-OID-range
|
|
15
15
|
* `oids_<start>-<end>.ndjson` with a sibling `.manifest.json`. Adapter iterates every `.ndjson`
|
|
16
16
|
* in the input directory, skipping the `quarantined-bash-bug/` subdir (legacy of the bash-
|
|
17
17
|
* fetcher's silent-page-failure bug).
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
*
|
|
21
21
|
* License: stamped `"Public Domain"` per CMS's federal government distribution terms.
|
|
22
22
|
*/
|
|
23
|
-
import type { CorpusAdapter } from "../../types.
|
|
23
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
24
24
|
export declare const USGOV_NPPES_ADAPTER_ID = "usgov-nppes";
|
|
25
25
|
export declare const USGOV_NPPES_DEFAULT_LICENSE = "Public Domain";
|
|
26
26
|
export declare function createUsgovNPPESAdapter(): CorpusAdapter;
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* License: stamped `"Public Domain"` per the SAMHSA Open Data Foundry's federal-government
|
|
28
28
|
* distribution terms.
|
|
29
29
|
*/
|
|
30
|
-
import type { CorpusAdapter } from "../../types.
|
|
30
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
31
31
|
export declare const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator";
|
|
32
32
|
export declare const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain";
|
|
33
33
|
export declare function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter;
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
* sub-blocks require a different data source (JP postcode database or real-world address dumps
|
|
29
29
|
* from MLIT/JapanPost).
|
|
30
30
|
*/
|
|
31
|
-
import type { CanonicalRow, CorpusAdapter } from "../../types.
|
|
31
|
+
import type { CanonicalRow, CorpusAdapter } from "../../types.ts";
|
|
32
32
|
export declare const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp";
|
|
33
33
|
interface PlaceRow {
|
|
34
34
|
id: number;
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
* License: CC0. The adapter stamps every row with `CC0-1.0`.
|
|
40
40
|
*/
|
|
41
41
|
import type { ComponentTag } from "@mailwoman/core/types";
|
|
42
|
-
import type { CorpusAdapter } from "../../types.
|
|
43
|
-
import { type WOFRecord } from "../../wof-json.
|
|
42
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
43
|
+
import { type WOFRecord } from "../../wof-json.ts";
|
|
44
44
|
interface VariantSpec {
|
|
45
45
|
/** Hierarchy-variant id appended to `source_id`. */
|
|
46
46
|
suffix: string;
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
* License: CC0.
|
|
38
38
|
*/
|
|
39
39
|
import type { ComponentTag } from "@mailwoman/core/types";
|
|
40
|
-
import type { CorpusAdapter } from "../../types.
|
|
41
|
-
import { type WOFRecord } from "../../wof-json.
|
|
40
|
+
import type { CorpusAdapter } from "../../types.ts";
|
|
41
|
+
import { type WOFRecord } from "../../wof-json.ts";
|
|
42
42
|
interface VariantSpec {
|
|
43
43
|
suffix: string;
|
|
44
44
|
components: Partial<Record<ComponentTag, string>>;
|
package/out/src/align.d.ts
CHANGED
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
* — NFD `é` occupies two code units where NFC `é` occupies one — and silently so).
|
|
34
34
|
*/
|
|
35
35
|
import type { ComponentTag } from "@mailwoman/core/types";
|
|
36
|
-
import { type Tokenizer } from "./tokenize.
|
|
37
|
-
import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.
|
|
36
|
+
import { type Tokenizer } from "./tokenize.ts";
|
|
37
|
+
import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.ts";
|
|
38
38
|
/** Options for `alignRow`. */
|
|
39
39
|
export interface AlignOptions {
|
|
40
40
|
/** Tokenizer to use. Defaults to `whitespaceTokenizer()`. */
|
package/out/src/build.d.ts
CHANGED
|
@@ -46,10 +46,10 @@
|
|
|
46
46
|
* can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
|
|
47
47
|
* self-contained.
|
|
48
48
|
*/
|
|
49
|
-
import { type ShardManifest } from "./parquet.
|
|
50
|
-
import { type AdapterRunManifest } from "./runner.
|
|
51
|
-
import { type SplitManifest } from "./split.
|
|
52
|
-
import type { AdapterOptions, CorpusAdapter } from "./types.
|
|
49
|
+
import { type ShardManifest } from "./parquet.ts";
|
|
50
|
+
import { type AdapterRunManifest } from "./runner.ts";
|
|
51
|
+
import { type SplitManifest } from "./split.ts";
|
|
52
|
+
import type { AdapterOptions, CorpusAdapter } from "./types.ts";
|
|
53
53
|
/** Stage tags surfaced to `onProgress`. */
|
|
54
54
|
export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest";
|
|
55
55
|
/** Per-invocation options for `buildCorpus`. */
|
package/out/src/index.d.ts
CHANGED
|
@@ -3,25 +3,26 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
-
export * from "./adapter.
|
|
7
|
-
export * from "./adapters/gnaf/assemble.
|
|
8
|
-
export * from "./adapters/index.
|
|
9
|
-
export * from "./align.
|
|
10
|
-
export * from "./build.
|
|
11
|
-
export * from "./format.
|
|
12
|
-
export * from "./golden.
|
|
13
|
-
export * from "./parquet.
|
|
14
|
-
export * from "./
|
|
15
|
-
export * from "./
|
|
16
|
-
export * from "./
|
|
17
|
-
export * from "./
|
|
18
|
-
export * from "./synthesize-
|
|
19
|
-
export * from "./synthesize-
|
|
20
|
-
export * from "./synthesize-
|
|
21
|
-
export * from "./synthesize-
|
|
22
|
-
export * from "./synthesize-
|
|
23
|
-
export * from "./synthesize-
|
|
24
|
-
export * from "./synthesize.
|
|
25
|
-
export * from "./
|
|
26
|
-
export * from "./
|
|
6
|
+
export * from "./adapter.ts";
|
|
7
|
+
export * from "./adapters/gnaf/assemble.ts";
|
|
8
|
+
export * from "./adapters/index.ts";
|
|
9
|
+
export * from "./align.ts";
|
|
10
|
+
export * from "./build.ts";
|
|
11
|
+
export * from "./format.ts";
|
|
12
|
+
export * from "./golden.ts";
|
|
13
|
+
export * from "./parquet.ts";
|
|
14
|
+
export * from "./parquet-wrapper/index.ts";
|
|
15
|
+
export * from "./runner.ts";
|
|
16
|
+
export * from "./shard-recipes/index.ts";
|
|
17
|
+
export * from "./split.ts";
|
|
18
|
+
export * from "./synthesize-anchor-absorption.ts";
|
|
19
|
+
export * from "./synthesize-german.ts";
|
|
20
|
+
export * from "./synthesize-house-venue.ts";
|
|
21
|
+
export * from "./synthesize-intersection.ts";
|
|
22
|
+
export * from "./synthesize-no-street.ts";
|
|
23
|
+
export * from "./synthesize-po-box.ts";
|
|
24
|
+
export * from "./synthesize-street.ts";
|
|
25
|
+
export * from "./synthesize.ts";
|
|
26
|
+
export * from "./tokenize.ts";
|
|
27
|
+
export * from "./types.ts";
|
|
27
28
|
//# sourceMappingURL=index.d.ts.map
|
package/out/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAC5B,cAAc,aAAa,CAAA;AAC3B,cAAc,0BAA0B,CAAA;AACxC,cAAc,YAAY,CAAA;AAC1B,cAAc,mCAAmC,CAAA;AACjD,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,2BAA2B,CAAA;AACzC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,eAAe,CAAA;AAC7B,cAAc,YAAY,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAC5B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,aAAa,CAAA;AAC3B,cAAc,0BAA0B,CAAA;AACxC,cAAc,YAAY,CAAA;AAC1B,cAAc,mCAAmC,CAAA;AACjD,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,2BAA2B,CAAA;AACzC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,eAAe,CAAA;AAC7B,cAAc,YAAY,CAAA"}
|
package/out/src/index.js
CHANGED
|
@@ -11,6 +11,7 @@ export * from "./build.js";
|
|
|
11
11
|
export * from "./format.js";
|
|
12
12
|
export * from "./golden.js";
|
|
13
13
|
export * from "./parquet.js";
|
|
14
|
+
export * from "./parquet-wrapper/index.js";
|
|
14
15
|
export * from "./runner.js";
|
|
15
16
|
export * from "./shard-recipes/index.js";
|
|
16
17
|
export * from "./split.js";
|
package/out/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAC5B,cAAc,aAAa,CAAA;AAC3B,cAAc,0BAA0B,CAAA;AACxC,cAAc,YAAY,CAAA;AAC1B,cAAc,mCAAmC,CAAA;AACjD,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,2BAA2B,CAAA;AACzC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,eAAe,CAAA;AAC7B,cAAc,YAAY,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,cAAc,CAAA;AAC5B,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,aAAa,CAAA;AAC3B,cAAc,cAAc,CAAA;AAC5B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,aAAa,CAAA;AAC3B,cAAc,0BAA0B,CAAA;AACxC,cAAc,YAAY,CAAA;AAC1B,cAAc,mCAAmC,CAAA;AACjD,cAAc,wBAAwB,CAAA;AACtC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,8BAA8B,CAAA;AAC5C,cAAc,2BAA2B,CAAA;AACzC,cAAc,wBAAwB,CAAA;AACtC,cAAc,wBAAwB,CAAA;AACtC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,eAAe,CAAA;AAC7B,cAAc,YAAY,CAAA"}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* @license AGPL-3.0
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*/
|
|
6
|
-
export * from "./reader.
|
|
7
|
-
export * from "./schema.
|
|
8
|
-
export * from "./writer.
|
|
6
|
+
export * from "./reader.ts";
|
|
7
|
+
export * from "./schema.ts";
|
|
8
|
+
export * from "./writer.ts";
|
|
9
9
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs";
|
|
11
11
|
import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js";
|
|
12
12
|
import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js";
|
|
13
|
-
import { type ParquetRecordLike, ParquetSchema } from "./schema.
|
|
13
|
+
import { type ParquetRecordLike, ParquetSchema } from "./schema.ts";
|
|
14
14
|
/** A typed Parquet reader, wrapping the base Parquet reader. */
|
|
15
15
|
export declare class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
|
|
16
16
|
schema: ParquetSchema<T>;
|
|
@@ -15,8 +15,10 @@
|
|
|
15
15
|
import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs";
|
|
16
16
|
import type { WriterOptions } from "@dsnp/parquetjs/dist/lib/declare.js";
|
|
17
17
|
import { type WriteStreamMinimal } from "@dsnp/parquetjs/dist/lib/util.js";
|
|
18
|
-
import { type ParquetRecordLike, ParquetSchema, type ParquetSchemaDefinition, ParquetSchemaDefinitionCache } from "./schema.
|
|
19
|
-
/**
|
|
18
|
+
import { type ParquetRecordLike, ParquetSchema, type ParquetSchemaDefinition, ParquetSchemaDefinitionCache } from "./schema.ts";
|
|
19
|
+
/**
|
|
20
|
+
* A typed Parquet writer, wrapping the base Parquet writer.
|
|
21
|
+
*/
|
|
20
22
|
export declare class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWriter implements AsyncDisposable {
|
|
21
23
|
#private;
|
|
22
24
|
schema: ParquetSchema<T>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,+BAAqC;WAG9D,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAC3D,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,YAAY,EAAE,kBAAkB,EAChC,IAAI,GAAE,aAAkB,GACtB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B,oGAAoG;WAC9E,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,GAAG,EACjC,IAAI,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B,mDAAmD;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7D,2FAA2F;IACrE,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD,8EAA8E;IACxD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|
|
@@ -18,7 +18,9 @@ import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs";
|
|
|
18
18
|
import { osopen } from "@dsnp/parquetjs/dist/lib/util.js";
|
|
19
19
|
import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js";
|
|
20
20
|
import { ParquetSchema, ParquetSchemaDefinitionCache, } from "./schema.js";
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* A typed Parquet writer, wrapping the base Parquet writer.
|
|
23
|
+
*/
|
|
22
24
|
export class ParquetWriter extends BaseParquetWriter {
|
|
23
25
|
static SchemaDefinitionCache = new ParquetSchemaDefinitionCache();
|
|
24
26
|
#flushing = Promise.resolve();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAEtE,MAAM,CAAU,qBAAqB,GAAG,IAAI,4BAA4B,EAAE,CAAA;IACpF,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;IAE5C,MAAM,CAAU,KAAK,CAAC,UAAU,CAC/B,UAAyD,EACzD,YAAgC,EAChC,OAAsB,EAAE;QAExB,MAAM,MAAM,GACX,UAAU,YAAY,aAAa;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;QAEtE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;QAEzF,OAAO,IAAI,aAAa,CAAI,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED,oGAAoG;IACpG,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,UAAyD,EACzD,UAAiC,EACjC,IAAoB;QAEpB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QAEnD,OAAO,aAAa,CAAC,UAAU,CAAI,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED,6DAA6D;IAC7D,mDAAmD;IACnC,WAAW,CAAC,GAAW,EAAE,KAAa;QACrD,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrC,CAAC;IAED,2FAA2F;IAC3E,KAAK,CAAC,SAAS,CAAC,GAAM;QACrC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,8EAA8E;IAC9D,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,SAAS,CAAA;QAEpB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAM;QAEvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;QAElE,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAEnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAA;QAExB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC"}
|
package/out/src/parquet.d.ts
CHANGED
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
* shard's path, row count, byte size, and SHA-256 (computed by re-reading the shard once after
|
|
37
37
|
* close — cheap relative to writing it).
|
|
38
38
|
*/
|
|
39
|
-
import { type ParquetSchemaDefinition } from "./parquet-wrapper/index.
|
|
40
|
-
import type { SplitName } from "./split.
|
|
41
|
-
import type { LabeledRow } from "./types.
|
|
39
|
+
import { type ParquetSchemaDefinition } from "./parquet-wrapper/index.ts";
|
|
40
|
+
import type { SplitName } from "./split.ts";
|
|
41
|
+
import type { LabeledRow } from "./types.ts";
|
|
42
42
|
/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
|
|
43
43
|
export declare const ROW_GROUP_SIZE = 50000;
|
|
44
44
|
/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
|
package/out/src/runner.d.ts
CHANGED
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
* The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
|
|
27
27
|
* steps run later, consuming the JSONL shards this writes.
|
|
28
28
|
*/
|
|
29
|
-
import { type AdapterRegistry } from "./adapter.
|
|
30
|
-
import type { AdapterOptions, CorpusAdapter } from "./types.
|
|
29
|
+
import { type AdapterRegistry } from "./adapter.ts";
|
|
30
|
+
import type { AdapterOptions, CorpusAdapter } from "./types.ts";
|
|
31
31
|
/** Snapshot of the runner's state, emitted on every progress tick. */
|
|
32
32
|
export interface RunnerProgress {
|
|
33
33
|
/** Adapter being driven. */
|
|
@@ -11,6 +11,6 @@
|
|
|
11
11
|
* teaching the model to OVERRIDE a present anchor from context. Ported from
|
|
12
12
|
* scripts/build-anchor-absorption-shard.mjs.
|
|
13
13
|
*/
|
|
14
|
-
import { type ShardRecipe } from "./scaffold.
|
|
14
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
15
15
|
export declare const anchorAbsorptionRecipe: ShardRecipe;
|
|
16
16
|
//# sourceMappingURL=anchor-absorption.d.ts.map
|
|
@@ -11,6 +11,6 @@
|
|
|
11
11
|
*
|
|
12
12
|
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
13
|
*/
|
|
14
|
-
import { type ShardRecipe } from "./scaffold.
|
|
14
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
15
15
|
export declare const boundaryStressRecipe: ShardRecipe;
|
|
16
16
|
//# sourceMappingURL=boundary-stress.d.ts.map
|
|
@@ -23,6 +23,6 @@
|
|
|
23
23
|
* the input. The passed `random` (the framework LCG) is consumed in the exact call order the
|
|
24
24
|
* legacy script used.
|
|
25
25
|
*/
|
|
26
|
-
import { type ShardRecipe } from "./scaffold.
|
|
26
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
27
27
|
export declare const countryBalancedRecipe: ShardRecipe;
|
|
28
28
|
//# sourceMappingURL=country-balanced.d.ts.map
|
|
@@ -13,6 +13,6 @@
|
|
|
13
13
|
* model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
|
|
14
14
|
* cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
|
|
15
15
|
*/
|
|
16
|
-
import { type ShardRecipe } from "./scaffold.
|
|
16
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
17
17
|
export declare const czPcFirstPrepositionRecipe: ShardRecipe;
|
|
18
18
|
//# sourceMappingURL=cz-pcfirst-preposition.d.ts.map
|
|
@@ -29,6 +29,6 @@
|
|
|
29
29
|
* anchor REINFORCES the FR split (FR postcode is trailing, unlike German PLZ-leading — the v0.9.2
|
|
30
30
|
* scar is positional, not universal).
|
|
31
31
|
*/
|
|
32
|
-
import { type ShardRecipe } from "./scaffold.
|
|
32
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
33
33
|
export declare const frAdminSplitRecipe: ShardRecipe;
|
|
34
34
|
//# sourceMappingURL=fr-admin-split.d.ts.map
|
|
@@ -19,6 +19,6 @@
|
|
|
19
19
|
* for FR and will `-inf` these gold labels (the v1.6.0 ~7M-loss blow-up). Disable that mask for any
|
|
20
20
|
* run including this shard.
|
|
21
21
|
*/
|
|
22
|
-
import { type ShardRecipe } from "./scaffold.
|
|
22
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
23
23
|
export declare const frBareStreetRecipe: ShardRecipe;
|
|
24
24
|
//# sourceMappingURL=fr-bare-street.d.ts.map
|
|
@@ -29,6 +29,6 @@
|
|
|
29
29
|
* stream is still open during `run`), and the dead `renderReversed` helper (the legacy `main`
|
|
30
30
|
* inlined the variant logic and never called it).
|
|
31
31
|
*/
|
|
32
|
-
import { type ShardRecipe } from "./scaffold.
|
|
32
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
33
33
|
export declare const frOrderRecipe: ShardRecipe;
|
|
34
34
|
//# sourceMappingURL=fr-order.d.ts.map
|
|
@@ -14,6 +14,6 @@
|
|
|
14
14
|
* German order so well it reads the US/feed-order eval as a "collapse". See
|
|
15
15
|
* docs/articles/evals/2026-06-06-anchor-pilot.md (the order-artifact correction).
|
|
16
16
|
*/
|
|
17
|
-
import { type ShardRecipe } from "./scaffold.
|
|
17
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
18
18
|
export declare const germanRecipe: ShardRecipe;
|
|
19
19
|
//# sourceMappingURL=german.d.ts.map
|
|
@@ -9,6 +9,6 @@
|
|
|
9
9
|
* signal that no-street's distributional shift cost the model. Ported from
|
|
10
10
|
* scripts/build-house-venue-shard.mjs.
|
|
11
11
|
*/
|
|
12
|
-
import { type ShardRecipe } from "./scaffold.
|
|
12
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
13
13
|
export declare const houseVenueRecipe: ShardRecipe;
|
|
14
14
|
//# sourceMappingURL=house-venue.d.ts.map
|