@mailwoman/corpus 9.3.0 → 10.0.0

package/out/adapters/fcc-bdc/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAQH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,kBAAkB,YAAY,CAAA;AAC3C;;;GAGG;AACH,eAAO,MAAM,uBAAuB,kBAAkB,CAAA;AActD;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAWxE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,aAAa,CAyEnD;AAED;;GAEG;AACH,eAAO,MAAM,aAAa,eAAwB,CAAA"}

package/out/adapters/fcc-bdc/adapter.js

@@ -1,144 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `fcc-bdc`: FCC Broadband Data Collection (BDC) — Fabric-derived location consumer.
- *
- *   The first member of Phase 1.6's "adversarial sources" class. BDC ships the public-domain US
- *   broadband-serviceable-location (BSL) fabric: ~120M addresses keyed by stable `location_id`,
- *   carrying `address_primary` + `city` + `state` + `zip` + `zip_suffix`. Compared to the clean
- *   gazetteer rows from WOF / TIGER / BAN, BDC carries the chaos of address data that has passed
- *   through several layers of human entry + automated geocoding + revision: abbreviation drift,
- *   inconsistent unit designators, "RR" / "HC" / "PSC" rural-route shapes, embedded apartment /
- *   suite numbers that did not survive the address parser cleanly. This is the highest-signal,
- *   hardest-to-normalize address corpus in the federal public-domain catalog.
- *
- *   Following the `tiger` / `wof-admin` pattern, this adapter consumes a SQLite database the operator
- *   pre-builds via the isp-nexus BDC ETL (`/srv/isp-nexus/sync/fcc/bdc/`) or any equivalent
- *   host-side pipeline. The mailwoman side does not download or parse the raw CSV/ZIP distribution
- *   directly — that keeps the adapter narrow and the BDC ingest pluggable.
- *
- *   The SQLite schema is documented in README.md and modeled after `NTIARecord`
- *   (`isp-nexus/fcc/bdc/data-collection.ts`): one row per `location_id`. The adapter splits
- *   `address_primary` into `house_number` (leading numeric prefix, if any) + `street` (everything
- *   after), and combines `zip` + `zip_suffix` into the canonical USPS `postcode` slot.
- *
- *   One CanonicalRow per fabric record. Unlike `tiger` (multiple postcode variants per segment) or
- *   `wof-admin` (multiple hierarchy variants per place), BDC records already represent fully
- *   specified addresses; no fan-out is warranted. Adversarial composition (Phase 1.6 §2.1) is the
- *   mechanism for deriving multiple training rows per BDC record.
- *
- *   License: stamped `"Public Domain"` per the BDC fabric's US federal-government distribution terms.
- *   The CostQuest Fabric source data has its own license; consumers who substitute that path should
- *   re-stamp accordingly.
- */
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter";
-import { DatabaseClient } from "@mailwoman/sqlite/client";
-import { splitStreetLine } from "#adapters/utils";
-import { lookupStateAbbreviation } from "#codex/us-fips-state";
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const FCC_BDC_ADAPTER_ID = "fcc-bdc";
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const FCC_BDC_DEFAULT_LICENSE = "Public Domain";
-/**
- * Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
- *
- * NTIARecord doc is ambiguous about whether `zip_suffix` is the 4-digit extension alone or the full ZIP+4 string. This
- * handles both:
- *
- * - Bare 4-digit extension (`zip="94103"`, `zip_suffix="1234"`) → `"94103-1234"`
- * - Already-joined form (`zip_suffix="94103-1234"`) → returned as-is
- * - No suffix → bare `zip`
- *
- * Empty / whitespace-only suffix is treated as missing.
- */
-export function buildPostcode(zip, suffix) {
-    const z = zip.trim();
-    if (!z)
-        return "";
-    const s = suffix?.trim() ?? "";
-    if (!s)
-        return z;
-    if (s.includes("-"))
-        return s;
-    return `${z}-${s}`;
-}
-/**
- * Build a BDC adapter. Pure factory so multiple instances can be created in tests.
- */
-export function createFccBdcAdapter() {
-    return {
-        id: FCC_BDC_ADAPTER_ID,
-        defaultLicense: FCC_BDC_DEFAULT_LICENSE,
-        description: "FCC Broadband Data Collection — Fabric-derived BSL addresses (public-domain); SQLite DB the operator builds via the isp-nexus BDC ETL.",
-        async *rows(opts) {
-            if (opts.country && opts.country !== "US") {
-                throw new Error(`fcc-bdc adapter: only US supported, got country=${opts.country}`);
-            }
-            const db = new DatabaseClient(opts.inputPath, { readOnly: true });
-            let emitted = 0;
-            try {
-                const stmt = db.prepare(`SELECT location_id, address_primary, city, state, zip, zip_suffix
-					 FROM bdc_locations
-					 ORDER BY location_id`);
-                for (const row of stmt.iterate()) {
-                    if (opts.signal?.aborted)
-                        return;
-                    if (opts.limit !== undefined && emitted >= opts.limit)
-                        return;
-                    const split = splitStreetLine(row.address_primary ?? "");
-                    if (!split)
-                        continue;
-                    const state = lookupStateAbbreviation(row.state);
-                    if (!state)
-                        continue;
-                    const locality = row.city?.trim();
-                    if (!locality)
-                        continue;
-                    const postcode = buildPostcode(row.zip ?? "", row.zip_suffix ?? null);
-                    if (!postcode)
-                        continue;
-                    const components = {
-                        ...(split.house_number ? { house_number: split.house_number } : {}),
-                        street: split.street,
-                        locality,
-                        region: state.abbreviation,
-                        postcode,
-                    };
-                    const raw = formatAddress(components, "US", { separator: ", " });
-                    if (!raw)
-                        continue;
-                    const aligned = reconcileComponents(components, raw);
-                    if (!Object.keys(aligned).length)
-                        continue;
-                    yield {
-                        raw,
-                        components: aligned,
-                        country: "US",
-                        locale: "en-US",
-                        source: FCC_BDC_ADAPTER_ID,
-                        source_id: `${FCC_BDC_ADAPTER_ID}-${row.location_id}`,
-                        corpus_version: "",
-                        license: FCC_BDC_DEFAULT_LICENSE,
-                    };
-                    emitted++;
-                }
-            }
-            finally {
-                await db.destroy();
-            }
-        },
-    };
-}
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const fccBdcAdapter = createFccBdcAdapter();
-//# sourceMappingURL=adapter.js.map

package/out/adapters/fcc-bdc/adapter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAGH,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AACzE,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAEzD,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAA;AAG9D;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAC3C;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAA;AActD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAqB;IAC/D,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAEpB,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IACjB,MAAM,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAE9B,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAA;IAEhB,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAA;IAE7B,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,CAAA;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IAClC,OAAO;QACN,EAAE,EAAE,kBAAkB;QACtB,cAAc,EAAE,uBAAuB;QACvC,WAAW,EACV,wIAAwI;QAEzI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,mDAAmD,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9E,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACtB;;2BAEsB,CACtB,CAAA;gBAED,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAsC,EAAE,CAAC;oBACtE,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,OAAM;oBAEhC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,OAAM;oBAE7D,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAA;oBAExD,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBACpB,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;oBAEhD,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBACpB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAA;oBAEjC,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBACvB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAA;oBAErE,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEvB,MAAM,UAAU,GAA+B;wBAC9C,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACnE,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ;wBACR,MAAM,EAAE,KAAK,CAAC,YAAY;wBAC1B,QAAQ;qBACR,CAAA;oBAED,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBAEhE,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAClB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBAEpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;wBAAE,SAAQ;oBAE1C,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,kBAAkB;wBAC1B,SAAS,EAAE,GAAG,kBAAkB,IAAI,GAAG,CAAC,WAAW,EAAE;wBACrD,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,uBAAuB;qBAChC,CAAA;oBAED,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,MAAM,EAAE,CAAC,OAAO,EAAE,CAAA;YACnB,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAA"}

package/out/adapters/geonames-postal/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/geonames-postal/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAMH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,0BAA0B,oBAAoB,CAAA;AAC3D;;;GAGG;AACH,eAAO,MAAM,+BAA+B,cAAc,CAAA;AAE1D;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;;CAA+E,CAAA;AAEnH,wBAAgB,2BAA2B,IAAI,aAAa,CAkE3D;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,eAAgC,CAAA"}

package/out/adapters/geonames-postal/adapter.js

@@ -1,107 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `geonames-postal`: GeoNames postal-code dump consumer (https://www.geonames.org/, CC-BY-4.0).
- *
- *   The GeoNames postal export (`https://download.geonames.org/export/zip/<CC>.zip`) is a clean,
- *   per-country `postcode → place → admin1` table with the place + region NAMES inline (no aux-file
- *   join needed). It broadens the corpus's postcode→locality→region coverage to ~80 countries, well
- *   beyond `wof-postalcode`/the coordinate-first table — forward coverage for the multi-locale
- *   goal.
- *
- *   Input: a per-country postal dump (`<CC>.txt`, 12 tab-separated columns, no header): country,
- *   postcode, place, admin1_name, admin1_code, admin2__, admin3__, lat, lon, accuracy.
- *
- *   Output: per row, postcode-FIRST (international) variants — the common order for the non-US
- *   locales this fills (US postcodes are already covered by TIGER/WOF, which use postcode-LAST):
- *
- *   1. `{ postcode, locality }` → "AD100 Canillo"
- *   2. `{ postcode, locality, region }` → "AD100 Canillo, Canillo" Prefer configuring this adapter for
- *        non-US countries; for US, the postcode-last sources are the right order. License:
- *        `"CC-BY-4.0"` per row (attribute "GeoNames").
- */
-import { reconcileComponents } from "@mailwoman/formatter";
-import { TSVSpliterator } from "spliterator";
-import { stableSourceID } from "#adapters/utils";
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal";
-/**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0";
-/**
- * GeoNames postal-dump columns (0-based): country, postcode, place, admin1_name, admin1_code, admin2_name, ….
- *
- * Shared with `tools/postcode-triples.ts`, which additionally reads `admin2Name` — the CITY for the IN/MX/PT-shaped
- * exports whose `place` column is a street or colonia.
- */
-export const GEONAMES_POSTAL_COLUMNS = { country: 0, postcode: 1, place: 2, admin1Name: 3, admin2Name: 5 };
-export function createGeonamesPostalAdapter() {
-    return {
-        id: GEONAMES_POSTAL_ADAPTER_ID,
-        defaultLicense: GEONAMES_POSTAL_DEFAULT_LICENSE,
-        description: "GeoNames postcodes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
-        async *rows(opts) {
-            // `header: false` — the GeoNames postal dump is headerless, and the spliterator would
-            // otherwise consume row 1 as column names and lose its first postcode.
-            const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false });
-            let emitted = 0;
-            for await (const rec of rows) {
-                if (opts.signal?.aborted)
-                    break;
-                if (opts.limit !== undefined && emitted >= opts.limit)
-                    break;
-                const cc = (rec[GEONAMES_POSTAL_COLUMNS.country] ?? "").trim();
-                if (!cc)
-                    continue;
-                if (opts.country && cc !== opts.country)
-                    continue;
-                const postcode = (rec[GEONAMES_POSTAL_COLUMNS.postcode] ?? "").trim();
-                const locality = (rec[GEONAMES_POSTAL_COLUMNS.place] ?? "").trim();
-                if (!postcode || !locality)
-                    continue;
-                const region = (rec[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? "").trim();
-                // Postcode-first (international) variants. Skip the region variant when admin1 just
-                // repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
-                const variants = [
-                    { slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
-                ];
-                if (region && region.toLowerCase() !== locality.toLowerCase()) {
-                    variants.push({
-                        slot: "plr",
-                        comp: { postcode, locality, region },
-                        raw: `${postcode} ${locality}, ${region}`,
-                    });
-                }
-                for (const v of variants) {
-                    if (opts.limit !== undefined && emitted >= opts.limit)
-                        break;
-                    const aligned = reconcileComponents(v.comp, v.raw);
-                    if (Object.keys(aligned).length < 2)
-                        continue;
-                    yield {
-                        raw: v.raw,
-                        components: aligned,
-                        country: cc,
-                        source: GEONAMES_POSTAL_ADAPTER_ID,
-                        source_id: `${stableSourceID(GEONAMES_POSTAL_ADAPTER_ID, aligned)}-${v.slot}`,
-                        corpus_version: "",
-                        license: GEONAMES_POSTAL_DEFAULT_LICENSE,
-                    };
-                    emitted++;
-                }
-            }
-        },
-    };
-}
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const geonamesPostalAdapter = createGeonamesPostalAdapter();
-//# sourceMappingURL=adapter.js.map

package/out/adapters/geonames-postal/adapter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/geonames-postal/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAGhD;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,iBAAiB,CAAA;AAC3D;;;GAGG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAG,WAAW,CAAA;AAE1D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAW,CAAA;AAEnH,MAAM,UAAU,2BAA2B;IAC1C,OAAO;QACN,EAAE,EAAE,0BAA0B;QAC9B,cAAc,EAAE,+BAA+B;QAC/C,WAAW,EACV,2HAA2H;QAE5H,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,sFAAsF;YACtF,uEAAuE;YACvE,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;YAExE,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAA+B,EAAE,CAAC;gBACzD,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAE9D,IAAI,CAAC,EAAE;oBAAE,SAAQ;gBAEjB,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,KAAK,IAAI,CAAC,OAAO;oBAAE,SAAQ;gBAEjD,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBACrE,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAElE,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBACpC,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,uBAAuB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAErE,oFAAoF;gBACpF,iFAAiF;gBACjF,MAAM,QAAQ,GAA2E;oBACxF,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,GAAG,QAAQ,IAAI,QAAQ,EAAE,EAAE;iBAC5E,CAAA;gBAED,IAAI,MAAM,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC/D,QAAQ,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,KAAK;wBACX,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE;wBACpC,GAAG,EAAE,GAAG,QAAQ,IAAI,QAAQ,KAAK,MAAM,EAAE;qBACzC,CAAC,CAAA;gBACH,CAAC;gBAED,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBAC1B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,MAAK;oBAC5D,MAAM,OAAO,GAAG,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAA;oBAElD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;wBAAE,SAAQ;oBAE7C,MAAM;wBACL,GAAG,EAAE,CAAC,CAAC,GAAG;wBACV,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,EAAE;wBACX,MAAM,EAAE,0BAA0B;wBAClC,SAAS,EAAE,GAAG,cAAc,CAAC,0BAA0B,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;wBAC7E,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,+BAA+B;qBACxC,CAAA;oBAED,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,2BAA2B,EAAE,CAAA"}

package/out/adapters/gnaf/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/gnaf/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAOH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,eAAe,SAAS,CAAA;AACrC;;GAEG;AACH,eAAO,MAAM,oBAAoB,cAAc,CAAA;AA4B/C;;;GAGG;AACH,wBAAgB,iBAAiB,IAAI,aAAa,CAkEjD;AAED;;GAEG;AACH,eAAO,MAAM,WAAW,eAAsB,CAAA"}

package/out/adapters/gnaf/adapter.js

@@ -1,124 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   G-NAF (Australia) corpus adapter — the parser-teaching half of #208.
- *
- *   The model mis-parses Australian addresses in their native postcode-first / house-number-last
- *   order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street
- *   ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap,
- *   not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the
- *   parse were order-robust). EU survives the same eval because its postcodes are
- *   format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only
- *   disambiguates by position.
- *
- *   So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real
- *   AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first,
- *   locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every
- *   position across the slice. This is the exact mechanism that fixed #148's v1.9.0 order-overfit
- *   for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0);
- *   AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes
- *   unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a
- *   clean single-variable extension of the proven recipe + matches its source-mass structure. The
- *   corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so
- *   alignment lands).
- *
- *   Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per
- *   line). Open G-NAF licence — attribute "Geoscape Australia".
- */
-import { tryParsingJSON } from "@mailwoman/core/json";
-import { reconcileComponents } from "@mailwoman/formatter";
-import { TextSpliterator } from "spliterator";
-import { stableSourceID } from "#adapters/utils";
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const GNAF_ADAPTER_ID = "gnaf";
-/**
- * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style).
- */
-export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0";
-/**
- * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the
- * two postcode-LEADING forms are the ones it fails, so they carry the change. We keep the canonical form too so the
- * retrain doesn't forget it.
- */
-function renderOrders(c) {
-    const region = c.region ? ` ${c.region}` : "";
-    return [
-        // real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000"
-        `${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`,
-        // postcode-first (the dominant failure): "2000 Carlton, Barry Street 50"
-        `${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`,
-        // locality-first: "Carlton, 2000, Barry Street 50"
-        `${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`,
-    ];
-}
-/**
- * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned
- * to AU regardless of `opts.country` (G-NAF is Australia-only).
- */
-export function createGNAFAdapter() {
-    return {
-        id: GNAF_ADAPTER_ID,
-        defaultLicense: GNAF_DEFAULT_LICENSE,
-        description: "G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.",
-        async *rows(opts) {
-            let emitted = 0;
-            let idx = 0;
-            // Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on
-            // loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is
-            // gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks.
-            // The render order rotates (i % 3), matching v1.9.1's rerender.
-            for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
-                if (opts.signal?.aborted)
-                    break;
-                if (opts.limit !== undefined && emitted >= opts.limit)
-                    break;
-                if (!line.trim())
-                    continue;
-                const t = tryParsingJSON(line);
-                if (t === null)
-                    continue;
-                if (!t.house_number || !t.street || !t.locality || !t.postcode)
-                    continue;
-                const orders = renderOrders(t);
-                const order = idx % orders.length;
-                idx++;
-                const raw = orders[order];
-                const components = {
-                    house_number: t.house_number,
-                    street: t.street,
-                    locality: t.locality,
-                    postcode: t.postcode,
-                };
-                // region (state) rides only the canonical render (order 0); the postcode-leading layouts
-                // omit it (matching the eval's serialization) so it never breaks verbatim alignment.
-                if (order === 0 && t.region) {
-                    components.region = t.region;
-                }
-                const aligned = reconcileComponents(components, raw);
-                if (!Object.keys(aligned).length)
-                    continue;
-                yield {
-                    raw,
-                    components: aligned,
-                    country: "AU",
-                    locale: "en-AU",
-                    source: GNAF_ADAPTER_ID,
-                    source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`,
-                    corpus_version: "",
-                    license: GNAF_DEFAULT_LICENSE,
-                };
-                emitted++;
-            }
-        },
-    };
-}
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const gnafAdapter = createGNAFAdapter();
-//# sourceMappingURL=adapter.js.map

package/out/adapters/gnaf/adapter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/gnaf/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAGhD;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,MAAM,CAAA;AACrC;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAA;AAU/C;;;;GAIG;AACH,SAAS,YAAY,CAAC,CAAY;IACjC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAE7C,OAAO;QACN,yGAAyG;QACzG,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE;QACrE,yEAAyE;QACzE,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,EAAE;QAC5D,mDAAmD;QACnD,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,YAAY,EAAE;KAC7D,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB;IAChC,OAAO;QACN,EAAE,EAAE,eAAe;QACnB,cAAc,EAAE,oBAAoB;QACpC,WAAW,EACV,4KAA4K;QAE7K,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;YACf,IAAI,GAAG,GAAG,CAAC,CAAA;YAEX,gGAAgG;YAChG,kGAAkG;YAClG,qGAAqG;YACrG,gEAAgE;YAChE,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpE,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBAAE,SAAQ;gBAE1B,MAAM,CAAC,GAAG,cAAc,CAAY,IAAI,CAAC,CAAA;gBAEzC,IAAI,CAAC,KAAK,IAAI;oBAAE,SAAQ;gBAExB,IAAI,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ;oBAAE,SAAQ;gBAExE,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;gBAC9B,MAAM,KAAK,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;gBAEjC,GAAG,EAAE,CAAA;gBACL,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAE,CAAA;gBAE1B,MAAM,UAAU,GAA+B;oBAC9C,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;iBACpB,CAAA;gBAED,yFAAyF;gBACzF,qFAAqF;gBACrF,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;oBAC7B,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAA;gBAC7B,CAAC;gBAED,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAEpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;oBAAE,SAAQ;gBAE1C,MAAM;oBACL,GAAG;oBACH,UAAU,EAAE,OAAO;oBACnB,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,eAAe;oBACvB,SAAS,EAAE,GAAG,cAAc,CAAC,eAAe,EAAE,OAAO,CAAC,KAAK,KAAK,EAAE;oBAClE,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,oBAAoB;iBAC7B,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,iBAAiB,EAAE,CAAA"}

package/out/adapters/gnaf/assemble.d.ts

@@ -1,65 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
- *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
- *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
- *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
- *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
- *   (ACT/NSW/…).
- *
- *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
- *   keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
- *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
- *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
- *   bounded (the OOM lesson from the Overture ingest).
- *
- *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
- *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
- *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
- *   component labels, not lat/lon.
- *
- *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
- *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
- *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
- */
-export interface GNAFAssembleOptions {
-    /**
-     * G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables).
-     */
-    standardDir: string;
-    /**
-     * Target sample size (uniform reservoir → population-proportional across states).
-     */
-    sampleSize: number;
-    /**
-     * Output JSONL path.
-     */
-    out: string;
-    /**
-     * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
-     */
-    holdoutPath?: string;
-    /**
-     * Progress sink (the CLI passes a setter).
-     */
-    onProgress?: (message: string) => void;
-}
-export interface GNAFAssembleResult {
-    written: number;
-    seen: number;
-    heldOut: number;
-    byState: Record<string, number>;
-}
-/**
- * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
- */
-export declare function titlecase(s: string): string;
-/**
- * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
- */
-export declare function gnafHoldoutKey(street: string, locality: string, postcode: string): string;
-export declare function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult>;
-//# sourceMappingURL=assemble.d.ts.map

package/out/adapters/gnaf/assemble.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"assemble.d.ts","sourceRoot":"","sources":["../../../lib/adapters/gnaf/assemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAQH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,GAAG,EAAE,MAAM,CAAA;IACX;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACtC;AAED,MAAM,WAAW,kBAAkB;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,wBAAgB,SAAS,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAK3C;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEzF;AA8CD,wBAAsB,YAAY,CAAC,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAmGzF"}

package/out/adapters/gnaf/assemble.js

@@ -1,154 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
- *   Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
- *   "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
- *   reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
- *   PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
- *   (ACT/NSW/…).
- *
- *   Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
- *   keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
- *   which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
- *   ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
- *   bounded (the OOM lesson from the Overture ingest).
- *
- *   No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
- *   AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
- *   pinned (65%→87% if the parse were order-robust). The parser needs the address string +
- *   component labels, not lat/lon.
- *
- *   Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
- *   multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
- *   excluded by (street, locality, postcode) so the training slice never overlaps the benchmark.
- */
-import { readDirectory } from "@mailwoman/core/fs/readers";
-import { openWriteStream } from "@mailwoman/core/fs/streams";
-import { tryParsingJSON } from "@mailwoman/core/json";
-import { join } from "path-ts";
-import { PSVSpliterator, TextSpliterator } from "spliterator";
-/**
- * UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour).
- */
-export function titlecase(s) {
-    return s
-        .toLowerCase()
-        .replaceAll(/(^|[\s'\-/])([a-z])/g, (_m, p, c) => p + c.toUpperCase())
-        .trim();
-}
-/**
- * Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative).
- */
-export function gnafHoldoutKey(street, locality, postcode) {
-    return `${street}|${locality}|${postcode}`.toLowerCase();
-}
-async function* psvObjects(path) {
-    yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true });
-}
-/**
- * Load a small lookup table fully into a Map keyed by `keyCol`.
- */
-async function loadMap(paths, keyCol, pick) {
-    const m = new Map();
-    for (const p of paths) {
-        for await (const r of psvObjects(p)) {
-            const k = r[keyCol];
-            if (k != null && k !== "") {
-                m.set(String(k), pick(r));
-            }
-        }
-    }
-    return m;
-}
-/**
- * Build the held-out key set from an eval JSONL whose rows carry a `components` object.
- */
-async function loadHoldout(path) {
-    const keys = new Set();
-    for await (const line of TextSpliterator.fromAsync(path)) {
-        if (!line.trim())
-            continue;
-        const c = tryParsingJSON(line)?.components;
-        if (c?.street && c?.locality && c?.postcode) {
-            keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode));
-        }
-    }
-    return keys;
-}
-export async function assembleGNAF(opts) {
-    const progress = opts.onProgress ?? (() => { });
-    const files = await readDirectory(opts.standardDir);
-    const pick = (re, exclude) => files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f));
-    // `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
-    const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/);
-    const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/);
-    const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/);
-    const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set();
-    if (opts.holdoutPath) {
-        progress(`held-out eval keys: ${holdout.size}`);
-    }
-    progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`);
-    const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
-        name: String(r.STREET_NAME ?? ""),
-        type: String(r.STREET_TYPE_CODE ?? ""),
-        suffix: String(r.STREET_SUFFIX_CODE ?? ""),
-    }));
-    const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""));
-    progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`);
-    const reservoir = [];
-    let seen = 0;
-    let heldOut = 0;
-    for (const p of addressPaths) {
-        const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? "";
-        for await (const r of psvObjects(p)) {
-            const numberFirst = String(r.NUMBER_FIRST ?? "");
-            if (!numberFirst || r.DATE_RETIRED || !r.POSTCODE)
-                continue;
-            const st = streetMap.get(String(r.STREET_LOCALITY_PID ?? ""));
-            const suburbRaw = localityMap.get(String(r.LOCALITY_PID ?? ""));
-            if (!st?.name || !suburbRaw)
-                continue;
-            const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim();
-            const locality = titlecase(suburbRaw);
-            const postcode = String(r.POSTCODE);
-            if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
-                heldOut++;
-                continue;
-            }
-            let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "");
-            if (r.NUMBER_LAST) {
-                house = `${house}-${String(r.NUMBER_LAST)}`;
-            }
-            if (r.FLAT_NUMBER) {
-                house = `${String(r.FLAT_NUMBER)}/${house}`;
-            }
-            const tuple = { house_number: house, street, locality, region: state, postcode };
-            seen++;
-            if (reservoir.length < opts.sampleSize) {
-                reservoir.push(tuple);
-            }
-            else {
-                const j = Math.floor(Math.random() * seen);
-                if (j < opts.sampleSize) {
-                    reservoir[j] = tuple;
-                }
-            }
-        }
-        progress(`${state}: ${seen.toLocaleString()} valid joinable seen`);
-    }
-    const out = openWriteStream(opts.out);
-    const byState = {};
-    for (const t of reservoir) {
-        out.write(JSON.stringify(t) + "\n");
-        byState[t.region] = (byState[t.region] ?? 0) + 1;
-    }
-    await new Promise((res) => {
-        out.end(res);
-    });
-    progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`);
-    return { written: reservoir.length, seen, heldOut, byState };
-}
-//# sourceMappingURL=assemble.js.map

package/out/adapters/gnaf/assemble.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"assemble.js","sourceRoot":"","sources":["../../../lib/adapters/gnaf/assemble.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAgC7D;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,CAAS;IAClC,OAAO,CAAC;SACN,WAAW,EAAE;SACb,UAAU,CAAC,sBAAsB,EAAE,CAAC,EAAE,EAAE,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;SACrF,IAAI,EAAE,CAAA;AACT,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,QAAgB;IAChF,OAAO,GAAG,MAAM,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;AACzD,CAAC;AAID,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,IAAY;IACtC,KAAK,CAAC,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,CAAuB,CAAA;AAC9F,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,OAAO,CAAI,KAAe,EAAE,MAAc,EAAE,IAAmB;IAC7E,MAAM,CAAC,GAAG,IAAI,GAAG,EAAa,CAAA;IAE9B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;YAEnB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC3B,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAC1B,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,SAAQ;QAE1B,MAAM,CAAC,GAAG,cAAc,CAA0C,IAAI,CAAC,EAAE,UAAU,CAAA;QAEnF,IAAI,CAAC,EAAE,MAAM,IAAI,CAAC,EAAE,QAAQ,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC7C,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAyB;IAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IAC9C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAEnD,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,OAAgB,EAAE,EAAE,CAC7C,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAA;IAEvG,+FAA+F;IAC/F,MAAM,WAAW,GAAG,IAAI,CAAC,4BAA4B,CAAC,CAAA;IACtD,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,EAAE,4BAA4B,CAAC,CAAA;IAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,2BAA2B,CAAC,CAAA;IAEtD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,EAAU,CAAA;IAE1F,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACtB,QAAQ,CAAC,uBAAuB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;IAChD,CAAC;IAED,QAAQ,CAAC,4BAA4B,WAAW,CAAC,MAAM,uBAAuB,aAAa,CAAC,MAAM,IAAI,CAAC,CAAA;IAEvG,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,qBAAqB,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3E,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;QACjC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,CAAC;QACtC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,kBAAkB,IAAI,EAAE,CAAC;KAC1C,CAAC,CAAC,CAAA;IAEH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,aAAa,EAAE,cAAc,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,CAAA;IACtG,QAAQ,CAAC,WAAW,SAAS,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,WAAW,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;IAEtG,MAAM,SAAS,GACd,EAAE,CAAA;IAEH,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAEnE,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAA;YAEhD,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,QAAQ;gBAAE,SAAQ;YAC3D,MAAM,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,CAAA;YAC7D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAA;YAE/D,IAAI,CAAC,EAAE,EAAE,IAAI,IAAI,CAAC,SAAS;gBAAE,SAAQ;YACrC,MAAM,MAAM,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAA;YACjH,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;YACrC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC7D,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,IAAI,KAAK,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAEtF,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnB,KAAK,GAAG,GAAG,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAA;YAC5C,CAAC;YAED,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBACnB,KAAK,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,KAAK,EAAE,CAAA;YAC5C,CAAC;YAED,MAAM,KAAK,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;YAEhF,IAAI,EAAE,CAAA;YAEN,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAA;gBAE1C,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBACzB,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAA;gBACrB,CAAC;YACF,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,GAAG,KAAK,KAAK,IAAI,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAE1C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3B,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;QACnC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,SAAS,CAAC,MAAM,CAAC,cAAc,EAAE,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IAE3E,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;AAC7D,CAAC"}

package/out/adapters/state-hi-schools/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/state-hi-schools/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,2BAA2B,qBAAqB,CAAA;AAC7D;;;GAGG;AACH,eAAO,MAAM,gCAAgC,kBAAkB,CAAA;AAqC/D,wBAAgB,2BAA2B,IAAI,aAAa,CA+E3D;AAED;;GAEG;AACH,eAAO,MAAM,qBAAqB,eAAgC,CAAA"}