@mailwoman/corpus 9.4.0 → 10.1.0

package/out/adapters/utils/index.js

@@ -1,206 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Adapter framework helpers — the pieces every corpus adapter and the runner share.
- *
- *   This file does **not** define `CorpusAdapter` (that lives in `./types.ts`, which is the single
- *   canonical schema module). It exposes:
- *
- *   - `AdapterRegistry`: a tiny lookup table the CLI + build pipeline use to find adapters by id.
- *   - `InMemoryAdapterRegistry`: the default implementation.
- *   - `stableSourceID(adapterID, components)`: deterministic content-addressed id for adapters whose
- *       source data has no native primary key (CSV, GeoJSON).
- *   - `HOUSE_NUMBER_PREFIX` + `splitStreetLine(line)`: the one house-number/street split every
- *       US CSV adapter uses.
- *   - `composeRaw(parts)`: the envelope-style raw line every US CSV adapter renders.
- *   - `canonicalDedupKey(row)`: normalized signature used to drop near-identical rows during a run.
- *       Adapter-internal dedup; cross-adapter dedup is the runner's job.
- *   - `streamingSha256()`: thin wrapper around `node:crypto` so the runner can hash JSONL output as it
- *       streams (avoids re-reading the slice for the manifest checksum).
- *   - `loadLibpostalDictionary(language, filename)`: the curated libpostal dictionary reader the
- *       street-decompose modules share — the one read this module performs; everything else here is
- *       pure, and other side-effecting code goes in `./runner.ts`.
- */
-import { readLocalTextFile } from "@mailwoman/core/fs/readers";
-import { sha256Hex, createHash } from "@mailwoman/core/hash";
-import { isPresent } from "@mailwoman/core/objects";
-import { resourceDictionaryPath } from "@mailwoman/core/paths";
-import { TextSpliterator } from "spliterator";
-/**
- * Default in-memory registry. The runner constructs one per invocation; the CLI re-uses a shared singleton
- * (`defaultAdapterRegistry`) populated by `./adapters/index.ts` as adapters come online.
- */
-export class InMemoryAdapterRegistry {
-    #byID = new Map();
-    register(adapter) {
-        if (this.#byID.has(adapter.id)) {
-            throw new Error(`AdapterRegistry: id ${JSON.stringify(adapter.id)} already registered`);
-        }
-        this.#byID.set(adapter.id, adapter);
-    }
-    get(id) {
-        return this.#byID.get(id);
-    }
-    list() {
-        return Array.from(this.#byID.values());
-    }
-    ids() {
-        return Array.from(this.#byID.keys());
-    }
-}
-/**
- * Process-wide default registry. Populated by `./adapters/index.ts` as adapters are built; imported by the CLI. Tests
- * should construct their own `InMemoryAdapterRegistry` to avoid cross-test pollution.
- */
-export const defaultAdapterRegistry = new InMemoryAdapterRegistry();
-/**
- * Deterministic content-addressed source id.
- *
- * For adapters whose upstream source has no native primary key (CSV rows, GeoJSON features), the runner expects a
- * stable id so dedup, holdout manifests, and resumability work across reruns. This helper produces one by hashing the
- * adapter id and a canonical serialization of the components dict (keys sorted, values verbatim).
- *
- * Output format: `<adapterID>-<first-12-hex-chars-of-sha256>`. 48 bits of entropy is enough for ~17M rows per adapter
- * before the expected collision count exceeds 1 (birthday paradox); adapters with more rows should extend the prefix
- * length.
- */
-export function stableSourceID(adapterID, components) {
-    return stableSourceIDFromParts(adapterID, components);
-}
-/**
- * {@link stableSourceID} over arbitrary disambiguator keys — a variant index, a slot number, anything that is not a
- * `ComponentTag`. Every key handed in is sorted and hashed either way; only the key vocabulary differs, and the narrow
- * signature above is what stops an adapter hashing a misspelled component name.
- */
-export function stableSourceIDFromParts(adapterID, parts) {
-    const sortedKeys = Object.keys(parts).toSorted();
-    const payload = sortedKeys.map((k) => `${k}=${parts[k] ?? ""}`).join("\u001F");
-    // One `update` over the joined string, not three chained ones — identical byte stream either way,
-    // but the separator has to stay inline or every id in every slice changes.
-    const digest = sha256Hex(`${adapterID}\u001E${payload}`);
-    return `${adapterID}-${digest.slice(0, 12)}`;
-}
-/**
- * Leading house number on a US-style street line: digits, an optional hyphenated range (Queens `40-12`), an optional
- * single alpha suffix (`101A`), then whitespace, then the rest.
- *
- * This decides where the house number ends and the street begins for every US CSV adapter, so an edge case fixed here
- * is fixed for all of them.
- *
- * The remainder must stay `(\S.*)` and must NOT become `(.+)`: `\s+` and `.` both match a tab, so `\s+(.+)$` lets the
- * engine split a run of tabs between the two groups every possible way before failing — quadratic backtracking on
- * attacker-shaped input. Requiring a non-space start removes the overlap. Group 2 is trimmed by the caller either way,
- * so the two forms are indistinguishable on real input; only the failure cost differs.
- */
-export const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(\S.*)$/;
-/**
- * Split a US-style street line on {@link HOUSE_NUMBER_PREFIX}.
- *
- * The US CSV sources follow USPS Publication 28 conventions with hand-entry drift. The leading digit run is the house
- * number (`"123 Main St"`, `"6450 W Indian School Rd"`), and the regex tolerates one trailing letter (`"123A Main St"`)
- * plus an optional hyphenated half (`"40-12 Bell Blvd"`, common in NYC and suburban garden-apartment numbering; Hawaii
- * uses it island-wide — `"47-470 Hui Aeko Place"`).
- *
- * Returns `null` for blank input. Anything that does not match the prefix shape (`"PO Box 1234"`, `"RR 2 Box 67"`, `"HC
- * 1"`) becomes a single `street` value rather than being mangled — the model sees the original surface form and
- * downstream classifiers pick it up. Callers that need those forms recognized as something other than a street (see
- * `usgov-irs-bmf`) test for them BEFORE calling this.
- */
-export function splitStreetLine(line) {
-    const trimmed = line.trim();
-    if (!trimmed)
-        return null;
-    const match = HOUSE_NUMBER_PREFIX.exec(trimmed);
-    if (match)
-        return { house_number: match[1], street: match[2].trim() };
-    return { street: trimmed };
-}
-/**
- * Compose the envelope-style raw address line the US CSV adapters share:
- *
- * `"<venue>, <house> <street> <unit>, <city>, <state> <postcode>"`
- *
- * The venue leads (US conventional addressee-then-address ordering) so a downstream model sees the
- * venue-prefix-then-address shape users actually type into geocoders. Empty segments drop out rather than leaving a
- * dangling separator.
- */
-export function composeRaw(parts) {
-    const streetLine = parts.streetLine ?? [parts.houseNumber, parts.street, parts.unit].filter(isPresent).join(" ").trim();
-    const tail = [parts.locality.trim(), [parts.region, parts.postcode].filter(isPresent).join(" ").trim()]
-        .filter(isPresent)
-        .join(", ");
-    return [parts.venue, streetLine || undefined, tail].filter(isPresent).join(", ");
-}
-/**
- * Load one curated libpostal dictionary (`core/data/libpostal/dictionaries/<language>/<filename>`) as a lower-cased
- * form set. libpostal format: `canonical|abbr|abbr|...` — every form is indexed.
- *
- * `resourceDictionaryPath` already resolves both layouts — `core/data/...` from source and from the packaged `out/`
- * tree. The candidate list this replaced named it TWICE and then guessed a third path off `process.cwd()`, and
- * swallowed every error while probing, so a corrupt dictionary reported as a missing one.
- *
- * The largest libpostal dictionary is 8.4 KB, and each caller runs this once per process at module load.
- */
-export async function loadLibpostalDictionary(language, filename) {
-    const text = await readLocalTextFile(resourceDictionaryPath("libpostal", language, filename));
-    const set = new Set();
-    for (const line of TextSpliterator.from(text)) {
-        const trimmed = line.trim();
-        if (!trimmed || trimmed.startsWith("#"))
-            continue;
-        for (const form of trimmed.split("|")) {
-            const f = form.trim().toLowerCase();
-            if (f) {
-                set.add(f);
-            }
-        }
-    }
-    return set;
-}
-/**
- * Canonical dedup key for a row.
- *
- * Two rows that share this key are treated as duplicates and only the first wins. The key is built from `country`, the
- * sorted `components` dict, and a normalized `raw` (lower-cased, whitespace collapsed). License and provenance fields
- * are intentionally excluded so the same address from multiple adapters is recognized as a duplicate.
- *
- * Synthetic rows are never deduplicated against natural rows: `synth.method` is folded into the key when present,
- * ensuring each augmentation variant survives.
- */
-export function canonicalDedupKey(row) {
-    const sortedKeys = Object.keys(row.components).toSorted();
-    const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\u001F");
-    const rawNorm = row.raw.toLowerCase().replaceAll(/\s+/g, " ").trim();
-    const synthPart = row.synth ? `\u001E${row.synth.method}` : "";
-    return `${row.country}\u001E${rawNorm}\u001E${compPart}${synthPart}`;
-}
-/**
- * Default `StreamingHasher` (SHA-256, hex).
- */
-/**
- * @see {@link StreamingHasher} — this is the incremental counterpart to `sha256Hex`, not a copy of
- *   it. `@mailwoman/core/utils` exposes one-shot digests (`sha256Hex`) and whole-file digests
- *   (`sha256File`); the runner needs neither, because it hashes JSONL lines as they stream past.
- */
-export function streamingSha256() {
-    const h = createHash("sha256");
-    let finalized = false;
-    let digestHex = "";
-    return {
-        update(chunk) {
-            if (finalized)
-                throw new Error("streamingSha256: update() called after digest()");
-            h.update(typeof chunk === "string" ? chunk : chunk);
-        },
-        digest() {
-            if (!finalized) {
-                digestHex = h.digest("hex");
-                finalized = true;
-            }
-            return digestHex;
-        },
-    };
-}
-//# sourceMappingURL=index.js.map

package/out/adapters/utils/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../lib/adapters/utils/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAa,MAAM,sBAAsB,CAAA;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAiC7C;;;GAGG;AACH,MAAM,OAAO,uBAAuB;IACnC,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAA;IAExC,QAAQ,CAAC,OAAsB;QAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAAA;QACxF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IACpC,CAAC;IAED,GAAG,CAAC,EAAU;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,IAAI;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,GAAG;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACrC,CAAC;CACD;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,uBAAuB,EAAE,CAAA;AAEnE;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB,EAAE,UAAiD;IAClG,OAAO,uBAAuB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAA;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACtC,SAAiB,EACjB,KAAmD;IAEnD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAA;IAChD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC9E,kGAAkG;IAClG,2EAA2E;IAC3E,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,EAAE,CAAC,CAAA;IAExD,OAAO,GAAG,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,oCAAoC,CAAA;AAUvE;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAE3B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAE/C,IAAI,KAAK;QAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,EAAE,CAAA;IAEtE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AAC3B,CAAC;AAqBD;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CAAC,KAAsB;IAChD,MAAM,UAAU,GACf,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAErG,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;SACrG,MAAM,CAAC,SAAS,CAAC;SACjB,IAAI,CAAC,IAAI,CAAC,CAAA;IAEZ,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,IAAI,SAAS,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACjF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,QAAgB,EAAE,QAAgB;IAC/E,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,sBAAsB,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;IAC7F,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAQ;QAEjD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAEnC,IAAI,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACX,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAiB;IAClD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAoB,CAAA;IAC3E,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACxF,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpE,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAE9D,OAAO,GAAG,GAAG,CAAC,OAAO,SAAS,OAAO,SAAS,QAAQ,GAAG,SAAS,EAAE,CAAA;AACrE,CAAC;AAcD;;GAEG;AACH;;;;GAIG;AACH,MAAM,UAAU,eAAe;IAC9B,MAAM,CAAC,GAAS,UAAU,CAAC,QAAQ,CAAC,CAAA;IACpC,IAAI,SAAS,GAAG,KAAK,CAAA;IACrB,IAAI,SAAS,GAAG,EAAE,CAAA;IAElB,OAAO;QACN,MAAM,CAAC,KAAK;YACX,IAAI,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;YACjF,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACpD,CAAC;QACD,MAAM;YACL,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC3B,SAAS,GAAG,IAAI,CAAA;YACjB,CAAC;YAED,OAAO,SAAS,CAAA;QACjB,CAAC;KACD,CAAA;AACF,CAAC"}

package/out/adapters/wof-admin-jp/adapter.d.ts

@@ -1,65 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-admin-jp`: Japanese admin-hierarchy adapter.
- *
- *   PROTOTYPE — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
- *   synthesizes BIO-labeled training rows from the global WOF SQLite.
- *
- *   JP addresses differ from US/EU in three ways:
- *
- *   1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
- *        "1-2-3 Minamicho, Setagaya, Tokyo"
- *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighbourhood"
- *        placetype (丁目) is the closest analog to a street but is actually a grid cell.
- *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-NNNN`.
- *
- *   Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
- *
- *   | JP concept | WOF placetype | ComponentTag (Phase 6) |
- *   |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
- *   region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
- *   丁目 (chome) | neighbourhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
- *   (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
- *
- *   This adapter currently emits only the admin chain (region → locality → block). House numbers and
- *   sub-blocks require a different data source (JP postcode database or real-world address dumps
- *   from MLIT/JapanPost).
- */
-import type { CanonicalRow, CorpusAdapter } from "#types";
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export declare const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp";
-interface PlaceRow {
-    id: number;
-    name: string;
-    placetype: string;
-    parent_id: number;
-    country: string;
-}
-/**
- * Synthesize a JP address from a hierarchy chain.
- *
- * Format: `〒<postcode>? <region><locality><neighbourhood>?`
- *
- * No house numbers yet — needs MLIT data.
- */
-export declare function synthesizeJpAddress(chain: PlaceRow[], jpnNames: Map<number, string>): {
-    raw: string;
-    components: CanonicalRow["components"];
-} | null;
-/**
- * Build the JP adapter. Reads from the unified global WOF SQLite, walks admin chains starting from neighbourhoods, and
- * yields canonical rows.
- */
-export declare function createWOFAdminJpAdapter(): CorpusAdapter;
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export declare const wofAdminJpAdapter: CorpusAdapter;
-export {};
-//# sourceMappingURL=adapter.d.ts.map

package/out/adapters/wof-admin-jp/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/wof-admin-jp/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAOH,OAAO,KAAK,EAAkB,YAAY,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAA;AAOzE;;;GAGG;AACH,eAAO,MAAM,uBAAuB,iBAAiB,CAAA;AAErD,UAAU,QAAQ;IACjB,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CACf;AAoCD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAClC,KAAK,EAAE,QAAQ,EAAE,EACjB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAC3B;IACF,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;CACtC,GAAG,IAAI,CAyBP;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,aAAa,CAyEvD;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,eAA4B,CAAA"}

package/out/adapters/wof-admin-jp/adapter.js

@@ -1,161 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-admin-jp`: Japanese admin-hierarchy adapter.
- *
- *   PROTOTYPE — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
- *   synthesizes BIO-labeled training rows from the global WOF SQLite.
- *
- *   JP addresses differ from US/EU in three ways:
- *
- *   1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
- *        "1-2-3 Minamicho, Setagaya, Tokyo"
- *   2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighbourhood"
- *        placetype (丁目) is the closest analog to a street but is actually a grid cell.
- *   3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-NNNN`.
- *
- *   Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
- *
- *   | JP concept | WOF placetype | ComponentTag (Phase 6) |
- *   |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
- *   region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
- *   丁目 (chome) | neighbourhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
- *   (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
- *
- *   This adapter currently emits only the admin chain (region → locality → block). House numbers and
- *   sub-blocks require a different data source (JP postcode database or real-world address dumps
- *   from MLIT/JapanPost).
- */
-import { isPresent } from "@mailwoman/core/objects";
-import { allRows, getRow } from "@mailwoman/core/utils";
-import { DatabaseClient } from "@mailwoman/sqlite/client";
-/**
- * Ancestry hops walked upward from a WOF record before giving up — deeper than any real JP admin chain.
- */
-const MAX_ANCESTRY_DEPTH = 6;
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp";
-/**
- * Walk parent chain up to 6 levels, over a place table already resident in memory.
- *
- * `resolve` is the override for a parent outside the preloaded set (a chain that leaves the country); it is a point
- * query, so keeping the common case out of it is the whole point. The GeoJSON adapters' in-memory analogue is
- * `buildAncestryIndex` (`#utils`); the two stay separate because this one's rows come from SQLite and a chain can leave
- * the preloaded country set.
- */
-function chainOf(byID, resolve, startID) {
-    const out = [];
-    let id = startID;
-    for (let i = 0; i < MAX_ANCESTRY_DEPTH && id > 0; i++) {
-        const row = byID.get(id) ?? resolve(id);
-        if (!row)
-            break;
-        out.push(row);
-        id = row.parent_id;
-    }
-    return out;
-}
-/**
- * Pick the best display name for a place: prefer Japanese variant, fall back to English.
- */
-function pickName(row, jpnNames) {
-    return jpnNames.get(row.id) ?? row.name;
-}
-/**
- * Synthesize a JP address from a hierarchy chain.
- *
- * Format: `〒<postcode>? <region><locality><neighbourhood>?`
- *
- * No house numbers yet — needs MLIT data.
- */
-export function synthesizeJpAddress(chain, jpnNames) {
-    const region = chain.find((r) => r.placetype === "region");
-    const locality = chain.find((r) => r.placetype === "locality" || r.placetype === "county");
-    if (!region || !locality)
-        return null;
-    const neighbourhood = chain.find((r) => r.placetype === "neighbourhood");
-    const regionName = pickName(region, jpnNames);
-    const localityName = pickName(locality, jpnNames);
-    const neighbourhoodName = neighbourhood ? pickName(neighbourhood, jpnNames) : null;
-    const components = {
-        region: regionName,
-        locality: localityName,
-        country: "JP",
-    };
-    if (neighbourhoodName) {
-        components.dependent_locality = neighbourhoodName;
-    }
-    const raw = [regionName, localityName, neighbourhoodName].filter(isPresent).join("");
-    return { raw, components };
-}
-/**
- * Build the JP adapter. Reads from the unified global WOF SQLite, walks admin chains starting from neighbourhoods, and
- * yields canonical rows.
- */
-export function createWOFAdminJpAdapter() {
-    return {
-        id: WOF_ADMIN_JP_ADAPTER_ID,
-        defaultLicense: "CC-BY-4.0",
-        description: "Japanese admin hierarchy from WOF (synthesized addresses without house numbers).",
-        async *rows(opts) {
-            if (opts.country && opts.country !== "JP") {
-                throw new Error(`wof-admin-jp adapter: only JP supported, got country=${opts.country}`);
-            }
-            using db = new DatabaseClient(opts.inputPath, { readOnly: true });
-            const jpnNamesStmt = db.prepare(`SELECT id, name FROM names WHERE language = 'jpn'`);
-            const jpnNames = new Map();
-            for (const row of jpnNamesStmt.all()) {
-                if (!jpnNames.has(row.id)) {
-                    jpnNames.set(row.id, row.name);
-                }
-            }
-            // One read of the JP place table instead of a fresh `prepare` + up to six point queries per
-            // seed; there are tens of thousands of neighbourhood seeds.
-            const byID = new Map();
-            for (const row of allRows(db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE country='JP'`))) {
-                byID.set(row.id, row);
-            }
-            const outsideStmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`);
-            const resolveOutside = (id) => {
-                const row = getRow(outsideStmt, id);
-                if (row) {
-                    byID.set(id, row);
-                }
-                return row;
-            };
-            const seeds = [...byID.values()].filter((row) => row.placetype === "neighbourhood");
-            let emitted = 0;
-            for (const seed of seeds) {
-                if (opts.signal?.aborted)
-                    break;
-                if (opts.limit !== undefined && emitted >= opts.limit)
-                    break;
-                const chain = chainOf(byID, resolveOutside, seed.id);
-                const synth = synthesizeJpAddress(chain, jpnNames);
-                if (!synth)
-                    continue;
-                yield {
-                    raw: synth.raw,
-                    components: synth.components,
-                    country: "JP",
-                    locale: "ja-JP",
-                    source: WOF_ADMIN_JP_ADAPTER_ID,
-                    source_id: `${WOF_ADMIN_JP_ADAPTER_ID}-${seed.id}`,
-                    corpus_version: "",
-                    license: "CC-BY-4.0",
-                };
-                emitted++;
-            }
-        },
-    };
-}
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const wofAdminJpAdapter = createWOFAdminJpAdapter();
-//# sourceMappingURL=adapter.js.map

package/out/adapters/wof-admin-jp/adapter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../lib/adapters/wof-admin-jp/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAA;AAEvD,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAIzD;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,cAAc,CAAA;AAUrD;;;;;;;GAOG;AACH,SAAS,OAAO,CACf,IAA2B,EAC3B,OAA6C,EAC7C,OAAe;IAEf,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,IAAI,EAAE,GAAG,OAAO,CAAA;IAEhB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,kBAAkB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,EAAE,CAAC,CAAA;QAEvC,IAAI,CAAC,GAAG;YAAE,MAAK;QACf,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACb,EAAE,GAAG,GAAG,CAAC,SAAS,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAa,EAAE,QAA6B;IAC7D,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAA;AACxC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAClC,KAAiB,EACjB,QAA6B;IAK7B,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAA;IAE1F,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;IAExE,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IAC7C,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IACjD,MAAM,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAElF,MAAM,UAAU,GAA+B;QAC9C,MAAM,EAAE,UAAU;QAClB,QAAQ,EAAE,YAAY;QACtB,OAAO,EAAE,IAAI;KACb,CAAA;IAED,IAAI,iBAAiB,EAAE,CAAC;QACvB,UAAU,CAAC,kBAAkB,GAAG,iBAAiB,CAAA;IAClD,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,UAAU,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEpF,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB;IACtC,OAAO;QACN,EAAE,EAAE,uBAAuB;QAC3B,cAAc,EAAE,WAAW;QAC3B,WAAW,EAAE,kFAAkF;QAE/F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,wDAAwD,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YACxF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,cAAc,CAAc,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;YAE9E,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,CAAC,mDAAmD,CAAC,CAAA;YACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;YAE1C,KAAK,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,EAAoC,EAAE,CAAC;gBACxE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC3B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAA;gBAC/B,CAAC;YACF,CAAC;YAED,4FAA4F;YAC5F,4DAA4D;YAC5D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAA;YAExC,KAAK,MAAM,GAAG,IAAI,OAAO,CACxB,EAAE,CAAC,OAAO,CAAC,4EAA4E,CAAC,CACxF,EAAE,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,sEAAsE,CAAC,CAAA;YAEtG,MAAM,cAAc,GAAG,CAAC,EAAU,EAAwB,EAAE;gBAC3D,MAAM,GAAG,GAAG,MAAM,CAAW,WAAW,EAAE,EAAE,CAAC,CAAA;gBAE7C,IAAI,GAAG,EAAE,CAAC;oBACT,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;gBAClB,CAAC;gBAED,OAAO,GAAG,CAAA;YACX,CAAC,CAAA;YAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,eAAe,CAAC,CAAA;YAEnF,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAA;gBACpD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;gBAElD,IAAI,CAAC,KAAK;oBAAE,SAAQ;gBAEpB,MAAM;oBACL,GAAG,EAAE,KAAK,CAAC,GAAG;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,uBAAuB;oBAC/B,SAAS,EAAE,GAAG,uBAAuB,IAAI,IAAI,CAAC,EAAE,EAAE;oBAClD,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,WAAW;iBACpB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,uBAAuB,EAAE,CAAA"}

package/out/adapters/wof-admin-json/adapter.d.ts

@@ -1,79 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-admin`: Who's On First admin GeoJSON-bundle adapter.
- *
- *   **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at
- *   `packages/corpus/lib/adapters/wof-admin/`, removed in this same change) was replaced by this
- *   one because the SQLite distribution path was unworkable for the real corpus build:
- *
- *   1. `dist.whosonfirst.org/sqlite/` is dead (NXDOMAIN); the Geocode-Earth mirror is the only one.
- *   2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated
- *        as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows.
- *   3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a
- *        separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases
- *        (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a
- *        patched `is_current` predicate.
- *
- *   Input: a directory containing one or more cloned `whosonfirst-data-admin-<cc>` GitHub repos. Each
- *   repo has `data/XXX/YYY/ZZZ/<wof-id>.geojson` files; `**\/*.geojson` walks the tree recursively.
- *   Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same
- *   record, not new records.
- *
- *   Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair:
- *
- *   - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized
- *       variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`,
- *       `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case:
- *       `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become
- *       training rows for the same WOF id.
- *   - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2,
- *       country → 1, county → 1.
- *
- *   `source_id` is `wof-admin-<wof_id>-<name-slot>-<hierarchy-variant>`. The previous SQLite adapter
- *   used `wof-admin-<wof_id>-<hierarchy-variant>` (no name slot); the new format adds a name-slot
- *   segment so the colloquial / preferred / per-locale variants survive dedup independently.
- *
- *   License: CC0. The adapter stamps every row with `CC0-1.0`.
- */
-import { type WOFVariantSpec } from "#adapters/wof-json-rows";
-import type { CorpusAdapter } from "#types";
-import { type WOFRecord } from "#utils";
-/**
- * Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
- *
- * `selfName` is the surface form to use for the record's own component (locality / region / country / subregion).
- * Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` localized value for variant slots;
- * ancestor names always come from the ancestor's canonical `wof:name`.
- *
- * Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
- * canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
- */
-export declare function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[];
-/**
- * Build the per-record name-slot list. The canonical `"default"` slot uses the OpenCage-canonical country form when the
- * record is itself a country (matches SQLite-adapter behavior); every other placetype's default slot uses `wof:name`
- * verbatim.
- */
-export declare function nameSlotsFor(rec: WOFRecord): Array<{
-    key: string;
-    value: string;
-}>;
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export declare const WOF_ADMIN_ADAPTER_ID = "wof-admin";
-/**
- * Construct the wof-admin JSON-bundle adapter. The adapter is stateless across runs; calling this twice with the same
- * input directory produces byte-identical `canonical.jsonl` (records are emitted in sorted `wof:id` order to be
- * insensitive to filesystem walk ordering).
- */
-export declare function createWOFAdminAdapter(): CorpusAdapter;
-/**
- * Single shared instance, suitable for `defaultAdapterRegistry`.
- */
-export declare const wofAdminAdapter: CorpusAdapter;
-//# sourceMappingURL=adapter.d.ts.map

package/out/adapters/wof-admin-json/adapter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/wof-admin-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAKH,OAAO,EAKN,KAAK,cAAc,EACnB,MAAM,yBAAyB,CAAA;AAChC,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AACzE,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,QAAQ,CAAA;AAgCzE;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAkErG;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAOlF;AAED;;;GAGG;AACH,eAAO,MAAM,oBAAoB,cAAc,CAAA;AAE/C;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,aAAa,CAsCrD;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,eAA0B,CAAA"}