@mailwoman/corpus 2.0.0

package/out/src/types.d.ts

@@ -0,0 +1,168 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *
+ *   The corpus pipeline produces two row shapes:
+ *
+ *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
+ *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
+ *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
+ *        suitable for direct ingestion by the neural training loop.
+ *
+ *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
+ *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ */
+import type { BioLabel, ComponentTag } from "@mailwoman/core/types";
+/**
+ * Provenance + augmentation metadata that travels with every corpus row.
+ *
+ * `synth` is `undefined` for natural (un-augmented) rows; present only when a row was produced by
+ * the synthesis pipeline (see `synthesize.ts`).
+ */
+export interface SourceProvenance {
+    /** Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`. */
+    source: string;
+    /**
+     * Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary
+     * key; for CSV/GeoJSON, a hash of the canonical components. Must be stable across reruns so that
+     * dedup and holdout manifests are reproducible.
+     */
+    source_id: string;
+    /**
+     * Corpus version string. Stamped by the runner, not the adapter. Locked together with the
+     * tokenizer version: `corpus-v0.1.0` ships with `tokenizer-v0.1.0`.
+     */
+    corpus_version: string;
+    /**
+     * Short license label or SPDX id for _this_ row. Defaults to the adapter's `defaultLicense`, but
+     * per-row sources (OpenAddresses) override.
+     */
+    license: string;
+}
+/**
+ * Marker placed on rows produced by `synthesize.ts`. Allows downstream code to weight, stratify, or
+ * exclude augmentations.
+ */
+export interface SynthMarker {
+    /**
+     * Pipeline id describing what augmentation produced this row. Free-form but stable — e.g.
+     * `"case-perturb"`, `"accent-strip"`, `"abbrev-swap"`, `"compose:case-perturb+typo"`.
+     */
+    method: string;
+    /**
+     * `source_id` of the un-augmented row this was derived from. Allows tracing every synthetic row
+     * back to its natural ancestor.
+     */
+    base_source_id: string;
+}
+/**
+ * One address row, before tokenization + BIO labeling.
+ *
+ * `raw` is what a parser would see in the wild — possibly multi-line, with arbitrary whitespace.
+ * `components` is the ground-truth tagging: every `ComponentTag` present in the source data, mapped
+ * to its surface form _as it appears in `raw`_. Alignment uses this to assign BIO labels.
+ *
+ * Country is ISO 3166-1 alpha-2 (`"US"`, `"FR"`). Locale is BCP-47 (`"en-US"`, `"fr-FR"`) and is
+ * optional; adapters that can't be sure leave it empty and let the runner default by country.
+ */
+export interface CanonicalRow extends SourceProvenance {
+    /** Address string as it might appear in source data. */
+    raw: string;
+    /**
+     * Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit
+     * distance threshold) or the row will land in the quarantine pile.
+     */
+    components: Partial<Record<ComponentTag, string>>;
+    /** ISO 3166-1 alpha-2 country code. */
+    country: string;
+    /** Optional BCP-47 locale. Defaulted by country if absent. */
+    locale?: string;
+    /** Present only on synthetic rows. */
+    synth?: SynthMarker;
+}
+/**
+ * Output of `align.ts`. Carries everything `CanonicalRow` does, plus parallel `tokens` and `labels`
+ * arrays of identical length. `labels[i]` is the BIO tag for `tokens[i]`.
+ */
+export interface LabeledRow extends CanonicalRow {
+    /** SentencePiece subword tokens for `raw`. */
+    tokens: readonly string[];
+    /** BIO labels, one per token. Same length as `tokens`. */
+    labels: readonly BioLabel[];
+}
+/**
+ * A row that alignment refused to label. Lands in `/data/corpus/quarantine/` for human review.
+ *
+ * The `reason` is human-readable; common values are `"component-not-found:<tag>"`,
+ * `"edit-distance-exceeded:<tag>"`, `"raw-empty"`. Re-running alignment after a fix should re-emit
+ * the quarantined rows; the runner keys them by `source_id`.
+ */
+export interface QuarantinedRow {
+    row: CanonicalRow;
+    reason: string;
+}
+/**
+ * Per-invocation knobs handed to an adapter by the runner.
+ *
+ * `inputPath` is interpreted by the adapter — it might be a single file path, a directory of
+ * shards, or even an HTTPS URL. Each adapter documents its own expected shape in its README.
+ *
+ * `country` filters to a single ISO 3166-1 alpha-2 country _at the adapter level_. Adapters that
+ * hold multi-country data (OSM PBF, OpenAddresses) MUST honor this; single-country adapters (BAN)
+ * may ignore it but should reject mismatches.
+ *
+ * `limit` is a soft cap on rows emitted; useful for fixture-driven tests and smoke runs.
+ *
+ * `signal` allows the runner to cancel a long-running scan cleanly.
+ */
+export interface AdapterOptions {
+    /** Path to the adapter's input data (file, directory, or URL — adapter-specific). */
+    inputPath: string;
+    /** Optional output directory, available to adapters that maintain side state (rare). */
+    outputDir?: string;
+    /** ISO 3166-1 alpha-2 country filter. */
+    country?: string;
+    /** Soft row cap. Adapters should stop iterating once this is reached. */
+    limit?: number;
+    /** Cancellation hook. Adapters should respect this on every iteration boundary. */
+    signal?: AbortSignal;
+}
+/**
+ * The contract every data source implements.
+ *
+ * Adapters are async generators: they yield `CanonicalRow`s one at a time, the runner consumes them
+ * (writing JSONL + maintaining checksums + driving alignment). Streaming is mandatory — many
+ * sources are tens of millions of rows and cannot be buffered.
+ *
+ * `defaultLicense` is stamped onto every emitted row's `license` field unless the adapter sets
+ * `license` explicitly (e.g. OpenAddresses, which carries per-source licenses).
+ */
+export interface CorpusAdapter {
+    /** Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`. */
+    readonly id: string;
+    /** Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed. */
+    readonly defaultLicense: string;
+    /** One-sentence description shown by `npx mailwoman corpus list`. */
+    readonly description: string;
+    /**
+     * Async iterable of canonical rows.
+     *
+     * Implementations MUST:
+     *
+     * - Honor `opts.country` (filter or reject mismatches).
+     * - Honor `opts.limit` (stop after N rows).
+     * - Respect `opts.signal` on every iteration.
+     * - Set `source` to `this.id` on every emitted row.
+     * - Set `license` to `this.defaultLicense` unless overriding per-row.
+     *
+     * Implementations MUST NOT:
+     *
+     * - Set `corpus_version` (the runner stamps it).
+     * - Mutate previously-yielded rows.
+     */
+    rows(opts: AdapterOptions): AsyncIterable<CanonicalRow>;
+}
+//# sourceMappingURL=types.d.ts.map

package/out/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEnE;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC,wFAAwF;IACxF,MAAM,EAAE,MAAM,CAAA;IAEd;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;IAEtB;;;OAGG;IACH,OAAO,EAAE,MAAM,CAAA;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC3B;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAA;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IACrD,wDAAwD;IACxD,GAAG,EAAE,MAAM,CAAA;IAEX;;;OAGG;IACH,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEjD,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAA;IAEf,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf,sCAAsC;IACtC,KAAK,CAAC,EAAE,WAAW,CAAA;CACnB;AAED;;;GAGG;AACH,MAAM,WAAW,UAAW,SAAQ,YAAY;IAC/C,8CAA8C;IAC9C,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IAEzB,0DAA0D;IAC1D,MAAM,EAAE,SAAS,QAAQ,EAAE,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,YAAY,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,cAAc;IAC9B,qFAAqF;IACrF,SAAS,EAAE,MAAM,CAAA;IAEjB,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,CAAA;IAElB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,yEAAyE;IACzE,KAAK,CAAC,EAAE,MAAM,CAAA;IAEd,mFAAmF;IACnF,MAAM,CAAC,EAAE,WAAW,CAAA;CACpB;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC7B,kFAAkF;IAClF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IAEnB,4FAA4F;IAC5F,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAE/B,qEAAqE;IACrE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAE5B;;;;;;;;;;;;;;;OAeG;IACH,IAAI,CAAC,IAAI,EAAE,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAA;CACvD"}

package/out/src/types.js

@@ -0,0 +1,19 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *
+ *   The corpus pipeline produces two row shapes:
+ *
+ *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
+ *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
+ *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
+ *        suitable for direct ingestion by the neural training loop.
+ *
+ *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
+ *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/out/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG"}

package/out/src/wof-json.d.ts

@@ -0,0 +1,105 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
+ *
+ *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
+ *   `names` table) and onto the per-record GeoJSON bundles published as
+ *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
+ *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
+ *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
+ *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
+ *   about.
+ *
+ *   This module provides:
+ *
+ *   - `WofRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
+ *   - `walkFeatures`: streaming directory walk → parsed `WofRecord`s (skips alt files, bad JSON,
+ *       deprecated records).
+ *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
+ *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
+ *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
+ *       in `source_id` suffixes.
+ *
+ *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
+ *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
+ *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
+ *   silently emitted zero rows from the real corpus.
+ */
+/** A WOF GeoJSON feature, as published by the per-record bundles. */
+export interface WofFeature {
+    type?: string;
+    id?: number | string;
+    properties?: Record<string, unknown> | null;
+}
+/**
+ * Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped —
+ * it's 95% of the file weight and the adapters never consult it.
+ */
+export interface WofRecord {
+    id: number;
+    parent_id: number | null;
+    /** Canonical `wof:name` of the record. */
+    name: string;
+    placetype: string;
+    /** ISO 3166-1 alpha-2 from `wof:country`. */
+    country: string;
+    /**
+     * Localized name variants from `name:*` properties.
+     *
+     * Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the
+     * underlying array (WOF stores variants as arrays even when only one form is present). The
+     * canonical `wof:name` is NOT included here — adapters add a synthetic `"default"` slot for it.
+     */
+    nameVariants: Map<string, string>;
+}
+/**
+ * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
+ *
+ * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias
+ * importer accepts `-1` alongside `1`. The previous SpatiaLite-backed adapters filtered on `= 1`
+ * only and silently emitted zero rows from the corpus — this loosened predicate is the load-bearing
+ * fix.
+ */
+export declare function isCurrentFeature(props: Record<string, unknown>): boolean;
+/**
+ * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays
+ * (`["Saint Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare;
+ * usually historical aliases) are not split into separate rows by this helper — adapters can opt in
+ * by iterating the underlying array if they need it.
+ */
+export declare function extractNameVariants(props: Record<string, unknown>): Map<string, string>;
+/**
+ * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
+ *
+ * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both
+ * collide with the existing source_id separator vocabulary and downstream consumers split on `-`.
+ */
+export declare function normalizeNameKey(rawKey: string): string;
+/**
+ * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WofRecord`s.
+ *
+ * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding
+ * several such repos (the corpus pipeline clones all four into a shared `wof/repos/` root and runs
+ * the adapter against that root). `**\/*.geojson` walks the whole tree; `-alt-` siblings are
+ * skipped since they're alternate-geometry exports, not new records.
+ *
+ * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file
+ * doesn't poison a 3 GB walk. Adapters can add stricter validation downstream if they need it.
+ */
+export declare function walkFeatures(repoDir: string, opts?: {
+    signal?: AbortSignal;
+}): AsyncIterable<WofRecord>;
+/**
+ * Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id`
+ * upward and stopping at the first missing link. A cycle guard halts at any re-visit (defensive —
+ * WOF data is acyclic by construction but corrupt fixtures shouldn't infinite-loop the adapter).
+ *
+ * Records whose ancestors aren't in `byId` (e.g. an FR locality whose region wasn't included in the
+ * cloned repo set) get a shorter chain; the variant emission gracefully degrades.
+ */
+export type AncestryIndex = Map<number, WofRecord[]>;
+export declare function buildAncestryIndex(byId: Map<number, WofRecord>): AncestryIndex;
+//# sourceMappingURL=wof-json.d.ts.map

package/out/src/wof-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAKH,qEAAqE;AACrE,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAA;IACf;;;;;;OAMG;IACH,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAIxE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAYvF;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD;AAwCD;;;;;;;;;;GAUG;AACH,wBAAuB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CA8BlH;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;AAEpD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,aAAa,CAiB9E"}

package/out/src/wof-json.js

@@ -0,0 +1,174 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
+ *
+ *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
+ *   `names` table) and onto the per-record GeoJSON bundles published as
+ *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
+ *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
+ *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
+ *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
+ *   about.
+ *
+ *   This module provides:
+ *
+ *   - `WofRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
+ *   - `walkFeatures`: streaming directory walk → parsed `WofRecord`s (skips alt files, bad JSON,
+ *       deprecated records).
+ *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
+ *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
+ *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
+ *       in `source_id` suffixes.
+ *
+ *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
+ *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
+ *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
+ *   silently emitted zero rows from the real corpus.
+ */
+import FastGlob from "fast-glob";
+import { readFile } from "node:fs/promises";
+/**
+ * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
+ *
+ * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias
+ * importer accepts `-1` alongside `1`. The previous SpatiaLite-backed adapters filtered on `= 1`
+ * only and silently emitted zero rows from the corpus — this loosened predicate is the load-bearing
+ * fix.
+ */
+export function isCurrentFeature(props) {
+    const raw = props["mz:is_current"];
+    const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : 1;
+    return n !== 0;
+}
+/**
+ * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays
+ * (`["Saint Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare;
+ * usually historical aliases) are not split into separate rows by this helper — adapters can opt in
+ * by iterating the underlying array if they need it.
+ */
+export function extractNameVariants(props) {
+    const out = new Map();
+    for (const [key, value] of Object.entries(props)) {
+        if (!key.startsWith("name:"))
+            continue;
+        const candidate = Array.isArray(value)
+            ? value.find((v) => typeof v === "string" && v.trim().length > 0)
+            : typeof value === "string" && value.trim().length > 0
+                ? value
+                : undefined;
+        if (candidate)
+            out.set(key, candidate.trim());
+    }
+    return out;
+}
+/**
+ * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
+ *
+ * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both
+ * collide with the existing source_id separator vocabulary and downstream consumers split on `-`.
+ */
+export function normalizeNameKey(rawKey) {
+    return rawKey.replace(/[:_]/g, "-");
+}
+/** Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason). */
+function recordFromFeature(feature) {
+    if (!feature || feature.type !== "Feature" || !feature.properties)
+        return null;
+    const props = feature.properties;
+    const rawId = typeof feature.id === "number" ? feature.id : props["wof:id"];
+    const id = typeof rawId === "number" ? rawId : typeof rawId === "string" ? Number(rawId) : NaN;
+    if (!Number.isFinite(id))
+        return null;
+    const name = props["wof:name"];
+    if (typeof name !== "string" || !name.trim())
+        return null;
+    const placetype = props["wof:placetype"];
+    if (typeof placetype !== "string" || !placetype)
+        return null;
+    const country = props["wof:country"];
+    if (typeof country !== "string" || !country)
+        return null;
+    if (!isCurrentFeature(props))
+        return null;
+    const parentRaw = props["wof:parent_id"];
+    const parent_id = typeof parentRaw === "number"
+        ? parentRaw
+        : typeof parentRaw === "string" && parentRaw.trim()
+            ? Number(parentRaw)
+            : null;
+    return {
+        id,
+        parent_id: Number.isFinite(parent_id) ? parent_id : null,
+        name: name.trim(),
+        placetype,
+        country,
+        nameVariants: extractNameVariants(props),
+    };
+}
+/**
+ * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WofRecord`s.
+ *
+ * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding
+ * several such repos (the corpus pipeline clones all four into a shared `wof/repos/` root and runs
+ * the adapter against that root). `**\/*.geojson` walks the whole tree; `-alt-` siblings are
+ * skipped since they're alternate-geometry exports, not new records.
+ *
+ * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file
+ * doesn't poison a 3 GB walk. Adapters can add stricter validation downstream if they need it.
+ */
+export async function* walkFeatures(repoDir, opts = {}) {
+    const stream = FastGlob.stream(["**/*.geojson"], {
+        cwd: repoDir,
+        absolute: true,
+        onlyFiles: true,
+        suppressErrors: true,
+    });
+    for await (const entry of stream) {
+        if (opts.signal?.aborted)
+            return;
+        const filePath = String(entry);
+        if (filePath.includes("-alt-"))
+            continue;
+        let text;
+        try {
+            text = await readFile(filePath, "utf8");
+        }
+        catch {
+            continue;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(text);
+        }
+        catch {
+            continue;
+        }
+        const rec = recordFromFeature(parsed);
+        if (rec)
+            yield rec;
+    }
+}
+export function buildAncestryIndex(byId) {
+    const index = new Map();
+    for (const [id, rec] of byId) {
+        const chain = [];
+        const guard = new Set([id]);
+        let cur = rec.parent_id;
+        while (cur !== null && cur > 0) {
+            const parent = byId.get(cur);
+            if (!parent)
+                break;
+            if (guard.has(parent.id))
+                break;
+            chain.push(parent);
+            guard.add(parent.id);
+            cur = parent.parent_id;
+        }
+        index.set(id, chain);
+    }
+    return index;
+}
+//# sourceMappingURL=wof-json.js.map

package/out/src/wof-json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AA+B3C;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA8B;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnF,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;IACrC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAQ;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC9E,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACrD,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,SAAS,CAAA;QACb,IAAI,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9C,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;AACpC,CAAC;AAED,0FAA0F;AAC1F,SAAS,iBAAiB,CAAC,OAAmB;IAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,UAAU;QAAE,OAAO,IAAI,CAAA;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAA;IAEhC,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC3E,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IAC9F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;IAC9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAA;IAEzD,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IACxC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAA;IAE5D,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;IACpC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAExD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IACxC,MAAM,SAAS,GACd,OAAO,SAAS,KAAK,QAAQ;QAC5B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;YACnB,CAAC,CAAC,IAAI,CAAA;IAET,OAAO;QACN,EAAE;QACF,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAmB,CAAC,CAAC,CAAC,CAAE,SAAoB,CAAC,CAAC,CAAC,IAAI;QAC9E,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACjB,SAAS;QACT,OAAO;QACP,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC;KACxC,CAAA;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,OAAe,EAAE,OAAiC,EAAE;IACvF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE;QAChD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAQ;QAExC,IAAI,IAAY,CAAA;QAChB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,IAAI,MAAkB,CAAA;QACtB,IAAI,CAAC;YACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QACrC,IAAI,GAAG;YAAE,MAAM,GAAG,CAAA;IACnB,CAAC;AACF,CAAC;AAYD,MAAM,UAAU,kBAAkB,CAAC,IAA4B;IAC9D,MAAM,KAAK,GAAkB,IAAI,GAAG,EAAE,CAAA;IACtC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,CAAC,EAAE,CAAC,CAAC,CAAA;QACnC,IAAI,GAAG,GAAkB,GAAG,CAAC,SAAS,CAAA;QACtC,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC5B,IAAI,CAAC,MAAM;gBAAE,MAAK;YAClB,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAE,MAAK;YAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAClB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACpB,GAAG,GAAG,MAAM,CAAC,SAAS,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}

package/package.json

@@ -0,0 +1,36 @@
+{
+ "name": "@mailwoman/corpus",
+ "version": "2.0.0",
+ "description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
+ "license": "AGPL-3.0-only",
+ "repository": {
+  "type": "git",
+  "url": "https://github.com/sister-software/mailwoman.git",
+  "directory": "corpus"
+ },
+ "type": "module",
+ "exports": {
+  "./package.json": "./package.json",
+  ".": "./out/src/index.js",
+  "./types": "./out/src/types.js",
+  "./format": "./out/src/format.js"
+ },
+ "dependencies": {
+  "@dsnp/parquetjs": "1.7.0",
+  "@fragaria/address-formatter": "^6.3.0",
+  "@mailwoman/core": "workspace:*",
+  "better-sqlite3": "^11.5.0",
+  "csv-parse": "^5.5.6",
+  "fastest-levenshtein": "^1.0.16",
+  "lru-cache": "^10.0.1"
+ },
+ "files": [
+  "out/**/*.js",
+  "out/**/*.js.map",
+  "out/**/*.d.ts",
+  "out/**/*.d.ts.map"
+ ],
+ "publishConfig": {
+  "access": "public"
+ }
+}