@mailwoman/corpus 9.1.0 → 9.3.0

package/lib/utils/split.ts

@@ -0,0 +1,279 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Train / val / test split with **locality holdout** per the Phase 1 plan.
+ *
+ *   The corpus's val + test sets are not randomly sampled rows — they're entire low-density regions
+ *   held out so the model cannot memorize them at training time. Rationale (per the plan's "Common
+ *   pitfalls" section): random splits leak by neighborhood — a model fed "13 Main St, Springfield,
+ *   IL" in train and "15 Main St, Springfield, IL" in test generalizes via region/locality
+ *   memorization, not by learning the underlying schema.
+ *
+ *   Phase 1 holdouts (chosen for low data density + administrative isolation):
+ *
+ *   - **US**: Vermont, Wyoming, North Dakota
+ *   - **FR**: Corse, Lozère, Creuse
+ *
+ *   Held-out rows are deterministically split 50/50 between val and test by hashing the row's
+ *   `source_id`. Non-held-out rows go to train. The 90/5/5 ratio is approximate — what matters is
+ *   the locality boundary, not the exact split percentages.
+ *
+ *   The output is a `SplitManifest`: three `string[]` arrays of `source_id`. Manifests live in git
+ *   (under `corpus/splits/<version>/`) so reruns are reproducible bit-for-bit.
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { writeLocalJSONFile, writeLocalTextFile, makeDirectories, removePath } from "@mailwoman/core/fs/writers"
+import { createHash } from "@mailwoman/core/hash"
+import { spawnProcess } from "@mailwoman/core/process"
+import { childEnv } from "@mailwoman/core/scripting/utils"
+import { join, type PathBuilderLike } from "path-ts"
+import { JSONSpliterator } from "spliterator"
+
+import type { CanonicalRow, LabeledRow } from "#types"
+
+export type SplitName = "train" | "val" | "test"
+
+export interface SplitOptions {
+	/**
+	 * Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
+	 * splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
+	 * `defaultHoldouts()`.
+	 */
+	holdouts?: Record<string, readonly string[]>
+}
+
+/**
+ * Output manifest: source_id lists per split.
+ */
+export interface SplitManifest {
+	train: string[]
+	val: string[]
+	test: string[]
+	/**
+	 * Echoes the holdouts used, so the manifest is self-describing.
+	 */
+	holdouts: Record<string, readonly string[]>
+	/**
+	 * Corpus version stamped onto the manifest. Read from the first row.
+	 */
+	corpus_version: string
+	/**
+	 * Counts for quick sanity checks.
+	 */
+	counts: { train: number; val: number; test: number; total: number }
+}
+
+/**
+ * Phase 1 default holdouts (per plan).
+ *
+ * - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
+ * - FR: Corse, Lozère, Creuse (small departments / regions).
+ * - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
+ *   the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
+ *   shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
+ *   their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
+ */
+export function defaultHoldouts(): Record<string, readonly string[]> {
+	return {
+		US: ["Vermont", "VT", "Wyoming", "WY", "North Dakota", "ND"],
+		FR: ["Corse", "Lozère", "Lozere", "Creuse"],
+		DE: ["Saarland", "SL", "Mecklenburg-Vorpommern", "MV"],
+	}
+}
+
+type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">
+
+/**
+ * Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
+ * (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
+ * array-based path so the decision is stable regardless of caller.
+ */
+export function splitForRow(
+	row: Pick<SplitInputRow, "source_id" | "country" | "components">,
+	holdouts: Record<string, readonly string[]> = defaultHoldouts()
+): SplitName {
+	const region = row.components.region
+	const countryHoldouts = holdouts[row.country] ?? []
+	const isHeldOut = region !== undefined && countryHoldouts.includes(region)
+
+	if (!isHeldOut) return "train"
+
+	// 50/50 deterministic by source_id hash. Same input always lands in the same split.
+	return hashBucket(row.source_id, 2) === 0 ? "val" : "test"
+}
+
+/**
+ * Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
+ * `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
+ *
+ * Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
+ * streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
+ * split membership in heap.
+ */
+export function splitRows(rows: Iterable<SplitInputRow>, opts: SplitOptions = {}): SplitManifest {
+	const holdouts = opts.holdouts ?? defaultHoldouts()
+	const train: string[] = []
+	const val: string[] = []
+	const test: string[] = []
+	let corpus_version = ""
+
+	for (const row of rows) {
+		if (!corpus_version && row.corpus_version) {
+			corpus_version = row.corpus_version
+		}
+
+		const split = splitForRow(row, holdouts)
+
+		if (split === "train") {
+			train.push(row.source_id)
+		} else if (split === "val") {
+			val.push(row.source_id)
+		} else {
+			test.push(row.source_id)
+		}
+	}
+
+	const total = train.length + val.length + test.length
+
+	return {
+		train,
+		val,
+		test,
+		holdouts,
+		corpus_version,
+		counts: { train: train.length, val: val.length, test: test.length, total },
+	}
+}
+
+/**
+ * Lightweight deterministic 0..(n-1) bucket from a string id.
+ */
+/**
+ * Deterministic bucket for a stable id.
+ *
+ * Stays on raw `createHash` rather than `sha256Hex` from `@mailwoman/core/utils`: it needs the digest BYTES, and the
+ * shared helper returns hex. Re-parsing hex back into bytes to reach the same four octets would cost more than the one
+ * line it saves.
+ */
+export function hashBucket(id: string, n: number): number {
+	const digest = createHash("sha256").update(id).digest()
+	// Read 4 bytes as uint32 to avoid bigint overhead.
+	const u = digest[0]! * 0x01_00_00_00 + digest[1]! * 0x01_00_00 + digest[2]! * 0x01_00 + digest[3]!
+
+	return u % n
+}
+
+/**
+ * Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
+ * id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
+ * including holdouts + counts + corpus version.
+ *
+ * Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
+ */
+export async function writeSplitManifests(manifest: SplitManifest, outputDir: PathBuilderLike): Promise<void> {
+	await makeDirectories(outputDir)
+
+	for (const name of ["train", "val", "test"] as const) {
+		const sorted = [...manifest[name]].toSorted()
+
+		await writeLocalTextFile(sorted.join("\n") + (sorted.length ? "\n" : ""), outputDir, `${name}.txt`)
+	}
+
+	const summary = {
+		corpus_version: manifest.corpus_version,
+		holdouts: manifest.holdouts,
+		counts: manifest.counts,
+	}
+
+	await writeLocalJSONFile(summary, outputDir, "SPLIT_MANIFEST.json")
+}
+
+/**
+ * Type re-export for callers that want to ingest LabeledRow specifically.
+ */
+export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">
+
+/**
+ * Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
+ * streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
+ * handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
+ *
+ * Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
+ * via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
+ */
+export async function writeSplitManifestsFromLabeledFiles(opts: {
+	labeledPaths: Record<SplitName, string>
+	outputDir: PathBuilderLike
+	corpusVersion: string
+	counts: Record<SplitName, number>
+	holdouts?: Record<string, readonly string[]>
+}): Promise<SplitManifest["counts"]> {
+	await makeDirectories(opts.outputDir)
+	const holdouts = opts.holdouts ?? defaultHoldouts()
+
+	for (const split of ["train", "val", "test"] as const) {
+		const labeledPath = opts.labeledPaths[split]
+		const outPath = join(opts.outputDir, `${split}.txt`)
+		await streamSortedSourceIDs(labeledPath, outPath)
+	}
+
+	const total = opts.counts.train + opts.counts.val + opts.counts.test
+
+	const summary = {
+		corpus_version: opts.corpusVersion,
+		holdouts,
+		counts: { ...opts.counts, total },
+	}
+
+	await writeLocalJSONFile(summary, opts.outputDir, "SPLIT_MANIFEST.json")
+
+	return summary.counts
+}
+
+/**
+ * Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
+ * absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
+ */
+async function streamSortedSourceIDs(labeledJsonlPath: string, outPath: string): Promise<void> {
+	const unsortedPath = `${outPath}.unsorted`
+	const out = openWriteStream(unsortedPath, { encoding: "utf8" })
+
+	const outClosed = new Promise<void>((resolve, reject) => {
+		out.on("close", () => resolve())
+		out.on("error", reject)
+	})
+
+	// JSONSpliterator parses each row (skipEmpty drops blank lines); a malformed row throws
+	// SyntaxError out of the loop, matching the prior `reject(err)` fail-loud behavior. `finally`
+	// always ends the write stream so `sort` reads a complete file even if the read throws.
+	try {
+		for await (const obj of JSONSpliterator.fromAsync<{ source_id?: string }>(labeledJsonlPath)) {
+			if (typeof obj.source_id === "string") {
+				out.write(`${obj.source_id}\n`)
+			}
+		}
+	} finally {
+		out.end()
+	}
+
+	await outClosed
+
+	await new Promise<void>((resolve, reject) => {
+		// LC_ALL=C: byte-sort, locale-independent → deterministic across hosts.
+		const proc = spawnProcess("sort", [unsortedPath, "-o", outPath], { env: childEnv({ LC_ALL: "C" }) })
+		proc.on("error", reject)
+
+		proc.on("exit", (code) => {
+			if (code === 0) {
+				resolve()
+			} else {
+				reject(new Error(`sort exited with code ${code}`))
+			}
+		})
+	})
+
+	await removePath(unsortedPath).catch(() => {})
+}

package/lib/utils/tokenize.ts

@@ -0,0 +1,120 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Tokenizer interface for alignment.
+ *
+ *   Two implementations live in the corpus package:
+ *
+ *   1. `whitespaceTokenizer()` (this file): pure-JS, depends on nothing. Splits a string into maximal
+ *        runs of letters/digits/marks, dropping whitespace and standalone punctuation. Used as the
+ *        default for in-container alignment tests and as a fallback when no SentencePiece model is
+ *        available.
+ *   2. `sentencePieceTokenizer(modelPath)` (Phase 1 task #11, deferred): wraps the SentencePiece model
+ *        trained on the corpus. Same interface, different splits. Locked against the corpus version
+ *        (`tokenizer-v0.1.0` ships with `corpus-v0.1.0`).
+ *
+ *   The interface is intentionally minimal — only what alignment needs. Each token comes back with
+ *   its (start, end) character offsets so BIO labels can be assigned by span overlap with component
+ *   spans, independent of how the tokenizer chose its splits.
+ */
+
+/**
+ * A token with its character span in the source string.
+ */
+export interface TokenSpan {
+	/**
+	 * The token text, possibly normalized (case unchanged here; tokenizers may differ).
+	 */
+	text: string
+
+	/**
+	 * Inclusive start offset (UTF-16 code-unit index) in the source string.
+	 */
+	start: number
+
+	/**
+	 * Exclusive end offset in the source string. `text === source.slice(start, end)`.
+	 */
+	end: number
+}
+
+/**
+ * A tokenizer that maps a string to a sequence of `TokenSpan`s.
+ */
+export interface Tokenizer {
+	tokenize(text: string): readonly TokenSpan[]
+}
+
+/**
+ * Whitespace + punctuation tokenizer (pure JS).
+ *
+ * Tokens are maximal runs of unicode word characters (`\p{L}` letters, `\p{N}` digits, `\p{M}` marks, plus `'`, `-`,
+ * `_`). Everything else — whitespace, punctuation, symbols — is treated as a separator and **not** emitted as a token.
+ * The resulting spans cover the original string only on token regions; in-between regions belong to no token.
+ *
+ * This is intentionally lossy at the edges (alignment can still label every meaningful span). A future SentencePiece
+ * tokenizer will preserve all bytes via byte-fallback.
+ */
+export function whitespaceTokenizer(): Tokenizer {
+	// Maximal runs of letters/digits/marks plus the joiners common to addresses
+	// (apostrophe, hyphen, underscore). Comma/space/period etc. are not in the set.
+	const tokenRe = /[\p{L}\p{N}\p{M}'_-]+/gu
+
+	return {
+		tokenize(text: string): readonly TokenSpan[] {
+			const out: TokenSpan[] = []
+			tokenRe.lastIndex = 0
+			let m: RegExpExecArray | null
+
+			while ((m = tokenRe.exec(text))) {
+				out.push({ text: m[0], start: m.index, end: m.index + m[0].length })
+			}
+
+			return out
+		},
+	}
+}
+
+/**
+ * Han characters, the script CJK addresses are written in and the one the whitespace tokenizer cannot split: a run like
+ * `三分场八队` is one "word" to `\p{L}+`, so the aligner could give it one label and never two.
+ */
+const HAN = /\p{Script=Han}/u
+
+/**
+ * Whitespace tokenizer for Latin runs, ONE TOKEN PER CHARACTER for Han runs.
+ *
+ * The CJK sibling model is character-level (CharCNN), so a per-character token is the unit it labels; the Latin tail of
+ * a mixed row (`赵光三分场二十九队, Heilongjiang, China`) keeps the word tokens the Latin aligner has always used. Spans still
+ * come back as `[start, end)` offsets over the source string, so the aligner's span-overlap rule needs no change.
+ */
+export function cjkAwareTokenizer(): Tokenizer {
+	const base = whitespaceTokenizer()
+
+	return {
+		tokenize(text: string): readonly TokenSpan[] {
+			const out: TokenSpan[] = []
+
+			for (const token of base.tokenize(text)) {
+				if (!HAN.test(token.text)) {
+					out.push(token)
+
+					continue
+				}
+
+				// A Han run may carry Latin letters or digits inside it (`3分场2队`); those stay glued to their neighbours
+				// only if they are Han too, so every code point of a Han-bearing token becomes its own token.
+				let offset = token.start
+
+				for (const codePoint of token.text) {
+					out.push({ text: codePoint, start: offset, end: offset + codePoint.length })
+					offset += codePoint.length
+				}
+			}
+
+			return out
+		},
+	}
+}

package/lib/utils/wof-json.ts

@@ -0,0 +1,247 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
+ *
+ *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
+ *   `names` table) and onto the per-record GeoJSON bundles published as
+ *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
+ *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
+ *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
+ *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
+ *   about.
+ *
+ *   This module provides:
+ *
+ *   - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
+ *   - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
+ *       deprecated records).
+ *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
+ *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
+ *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
+ *       in `source_id` suffixes.
+ *
+ *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
+ *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
+ *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
+ *   silently emitted zero rows from the real corpus.
+ */
+
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import FastGlob from "fast-glob"
+
+/**
+ * A WOF GeoJSON feature, as published by the per-record bundles.
+ */
+export interface WOFFeature {
+	type?: string
+	id?: number | string
+	properties?: Record<string, unknown> | null
+}
+
+/**
+ * Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped — it's 95% of the file
+ * weight and the adapters never consult it.
+ */
+export interface WOFRecord {
+	id: number
+	parent_id: number | null
+	/**
+	 * Canonical `wof:name` of the record.
+	 */
+	name: string
+	placetype: string
+	/**
+	 * ISO 3166-1 alpha-2 from `wof:country`.
+	 */
+	country: string
+	/**
+	 * Localized name variants from `name:*` properties.
+	 *
+	 * Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the underlying array (WOF
+	 * stores variants as arrays even when only one form is present). The canonical `wof:name` is NOT included here —
+	 * adapters add a synthetic `"default"` slot for it.
+	 */
+	nameVariants: Map<string, string>
+}
+
+/**
+ * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
+ *
+ * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
+ * alongside `1`. Tightening the predicate to `= 1` is the trap: the postalcode distros then contribute ZERO rows to the
+ * corpus, silently, with nothing raised to notice it by.
+ */
+export function isCurrentFeature(props: Record<string, unknown>): boolean {
+	const raw = props["mz:is_current"]
+	const n = typeof raw === "number" ? raw : typeof raw === "string" ? Number(raw) : 1
+
+	return n !== 0
+}
+
+/**
+ * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
+ * Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
+ * not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
+ */
+export function extractNameVariants(props: Record<string, unknown>): Map<string, string> {
+	const out = new Map<string, string>()
+
+	for (const [key, value] of Object.entries(props)) {
+		if (!key.startsWith("name:")) continue
+
+		const candidate = Array.isArray(value)
+			? value.find((v): v is string => typeof v === "string" && v.trim().length > 0)
+			: typeof value === "string" && value.trim().length
+				? value
+				: undefined
+
+		if (candidate) {
+			out.set(key, candidate.trim())
+		}
+	}
+
+	return out
+}
+
+/**
+ * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
+ *
+ * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
+ * existing source_id separator vocabulary and downstream consumers split on `-`.
+ */
+export function normalizeNameKey(rawKey: string): string {
+	return rawKey.replaceAll(/[:_]/g, "-")
+}
+
+/**
+ * Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason).
+ */
+function recordFromFeature(feature: WOFFeature): WOFRecord | null {
+	if (!feature || feature.type !== "Feature" || !feature.properties) return null
+	const props = feature.properties
+
+	const rawID = typeof feature.id === "number" ? feature.id : props["wof:id"]
+	const id = typeof rawID === "number" ? rawID : typeof rawID === "string" ? Number(rawID) : Number.NaN
+
+	if (!Number.isFinite(id)) return null
+
+	const name = props["wof:name"]
+
+	if (typeof name !== "string" || !name.trim()) return null
+
+	const placetype = props["wof:placetype"]
+
+	if (typeof placetype !== "string" || !placetype) return null
+
+	const country = props["wof:country"]
+
+	if (typeof country !== "string" || !country) return null
+
+	if (!isCurrentFeature(props)) return null
+
+	const parentRaw = props["wof:parent_id"]
+
+	const parent_id =
+		typeof parentRaw === "number"
+			? parentRaw
+			: typeof parentRaw === "string" && parentRaw.trim()
+				? Number(parentRaw)
+				: null
+
+	return {
+		id,
+		parent_id: Number.isFinite(parent_id as number) ? (parent_id as number) : null,
+		name: name.trim(),
+		placetype,
+		country,
+		nameVariants: extractNameVariants(props),
+	}
+}
+
+/**
+ * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
+ *
+ * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
+ * (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
+ * `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
+ * records.
+ *
+ * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
+ * walk. Adapters can add stricter validation downstream if they need it.
+ */
+export async function* walkFeatures(repoDir: string, opts: { signal?: AbortSignal } = {}): AsyncIterable<WOFRecord> {
+	const stream = FastGlob.stream(["**/*.geojson"], {
+		cwd: repoDir,
+		absolute: true,
+		onlyFiles: true,
+		suppressErrors: true,
+	})
+
+	for await (const entry of stream) {
+		if (opts.signal?.aborted) return
+		const filePath = String(entry)
+
+		if (filePath.includes("-alt-")) continue
+
+		let text: string
+
+		try {
+			text = await readLocalTextFile(filePath)
+		} catch {
+			continue
+		}
+
+		const parsed = tryParsingJSON<WOFFeature>(text)
+
+		if (parsed === null) continue
+
+		const rec = recordFromFeature(parsed)
+
+		if (rec) {
+			yield rec
+		}
+	}
+}
+
+/**
+ * Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id` upward and stopping
+ * at the first missing link. A cycle guard halts at any re-visit (defensive — WOF data is acyclic by construction but
+ * corrupt fixtures shouldn't infinite-loop the adapter).
+ *
+ * Records whose ancestors aren't in `byID` (e.g. an FR locality whose region wasn't included in the cloned repo set)
+ * get a shorter chain; the variant emission gracefully degrades.
+ *
+ * `wof-admin-jp`'s `chainOf` walks the same parent-child relation over SQLite `spr` rows with an
+ * outside-the-preloaded-set fallback query — a different substrate with a different escape hatch, kept separate on
+ * purpose.
+ */
+export type AncestryIndex = Map<number, WOFRecord[]>
+
+export function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex {
+	const index: AncestryIndex = new Map()
+
+	for (const [id, rec] of byID) {
+		const chain: WOFRecord[] = []
+		const guard = new Set<number>([id])
+		let cur: number | null = rec.parent_id
+
+		while (cur !== null && cur > 0) {
+			const parent = byID.get(cur)
+
+			if (!parent) break
+
+			if (guard.has(parent.id)) break
+			chain.push(parent)
+			guard.add(parent.id)
+			cur = parent.parent_id
+		}
+
+		index.set(id, chain)
+	}
+
+	return index
+}

package/out/adapters/ban/adapter.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `ban`: Base Adresse Nationale CSV adapter (FR street-level).
+ *
+ *   Input: a CSV dump from `adresse.data.gouv.fr` (semicolon-separated, ~25M rows nationally). The
+ *   adapter only reads the small set of columns needed for the corpus:
+ *
+ *   - `numero` → `house_number`
+ *   - `rep` → repetition index ("bis", "ter") appended to house_number
+ *   - `nom_voie` → `street` (full road name; includes the prefix "Rue", "Avenue", etc.)
+ *   - `code_postal` → `postcode`
+ *   - `nom_commune` → `locality`
+ *
+ *   `region` and `country` are not in BAN. The adapter stamps `country: "FR"` on every row; region is
+ *   left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
+ *   for Phase 1 the row's region is simply absent).
+ *
+ *   License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
+ *   attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
+ *   allowed for training with attribution; the ODbL option's share-alike obligation would defeat
+ *   the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
+ *   conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
+ *   The model card MUST carry the BAN attribution (Tier B obligation).
+ *
+ *   The adapter is streaming-aware: `CSVSpliterator.fromAsync` reads the `;`-delimited dump row by
+ *   row, so a 25M-row file never sits in memory. Honors `opts.limit` for fixture / smoke runs,
+ *   `opts.signal` for cancellation, and `opts.country` for a self-consistency check (errors if
+ *   country !== FR).
+ */
+import type { CorpusAdapter } from "#types";
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export declare const BAN_ADAPTER_ID = "ban";
+export declare function createBanAdapter(): CorpusAdapter;
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export declare const banAdapter: CorpusAdapter;
+//# sourceMappingURL=adapter.d.ts.map

package/out/adapters/ban/adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../lib/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAQH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;;GAGG;AACH,eAAO,MAAM,cAAc,QAAQ,CAAA;AAsDnC,wBAAgB,gBAAgB,IAAI,aAAa,CAqFhD;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,eAAqB,CAAA"}