@mailwoman/corpus 7.2.0 → 7.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mailwoman/corpus",
-  "version": "7.2.0",
+  "version": "7.2.1",
   "description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
   "license": "AGPL-3.0-only OR LicenseRef-Commercial",
   "repository": {
@@ -12,55 +12,87 @@
     "out/**/*.js",
     "out/**/*.js.map",
     "out/**/*.d.ts",
-    "out/**/*.d.ts.map"
+    "out/**/*.d.ts.map",
+    "*.ts",
+    "*.tsx",
+    "**/*.ts",
+    "**/*.tsx",
+    "!*.test.ts",
+    "!*.test.tsx",
+    "!**/*.test.ts",
+    "!**/*.test.tsx"
   ],
   "type": "module",
   "exports": {
     "./package.json": "./package.json",
     ".": {
-      "node": "./src/index.ts",
-      "default": "./out/src/index.js",
-      "types": "./out/src/index.d.ts"
+      "types": "./out/src/index.d.ts",
+      "default": "./out/src/index.js"
     },
     "./types": {
-      "node": "./src/types.ts",
-      "default": "./out/src/types.js",
-      "types": "./out/src/types.d.ts"
+      "types": "./out/src/types.d.ts",
+      "default": "./out/src/types.js"
     },
     "./format": {
-      "node": "./src/format.ts",
-      "default": "./out/src/format.js",
-      "types": "./out/src/format.d.ts"
+      "types": "./out/src/format.d.ts",
+      "default": "./out/src/format.js"
     },
     "./license": {
-      "node": "./src/license.ts",
-      "default": "./out/src/license.js",
-      "types": "./out/src/license.d.ts"
+      "types": "./out/src/license.d.ts",
+      "default": "./out/src/license.js"
     },
     "./build": {
-      "node": "./src/build.ts",
-      "default": "./out/src/build.js",
-      "types": "./out/src/build.d.ts"
+      "types": "./out/src/build.d.ts",
+      "default": "./out/src/build.js"
     },
     "./adapters": {
-      "node": "./src/adapters/index.ts",
-      "default": "./out/src/adapters/index.js",
-      "types": "./out/src/adapters/index.d.ts"
+      "types": "./out/src/adapters/index.d.ts",
+      "default": "./out/src/adapters/index.js"
     },
     "./tools": {
-      "node": "./src/tools/index.ts",
-      "default": "./out/src/tools/index.js",
-      "types": "./out/src/tools/index.d.ts"
+      "types": "./out/src/tools/index.d.ts",
+      "default": "./out/src/tools/index.js"
     }
   },
   "publishConfig": {
-    "access": "public"
+    "access": "public",
+    "exports": {
+      "./package.json": "./package.json",
+      ".": {
+        "types": "./out/src/index.d.ts",
+        "default": "./out/src/index.js"
+      },
+      "./types": {
+        "types": "./out/src/types.d.ts",
+        "default": "./out/src/types.js"
+      },
+      "./format": {
+        "types": "./out/src/format.d.ts",
+        "default": "./out/src/format.js"
+      },
+      "./license": {
+        "types": "./out/src/license.d.ts",
+        "default": "./out/src/license.js"
+      },
+      "./build": {
+        "types": "./out/src/build.d.ts",
+        "default": "./out/src/build.js"
+      },
+      "./adapters": {
+        "types": "./out/src/adapters/index.d.ts",
+        "default": "./out/src/adapters/index.js"
+      },
+      "./tools": {
+        "types": "./out/src/tools/index.d.ts",
+        "default": "./out/src/tools/index.js"
+      }
+    }
   },
   "dependencies": {
     "@dsnp/parquetjs": "1.8.8",
-    "@mailwoman/codex": "7.2.0",
-    "@mailwoman/core": "7.2.0",
-    "@mailwoman/formatter": "7.2.0",
+    "@mailwoman/codex": "7.2.1",
+    "@mailwoman/core": "7.2.1",
+    "@mailwoman/formatter": "7.2.1",
     "csv-parse": "^7.0.1",
     "fast-glob": "^3.3.3",
     "fastest-levenshtein": "^1.0.16",

package/src/adapter.ts

@@ -0,0 +1,153 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter framework helpers — the pieces every corpus adapter and the runner share.
+ *
+ *   This file does **not** define `CorpusAdapter` (that lives in `./types.ts`, which is the single
+ *   canonical schema module). It exposes:
+ *
+ *   - `AdapterRegistry`: a tiny lookup table the CLI + build pipeline use to find adapters by id.
+ *   - `InMemoryAdapterRegistry`: the default implementation.
+ *   - `stableSourceID(adapterID, components)`: deterministic content-addressed id for adapters whose
+ *       source data has no native primary key (CSV, GeoJSON).
+ *   - `canonicalDedupKey(row)`: normalized signature used to drop near-identical rows during a run.
+ *       Adapter-internal dedup; cross-adapter dedup is the runner's job.
+ *   - `streamingSha256()`: thin wrapper around `node:crypto` so the runner can hash JSONL output as it
+ *       streams (avoids re-reading the shard for the manifest checksum).
+ *
+ *   Everything here is pure (no I/O); side-effecting code goes in `./runner.ts`.
+ */
+
+import { createHash, type Hash } from "node:crypto"
+
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import type { CanonicalRow, CorpusAdapter } from "./types.ts"
+
+/**
+ * Lookup table for corpus adapters.
+ *
+ * The CLI's `npx mailwoman corpus run <adapter-id>` resolves `<adapter-id>` against this registry; the same registry is
+ * iterated by the `corpus build` pipeline. Adapters do not self-register at module load — they're added explicitly so
+ * the dependency graph stays traceable.
+ */
+export interface AdapterRegistry {
+	/** Add an adapter. Throws if `adapter.id` is already registered. */
+	register(adapter: CorpusAdapter): void
+
+	/** Return the adapter for `id`, or `undefined`. */
+	get(id: string): CorpusAdapter | undefined
+
+	/** All registered adapters, in insertion order. */
+	list(): readonly CorpusAdapter[]
+
+	/** Convenience: ids only, in insertion order. */
+	ids(): readonly string[]
+}
+
+/**
+ * Default in-memory registry. The runner constructs one per invocation; the CLI re-uses a shared singleton
+ * (`defaultAdapterRegistry`) populated by `./adapters/index.ts` as adapters come online.
+ */
+export class InMemoryAdapterRegistry implements AdapterRegistry {
+	#byID = new Map<string, CorpusAdapter>()
+
+	register(adapter: CorpusAdapter): void {
+		if (this.#byID.has(adapter.id)) {
+			throw new Error(`AdapterRegistry: id ${JSON.stringify(adapter.id)} already registered`)
+		}
+		this.#byID.set(adapter.id, adapter)
+	}
+
+	get(id: string): CorpusAdapter | undefined {
+		return this.#byID.get(id)
+	}
+
+	list(): readonly CorpusAdapter[] {
+		return Array.from(this.#byID.values())
+	}
+
+	ids(): readonly string[] {
+		return Array.from(this.#byID.keys())
+	}
+}
+
+/**
+ * Process-wide default registry. Populated by `./adapters/index.ts` as adapters are built; imported by the CLI. Tests
+ * should construct their own `InMemoryAdapterRegistry` to avoid cross-test pollution.
+ */
+export const defaultAdapterRegistry = new InMemoryAdapterRegistry()
+
+/**
+ * Deterministic content-addressed source id.
+ *
+ * For adapters whose upstream source has no native primary key (CSV rows, GeoJSON features), the runner expects a
+ * stable id so dedup, holdout manifests, and resumability work across reruns. This helper produces one by hashing the
+ * adapter id and a canonical serialization of the components dict (keys sorted, values verbatim).
+ *
+ * Output format: `<adapterID>-<first-12-hex-chars-of-sha256>`. 48 bits of entropy is enough for ~17M rows per adapter
+ * before the expected collision count exceeds 1 (birthday paradox); adapters with more rows should extend the prefix
+ * length.
+ */
+export function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string {
+	const sortedKeys = Object.keys(components).sort() as ComponentTag[]
+	const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\x1f")
+	const digest = createHash("sha256").update(adapterID).update("\x1e").update(payload).digest("hex")
+
+	return `${adapterID}-${digest.slice(0, 12)}`
+}
+
+/**
+ * Canonical dedup key for a row.
+ *
+ * Two rows that share this key are treated as duplicates and only the first wins. The key is built from `country`, the
+ * sorted `components` dict, and a normalized `raw` (lower-cased, whitespace collapsed). License and provenance fields
+ * are intentionally excluded so the same address from multiple adapters is recognized as a duplicate.
+ *
+ * Synthetic rows are never deduplicated against natural rows: `synth.method` is folded into the key when present,
+ * ensuring each augmentation variant survives.
+ */
+export function canonicalDedupKey(row: CanonicalRow): string {
+	const sortedKeys = Object.keys(row.components).sort() as ComponentTag[]
+	const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\x1f")
+	const rawNorm = row.raw.toLowerCase().replace(/\s+/g, " ").trim()
+	const synthPart = row.synth ? `\x1e${row.synth.method}` : ""
+
+	return `${row.country}\x1e${rawNorm}\x1e${compPart}${synthPart}`
+}
+
+/**
+ * Streaming SHA-256 hasher.
+ *
+ * The runner feeds every JSONL line into one of these so the per-shard checksum can be recorded in `MANIFEST.json`
+ * without a second pass over the shard. Implementation is a one-line wrapper, but giving it a name keeps the runner's
+ * hash-tracking intent obvious.
+ */
+export interface StreamingHasher {
+	update(chunk: string | Uint8Array): void
+	digest(): string
+}
+
+/** Default `StreamingHasher` (SHA-256, hex). */
+export function streamingSha256(): StreamingHasher {
+	const h: Hash = createHash("sha256")
+	let finalized = false
+	let digestHex = ""
+
+	return {
+		update(chunk) {
+			if (finalized) throw new Error("streamingSha256: update() called after digest()")
+			h.update(typeof chunk === "string" ? chunk : chunk)
+		},
+		digest() {
+			if (!finalized) {
+				digestHex = h.digest("hex")
+				finalized = true
+			}
+
+			return digestHex
+		},
+	}
+}

package/src/adapters/ban/adapter.ts

@@ -0,0 +1,189 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `ban`: Base Adresse Nationale CSV adapter (FR street-level).
+ *
+ *   Input: a CSV dump from `adresse.data.gouv.fr` (semicolon-separated, ~25M rows nationally). The
+ *   adapter only reads the small set of columns needed for the corpus:
+ *
+ *   - `numero` → `house_number`
+ *   - `rep` → repetition index ("bis", "ter") appended to house_number
+ *   - `nom_voie` → `street` (full road name; includes the prefix "Rue", "Avenue", etc.)
+ *   - `code_postal` → `postcode`
+ *   - `nom_commune` → `locality`
+ *
+ *   `region` and `country` are not in BAN. The adapter stamps `country: "FR"` on every row; region is
+ *   left for the wof-postalcode + wof-admin cross-reference at corpus build time (a future pass;
+ *   for Phase 1 the row's region is simply absent).
+ *
+ *   License: the official BAN (adresse.data.gouv.fr) is DUAL-licensed — Licence Ouverte 2.0 (Etalab,
+ *   attribution-only) OR ODbL (share-alike). We ELECT Licence Ouverte 2.0 (issue #26 Tier B:
+ *   allowed for training with attribution; the ODbL option's share-alike obligation would defeat
+ *   the proprietary-weights goal). Stamped onto every row as `Licence Ouverte 2.0` — NOT the older
+ *   conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
+ *   The model card MUST carry the BAN attribution (Tier B obligation).
+ *
+ *   The adapter is streaming-aware: it uses `csv-parse` in streaming mode so a 25M-row dump never
+ *   sits in memory. Honors `opts.limit` for fixture / smoke runs, `opts.signal` for cancellation,
+ *   and `opts.country` for a self-consistency check (errors if country !== FR).
+ */
+
+import { createReadStream } from "node:fs"
+
+import { parse as csvParse } from "csv-parse"
+
+import { stableSourceID } from "../../adapter.ts"
+import { reconcileComponents } from "../../format.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
+import { decomposeFrStreet } from "./street-decompose.ts"
+
+export const BAN_ADAPTER_ID = "ban"
+
+/**
+ * Subset of BAN CSV columns the adapter consults. Everything else is ignored; declaring the shape explicitly catches
+ * column-name drift early if BAN evolves its schema.
+ */
+interface BanRow {
+	id: string
+	numero: string
+	rep: string
+	nom_voie: string
+	code_postal: string
+	nom_commune: string
+}
+
+/**
+ * Compose `house_number` from `numero` + `rep`. BAN uses `rep` for repetition indices ("bis", "ter", "quater") that
+ * follow the house number. Result: `"10 bis"`, `"45"`, etc.
+ */
+function composeHouseNumber(numero: string, rep: string): string {
+	const n = numero.trim()
+	const r = rep.trim()
+
+	if (!n) return ""
+
+	return r ? `${n} ${r}` : n
+}
+
+/**
+ * Compose the raw FR-style address line. Two common BAN-derived shapes:
+ *
+ * "10 bis Avenue des Champs-Élysées, 75008 Paris" "45 Cours Lafayette, 69003 Lyon"
+ *
+ * FR convention puts postcode on the same line as the locality, comma-separated from the street. The adapter renders
+ * that directly rather than relying on OpenCage's template — BAN already gives us the canonical FR strings; the
+ * template would round-trip identically.
+ */
+function composeRaw(house: string, street: string, postcode: string, locality: string): string {
+	const parts: string[] = []
+	const streetPart = [house, street].filter(Boolean).join(" ").trim()
+
+	if (streetPart) {
+		parts.push(streetPart)
+	}
+	const cityPart = [postcode, locality].filter(Boolean).join(" ").trim()
+
+	if (cityPart) {
+		parts.push(cityPart)
+	}
+
+	return parts.join(", ").replace(/\s+/g, " ").trim()
+}
+
+export function createBanAdapter(): CorpusAdapter {
+	return {
+		id: BAN_ADAPTER_ID,
+		defaultLicense: "Licence Ouverte 2.0",
+		description: "Base Adresse Nationale (FR): house-number-level street addresses (~25M rows).",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && opts.country !== "FR") {
+				throw new Error(`ban adapter: only FR supported, got country=${opts.country}`)
+			}
+
+			const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
+			const parser = stream.pipe(
+				csvParse({
+					delimiter: ";",
+					columns: true,
+					skip_empty_lines: true,
+					relax_quotes: true,
+					relax_column_count: true,
+				})
+			)
+
+			let emitted = 0
+
+			try {
+				for await (const record of parser as AsyncIterable<BanRow>) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const house = composeHouseNumber(record.numero ?? "", record.rep ?? "")
+					const street = (record.nom_voie ?? "").trim()
+					const postcode = (record.code_postal ?? "").trim()
+					const locality = (record.nom_commune ?? "").trim()
+
+					if (!street || !locality) continue
+
+					if (!house && !postcode) continue
+
+					const decomposed = decomposeFrStreet(street)
+
+					const components: CanonicalRow["components"] = {}
+
+					if (house) {
+						components.house_number = house
+					}
+
+					if (decomposed.prefix) {
+						components.street_prefix = decomposed.prefix
+					}
+
+					if (decomposed.street) {
+						components.street = decomposed.street
+					}
+
+					if (postcode) {
+						components.postcode = postcode
+					}
+
+					if (locality) {
+						components.locality = locality
+					}
+
+					const raw = composeRaw(house, street, postcode, locality)
+
+					if (!raw) continue
+
+					const aligned = reconcileComponents(components, raw)
+
+					if (Object.keys(aligned).length === 0) continue
+
+					const sourceID = record.id?.trim()
+						? `${BAN_ADAPTER_ID}-${record.id.trim()}`
+						: stableSourceID(BAN_ADAPTER_ID, aligned)
+
+					yield {
+						raw,
+						components: aligned,
+						country: "FR",
+						locale: "fr-FR",
+						source: BAN_ADAPTER_ID,
+						source_id: sourceID,
+						corpus_version: "",
+						license: "Licence Ouverte 2.0",
+					}
+					emitted++
+				}
+			} finally {
+				stream.destroy()
+			}
+		},
+	}
+}
+
+export const banAdapter = createBanAdapter()

package/src/adapters/ban/street-decompose.ts

@@ -0,0 +1,98 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Decompose a French street name into Stage 3 components. French convention puts the street type as
+ *   a leading word: "Rue de Rivoli", "Avenue des Champs-Élysées", "Bd Voltaire".
+ *
+ *   The street type becomes street_prefix in our schema. The remaining tokens form the street name.
+ *
+ *   Examples: "Rue de Rivoli" → { prefix: "Rue", street: "de Rivoli" } "Avenue des Champs-Élysées" →
+ *   { prefix: "Avenue", street: "des Champs-Élysées" } "Boulevard Voltaire" → { prefix:
+ *   "Boulevard", street: "Voltaire" }
+ *
+ *   Sources street types from `core/data/libpostal/dictionaries/fr/street_types.txt`.
+ */
+
+import { readFileSync } from "node:fs"
+import { resolve } from "node:path"
+
+import { resourceDictionaryPath } from "@mailwoman/core/utils"
+
+function loadDictionary(filename: string): Set<string> {
+	const candidates = [
+		resourceDictionaryPath("libpostal", "fr", filename),
+		resourceDictionaryPath("libpostal", "fr", filename),
+		resolve(process.cwd(), "core/data/libpostal/dictionaries/fr", filename),
+	]
+
+	for (const path of candidates) {
+		try {
+			const text = readFileSync(path, "utf8")
+			const set = new Set<string>()
+
+			for (const line of text.split("\n")) {
+				const trimmed = line.trim()
+
+				if (!trimmed || trimmed.startsWith("#")) continue
+
+				for (const form of trimmed.split("|")) {
+					const f = form.trim().toLowerCase()
+
+					if (f) {
+						set.add(f)
+					}
+				}
+			}
+
+			return set
+		} catch {
+			// try next
+		}
+	}
+	throw new Error(`Could not load FR libpostal dictionary: ${filename}`)
+}
+
+const STREET_TYPES_FR = loadDictionary("street_types.txt")
+
+export interface DecomposedFrStreet {
+	prefix: string | null
+	street: string
+}
+
+/**
+ * Decompose a French street name into prefix (leading type word) and street name.
+ *
+ * If the first 1-2 tokens match a known street type (allowing for multi-word like "ancien chemin"), they become the
+ * prefix. Returns `{ prefix: null, street: original }` if no match.
+ */
+export function decomposeFrStreet(fullname: string): DecomposedFrStreet {
+	const trimmed = fullname.trim()
+
+	if (!trimmed) return { prefix: null, street: "" }
+
+	const tokens = trimmed.split(/\s+/)
+
+	if (tokens.length < 2) return { prefix: null, street: trimmed }
+
+	const norm = (s: string) => s.toLowerCase().replace(/[.,;]$/, "")
+
+	// Try 2-word prefix first (e.g. "ancien chemin")
+	if (tokens.length >= 3) {
+		const twoWord = norm(tokens[0]!) + " " + norm(tokens[1]!)
+
+		if (STREET_TYPES_FR.has(twoWord)) {
+			return { prefix: tokens.slice(0, 2).join(" "), street: tokens.slice(2).join(" ") }
+		}
+	}
+
+	// Then try 1-word prefix
+	const first = norm(tokens[0]!)
+
+	if (STREET_TYPES_FR.has(first)) {
+		return { prefix: tokens[0]!, street: tokens.slice(1).join(" ") }
+	}
+
+	return { prefix: null, street: trimmed }
+}