@mailwoman/corpus 7.2.0 → 7.3.0

package/src/build.ts

@@ -0,0 +1,348 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   End-to-end corpus build (Phase 1 task #10 in the plan).
+ *
+ *   `buildCorpus(opts)` orchestrates every stage of the pipeline:
+ *
+ *   1. **Adapter runs** — drives every adapter in turn (via `runAdapter`), writing
+ *        `<intermediate>/<adapter.id>/canonical.jsonl` shards.
+ *   2. **Synthesis** — optional. For each canonical row, every applicable augmentation in the row's
+ *        country-default policy emits an augmented row alongside the original.
+ *   3. **Alignment** — every row (original + augmented) is aligned via `alignRow`. Successes go to
+ *        `labeled.jsonl`; quarantines are appended to `quarantine.jsonl` with reasons.
+ *   4. **Splits** — `splitRows` partitions labeled `source_id`s into train/val/test by locality holdout.
+ *        Manifest written to `splits/SPLIT_MANIFEST.json` + per-split `train.txt` / `val.txt` /
+ *        `test.txt`.
+ *   5. **Parquet shards** — `writeShards` streams labeled rows into 1M-row `.parquet` shards per split
+ *        under `corpus-v<version>/{train,val,test}/part-NNNN.parquet` (SNAPPY-compressed, 50k-row
+ *        row groups), with per-shard checksums + per-stage manifest in
+ *        `corpus-v<version>/MANIFEST.json`.
+ *   6. **Top-level manifest** — `<outputDir>/MANIFEST.json` ties every per-stage manifest together with
+ *        a top-level corpus_version, built_at, and aggregate counts.
+ *
+ *   Output layout:
+ *
+ *   ```
+ *   <outputDir>/
+ *   MANIFEST.json
+ *   intermediate/
+ *     <adapter.id>/canonical.jsonl   # one per adapter
+ *     labeled.jsonl                  # post-alignment, pre-shard
+ *     quarantine.jsonl               # rows that failed alignment
+ *   splits/
+ *     SPLIT_MANIFEST.json
+ *     train.txt / val.txt / test.txt
+ *   corpus-v<version>/
+ *     MANIFEST.json
+ *     train/part-NNNN.parquet
+ *     val/part-NNNN.parquet
+ *     test/part-NNNN.parquet
+ * ```
+ *
+ *   The intermediate files live alongside the final shards for reproducibility + debugging. Operators
+ *   can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
+ *   self-contained.
+ */
+
+import { createWriteStream, existsSync, readFileSync, type WriteStream } from "node:fs"
+import { mkdir, writeFile } from "node:fs/promises"
+import { join } from "node:path"
+
+import { $public } from "@mailwoman/core/env"
+import { JSONSpliterator } from "spliterator"
+
+import { defaultAdapterRegistry } from "./adapter.ts"
+import { alignRow } from "./align.ts"
+import { licenseExcluded } from "./license.ts"
+import { writeShards, type ShardManifest } from "./parquet.ts"
+import { runAdapter, type AdapterRunManifest } from "./runner.ts"
+import {
+	defaultHoldouts,
+	splitForRow,
+	writeSplitManifestsFromLabeledFiles,
+	type SplitManifest,
+	type SplitName,
+} from "./split.ts"
+import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "./types.ts"
+
+/** Stage tags surfaced to `onProgress`. */
+export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest"
+
+/** Per-invocation options for `buildCorpus`. */
+export interface BuildCorpusOptions {
+	/** Root output directory. All build artifacts land beneath it. */
+	outputDir: string
+
+	/** Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name. */
+	corpusVersion: string
+
+	/**
+	 * Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
+	 * `[wofAdminAdapter]` for a smoke run).
+	 */
+	adapters?: readonly CorpusAdapter[]
+
+	/**
+	 * Per-adapter `AdapterOptions` — looked up by adapter id. Adapters whose id is missing from this map are skipped (and
+	 * noted in the manifest).
+	 */
+	adapterInputs: Record<string, AdapterOptions>
+
+	/** Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests. */
+	synthesize?: boolean
+
+	/** Forwarded to `writeShards`. Default 1_000_000. */
+	rowsPerShard?: number
+
+	/** Progress hook. Errors thrown abort the build. */
+	onProgress?: (stage: BuildStage, message: string) => void
+
+	/**
+	 * License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
+	 * `SHARE_ALIKE_PATTERN` in `license.ts`); a row whose `license` matches any is dropped at ingest. Default (omitted)
+	 * includes EVERYTHING — exclusion is a deliberate act, not a silent default. A proprietary-weights build passes the
+	 * share-alike set (`--exclude-share-alike`).
+	 */
+	excludeLicenses?: readonly RegExp[]
+}
+
+/** Top-level manifest tying every stage together. */
+export interface BuildCorpusManifest {
+	corpus_version: string
+	built_at: string
+	adapters: AdapterRunManifest[]
+	skipped_adapters: string[]
+	splits: { counts: SplitManifest["counts"]; holdouts: SplitManifest["holdouts"] }
+	shards: { counts: ShardManifest["counts"]; total_rows: number }
+	quarantine_count: number
+	total_aligned_rows: number
+	/**
+	 * Resolved license set across all INCLUDED rows (license string → row count), + the count dropped by
+	 * `excludeLicenses` (#26). The model card derives its data-attribution table from `licenses`.
+	 */
+	licenses: Record<string, number>
+	excluded_by_license: number
+}
+
+/**
+ * Drive the full corpus build to completion.
+ *
+ * Memory profile: the function maintains an in-memory `Map<source_id, SplitName>` to bridge the align → shard hand-off.
+ * For Phase 1 fixture-scale runs (≤ 10⁴ rows) this is trivial. For real 5M+ runs, the map fits comfortably in a few
+ * hundred MB; the canonical.jsonl and labeled.jsonl payloads stream and never sit in memory.
+ */
+export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpusManifest> {
+	const adapters = opts.adapters ?? defaultAdapterRegistry.list()
+	const synthesize = opts.synthesize ?? true
+	const rowsPerShard = opts.rowsPerShard ?? 1_000_000
+	const built_at = new Date().toISOString()
+
+	await mkdir(opts.outputDir, { recursive: true })
+	const intermediateDir = join(opts.outputDir, "intermediate")
+	await mkdir(intermediateDir, { recursive: true })
+
+	// 1. Adapter runs.
+	const adapterRuns: AdapterRunManifest[] = []
+	const skipped: string[] = []
+
+	for (const adapter of adapters) {
+		const adapterOptions = opts.adapterInputs[adapter.id]
+
+		if (!adapterOptions) {
+			skipped.push(adapter.id)
+			opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`)
+			continue
+		}
+		// Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
+		// already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
+		// canonical is fully flushed, so its presence guarantees completeness; row order is identical,
+		// so downstream holdout-split determinism is preserved. Recovers an align-phase crash without
+		// redoing the (expensive) emit phase. Default (unset) re-emits, preserving correctness. (2026-06-12.)
+		const adapterDir = join(intermediateDir, adapter.id)
+		const cachedManifest = join(adapterDir, "MANIFEST.json")
+
+		if (
+			$public.MAILWOMAN_RESUME === "1" &&
+			existsSync(cachedManifest) &&
+			existsSync(join(adapterDir, "canonical.jsonl"))
+		) {
+			const cached = JSON.parse(readFileSync(cachedManifest, "utf8")) as AdapterRunManifest
+			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
+			adapterRuns.push(cached)
+			continue
+		}
+		opts.onProgress?.("adapter-run", `running ${adapter.id}`)
+		const m = await runAdapter({
+			adapter,
+			adapterOptions,
+			outputDir: intermediateDir,
+			corpusVersion: opts.corpusVersion,
+		})
+		adapterRuns.push(m)
+	}
+
+	// 2 + 3. Synthesis + alignment: stream every canonical.jsonl, optionally augment, align,
+	// and route each labeled row directly to its split-specific JSONL (`labeled-{train,val,test}.
+	// jsonl`). Memory cost is O(1) — the prior in-memory `splitInputs` array + `splitByIDMap`
+	// + `SplitManifest.{train,val,test}` arrays are gone; per-row split is decided inline via
+	// `splitForRow` (a pure function of source_id + region + holdout policy).
+	const labeledPaths: Record<SplitName, string> = {
+		train: join(intermediateDir, "labeled-train.jsonl"),
+		val: join(intermediateDir, "labeled-val.jsonl"),
+		test: join(intermediateDir, "labeled-test.jsonl"),
+	}
+	const labeledStreams: Record<SplitName, WriteStream> = {
+		train: createWriteStream(labeledPaths.train, { encoding: "utf8" }),
+		val: createWriteStream(labeledPaths.val, { encoding: "utf8" }),
+		test: createWriteStream(labeledPaths.test, { encoding: "utf8" }),
+	}
+	const quarantinePath = join(intermediateDir, "quarantine.jsonl")
+	const quarantineStream = createWriteStream(quarantinePath, { encoding: "utf8" })
+
+	let aligned = 0
+	let quarantined = 0
+	const counts: Record<SplitName, number> = { train: 0, val: 0, test: 0 }
+	const holdouts = defaultHoldouts()
+	// License accounting + the deliberate exclusion filter (#26). `licenseCounts` is the resolved
+	// license set (→ manifest + model-card attribution); `excludeLicenses` (empty by default → include
+	// everything) is the operator's PURPOSEFUL exclusion, never a silent drop.
+	const excludeLicenses = opts.excludeLicenses ?? []
+	const licenseCounts = new Map<string, number>()
+	let excludedByLicense = 0
+
+	const writeQuarantine = (row: CanonicalRow, reason: string): void => {
+		quarantineStream.write(`${JSON.stringify({ row, reason })}\n`)
+	}
+
+	for (const adapterRun of adapterRuns) {
+		opts.onProgress?.("align", `aligning ${adapterRun.adapter_id}`)
+
+		for await (const row of streamJSONL<CanonicalRow>(adapterRun.jsonl_path)) {
+			licenseCounts.set(row.license, (licenseCounts.get(row.license) ?? 0) + 1)
+
+			// Deliberate license exclusion (#26): drop a row ONLY when the operator named its license
+			// kind via `excludeLicenses`. Default (no patterns) keeps everything — exclusion is a
+			// purposeful act, not a silent default. Counted BEFORE the drop so the manifest's license
+			// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
+			if (licenseExcluded(row.license, excludeLicenses)) {
+				excludedByLicense++
+				continue
+			}
+			const fanned: CanonicalRow[] = [row]
+
+			if (synthesize) {
+				for (const aug of synthesizeRow(row, defaultAugmentationsForCountry(row.country))) {
+					fanned.push(aug)
+				}
+			}
+
+			for (const r of fanned) {
+				let result: ReturnType<typeof alignRow>
+
+				try {
+					result = alignRow(r)
+				} catch (err) {
+					// Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
+					// alignRow's targeted paths normalize/quarantine known issues with specific reasons;
+					// this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
+					// shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
+					writeQuarantine(r, `align-threw:${(err as Error).message.slice(0, 160)}`)
+					quarantined++
+					continue
+				}
+
+				if (result.kind === "labeled") {
+					const split = splitForRow(result.row, holdouts)
+					labeledStreams[split].write(`${JSON.stringify(result.row)}\n`)
+					counts[split]++
+					aligned++
+				} else {
+					writeQuarantine(r, result.row.reason)
+					quarantined++
+				}
+			}
+		}
+	}
+
+	for (const s of Object.values(labeledStreams)) {
+		s.end()
+	}
+	quarantineStream.end()
+	await Promise.all([...Object.values(labeledStreams).map(streamEnd), streamEnd(quarantineStream)])
+
+	// 4. Splits — manifest derived by streaming the per-split labeled files; no in-memory
+	// source-id arrays. `sort(1)` from coreutils produces the deterministic per-split .txt
+	// manifests with disk spill for splits that exceed in-memory thresholds.
+	opts.onProgress?.("split", `splitting ${aligned} aligned rows`)
+	const splitsDir = join(opts.outputDir, "splits")
+	const splitCounts = await writeSplitManifestsFromLabeledFiles({
+		labeledPaths,
+		outputDir: splitsDir,
+		corpusVersion: opts.corpusVersion,
+		counts,
+		holdouts,
+	})
+
+	// 5. Parquet shards — per-split labeled JSONL streams in, sharded `.parquet` out. The prior
+	// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
+	opts.onProgress?.("shard", "writing parquet shards")
+	const shardManifest = await writeShards(
+		{
+			train: streamJSONL<LabeledRow>(labeledPaths.train),
+			val: streamJSONL<LabeledRow>(labeledPaths.val),
+			test: streamJSONL<LabeledRow>(labeledPaths.test),
+		},
+		{
+			outputDir: opts.outputDir,
+			corpusVersion: opts.corpusVersion,
+			rowsPerShard,
+		}
+	)
+
+	// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
+	// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
+	const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1])
+	opts.onProgress?.(
+		"manifest",
+		`license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
+			(excludedByLicense > 0
+				? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
+				: " | NO license exclusion applied (all rows kept)")
+	)
+
+	// 6. Top-level manifest.
+	opts.onProgress?.("manifest", "writing top-level MANIFEST.json")
+	const manifest: BuildCorpusManifest = {
+		corpus_version: opts.corpusVersion,
+		built_at,
+		adapters: adapterRuns,
+		skipped_adapters: skipped,
+		splits: { counts: splitCounts, holdouts },
+		shards: { counts: shardManifest.counts, total_rows: shardManifest.total_rows },
+		quarantine_count: quarantined,
+		total_aligned_rows: aligned,
+		licenses: Object.fromEntries(licenseSummary),
+		excluded_by_license: excludedByLicense,
+	}
+	await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
+
+	return manifest
+}
+
+async function* streamJSONL<T>(path: string): AsyncIterable<T> {
+	// JSONSpliterator yields already-parsed rows (skipEmpty is on by default, so blank
+	// lines are dropped at the row level) and throws SyntaxError on a malformed row —
+	// same fail-loud behavior as the prior readline + bare `JSON.parse`.
+	yield* JSONSpliterator.fromAsync<T>(path)
+}
+
+function streamEnd(s: WriteStream): Promise<void> {
+	return new Promise((resolve, reject) => {
+		s.once("close", resolve)
+		s.once("error", reject)
+	})
+}

package/src/codex/us-fips-state.ts

@@ -0,0 +1,109 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   US Census FIPS state codes → two-letter abbreviation + full display name.
+ *
+ *   Covers all 50 states + DC + the five primary territories (PR, GU, VI, MP, AS).
+ *
+ *   The data is public-domain US Census reference (FIPS PUB 5-2, withdrawn but still the canonical
+ *   numeric ID used by every TIGER product). It is reproduced here so the TIGER adapter can resolve
+ *   `statefp` columns (e.g. `"50"`) into a `region` component (e.g. `"VT"`) without an extra DB
+ *   join.
+ */
+
+/** Per-state record: two-letter postal abbreviation + full canonical display name. */
+export interface USStateInfo {
+	abbreviation: string
+	name: string
+}
+
+/**
+ * FIPS state-or-territory code → `{ abbreviation, name }`. Includes all 50 states, DC, and the five primary territories
+ * (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER column `statefp`.
+ */
+export const US_FIPS_STATE: Readonly<Record<string, USStateInfo>> = Object.freeze({
+	"01": { abbreviation: "AL", name: "Alabama" },
+	"02": { abbreviation: "AK", name: "Alaska" },
+	"04": { abbreviation: "AZ", name: "Arizona" },
+	"05": { abbreviation: "AR", name: "Arkansas" },
+	"06": { abbreviation: "CA", name: "California" },
+	"08": { abbreviation: "CO", name: "Colorado" },
+	"09": { abbreviation: "CT", name: "Connecticut" },
+	"10": { abbreviation: "DE", name: "Delaware" },
+	"11": { abbreviation: "DC", name: "District of Columbia" },
+	"12": { abbreviation: "FL", name: "Florida" },
+	"13": { abbreviation: "GA", name: "Georgia" },
+	"15": { abbreviation: "HI", name: "Hawaii" },
+	"16": { abbreviation: "ID", name: "Idaho" },
+	"17": { abbreviation: "IL", name: "Illinois" },
+	"18": { abbreviation: "IN", name: "Indiana" },
+	"19": { abbreviation: "IA", name: "Iowa" },
+	"20": { abbreviation: "KS", name: "Kansas" },
+	"21": { abbreviation: "KY", name: "Kentucky" },
+	"22": { abbreviation: "LA", name: "Louisiana" },
+	"23": { abbreviation: "ME", name: "Maine" },
+	"24": { abbreviation: "MD", name: "Maryland" },
+	"25": { abbreviation: "MA", name: "Massachusetts" },
+	"26": { abbreviation: "MI", name: "Michigan" },
+	"27": { abbreviation: "MN", name: "Minnesota" },
+	"28": { abbreviation: "MS", name: "Mississippi" },
+	"29": { abbreviation: "MO", name: "Missouri" },
+	"30": { abbreviation: "MT", name: "Montana" },
+	"31": { abbreviation: "NE", name: "Nebraska" },
+	"32": { abbreviation: "NV", name: "Nevada" },
+	"33": { abbreviation: "NH", name: "New Hampshire" },
+	"34": { abbreviation: "NJ", name: "New Jersey" },
+	"35": { abbreviation: "NM", name: "New Mexico" },
+	"36": { abbreviation: "NY", name: "New York" },
+	"37": { abbreviation: "NC", name: "North Carolina" },
+	"38": { abbreviation: "ND", name: "North Dakota" },
+	"39": { abbreviation: "OH", name: "Ohio" },
+	"40": { abbreviation: "OK", name: "Oklahoma" },
+	"41": { abbreviation: "OR", name: "Oregon" },
+	"42": { abbreviation: "PA", name: "Pennsylvania" },
+	"44": { abbreviation: "RI", name: "Rhode Island" },
+	"45": { abbreviation: "SC", name: "South Carolina" },
+	"46": { abbreviation: "SD", name: "South Dakota" },
+	"47": { abbreviation: "TN", name: "Tennessee" },
+	"48": { abbreviation: "TX", name: "Texas" },
+	"49": { abbreviation: "UT", name: "Utah" },
+	"50": { abbreviation: "VT", name: "Vermont" },
+	"51": { abbreviation: "VA", name: "Virginia" },
+	"53": { abbreviation: "WA", name: "Washington" },
+	"54": { abbreviation: "WV", name: "West Virginia" },
+	"55": { abbreviation: "WI", name: "Wisconsin" },
+	"56": { abbreviation: "WY", name: "Wyoming" },
+	// Territories
+	"60": { abbreviation: "AS", name: "American Samoa" },
+	"66": { abbreviation: "GU", name: "Guam" },
+	"69": { abbreviation: "MP", name: "Northern Mariana Islands" },
+	"72": { abbreviation: "PR", name: "Puerto Rico" },
+	"78": { abbreviation: "VI", name: "Virgin Islands" },
+})
+
+/** Lookup helper. Returns null when the FIPS code isn't recognized. */
+export function lookupFipsState(statefp: string | null | undefined): USStateInfo | null {
+	if (!statefp) return null
+
+	return US_FIPS_STATE[statefp] ?? null
+}
+
+/**
+ * Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
+ * source data ships the abbreviation rather than the FIPS code (FCC BDC, most federal CSVs).
+ */
+export const US_STATE_BY_ABBREVIATION: Readonly<Record<string, USStateInfo>> = Object.freeze(
+	Object.fromEntries(Object.values(US_FIPS_STATE).map((info) => [info.abbreviation, info]))
+)
+
+/**
+ * Lookup helper for adapters carrying 2-char USPS abbreviations (`"CA"`, `"VT"`). Case-folded; null for any value
+ * outside the 50 states + DC + the five primary territories.
+ */
+export function lookupStateAbbreviation(abbreviation: string | null | undefined): USStateInfo | null {
+	if (!abbreviation) return null
+
+	return US_STATE_BY_ABBREVIATION[abbreviation.toUpperCase()] ?? null
+}

package/src/format.ts

@@ -0,0 +1,14 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Thin re-export of `@mailwoman/formatter`.
+ *
+ *   The formatter implementation moved to its own workspace (`@mailwoman/formatter`) so the parser,
+ *   the corpus pipeline, and the record matcher can share one locale-aware renderer plus the
+ *   canonical match key. This module stays put as the stable `@mailwoman/corpus/format` entry point
+ *   the synthesis adapters import `formatAddress` / `reconcileComponents` from.
+ */
+
+export * from "@mailwoman/formatter"

package/src/golden.ts

@@ -0,0 +1,147 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Golden eval-set validator (Phase 1 task #9 in the plan).
+ *
+ *   The golden set is hand-labeled ground truth for the neural classifier. Each entry must carry
+ *   components whose surface forms actually occur in `raw` — otherwise the entry will silently rot
+ *   the eval signal. This module:
+ *
+ *   - Defines `GoldenEntry` (schema check).
+ *   - Loads `.jsonl` files (one entry per line).
+ *   - Validates every entry: schema shape, ComponentTag membership, reachability of each component in
+ *       `raw` via the same `reconcileComponents` helper alignment uses.
+ *   - Returns a structured report of per-entry errors so the CLI / CI surface can act on it.
+ *
+ *   The 1000-entry target (500 US + 500 FR) is a human task. This module catches the regressions that
+ *   creep in over time as new entries land.
+ */
+
+import { readdir, readFile } from "node:fs/promises"
+import { extname, join } from "node:path"
+
+import { COMPONENT_TAGS, type ComponentTag } from "@mailwoman/core/types"
+
+import { reconcileComponents } from "./format.ts"
+
+const TAG_SET = new Set<string>(COMPONENT_TAGS as readonly string[])
+
+/** One entry in a golden `.jsonl` file. */
+export interface GoldenEntry {
+	raw: string
+	components: Partial<Record<ComponentTag, string>>
+	country: string
+	source: "golden"
+	notes?: string
+}
+
+/** Per-entry validation failure. */
+export interface GoldenIssue {
+	file: string
+	line: number
+	reason: string
+}
+
+/** Aggregate report from `validateGoldenDir`. */
+export interface GoldenReport {
+	entries: number
+	files: number
+	issues: GoldenIssue[]
+}
+
+/** Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations. */
+export function parseGoldenLine(line: string): GoldenEntry {
+	const obj = JSON.parse(line) as Partial<GoldenEntry> & Record<string, unknown>
+
+	if (typeof obj.raw !== "string" || obj.raw.length === 0) {
+		throw new Error("missing/empty raw")
+	}
+
+	if (typeof obj.country !== "string" || !/^[A-Z]{2}$/u.test(obj.country)) {
+		throw new Error(`country must be ISO 3166-1 alpha-2 (got ${JSON.stringify(obj.country)})`)
+	}
+
+	if (obj.source !== "golden") {
+		throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`)
+	}
+	const components = (obj.components ?? {}) as Record<string, unknown>
+
+	for (const [k, v] of Object.entries(components)) {
+		if (!TAG_SET.has(k)) throw new Error(`unknown ComponentTag: ${k}`)
+
+		if (typeof v !== "string" || v.length === 0) {
+			throw new Error(`components.${k} must be a non-empty string`)
+		}
+	}
+
+	return {
+		raw: obj.raw,
+		components: components as GoldenEntry["components"],
+		country: obj.country,
+		source: "golden",
+		notes: typeof obj.notes === "string" ? obj.notes : undefined,
+	}
+}
+
+/** Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent). */
+export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
+	const reconciled = reconcileComponents(entry.components, entry.raw)
+	const missing: ComponentTag[] = []
+
+	for (const tag of Object.keys(entry.components) as ComponentTag[]) {
+		if (!(tag in reconciled)) {
+			missing.push(tag)
+		}
+	}
+
+	return missing
+}
+
+/** Validate one `.jsonl` file end-to-end, returning a list of issues. */
+export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
+	const text = await readFile(path, "utf8")
+	const lines = text.split("\n")
+	const issues: GoldenIssue[] = []
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i]!.trim()
+
+		if (!line) continue
+
+		try {
+			const entry = parseGoldenLine(line)
+			const unreachable = unreachableComponents(entry)
+
+			if (unreachable.length > 0) {
+				issues.push({
+					file: path,
+					line: i + 1,
+					reason: `components not reachable in raw: ${unreachable.join(", ")}`,
+				})
+			}
+		} catch (err) {
+			issues.push({ file: path, line: i + 1, reason: (err as Error).message })
+		}
+	}
+
+	return issues
+}
+
+/** Validate every `.jsonl` in a golden directory. */
+export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
+	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").sort()
+	const issues: GoldenIssue[] = []
+	let entries = 0
+
+	for (const name of files) {
+		const fullPath = join(dir, name)
+		const fileIssues = await validateGoldenFile(fullPath)
+		issues.push(...fileIssues)
+		const text = await readFile(fullPath, "utf8")
+		entries += text.split("\n").filter((l) => l.trim()).length
+	}
+
+	return { entries, files: files.length, issues }
+}

package/src/index.ts

@@ -0,0 +1,28 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+export * from "./adapter.ts"
+export * from "./adapters/gnaf/assemble.ts"
+export * from "./adapters/index.ts"
+export * from "./align.ts"
+export * from "./build.ts"
+export * from "./format.ts"
+export * from "./golden.ts"
+export * from "./parquet.ts"
+export * from "./parquet-wrapper/index.ts"
+export * from "./runner.ts"
+export * from "./shard-recipes/index.ts"
+export * from "./split.ts"
+export * from "./synthesize-anchor-absorption.ts"
+export * from "./synthesize-german.ts"
+export * from "./synthesize-house-venue.ts"
+export * from "./synthesize-intersection.ts"
+export * from "./synthesize-no-street.ts"
+export * from "./synthesize-po-box.ts"
+export * from "./synthesize-street.ts"
+export * from "./synthesize.ts"
+export * from "./tokenize.ts"
+export * from "./types.ts"