@mailwoman/corpus 7.2.0 → 7.3.0

package/src/tools/shard-kryptonite.ts

@@ -0,0 +1,131 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build a parquet shard from the DeepSeek-generated kryptonite JSONL and emit the corpus-v0.4.0
+ *   MANIFEST. corpus-v0.4.0 is a pure adapter-addition revision: it points at every shard from
+ *   v0.3.0 plus the new kryptonite shard(s). No v0.3.0 bytes are touched or re-shuffled.
+ *
+ *   See docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for the why; that doc also pins the
+ *   DeepSeek model version + prompt versions used to produce the JSONL.
+ *
+ *   Invoke via `mailwoman corpus shard kryptonite \
+ *   --jsonl /data/corpus/versioned/v0.4.0/kryptonite/canonical-kryptonite.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.3.0/corpus-v0.3.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { mkdir } from "node:fs/promises"
+import { join } from "node:path"
+
+import { iterateJSONL } from "@mailwoman/core/utils"
+
+import { alignRow } from "../align.ts"
+import type { ShardManifest } from "../parquet.ts"
+import { PARQUET_COLUMNS, ROW_GROUP_SIZE, SHARD_COMPRESSION, writeShards } from "../parquet.ts"
+import type { CanonicalRow, LabeledRow } from "../types.ts"
+
+export interface ShardKryptoniteOptions {
+	jsonl: string
+	baseManifest: string
+	outDir: string
+	/** Default `"0.4.0"`. */
+	corpusVersion?: string
+	/** Default `"deepseek-kryptonite"`. */
+	source?: string
+}
+
+async function* canonicalRows(jsonl: string, corpusVersion: string): AsyncIterable<CanonicalRow> {
+	for await (const raw of iterateJSONL<Record<string, unknown>>(jsonl)) {
+		// Strip sidecar underscore-prefixed fields the generator left behind for debugging.
+		const components = raw["components"] as Record<string, string>
+		yield {
+			raw: raw["raw"] as string,
+			components,
+			country: (raw["country"] as string) ?? "US",
+			locale: (raw["locale"] as string) ?? undefined,
+			source: (raw["source"] as string) ?? "deepseek-kryptonite",
+			source_id: raw["source_id"] as string,
+			corpus_version: corpusVersion,
+			license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+			synth: raw["synth"] as CanonicalRow["synth"],
+		}
+	}
+}
+
+async function* labeledRows(jsonl: string, corpusVersion: string, quarantineLog: string[]): AsyncIterable<LabeledRow> {
+	for await (const row of canonicalRows(jsonl, corpusVersion)) {
+		const result = alignRow(row)
+
+		if (result.kind === "labeled") {
+			yield result.row
+		} else {
+			quarantineLog.push(`${row.source_id}\t${result.row.reason}`)
+		}
+	}
+}
+
+export async function buildKryptoniteShard(
+	options: ShardKryptoniteOptions,
+	report?: (line: string) => void
+): Promise<void> {
+	const corpusVersion = options.corpusVersion ?? "0.4.0"
+	const source = options.source ?? "deepseek-kryptonite"
+
+	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
+
+	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
+
+	await mkdir(options.outDir, { recursive: true })
+
+	const quarantine: string[] = []
+	const newManifest = await writeShards(
+		{ train: labeledRows(options.jsonl, corpusVersion, quarantine) },
+		{ outputDir: options.outDir, corpusVersion }
+	)
+
+	report?.(
+		`wrote ${newManifest.total_rows} rows into ${newManifest.shards.length} shard(s); ` +
+			`quarantined ${quarantine.length}`
+	)
+
+	if (quarantine.length > 0) {
+		const qPath = join(options.outDir, `corpus-v${corpusVersion}`, "quarantine-kryptonite.tsv")
+		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
+		report?.(`quarantine log → ${qPath}`)
+	}
+
+	// Stamp the new shard's source field for audit.ts (which prefers shard.source over
+	// first_source_id-prefix inference). Without this, deepseek-kryptonite IDs would have
+	// to match a prefix in KNOWN_SOURCE_PREFIXES — we add it there too as a belt-and-braces.
+	for (const sh of newManifest.shards) {
+		;(sh as unknown as { source: string }).source = source
+	}
+
+	// Compose the final corpus-v0.4.0 manifest: every shard from base + the new shard(s).
+	const base = JSON.parse(readFileSync(options.baseManifest, "utf8")) as ShardManifest
+	const combined: ShardManifest = {
+		corpus_version: corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_shard: base.rows_per_shard,
+		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+		shards: [...base.shards, ...newManifest.shards],
+		counts: {
+			train: base.counts.train + (newManifest.counts.train ?? 0),
+			val: base.counts.val,
+			test: base.counts.test,
+		},
+		total_rows: base.total_rows + newManifest.total_rows,
+	}
+	// Stamp source on the legacy v0.3.0 shards too, so audit's shard.source path is the
+	// authoritative one. v0.3.0 shards mix sources; we use the first_source_id-prefix
+	// inference for them (audit.ts will re-derive on its own when shard.source is absent).
+	const combinedPath = join(options.outDir, `corpus-v${corpusVersion}`, "MANIFEST.json")
+	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
+	report?.(`wrote combined manifest → ${combinedPath}`)
+	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newManifest.total_rows})`)
+	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newManifest.shards.length})`)
+	report?.(`  compression=${SHARD_COMPRESSION}`)
+}

package/src/tools/shard-translit.ts

@@ -0,0 +1,244 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build per-script parquet shards from the DeepSeek-generated transliteration JSONL and emit the
+ *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 shards.
+ *
+ *   Sibling to `shard-kryptonite.ts`. The two modules share the same composition pattern: take a
+ *   base MANIFEST, append new shards, write a combined MANIFEST. Differences specific to
+ *   transliteration:
+ *
+ *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
+ *       `source` and write one shard per script so `audit.ts` can attribute each shard to its
+ *       synthetic source without relying on filename-prefix inference.
+ *   - Each shard is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
+ *       `part-0000.parquet`, which v0.4.0's first builder already produced).
+ *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 shard paths are
+ *       rewritten from `/mnt/playpen/mailwoman-data/...` to `/data/...` in the combined MANIFEST so
+ *       all paths share one container-friendly form.
+ *
+ *   See docs/articles/plan/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
+ *   reproducibility contract.
+ *
+ *   Invoke via `mailwoman corpus shard translit \
+ *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
+ *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
+ *   --out-dir /data/corpus/versioned/v0.4.0`
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { mkdir, stat } from "node:fs/promises"
+import { join } from "node:path"
+
+import { iterateJSONL, sha256File } from "@mailwoman/core/utils"
+
+import { alignRow } from "../align.ts"
+import { ParquetWriter } from "../parquet-wrapper/index.ts"
+import type { ParquetRow, ShardDescriptor, ShardManifest } from "../parquet.ts"
+import { LABELED_ROW_SCHEMA, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SHARD_COMPRESSION } from "../parquet.ts"
+import type { CanonicalRow, LabeledRow } from "../types.ts"
+
+export interface ShardTranslitOptions {
+	jsonl: string
+	baseManifest: string
+	outDir: string
+	/** Default `"0.4.0"`. */
+	corpusVersion?: string
+	/** Default `"/data/"`. */
+	canonicalPathPrefix?: string
+	/** Default `"/mnt/playpen/mailwoman-data/"`. */
+	legacyPathPrefix?: string
+}
+
+function toCanonicalRow(raw: Record<string, unknown>, corpusVersion: string): CanonicalRow {
+	return {
+		raw: raw["raw"] as string,
+		components: raw["components"] as Record<string, string>,
+		country: (raw["country"] as string) ?? "US",
+		locale: (raw["locale"] as string) ?? undefined,
+		source: raw["source"] as string,
+		source_id: raw["source_id"] as string,
+		corpus_version: corpusVersion,
+		license: (raw["license"] as string) ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
+		synth: raw["synth"] as CanonicalRow["synth"],
+	}
+}
+
+function appendShape(row: ParquetRow): Record<string, unknown> {
+	const out: Record<string, unknown> = {
+		raw: row.raw,
+		tokens: row.tokens,
+		labels: row.labels,
+		country: row.country,
+		source: row.source,
+		source_id: row.source_id,
+		corpus_version: row.corpus_version,
+		license: row.license,
+	}
+
+	if (row.locale !== null) {
+		out.locale = row.locale
+	}
+
+	if (row.synth_method !== null) {
+		out.synth_method = row.synth_method
+	}
+
+	if (row.synth_base_id !== null) {
+		out.synth_base_id = row.synth_base_id
+	}
+
+	return out
+}
+
+/**
+ * Write one shard for a single source slug. Returns the populated ShardDescriptor + a list of quarantine reasons for
+ * rows that failed alignment.
+ */
+async function writeOneShard(
+	rows: readonly LabeledRow[],
+	outPath: string,
+	source: string,
+	corpusVersion: string
+): Promise<ShardDescriptor> {
+	const writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, outPath, {
+		rowGroupSize: ROW_GROUP_SIZE,
+	})
+	writer.setMetadata("mailwoman.corpus_version", corpusVersion)
+	writer.setMetadata("mailwoman.split", "train")
+	writer.setMetadata("mailwoman.shard_source", source)
+
+	let firstSourceID = ""
+	let lastSourceID = ""
+
+	for (const row of rows) {
+		const pq = rowToParquet(row)
+		await writer.appendRow(appendShape(pq) as unknown as ParquetRow)
+
+		if (firstSourceID === "") {
+			firstSourceID = row.source_id
+		}
+		lastSourceID = row.source_id
+	}
+	await writer.close()
+
+	const fileStat = await stat(outPath)
+	const sha256 = await sha256File(outPath)
+
+	return {
+		split: "train",
+		path: outPath,
+		format: "parquet",
+		compression: SHARD_COMPRESSION,
+		rows: rows.length,
+		bytes: fileStat.size,
+		sha256,
+		first_source_id: firstSourceID,
+		last_source_id: lastSourceID,
+		// Stamp source so audit.ts attributes the shard without falling back to filename-prefix
+		// inference. Cast widens ShardDescriptor; the field is read by audit.ts.
+		...({ source } as Record<string, string>),
+	}
+}
+
+function canonicalizeShardPath(path: string, legacyPrefix: string, canonicalPrefix: string): string {
+	if (path.startsWith(legacyPrefix)) return canonicalPrefix + path.slice(legacyPrefix.length)
+
+	return path
+}
+
+export async function buildTranslitShard(
+	options: ShardTranslitOptions,
+	report?: (line: string) => void
+): Promise<void> {
+	const corpusVersion = options.corpusVersion ?? "0.4.0"
+	const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/"
+	const legacyPathPrefix = options.legacyPathPrefix ?? "/mnt/playpen/mailwoman-data/"
+
+	if (!existsSync(options.jsonl)) throw new Error(`jsonl not found: ${options.jsonl}`)
+
+	if (!existsSync(options.baseManifest)) throw new Error(`base-manifest not found: ${options.baseManifest}`)
+
+	const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`)
+	const trainDir = join(corpusDir, "train")
+	await mkdir(trainDir, { recursive: true })
+
+	// Bucket canonical rows by source. Quarantined rows are logged.
+	const buckets = new Map<string, LabeledRow[]>()
+	const quarantine: string[] = []
+	let totalIn = 0
+
+	for await (const raw of iterateJSONL<Record<string, unknown>>(options.jsonl)) {
+		totalIn++
+		const canon = toCanonicalRow(raw, corpusVersion)
+		const result = alignRow(canon)
+
+		if (result.kind !== "labeled") {
+			quarantine.push(`${canon.source_id}\t${result.row.reason}`)
+			continue
+		}
+		const bucket = buckets.get(canon.source)
+
+		if (bucket) {
+			bucket.push(result.row)
+		} else {
+			buckets.set(canon.source, [result.row])
+		}
+	}
+	report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`)
+
+	const newShards: ShardDescriptor[] = []
+	const sortedKeys = [...buckets.keys()].sort()
+
+	for (const source of sortedKeys) {
+		const rows = buckets.get(source)!
+		const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source
+		const outPath = join(trainDir, `part-translit-${slug}.parquet`)
+		const descriptor = await writeOneShard(rows, outPath, source, corpusVersion)
+		newShards.push(descriptor)
+		report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`)
+	}
+
+	if (quarantine.length > 0) {
+		const qPath = join(corpusDir, "quarantine-transliteration.tsv")
+		writeFileSync(qPath, quarantine.join("\n") + "\n", "utf8")
+		report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`)
+	}
+
+	// Compose final MANIFEST: rewrite base.shards paths from /mnt/playpen/... → /data/... and append
+	// the new translit shards. Kryptonite shard already lives in the base manifest (it was written
+	// there by Thread B).
+	const base = JSON.parse(readFileSync(options.baseManifest, "utf8")) as ShardManifest
+	const rewrittenBase = base.shards.map((sh) => ({
+		...sh,
+		path: canonicalizeShardPath(sh.path, legacyPathPrefix, canonicalPathPrefix),
+	}))
+	const newTrainRows = newShards.reduce((sum, sh) => sum + sh.rows, 0)
+	const combined: ShardManifest = {
+		corpus_version: corpusVersion,
+		schema: PARQUET_COLUMNS,
+		rows_per_shard: base.rows_per_shard,
+		row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
+		shards: [...rewrittenBase, ...newShards],
+		counts: {
+			train: base.counts.train + newTrainRows,
+			val: base.counts.val,
+			test: base.counts.test,
+		},
+		total_rows: base.total_rows + newTrainRows,
+	}
+
+	const combinedPath = join(corpusDir, "MANIFEST.json")
+	writeFileSync(combinedPath, JSON.stringify(combined, null, 2) + "\n", "utf8")
+	report?.(`wrote combined manifest → ${combinedPath}`)
+	report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`)
+	report?.(`  shards=${combined.shards.length} (base=${base.shards.length}, added=${newShards.length})`)
+	report?.(`  compression=${SHARD_COMPRESSION}`)
+	const pathFix = rewrittenBase.filter((s, i) => s.path !== base.shards[i]!.path).length
+
+	if (pathFix > 0) {
+		report?.(`  path-canonicalized base shards: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`)
+	}
+}

package/src/types.ts

@@ -0,0 +1,213 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Canonical row schemas for the corpus pipeline (per #6 / Phase 1 plan).
+ *
+ *   The corpus pipeline produces two row shapes:
+ *
+ *   1. `CanonicalRow`: an adapter's raw output. Carries a free-form `raw` string, a per-component
+ *        ground-truth dict, provenance, and an optional augmentation marker. Adapters emit these.
+ *   2. `LabeledRow`: alignment's output. Adds a SentencePiece token list and a parallel BIO label list,
+ *        suitable for direct ingestion by the neural training loop.
+ *
+ *   `CorpusAdapter` is the contract every data source implements; `AdapterOptions` is the
+ *   per-invocation knob set (input path, optional country filter, row cap, abort signal).
+ */
+
+import type { BIOLabel, ComponentTag } from "@mailwoman/core/types"
+
+/**
+ * Provenance + augmentation metadata that travels with every corpus row.
+ *
+ * `synth` is `undefined` for natural (un-augmented) rows; present only when a row was produced by the synthesis
+ * pipeline (see `synthesize.ts`).
+ */
+export interface SourceProvenance {
+	/** Adapter id that emitted this row, e.g. `"wof-admin"`, `"ban"`, `"openaddresses"`. */
+	source: string
+
+	/**
+	 * Stable id within the adapter's source. For SQLite-backed adapters this is the row's primary key; for CSV/GeoJSON, a
+	 * hash of the canonical components. Must be stable across reruns so that dedup and holdout manifests are
+	 * reproducible.
+	 */
+	source_id: string
+
+	/**
+	 * Corpus version string. Stamped by the runner, not the adapter. Locked together with the tokenizer version:
+	 * `corpus-v0.1.0` ships with `tokenizer-v0.1.0`.
+	 */
+	corpus_version: string
+
+	/**
+	 * Short license label or SPDX id for _this_ row. Defaults to the adapter's `defaultLicense`, but per-row sources
+	 * (OpenAddresses) override.
+	 */
+	license: string
+}
+
+/**
+ * Marker placed on rows produced by `synthesize.ts`. Allows downstream code to weight, stratify, or exclude
+ * augmentations.
+ */
+export interface SynthMarker {
+	/**
+	 * Pipeline id describing what augmentation produced this row. Free-form but stable — e.g. `"case-perturb"`,
+	 * `"accent-strip"`, `"abbrev-swap"`, `"compose:case-perturb+typo"`.
+	 */
+	method: string
+
+	/**
+	 * `source_id` of the un-augmented row this was derived from. Allows tracing every synthetic row back to its natural
+	 * ancestor.
+	 */
+	base_source_id: string
+}
+
+/**
+ * One address row, before tokenization + BIO labeling.
+ *
+ * `raw` is what a parser would see in the wild — possibly multi-line, with arbitrary whitespace. `components` is the
+ * ground-truth tagging: every `ComponentTag` present in the source data, mapped to its surface form _as it appears in
+ * `raw`_. Alignment uses this to assign BIO labels.
+ *
+ * Country is ISO 3166-1 alpha-2 (`"US"`, `"FR"`). Locale is BCP-47 (`"en-US"`, `"fr-FR"`) and is optional; adapters
+ * that can't be sure leave it empty and let the runner default by country.
+ */
+export interface CanonicalRow extends SourceProvenance {
+	/** Address string as it might appear in source data. */
+	raw: string
+
+	/**
+	 * Component-by-tag ground truth. Surface forms must occur in `raw` (within the alignment edit distance threshold) or
+	 * the row will land in the quarantine pile.
+	 */
+	components: Partial<Record<ComponentTag, string>>
+
+	/** ISO 3166-1 alpha-2 country code. */
+	country: string
+
+	/** Optional BCP-47 locale. Defaulted by country if absent. */
+	locale?: string
+
+	/** Present only on synthetic rows. */
+	synth?: SynthMarker
+}
+
+/**
+ * Output of `align.ts`. Carries everything `CanonicalRow` does, plus parallel `tokens` and `labels` arrays of identical
+ * length (`labels[i]` is the BIO tag for `tokens[i]`) and — as of the v0.5.0 char-offset migration (#519) — parallel
+ * char-span arrays addressing `raw` directly.
+ *
+ * The span triple is the v0.5.0 source of truth; `tokens`/`labels` remain emitted during the transition (and stay
+ * derivable afterwards: whitespace split + span lookup). The reverse derivation — today's token labels — is the lossy
+ * direction (punctuation-mute).
+ */
+export interface LabeledRow extends CanonicalRow {
+	/** SentencePiece subword tokens for `raw`. */
+	tokens: readonly string[]
+
+	/** BIO labels, one per token. Same length as `tokens`. */
+	labels: readonly BIOLabel[]
+
+	/**
+	 * Char-offset label spans over `raw` (parallel arrays, per the #519 ruling): `span_starts[i]` is the inclusive start
+	 * offset (UTF-16 code units) of span `i`, `span_ends[i]` its exclusive end, `span_tags[i]` its component tag.
+	 * Invariants — enforced loudly by `alignRow`, documented for every other producer: sorted ascending by start,
+	 * non-overlapping. `raw` must be NFC-normalized or the offsets are ambiguous (also enforced by `alignRow`).
+	 *
+	 * Optional during the v0.4.x → v0.5.0 transition only: alignment always emits the triple; frozen historical corpora
+	 * and not-yet-migrated synthesis paths may lack it. Required once v0.5.0 lands and the token path is deleted.
+	 */
+	span_starts?: readonly number[]
+
+	/** Exclusive end offsets, parallel to `span_starts`. */
+	span_ends?: readonly number[]
+
+	/** Component tags, parallel to `span_starts`. */
+	span_tags?: readonly ComponentTag[]
+}
+
+/**
+ * A row that alignment refused to label. Lands in `/data/corpus/quarantine/` for human review.
+ *
+ * The `reason` is human-readable; common values are `"component-not-found:<tag>"`, `"edit-distance-exceeded:<tag>"`,
+ * `"raw-empty"`. Re-running alignment after a fix should re-emit the quarantined rows; the runner keys them by
+ * `source_id`.
+ */
+export interface QuarantinedRow {
+	row: CanonicalRow
+	reason: string
+}
+
+/**
+ * Per-invocation knobs handed to an adapter by the runner.
+ *
+ * `inputPath` is interpreted by the adapter — it might be a single file path, a directory of shards, or even an HTTPS
+ * URL. Each adapter documents its own expected shape in its README.
+ *
+ * `country` filters to a single ISO 3166-1 alpha-2 country _at the adapter level_. Adapters that hold multi-country
+ * data (OSM PBF, OpenAddresses) MUST honor this; single-country adapters (BAN) may ignore it but should reject
+ * mismatches.
+ *
+ * `limit` is a soft cap on rows emitted; useful for fixture-driven tests and smoke runs.
+ *
+ * `signal` allows the runner to cancel a long-running scan cleanly.
+ */
+export interface AdapterOptions {
+	/** Path to the adapter's input data (file, directory, or URL — adapter-specific). */
+	inputPath: string
+
+	/** Optional output directory, available to adapters that maintain side state (rare). */
+	outputDir?: string
+
+	/** ISO 3166-1 alpha-2 country filter. */
+	country?: string
+
+	/** Soft row cap. Adapters should stop iterating once this is reached. */
+	limit?: number
+
+	/** Cancellation hook. Adapters should respect this on every iteration boundary. */
+	signal?: AbortSignal
+}
+
+/**
+ * The contract every data source implements.
+ *
+ * Adapters are async generators: they yield `CanonicalRow`s one at a time, the runner consumes them (writing JSONL +
+ * maintaining checksums + driving alignment). Streaming is mandatory — many sources are tens of millions of rows and
+ * cannot be buffered.
+ *
+ * `defaultLicense` is stamped onto every emitted row's `license` field unless the adapter sets `license` explicitly
+ * (e.g. OpenAddresses, which carries per-source licenses).
+ */
+export interface CorpusAdapter {
+	/** Stable, machine-friendly id used in paths and CLI args. E.g. `"wof-admin"`. */
+	readonly id: string
+
+	/** Default SPDX-ish license label for rows from this adapter. Per-row overrides allowed. */
+	readonly defaultLicense: string
+
+	/** One-sentence description shown by `npx mailwoman corpus list`. */
+	readonly description: string
+
+	/**
+	 * Async iterable of canonical rows.
+	 *
+	 * Implementations MUST:
+	 *
+	 * - Honor `opts.country` (filter or reject mismatches).
+	 * - Honor `opts.limit` (stop after N rows).
+	 * - Respect `opts.signal` on every iteration.
+	 * - Set `source` to `this.id` on every emitted row.
+	 * - Set `license` to `this.defaultLicense` unless overriding per-row.
+	 *
+	 * Implementations MUST NOT:
+	 *
+	 * - Set `corpus_version` (the runner stamps it).
+	 * - Mutate previously-yielded rows.
+	 */
+	rows(opts: AdapterOptions): AsyncIterable<CanonicalRow>
+}