@mailwoman/corpus 9.1.0 → 9.3.0

package/lib/runner.ts

@@ -0,0 +1,312 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
+ *   per-slice manifest.
+ *
+ *   Output layout under `outputDir`:
+ *
+ *   ```
+ *   <outputDir>/<adapter.id>/
+ *   canonical.jsonl       # one row per line, in emission order
+ *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
+ * ```
+ *
+ *   The runner is responsible for everything an adapter is **not** responsible for:
+ *
+ *   - Stamping `corpus_version` on every row (adapters must NOT set it).
+ *   - Applying `canonicalDedupKey` and skipping duplicates.
+ *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
+ *   - Honoring backpressure on the output write stream.
+ *   - Counting + emitting periodic progress to an optional callback.
+ *   - Honoring `signal` (delegates to adapter's iteration boundary).
+ *
+ *   The runner does NOT perform alignment, tokenization, synthesis, or slicing into Parquet. Those
+ *   steps run later, consuming the JSONL slices this writes.
+ */
+
+import { openWriteStream, type WriteStream } from "@mailwoman/core/fs/streams"
+import { writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers"
+import { join, type PathBuilderLike } from "path-ts"
+
+import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "#adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+
+/**
+ * Snapshot of the runner's state, emitted on every progress tick.
+ */
+export interface RunnerProgress {
+	/**
+	 * Adapter being driven.
+	 */
+	adapterID: string
+
+	/**
+	 * Total rows the adapter has yielded (before dedup).
+	 */
+	yielded: number
+
+	/**
+	 * Rows actually written to JSONL (after dedup).
+	 */
+	written: number
+
+	/**
+	 * Bytes written to JSONL so far.
+	 */
+	bytes: number
+
+	/**
+	 * Wall-clock milliseconds since the run started.
+	 */
+	elapsed_ms: number
+}
+
+/**
+ * Per-invocation options for `runAdapter`.
+ */
+export interface RunAdapterOptions {
+	/**
+	 * Adapter to drive.
+	 */
+	adapter: CorpusAdapter
+
+	/**
+	 * Options handed to the adapter (input path, country filter, limit, signal).
+	 */
+	adapterOptions: AdapterOptions
+
+	/**
+	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
+	 */
+	outputDir: PathBuilderLike
+
+	/**
+	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
+	 */
+	corpusVersion: string
+
+	/**
+	 * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
+	 * run. Errors thrown from this callback abort the run.
+	 */
+	onProgress?: (snapshot: RunnerProgress) => void
+
+	/**
+	 * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
+	 * of this value.
+	 */
+	progressEvery?: number
+}
+
+/**
+ * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
+ */
+export interface AdapterRunManifest {
+	adapter_id: string
+	corpus_version: string
+	default_license: string
+	description: string
+	yielded: number
+	written: number
+	deduped: number
+	bytes: number
+	sha256: string
+	jsonl_path: string
+	started_at: string
+	ended_at: string
+	elapsed_ms: number
+}
+
+/**
+ * Drive a single adapter to completion.
+ *
+ * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
+ * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
+ * signal fires.
+ */
+export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
+	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
+	const progressEvery = opts.progressEvery ?? 1000
+
+	const adapterDir = join(outputDir, adapter.id)
+	await makeDirectories(adapterDir)
+
+	const jsonlPath = join(adapterDir, "canonical.jsonl")
+	const manifestPath = join(adapterDir, "MANIFEST.json")
+
+	const startedAt = new Date()
+	const t0 = performance.now()
+
+	const stream = openWriteStream(jsonlPath, { encoding: "utf8" })
+	const hasher: StreamingHasher = streamingSha256()
+	const seen = new Set<string>()
+	const DEDUP_MAX_SIZE = 10_000_000
+	let dedupExhausted = false
+
+	let yielded = 0
+	let written = 0
+	let bytes = 0
+
+	const emitProgress = (): void => {
+		opts.onProgress?.({
+			adapterID: adapter.id,
+			yielded,
+			written,
+			bytes,
+			elapsed_ms: performance.now() - t0,
+		})
+	}
+
+	try {
+		for await (const row of adapter.rows(adapterOptions)) {
+			if (adapterOptions.signal?.aborted) {
+				throw new DOMException("Adapter run aborted by signal", "AbortError")
+			}
+
+			yielded++
+			assertEmittedRow(adapter, row)
+
+			const stamped: CanonicalRow = { ...row, corpus_version: corpusVersion }
+			const key = canonicalDedupKey(stamped)
+
+			if (!dedupExhausted) {
+				if (seen.has(key)) {
+					if (yielded % progressEvery === 0) {
+						emitProgress()
+					}
+
+					continue
+				}
+
+				if (seen.size >= DEDUP_MAX_SIZE) {
+					dedupExhausted = true
+
+					process.stderr.write(
+						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
+					)
+				} else {
+					seen.add(key)
+				}
+			}
+
+			const line = `${JSON.stringify(stamped)}\n`
+			hasher.update(line)
+			bytes += Buffer.byteLength(line, "utf8")
+
+			written++
+
+			if (!stream.write(line)) {
+				await once(stream, "drain")
+			}
+
+			if (yielded % progressEvery === 0) {
+				emitProgress()
+			}
+		}
+	} finally {
+		stream.end()
+		await once(stream, "close")
+	}
+
+	const endedAt = new Date()
+	const elapsed_ms = performance.now() - t0
+	emitProgress()
+
+	const manifest: AdapterRunManifest = {
+		adapter_id: adapter.id,
+		corpus_version: corpusVersion,
+		default_license: adapter.defaultLicense,
+		description: adapter.description,
+		yielded,
+		written,
+		deduped: yielded - written,
+		bytes,
+		sha256: hasher.digest(),
+		jsonl_path: jsonlPath,
+		started_at: startedAt.toISOString(),
+		ended_at: endedAt.toISOString(),
+		elapsed_ms,
+	}
+
+	await writeLocalJSONFile(manifest, manifestPath)
+
+	return manifest
+}
+
+/**
+ * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
+ * calling if partial-failure is desired).
+ *
+ * Returns the manifests in registry insertion order.
+ */
+export async function runAllAdapters(
+	registry: AdapterRegistry,
+	common: Omit<RunAdapterOptions, "adapter"> & { adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions }
+): Promise<AdapterRunManifest[]> {
+	const out: AdapterRunManifest[] = []
+
+	for (const adapter of registry.list()) {
+		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
+
+		out.push(
+			await runAdapter({
+				...common,
+				adapter,
+				adapterOptions,
+			})
+		)
+	}
+
+	return out
+}
+
+/**
+ * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
+ * half-malformed.
+ */
+function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
+	if (row.source !== adapter.id) {
+		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`)
+	}
+
+	if (!row.source_id) {
+		throw new Error(`adapter ${adapter.id}: row.source_id is empty`)
+	}
+
+	if (!row.raw) {
+		throw new Error(`adapter ${adapter.id}: row.raw is empty for source_id=${row.source_id}`)
+	}
+
+	if (!row.country) {
+		throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`)
+	}
+
+	if (!row.license) {
+		throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`)
+	}
+}
+
+/**
+ * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. Exported for `build.ts`,
+ * whose stage streams await `close` the same way; unlike a bare two-listener race, the loser listener is detached so a
+ * long-lived stream does not accumulate one orphan handler per wait.
+ */
+export function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
+	return new Promise((resolve, reject) => {
+		const onEvent = (): void => {
+			emitter.off("error", onError)
+			resolve()
+		}
+
+		const onError = (err: Error): void => {
+			emitter.off(event, onEvent)
+			reject(err)
+		}
+
+		emitter.once(event, onEvent)
+		emitter.once("error", onError)
+	})
+}

package/lib/synthesizers/anchor-absorption.ts

@@ -0,0 +1,288 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Anchor-absorption counter-augmentation (#220/#723, Probe A1). Teaches the model the CONTEXT-
+ *   DEPENDENT leading-5-digit disambiguation that the killed #723 override was faking, AND that the
+ *   `anchor_paint_mode=shaped` WHERE fix alone over-corrected on (Probe A0: it flipped the default
+ *   to house_number, recovering CASE-H but ERODING CASE-P — postcode F1 99.3→86.5 on
+ *   leading-postcode rows like "05764 Finel Hollow Road, VT").
+ *
+ *   THE DISCRIMINATOR the model must learn (from the CASE-H vs CASE-P contrast, NOT a flipped
+ *   default):
+ *
+ *   - A leading 5-digit WITH a trailing postcode + street context → it is the HOUSE NUMBER (CASE-H)
+ *   - A leading 5-digit with NO trailing postcode (US-rural / DE) → it IS the POSTCODE (CASE-P) Both
+ *       leading tokens are real ZIPs (so the painted anchor fires on both); only the surrounding
+ *       context separates them. The model attends to the trailing token to decide the leading one.
+ *
+ *   Slice mix (the A0 learning sets a HEAVY CASE-P floor so the default doesn't flip — DeepSeek's
+ *   ≥35% CASE-P; here CASE-P total = 35%): H-adversarial 30% US street, leading real-ZIP house# +
+ *   TRAILING postcode → house_number P-us-rural 20% US rural, leading postcode, NO trailing →
+ *   postcode (the A0-erosion fix) P-de 15% German leading postcode "{pc} {city}, {street} {hn}" →
+ *   postcode anchor-fp 10% leading 5-digit that is NOT a real ZIP + trailing postcode →
+ *   house_number locale-ambig 15% minimal context; the local token (street-type vs none) decides
+ *   standard 10% normal small house# + trailing postcode → house_number (baseline)
+ *
+ *   Real ZIPs for the leading-5-digit are sampled from the postcode anchor lookup at build time
+ *   (passed via opts.realZips) so the shaped anchor fires on them exactly as at inference; fake
+ *   ZIPs (anchor-fp) are 5-digit strings deliberately absent from the lookup.
+ */
+
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import { pick } from "#synthesizers/utils"
+
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
+export interface AnchorAbsorptionBaseTuple {
+	locality: string
+	region: string
+	postcode: string
+}
+
+export type AnchorAbsorptionTemplate =
+	| "h-adversarial"
+	| "h-no-trailing-locality"
+	| "p-us-rural"
+	| "p-de"
+	| "anchor-fp"
+	| "locale-ambig"
+	| "standard"
+
+export interface AnchorAbsorptionSynthesisOpts {
+	random?: () => number
+	forceTemplate?: AnchorAbsorptionTemplate
+	/**
+	 * Real US ZIPs (in the anchor lookup) to use as the LEADING 5-digit house number — so the painted anchor fires on it,
+	 * the CASE-H/anchor-fp trigger. Builder loads these from pilot-anchor-lookup.json.
+	 */
+	realZips?: ReadonlyArray<string>
+}
+
+export interface SynthesizedAnchorAbsorptionRow {
+	raw: string
+	components: Partial<Record<ComponentTag, string>>
+	locale: string
+	template: AnchorAbsorptionTemplate
+}
+
+/**
+ * A realistic US house number: mostly 1–4 digits (the common range), 25% a real 5-digit ZIP (the HARD case — a 5-digit
+ * leading number that is still a house number when a locality is present).
+ */
+function houseNum(random: () => number, realZips: ReadonlyArray<string>): string {
+	if (random() < 0.25) return pick(realZips, random)
+
+	return String(1 + Math.floor(random() * 9999))
+}
+
+/**
+ * Curated, provenance-light reference vocab (real US street/city/state + DE). Surface forms must appear in `raw` for
+ * alignRow; these are plain ASCII tokens that align cleanly.
+ */
+const STREET_NAMES = [
+	"Main",
+	"Oak",
+	"Elm",
+	"Maple",
+	"Cedar",
+	"Pine",
+	"Washington",
+	"Lincoln",
+	"Park",
+	"Hill",
+	"Finel Hollow",
+	"Mt Tabor",
+	"Swasey",
+	"Camperdown",
+	"Mellville",
+	"Rhone",
+	"Westpark",
+	"Crescent Meadow",
+]
+
+const STREET_TYPES = ["St", "Ave", "Rd", "Dr", "Ln", "Blvd", "Ct", "Way", "Road", "Drive"]
+
+const US_TUPLES: ReadonlyArray<AnchorAbsorptionBaseTuple> = [
+	{ locality: "Springfield", region: "IL", postcode: "62701" },
+	{ locality: "Portland", region: "OR", postcode: "97215" },
+	{ locality: "Houston", region: "TX", postcode: "77598" },
+	{ locality: "Dallas", region: "TX", postcode: "75229" },
+	{ locality: "Austin", region: "TX", postcode: "78748" },
+	{ locality: "Albuquerque", region: "NM", postcode: "87102" },
+	{ locality: "Rochester", region: "NY", postcode: "14606" },
+	{ locality: "Sacramento", region: "CA", postcode: "95823" },
+]
+
+/**
+ * US rural states where the leading-postcode "{ZIP} {Street}, {STATE}" form (no trailing ZIP) is real — the VT/rural
+ * format the #723 override broke and Probe A0 eroded.
+ */
+const RURAL_REGIONS = ["VT", "ND", "SD", "NH", "ME", "MT", "WY"]
+
+/**
+ * DE leading-postcode tuples: "{postcode} {locality}, {street} {house}".
+ */
+const DE_TUPLES = [
+	{ postcode: "10115", locality: "Berlin", street: "Hauptstraße" },
+	{ postcode: "80331", locality: "München", street: "Sendlinger Straße" },
+	{ postcode: "20095", locality: "Hamburg", street: "Mönckebergstraße" },
+	{ postcode: "50667", locality: "Köln", street: "Hohe Straße" },
+	{ postcode: "01067", locality: "Dresden", street: "Prager Straße" },
+]
+
+const HOUSE_NUMS = ["5", "12", "27", "100", "212", "1450", "8"]
+/**
+ * Fake 5-digit strings that are NOT real US ZIPs (so the anchor lookup MISSES them) — for anchor-fp.
+ */
+const FAKE_ZIPS = ["00000", "99998", "99997", "00001", "99996"]
+
+/**
+ * Build one anchor-absorption counter-augmentation row.
+ */
+export function synthesizeAnchorAbsorptionRow(
+	opts: AnchorAbsorptionSynthesisOpts = {}
+): SynthesizedAnchorAbsorptionRow {
+	const random = opts.random ?? Math.random
+	const realZips = opts.realZips && opts.realZips.length ? opts.realZips : US_TUPLES.map((t) => t.postcode)
+	const template = opts.forceTemplate ?? pick(ALL_TEMPLATES, random)
+	const street = `${pick(STREET_NAMES, random)} ${pick(STREET_TYPES, random)}`
+
+	if (template === "h-adversarial") {
+		// US street, leading real-ZIP house number, WITH a trailing postcode → the leading is house_number.
+		const zip = pick(realZips, random)
+		const t = pick(US_TUPLES, random)
+		const raw = `${zip} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
+
+		return {
+			raw,
+			components: { house_number: zip, street, locality: t.locality, region: t.region, postcode: t.postcode },
+			locale: "en-US",
+			template,
+		}
+	}
+
+	if (template === "p-us-rural") {
+		// US rural, leading postcode, NO trailing postcode → the leading IS the postcode (the A0-erosion fix).
+		const zip = pick(realZips, random)
+		const region = pick(RURAL_REGIONS, random)
+		const raw = `${zip} ${street}, ${region}`
+
+		return {
+			raw,
+			components: { postcode: zip, street, region },
+			locale: "en-US",
+			template,
+		}
+	}
+
+	if (template === "p-de") {
+		// German leading postcode "{pc} {city}, {street} {hn}" → the leading is the postcode.
+		const d = pick(DE_TUPLES, random)
+		const hn = pick(HOUSE_NUMS, random)
+		const raw = `${d.postcode} ${d.locality}, ${d.street} ${hn}`
+
+		return {
+			raw,
+			components: { postcode: d.postcode, locality: d.locality, street: d.street, house_number: hn },
+			locale: "de-DE",
+			template,
+		}
+	}
+
+	if (template === "anchor-fp") {
+		// Leading 5-digit that is NOT a real ZIP (anchor MISSES) + trailing postcode → still house_number.
+		const fake = pick(FAKE_ZIPS, random)
+		const t = pick(US_TUPLES, random)
+		const raw = `${fake} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
+
+		return {
+			raw,
+			components: { house_number: fake, street, locality: t.locality, region: t.region, postcode: t.postcode },
+			locale: "en-US",
+			template,
+		}
+	}
+
+	if (template === "locale-ambig") {
+		// Minimal context — the LOCAL token decides. Half: "{realZip} {street}" (street-type → house#);
+		// half: "{realZip} {locality}" (no street, leading postcode → postcode). No trailing, no region.
+		const zip = pick(realZips, random)
+
+		if (random() < 0.5) {
+			return { raw: `${zip} ${street}`, components: { house_number: zip, street }, locale: "en-US", template }
+		}
+
+		const t = pick(US_TUPLES, random)
+
+		return {
+			raw: `${zip} ${t.locality}`,
+			components: { postcode: zip, locality: t.locality },
+			locale: "en-US",
+			template,
+		}
+	}
+
+	if (template === "h-no-trailing-locality") {
+		// The A3 fix (#220): the common US format "{house#} {street}, {locality}, {STATE}" with NO trailing
+		// postcode → the leading number is the HOUSE NUMBER. The CONTRAST to p-us-rural (same no-trailing,
+		// state-bearing shape) is the LOCALITY: present here, absent there. The A2 slice lacked this slice,
+		// so p-us-rural's "leading-number + STATE → postcode" rule over-generalized to 98 golden house#
+		// rows ("36 Oxbow Dr, Bradford, VT" → postcode). The house# spans 1-4 digits AND real 5-digit ZIPs
+		// (the hard case: 5-digit + locality is STILL a house number, distinct from p-us-rural's no-locality).
+		const hn = houseNum(random, realZips)
+		const t = pick(US_TUPLES, random)
+		const region = random() < 0.5 ? pick(RURAL_REGIONS, random) : t.region
+		const raw = `${hn} ${street}, ${t.locality}, ${region}`
+
+		return {
+			raw,
+			components: { house_number: hn, street, locality: t.locality, region },
+			locale: "en-US",
+			template,
+		}
+	}
+
+	// standard: normal house number + trailing postcode → house_number (baseline, keeps the common case).
+	const hn = houseNum(random, realZips)
+	const t = pick(US_TUPLES, random)
+	const raw = `${hn} ${street}, ${t.locality}, ${t.region} ${t.postcode}`
+
+	return {
+		raw,
+		components: { house_number: hn, street, locality: t.locality, region: t.region, postcode: t.postcode },
+		locale: "en-US",
+		template,
+	}
+}
+
+// Weighted template bag — the slice mix. Expanded to a flat array so `pick` draws at the target
+// frequencies (matches the boundary-stress ALL_TEMPLATES idiom).
+//
+// A3 (#220, after the per-row diagnostic on the A2 probe): A1/A2 both held SLICE-H (100) + postcode
+// (~98) but cost house_number (95.8->92.8), and the A2 mix-rebalance did NOT move it — so it was never a
+// CASE-P-quantity problem. The row-by-row v192-vs-A2 diff (hn-regression-diff.ts) pinned it: 132/132 house#
+// regressions were house#->POSTCODE on "{house#} {street}, {locality}, {STATE}" no-trailing rows — the
+// p-us-rural rule ("leading-number + STATE + no-trailing -> postcode") OVER-GENERALIZED because the slice
+// had NO counter-slice for the common locality-bearing house# case. A3 ADDS h-no-trailing-locality (15%)
+// to teach the LOCALITY discriminator (present -> house#, absent + 5-digit -> postcode = p-us-rural) and
+// trims p-us-rural 16->13. Goal: house_number recovers WITHOUT re-eroding postcode/SLICE-H. CASE-P = 26%.
+/**
+ * Every anchor-absorption template, in one list for the slice runner to sample from. Each covers a way a venue or
+ * landmark name can swallow the street token that follows it.
+ */
+export const ALL_TEMPLATES: ReadonlyArray<AnchorAbsorptionTemplate> = [
+	...new Array<AnchorAbsorptionTemplate>(25).fill("h-adversarial"),
+	...new Array<AnchorAbsorptionTemplate>(15).fill("h-no-trailing-locality"),
+	...new Array<AnchorAbsorptionTemplate>(13).fill("p-us-rural"),
+	...new Array<AnchorAbsorptionTemplate>(13).fill("p-de"),
+	...new Array<AnchorAbsorptionTemplate>(8).fill("anchor-fp"),
+	...new Array<AnchorAbsorptionTemplate>(14).fill("locale-ambig"),
+	...new Array<AnchorAbsorptionTemplate>(12).fill("standard"),
+]