@mailwoman/corpus 7.2.0 → 7.3.0

package/src/runner.ts

@@ -0,0 +1,283 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
+ *   per-shard manifest.
+ *
+ *   Output layout under `outputDir`:
+ *
+ *   ```
+ *   <outputDir>/<adapter.id>/
+ *   canonical.jsonl       # one row per line, in emission order
+ *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
+ * ```
+ *
+ *   The runner is responsible for everything an adapter is **not** responsible for:
+ *
+ *   - Stamping `corpus_version` on every row (adapters must NOT set it).
+ *   - Applying `canonicalDedupKey` and skipping duplicates.
+ *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
+ *   - Honoring backpressure on the output write stream.
+ *   - Counting + emitting periodic progress to an optional callback.
+ *   - Honoring `signal` (delegates to adapter's iteration boundary).
+ *
+ *   The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
+ *   steps run later, consuming the JSONL shards this writes.
+ */
+
+import { createWriteStream, type WriteStream } from "node:fs"
+import { mkdir, writeFile } from "node:fs/promises"
+import { dirname, join } from "node:path"
+
+import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
+
+/** Snapshot of the runner's state, emitted on every progress tick. */
+export interface RunnerProgress {
+	/** Adapter being driven. */
+	adapterID: string
+
+	/** Total rows the adapter has yielded (before dedup). */
+	yielded: number
+
+	/** Rows actually written to JSONL (after dedup). */
+	written: number
+
+	/** Bytes written to JSONL so far. */
+	bytes: number
+
+	/** Wall-clock milliseconds since the run started. */
+	elapsed_ms: number
+}
+
+/** Per-invocation options for `runAdapter`. */
+export interface RunAdapterOptions {
+	/** Adapter to drive. */
+	adapter: CorpusAdapter
+
+	/** Options handed to the adapter (input path, country filter, limit, signal). */
+	adapterOptions: AdapterOptions
+
+	/** Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it. */
+	outputDir: string
+
+	/** Corpus version stamped onto every row. Locked together with the tokenizer version. */
+	corpusVersion: string
+
+	/**
+	 * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
+	 * run. Errors thrown from this callback abort the run.
+	 */
+	onProgress?: (snapshot: RunnerProgress) => void
+
+	/**
+	 * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
+	 * of this value.
+	 */
+	progressEvery?: number
+}
+
+/** Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk. */
+export interface AdapterRunManifest {
+	adapter_id: string
+	corpus_version: string
+	default_license: string
+	description: string
+	yielded: number
+	written: number
+	deduped: number
+	bytes: number
+	sha256: string
+	jsonl_path: string
+	started_at: string
+	ended_at: string
+	elapsed_ms: number
+}
+
+/**
+ * Drive a single adapter to completion.
+ *
+ * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
+ * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
+ * signal fires.
+ */
+export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
+	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
+	const progressEvery = opts.progressEvery ?? 1_000
+
+	const adapterDir = join(outputDir, adapter.id)
+	await mkdir(adapterDir, { recursive: true })
+
+	const jsonlPath = join(adapterDir, "canonical.jsonl")
+	const manifestPath = join(adapterDir, "MANIFEST.json")
+
+	const startedAt = new Date()
+	const t0 = performance.now()
+
+	const stream = createWriteStream(jsonlPath, { encoding: "utf8" })
+	const hasher: StreamingHasher = streamingSha256()
+	const seen = new Set<string>()
+	const DEDUP_MAX_SIZE = 10_000_000
+	let dedupExhausted = false
+
+	let yielded = 0
+	let written = 0
+	let bytes = 0
+
+	const emitProgress = (): void => {
+		opts.onProgress?.({
+			adapterID: adapter.id,
+			yielded,
+			written,
+			bytes,
+			elapsed_ms: performance.now() - t0,
+		})
+	}
+
+	try {
+		for await (const row of adapter.rows(adapterOptions)) {
+			if (adapterOptions.signal?.aborted) {
+				throw new DOMException("Adapter run aborted by signal", "AbortError")
+			}
+
+			yielded++
+			assertEmittedRow(adapter, row)
+
+			const stamped: CanonicalRow = { ...row, corpus_version: corpusVersion }
+			const key = canonicalDedupKey(stamped)
+
+			if (!dedupExhausted) {
+				if (seen.has(key)) {
+					if (yielded % progressEvery === 0) {
+						emitProgress()
+					}
+					continue
+				}
+
+				if (seen.size >= DEDUP_MAX_SIZE) {
+					dedupExhausted = true
+					process.stderr.write(
+						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
+					)
+				} else {
+					seen.add(key)
+				}
+			}
+
+			const line = `${JSON.stringify(stamped)}\n`
+			hasher.update(line)
+			bytes += Buffer.byteLength(line, "utf8")
+			written++
+
+			if (!stream.write(line)) {
+				await once(stream, "drain")
+			}
+
+			if (yielded % progressEvery === 0) {
+				emitProgress()
+			}
+		}
+	} finally {
+		stream.end()
+		await once(stream, "close")
+	}
+
+	const endedAt = new Date()
+	const elapsed_ms = performance.now() - t0
+	emitProgress()
+
+	const manifest: AdapterRunManifest = {
+		adapter_id: adapter.id,
+		corpus_version: corpusVersion,
+		default_license: adapter.defaultLicense,
+		description: adapter.description,
+		yielded,
+		written,
+		deduped: yielded - written,
+		bytes,
+		sha256: hasher.digest(),
+		jsonl_path: jsonlPath,
+		started_at: startedAt.toISOString(),
+		ended_at: endedAt.toISOString(),
+		elapsed_ms,
+	}
+
+	await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
+
+	return manifest
+}
+
+/**
+ * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
+ * calling if partial-failure is desired).
+ *
+ * Returns the manifests in registry insertion order.
+ */
+export async function runAllAdapters(
+	registry: AdapterRegistry,
+	common: Omit<RunAdapterOptions, "adapter"> & { adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions }
+): Promise<AdapterRunManifest[]> {
+	const out: AdapterRunManifest[] = []
+
+	for (const adapter of registry.list()) {
+		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
+		out.push(
+			await runAdapter({
+				...common,
+				adapter,
+				adapterOptions,
+			})
+		)
+	}
+
+	return out
+}
+
+/**
+ * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
+ * half-malformed.
+ */
+function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
+	if (row.source !== adapter.id) {
+		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`)
+	}
+
+	if (!row.source_id) {
+		throw new Error(`adapter ${adapter.id}: row.source_id is empty`)
+	}
+
+	if (!row.raw) {
+		throw new Error(`adapter ${adapter.id}: row.raw is empty for source_id=${row.source_id}`)
+	}
+
+	if (!row.country) {
+		throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`)
+	}
+
+	if (!row.license) {
+		throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`)
+	}
+}
+
+/** Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. */
+function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
+	return new Promise((resolve, reject) => {
+		const onEvent = (): void => {
+			emitter.off("error", onError)
+			resolve()
+		}
+		const onError = (err: Error): void => {
+			emitter.off(event, onEvent)
+			reject(err)
+		}
+		emitter.once(event, onEvent)
+		emitter.once("error", onError)
+	})
+}
+
+/** Convenience: ensure the parent directory of `filePath` exists. */
+export async function ensureParentDir(filePath: string): Promise<void> {
+	await mkdir(dirname(filePath), { recursive: true })
+}

package/src/shard-recipes/anchor-absorption.ts

@@ -0,0 +1,86 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `anchor-absorption` shard recipe (#220/#723, Probe A1) — self-generates `--count` rows from
+ *   {@link synthesizeAnchorAbsorptionRow}'s 6-slice mix (CASE-H / CASE-P-us-rural / CASE-P-de /
+ *   anchor-fp / locale-ambig / standard), aligns each to BIO, and emits a labeled JSONL. The
+ *   LEADING 5-digit on CASE-H/anchor-fp/locale-ambig is sampled from the REAL US ZIPs in the
+ *   postcode-anchor lookup, so the shaped-painted anchor fires on it exactly as inference does —
+ *   teaching the model to OVERRIDE a present anchor from context. Ported from
+ *   scripts/build-anchor-absorption-shard.mjs.
+ */
+
+import { readFileSync } from "node:fs"
+
+import { dataRootPath } from "@mailwoman/core/utils"
+
+import { stableSourceID } from "../adapter.ts"
+import { alignRow } from "../align.ts"
+import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
+import { makeLcg, type ShardRecipe } from "./scaffold.ts"
+
+// The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through
+// the data-root helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
+const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
+
+/** The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list). */
+function loadRealUsZips(path: string): string[] {
+	const d = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>
+	const zips: string[] = []
+
+	for (const [pc, v] of Object.entries(d)) {
+		if (Array.isArray(v) && v[0] && typeof v[0] === "object" && "US" in (v[0] as object) && /^\d{5}$/.test(pc)) {
+			zips.push(pc)
+		}
+	}
+
+	return zips
+}
+
+export const anchorAbsorptionRecipe: ShardRecipe = {
+	name: "anchor-absorption",
+	description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
+	mode: "generate",
+	async run(opts, write) {
+		// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
+		const random = makeLcg(opts.seed)
+		const source = opts.sourceName ?? "synth-anchor-absorption"
+		const count = opts.count ?? 50000
+		const realZips = loadRealUsZips(ANCHOR_LOOKUP)
+		console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`)
+
+		let written = 0
+		let quarantined = 0
+		const byTemplate: Record<string, number> = {}
+
+		for (let i = 0; i < count; i++) {
+			const synth = synthesizeAnchorAbsorptionRow({ random, realZips })
+			const country = synth.locale.split("-")[1] // "en-US" -> "US", "de-DE" -> "DE"
+			const canonical = {
+				raw: synth.raw,
+				components: synth.components,
+				country,
+				locale: synth.locale,
+				source,
+				source_id: stableSourceID(source, `${i}` as unknown as Parameters<typeof stableSourceID>[1]),
+			}
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
+
+			if (aligned.kind !== "labeled") {
+				quarantined++
+				continue
+			}
+			write(
+				JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n"
+			)
+			written++
+			byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1
+		}
+		console.error(`\nwrote ${written} rows (${quarantined} quarantined)`)
+		console.error("  by slice:", JSON.stringify(byTemplate))
+
+		return { emitted: written, skipped: quarantined }
+	},
+}

package/src/shard-recipes/boundary-stress.ts

@@ -0,0 +1,87 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `boundary-stress` shard recipe (#375) — the boundary-instability augmentation. Self-generates
+ *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
+ *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
+ *   lever for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
+ *   scripts/build-boundary-stress-shard.mjs.
+ *
+ *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
+ */
+
+import { alignRow } from "../align.ts"
+import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.ts"
+import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
+
+// Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality
+// drop on bare "City, STATE" rows WITHOUT becoming a locality-first majority), and
+// house-number-before:after = 7:3 (FR's dominant order is number-BEFORE; 30% after breaks the order-bias
+// shortcut without risking FR hn-before accuracy). The three original non-number shapes keep the bulk.
+// Weights sum to 1.0. Key order is load-bearing — it drives the cumulative thresholds below.
+const WEIGHTS: Record<BoundaryStressTemplate, number> = {
+	"street-eats-affix": 0.22,
+	"comma-less-city-state": 0.22,
+	"fr-prefix": 0.18,
+	"bare-locality": 0.11,
+	"house-number-before-street": 0.189,
+	"house-number-after-street": 0.081,
+}
+const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
+	let acc = 0
+
+	return (Object.entries(WEIGHTS) as Array<[BoundaryStressTemplate, number]>).map(
+		([t, w]) => [t, (acc += w)] as [BoundaryStressTemplate, number]
+	)
+})()
+function pickTemplate(r: () => number): BoundaryStressTemplate {
+	const x = r()
+
+	for (const [t, c] of CUM) if (x <= c) return t
+
+	return CUM[CUM.length - 1]![0]
+}
+
+export const boundaryStressRecipe: ShardRecipe = {
+	name: "boundary-stress",
+	description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
+	mode: "generate",
+	async run(opts, write) {
+		// Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
+		const random = makeMulberry32(opts.seed)
+		const count = opts.count ?? 20000
+		let emitted = 0
+		let skipped = 0
+
+		for (let i = 0; i < count; i++) {
+			const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) })
+			const country = row.locale.split("-")[1] ?? "US"
+			const source_id = shardSourceID("synth-boundary-stress", { ...row.components, v: String(i) })
+			const canonical = {
+				raw: row.raw,
+				components: row.components,
+				country,
+				locale: row.locale,
+				source: "synth-boundary-stress",
+				source_id,
+				corpus_version: "0.6.0",
+				license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
+			}
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
+
+			if (aligned.kind !== "labeled") {
+				skipped++
+				continue
+			}
+			// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
+			write(
+				JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n"
+			)
+			emitted++
+		}
+
+		return { emitted, skipped }
+	},
+}