@mailwoman/corpus 9.1.0 → 9.3.0

package/src/parquet-wrapper/writer.ts

@@ -1,110 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Typed Parquet writer. Two static constructors mirror the base class:
- *
- *   - `openStream`: wrap an existing writable stream.
- *   - `openFile`: open a path on disk, ensuring the parent directory exists first.
- *
- *   Implements `AsyncDisposable` so `await using writer = await ParquetWriter.openFile(...)` flushes
- *   and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
- *   dispose calls don't race.
- */
-
-import * as fs from "node:fs/promises"
-import * as path from "node:path"
-
-import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs"
-import type { WriterOptions } from "@dsnp/parquetjs/dist/lib/declare.js"
-import { osopen, type WriteStreamMinimal } from "@dsnp/parquetjs/dist/lib/util.js"
-import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js"
-
-import {
-	type ParquetRecordLike,
-	ParquetSchema,
-	type ParquetSchemaDefinition,
-	ParquetSchemaDefinitionCache,
-} from "./schema.ts"
-
-/**
- * A typed Parquet writer, wrapping the base Parquet writer.
- */
-export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWriter implements AsyncDisposable {
-	declare schema: ParquetSchema<T>
-	protected static readonly SchemaDefinitionCache = new ParquetSchemaDefinitionCache()
-	#flushing: Promise<void> = Promise.resolve()
-
-	static override async openStream<T extends ParquetRecordLike>(
-		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
-		outputStream: WriteStreamMinimal,
-		opts: WriterOptions = {}
-	): Promise<ParquetWriter<T>> {
-		const schema =
-			schemaLike instanceof ParquetSchema
-				? schemaLike
-				: ParquetWriter.SchemaDefinitionCache.findOrCreateSchema(schemaLike)
-
-		const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts)
-
-		return new ParquetWriter<T>(schema, envelopeWriter, opts)
-	}
-
-	/**
-	 * Convenience method to create a new buffered parquet writer that writes to the specified file.
-	 */
-	static override async openFile<T extends ParquetRecordLike>(
-		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
-		sourcePath: string | Buffer | URL,
-		opts?: WriterOptions
-	): Promise<ParquetWriter<T>> {
-		if (typeof sourcePath === "string") {
-			await fs.mkdir(path.dirname(sourcePath), { recursive: true })
-		}
-
-		const outputStream = await osopen(sourcePath, opts)
-
-		return ParquetWriter.openStream<T>(schemaLike, outputStream, opts)
-	}
-
-	// @note This fixes invalid Markdown in the base class JSDoc.
-	/**
-	 * Set a metadata key-value pair on the writer.
-	 */
-	public override setMetadata(key: string, value: string): void {
-		return super.setMetadata(key, value)
-	}
-
-	/**
-	 * Append a row to the buffer. If the buffer is full, the data will be written to disk.
-	 */
-	public override async appendRow(row: T): Promise<void> {
-		return super.appendRow(row)
-	}
-
-	/**
-	 * Flush all buffered data to disk, close the file, and release resources.
-	 */
-	public override async close(): Promise<void> {
-		await this.#flushing
-
-		if (this.closed) return
-
-		const { promise, resolve, reject } = Promise.withResolvers<void>()
-
-		super.close().then(resolve, reject)
-
-		this.#flushing = promise
-
-		return this.#flushing
-	}
-
-	public async [Symbol.asyncDispose]() {
-		return this.close()
-	}
-
-	public async dispose() {
-		return this[Symbol.asyncDispose]()
-	}
-}

package/src/runner.ts

@@ -1,322 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Adapter runner — drives a `CorpusAdapter` to completion and writes intermediate JSONL + a
- *   per-shard manifest.
- *
- *   Output layout under `outputDir`:
- *
- *   ```
- *   <outputDir>/<adapter.id>/
- *   canonical.jsonl       # one row per line, in emission order
- *   MANIFEST.json         # adapter id, version, row count, sha256, license, started_at, ended_at
- * ```
- *
- *   The runner is responsible for everything an adapter is **not** responsible for:
- *
- *   - Stamping `corpus_version` on every row (adapters must NOT set it).
- *   - Applying `canonicalDedupKey` and skipping duplicates.
- *   - Streaming sha256 over JSONL bytes so the manifest checksum doesn't require a re-read.
- *   - Honoring backpressure on the output write stream.
- *   - Counting + emitting periodic progress to an optional callback.
- *   - Honoring `signal` (delegates to adapter's iteration boundary).
- *
- *   The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
- *   steps run later, consuming the JSONL shards this writes.
- */
-
-import { createWriteStream, type WriteStream } from "node:fs"
-import { mkdir, writeFile } from "node:fs/promises"
-import { dirname, join } from "node:path"
-
-import {
-	canonicalDedupKey,
-	streamingSha256,
-	type AdapterRegistry,
-	type StreamingHasher,
-} from "@mailwoman/corpus/adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
-
-/**
- * Snapshot of the runner's state, emitted on every progress tick.
- */
-export interface RunnerProgress {
-	/**
-	 * Adapter being driven.
-	 */
-	adapterID: string
-
-	/**
-	 * Total rows the adapter has yielded (before dedup).
-	 */
-	yielded: number
-
-	/**
-	 * Rows actually written to JSONL (after dedup).
-	 */
-	written: number
-
-	/**
-	 * Bytes written to JSONL so far.
-	 */
-	bytes: number
-
-	/**
-	 * Wall-clock milliseconds since the run started.
-	 */
-	elapsed_ms: number
-}
-
-/**
- * Per-invocation options for `runAdapter`.
- */
-export interface RunAdapterOptions {
-	/**
-	 * Adapter to drive.
-	 */
-	adapter: CorpusAdapter
-
-	/**
-	 * Options handed to the adapter (input path, country filter, limit, signal).
-	 */
-	adapterOptions: AdapterOptions
-
-	/**
-	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
-	 */
-	outputDir: string
-
-	/**
-	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
-	 */
-	corpusVersion: string
-
-	/**
-	 * Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
-	 * run. Errors thrown from this callback abort the run.
-	 */
-	onProgress?: (snapshot: RunnerProgress) => void
-
-	/**
-	 * Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
-	 * of this value.
-	 */
-	progressEvery?: number
-}
-
-/**
- * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
- */
-export interface AdapterRunManifest {
-	adapter_id: string
-	corpus_version: string
-	default_license: string
-	description: string
-	yielded: number
-	written: number
-	deduped: number
-	bytes: number
-	sha256: string
-	jsonl_path: string
-	started_at: string
-	ended_at: string
-	elapsed_ms: number
-}
-
-/**
- * Drive a single adapter to completion.
- *
- * Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
- * Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
- * signal fires.
- */
-export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
-	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
-	const progressEvery = opts.progressEvery ?? 1000
-
-	const adapterDir = join(outputDir, adapter.id)
-	await mkdir(adapterDir, { recursive: true })
-
-	const jsonlPath = join(adapterDir, "canonical.jsonl")
-	const manifestPath = join(adapterDir, "MANIFEST.json")
-
-	const startedAt = new Date()
-	const t0 = performance.now()
-
-	const stream = createWriteStream(jsonlPath, { encoding: "utf8" })
-	const hasher: StreamingHasher = streamingSha256()
-	const seen = new Set<string>()
-	const DEDUP_MAX_SIZE = 10_000_000
-	let dedupExhausted = false
-
-	let yielded = 0
-	let written = 0
-	let bytes = 0
-
-	const emitProgress = (): void => {
-		opts.onProgress?.({
-			adapterID: adapter.id,
-			yielded,
-			written,
-			bytes,
-			elapsed_ms: performance.now() - t0,
-		})
-	}
-
-	try {
-		for await (const row of adapter.rows(adapterOptions)) {
-			if (adapterOptions.signal?.aborted) {
-				throw new DOMException("Adapter run aborted by signal", "AbortError")
-			}
-
-			yielded++
-			assertEmittedRow(adapter, row)
-
-			const stamped: CanonicalRow = { ...row, corpus_version: corpusVersion }
-			const key = canonicalDedupKey(stamped)
-
-			if (!dedupExhausted) {
-				if (seen.has(key)) {
-					if (yielded % progressEvery === 0) {
-						emitProgress()
-					}
-
-					continue
-				}
-
-				if (seen.size >= DEDUP_MAX_SIZE) {
-					dedupExhausted = true
-
-					process.stderr.write(
-						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
-					)
-				} else {
-					seen.add(key)
-				}
-			}
-
-			const line = `${JSON.stringify(stamped)}\n`
-			hasher.update(line)
-			bytes += Buffer.byteLength(line, "utf8")
-
-			written++
-
-			if (!stream.write(line)) {
-				await once(stream, "drain")
-			}
-
-			if (yielded % progressEvery === 0) {
-				emitProgress()
-			}
-		}
-	} finally {
-		stream.end()
-		await once(stream, "close")
-	}
-
-	const endedAt = new Date()
-	const elapsed_ms = performance.now() - t0
-	emitProgress()
-
-	const manifest: AdapterRunManifest = {
-		adapter_id: adapter.id,
-		corpus_version: corpusVersion,
-		default_license: adapter.defaultLicense,
-		description: adapter.description,
-		yielded,
-		written,
-		deduped: yielded - written,
-		bytes,
-		sha256: hasher.digest(),
-		jsonl_path: jsonlPath,
-		started_at: startedAt.toISOString(),
-		ended_at: endedAt.toISOString(),
-		elapsed_ms,
-	}
-
-	await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
-
-	return manifest
-}
-
-/**
- * Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
- * calling if partial-failure is desired).
- *
- * Returns the manifests in registry insertion order.
- */
-export async function runAllAdapters(
-	registry: AdapterRegistry,
-	common: Omit<RunAdapterOptions, "adapter"> & { adapterOptionsFor?: (a: CorpusAdapter) => AdapterOptions }
-): Promise<AdapterRunManifest[]> {
-	const out: AdapterRunManifest[] = []
-
-	for (const adapter of registry.list()) {
-		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
-
-		out.push(
-			await runAdapter({
-				...common,
-				adapter,
-				adapterOptions,
-			})
-		)
-	}
-
-	return out
-}
-
-/**
- * Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
- * half-malformed.
- */
-function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
-	if (row.source !== adapter.id) {
-		throw new Error(`adapter ${adapter.id}: row.source must equal adapter.id (got ${JSON.stringify(row.source)})`)
-	}
-
-	if (!row.source_id) {
-		throw new Error(`adapter ${adapter.id}: row.source_id is empty`)
-	}
-
-	if (!row.raw) {
-		throw new Error(`adapter ${adapter.id}: row.raw is empty for source_id=${row.source_id}`)
-	}
-
-	if (!row.country) {
-		throw new Error(`adapter ${adapter.id}: row.country is empty for source_id=${row.source_id}`)
-	}
-
-	if (!row.license) {
-		throw new Error(`adapter ${adapter.id}: row.license is empty for source_id=${row.source_id}`)
-	}
-}
-
-/**
- * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream.
- */
-function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
-	return new Promise((resolve, reject) => {
-		const onEvent = (): void => {
-			emitter.off("error", onError)
-			resolve()
-		}
-
-		const onError = (err: Error): void => {
-			emitter.off(event, onEvent)
-			reject(err)
-		}
-
-		emitter.once(event, onEvent)
-		emitter.once("error", onError)
-	})
-}
-
-/**
- * Convenience: ensure the parent directory of `filePath` exists.
- */
-export async function ensureParentDir(filePath: string): Promise<void> {
-	await mkdir(dirname(filePath), { recursive: true })
-}

package/src/shard-recipes/anchor-absorption.ts

@@ -1,103 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `anchor-absorption` shard recipe (#220/#723, Probe A1) — self-generates `--count` rows from
- *   {@link synthesizeAnchorAbsorptionRow}'s 6-slice mix (CASE-H / CASE-P-us-rural / CASE-P-de /
- *   anchor-fp / locale-ambig / standard), aligns each to BIO, and emits a labeled JSONL. The
- *   LEADING 5-digit on CASE-H/anchor-fp/locale-ambig is sampled from the REAL US ZIPs in the
- *   postcode-anchor lookup, so the shaped-painted anchor fires on it exactly as inference does —
- *   teaching the model to OVERRIDE a present anchor from context. Ported from
- *   scripts/build-anchor-absorption-shard.mjs.
- */
-
-import { readFileSync } from "node:fs"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { dataRootPath } from "@mailwoman/core/utils"
-import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
-import { synthesizeAnchorAbsorptionRow } from "@mailwoman/corpus/synthesizers/anchor-absorption"
-import { alignRow } from "@mailwoman/corpus/utils"
-
-import { makeLcg, type ShardRecipe } from "./scaffold.ts"
-
-/**
- * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
- * helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
- */
-const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
-
-/**
- * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
- */
-function loadRealUsZips(path: string): string[] {
-	const d = parseJSONStrict<Record<string, unknown>>(readFileSync(path, "utf8"))
-	const zips: string[] = []
-
-	for (const [pc, v] of Object.entries(d)) {
-		if (Array.isArray(v) && v[0] && typeof v[0] === "object" && "US" in (v[0] as object) && /^\d{5}$/.test(pc)) {
-			zips.push(pc)
-		}
-	}
-
-	return zips
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const anchorAbsorptionRecipe: ShardRecipe = {
-	name: "anchor-absorption",
-	description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
-	mode: "generate",
-	async run(opts, write) {
-		// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
-		const random = makeLcg(opts.seed)
-		const source = opts.sourceName ?? "synth-anchor-absorption"
-		const count = opts.count ?? 50_000
-		const realZips = loadRealUsZips(ANCHOR_LOOKUP)
-
-		console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`)
-
-		let written = 0
-		let quarantined = 0
-		const byTemplate: Record<string, number> = {}
-
-		for (let i = 0; i < count; i++) {
-			const synth = synthesizeAnchorAbsorptionRow({ random, realZips })
-			const country = synth.locale.split("-")[1]
-
-			// "en-US" -> "US", "de-DE" -> "DE"
-			const canonical = {
-				raw: synth.raw,
-				components: synth.components,
-				country,
-				locale: synth.locale,
-				source,
-				source_id: stableSourceID(source, `${i}` as unknown as Parameters<typeof stableSourceID>[1]),
-			}
-
-			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
-
-			if (aligned.kind !== "labeled") {
-				quarantined++
-
-				continue
-			}
-
-			write(
-				JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n"
-			)
-
-			written++
-			byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1
-		}
-
-		console.error(`\nwrote ${written} rows (${quarantined} quarantined)`)
-		console.error("  by slice:", JSON.stringify(byTemplate))
-
-		return { emitted: written, skipped: quarantined }
-	},
-}

package/src/shard-recipes/boundary-stress.ts

@@ -1,104 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `boundary-stress` shard recipe (#375) — the boundary-instability augmentation. Self-generates
- *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
- *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
- *   lever for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
- *   scripts/build-boundary-stress-shard.mjs.
- *
- *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
- */
-
-import {
-	type BoundaryStressTemplate,
-	synthesizeBoundaryStressRow,
-} from "@mailwoman/corpus/synthesizers/boundary-stress"
-import { alignRow } from "@mailwoman/corpus/utils"
-
-import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
-
-/**
- * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
- * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
- * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
- * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is load-bearing — it drives the cumulative
- * thresholds below.
- */
-const WEIGHTS: Record<BoundaryStressTemplate, number> = {
-	"street-eats-affix": 0.22,
-	"comma-less-city-state": 0.22,
-	"fr-prefix": 0.18,
-	"bare-locality": 0.11,
-	"house-number-before-street": 0.189,
-	"house-number-after-street": 0.081,
-}
-
-const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
-	let acc = 0
-
-	return (Object.entries(WEIGHTS) as Array<[BoundaryStressTemplate, number]>).map(
-		([t, w]) => [t, (acc += w)] as [BoundaryStressTemplate, number]
-	)
-})()
-
-function pickTemplate(r: () => number): BoundaryStressTemplate {
-	const x = r()
-
-	for (const [t, c] of CUM) if (x <= c) return t
-
-	return CUM.at(-1)![0]
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const boundaryStressRecipe: ShardRecipe = {
-	name: "boundary-stress",
-	description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
-	mode: "generate",
-	async run(opts, write) {
-		// Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
-		const random = makeMulberry32(opts.seed)
-		const count = opts.count ?? 20_000
-		let emitted = 0
-		let skipped = 0
-
-		for (let i = 0; i < count; i++) {
-			const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) })
-			const country = row.locale.split("-")[1] ?? "US"
-			const source_id = shardSourceID("synth-boundary-stress", { ...row.components, v: String(i) })
-
-			const canonical = {
-				raw: row.raw,
-				components: row.components,
-				country,
-				locale: row.locale,
-				source: "synth-boundary-stress",
-				source_id,
-				corpus_version: "0.6.0",
-				license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
-			}
-
-			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
-
-			if (aligned.kind !== "labeled") {
-				skipped++
-
-				continue
-			}
-
-			// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
-			write(
-				JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n"
-			)
-
-			emitted++
-		}
-
-		return { emitted, skipped }
-	},
-}