@mailwoman/corpus 8.0.0 → 8.2.0

package/src/parquet.ts

@@ -46,10 +46,14 @@ import { ParquetWriter, type ParquetSchemaDefinition } from "./parquet-wrapper/i
 import type { SplitName } from "./split.ts"
 import type { LabeledRow } from "./types.ts"
 
-/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
+/**
+ * Row groups flush at this many rows (parquetjs internal cadence within a shard).
+ */
 export const ROW_GROUP_SIZE = 50_000
 
-/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
+/**
+ * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
+ */
 export const SHARD_COMPRESSION = "SNAPPY" as const
 
 /**
@@ -74,7 +78,9 @@ export interface ParquetRow {
 	[key: string]: unknown
 }
 
-/** Column names emitted into every shard. Matches `ParquetRow`. */
+/**
+ * Column names emitted into every shard. Matches `ParquetRow`.
+ */
 export const PARQUET_COLUMNS = [
 	"raw",
 	"tokens",
@@ -92,6 +98,10 @@ export const PARQUET_COLUMNS = [
 	"synth_base_id",
 ] as const
 
+/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
+   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
+   and the rule has no way to tell the two apart. */
+
 /**
  * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
  * tokens/labels arrays. Compression is per-column SNAPPY.
@@ -116,7 +126,9 @@ export const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow> = {
 	synth_base_id: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
 }
 
-/** Per-shard metadata captured in `MANIFEST.json`. */
+/**
+ * Per-shard metadata captured in `MANIFEST.json`.
+ */
 export interface ShardDescriptor {
 	split: SplitName
 	path: string
@@ -140,13 +152,19 @@ export interface ShardManifest {
 }
 
 export interface WriteShardsOptions {
-	/** Root output directory; corpus version dir is created beneath. */
+	/**
+	 * Root output directory; corpus version dir is created beneath.
+	 */
 	outputDir: string
 
-	/** Corpus version stamped onto rows + into the output directory name. */
+	/**
+	 * Corpus version stamped onto rows + into the output directory name.
+	 */
 	corpusVersion: string
 
-	/** Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan. */
+	/**
+	 * Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan.
+	 */
 	rowsPerShard?: number
 }
 
@@ -274,9 +292,11 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			const splitDir = join(corpusDir, split)
 			await mkdir(splitDir, { recursive: true })
 			path = join(splitDir, `part-${String(shardIndex).padStart(4, "0")}.parquet`)
+
 			writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, path, {
 				rowGroupSize: ROW_GROUP_SIZE,
 			})
+
 			writer.setMetadata("mailwoman.corpus_version", opts.corpusVersion)
 			writer.setMetadata("mailwoman.split", split)
 			writer.setMetadata("mailwoman.shard_index", String(shardIndex))
@@ -292,6 +312,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			if (shardRows > 0) {
 				const fileStat = await stat(path)
 				const sha256 = await hashFile(path)
+
 				shards.push({
 					split,
 					path,
@@ -304,6 +325,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 					last_source_id: lastSourceID,
 				})
 			}
+
 			writer = null
 		}
 
@@ -311,19 +333,25 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			if (!writer) {
 				await openShard()
 			}
+
 			const pq = rowToParquet(row)
 			await writer!.appendRow(appendShape(pq) as unknown as ParquetRow)
 
 			if (shardRows === 0) {
 				firstSourceID = row.source_id
 			}
+
 			lastSourceID = row.source_id
+
 			shardRows++
+
 			counts[split]++
+
 			totalRows++
 
 			if (shardRows >= rowsPerShard) {
 				await closeShard()
+
 				shardIndex++
 			}
 		}
@@ -342,12 +370,15 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 		counts,
 		total_rows: totalRows,
 	}
+
 	await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
 
 	return manifest
 }
 
-/** Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput. */
+/**
+ * Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput.
+ */
 async function hashFile(path: string): Promise<string> {
 	const hash = createHash("sha256")
 	const stream = createReadStream(path)

package/src/runner.ts

@@ -34,36 +34,58 @@ import { dirname, join } from "node:path"
 import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
 
-/** Snapshot of the runner's state, emitted on every progress tick. */
+/**
+ * Snapshot of the runner's state, emitted on every progress tick.
+ */
 export interface RunnerProgress {
-	/** Adapter being driven. */
+	/**
+	 * Adapter being driven.
+	 */
 	adapterID: string
 
-	/** Total rows the adapter has yielded (before dedup). */
+	/**
+	 * Total rows the adapter has yielded (before dedup).
+	 */
 	yielded: number
 
-	/** Rows actually written to JSONL (after dedup). */
+	/**
+	 * Rows actually written to JSONL (after dedup).
+	 */
 	written: number
 
-	/** Bytes written to JSONL so far. */
+	/**
+	 * Bytes written to JSONL so far.
+	 */
 	bytes: number
 
-	/** Wall-clock milliseconds since the run started. */
+	/**
+	 * Wall-clock milliseconds since the run started.
+	 */
 	elapsed_ms: number
 }
 
-/** Per-invocation options for `runAdapter`. */
+/**
+ * Per-invocation options for `runAdapter`.
+ */
 export interface RunAdapterOptions {
-	/** Adapter to drive. */
+	/**
+	 * Adapter to drive.
+	 */
 	adapter: CorpusAdapter
 
-	/** Options handed to the adapter (input path, country filter, limit, signal). */
+	/**
+	 * Options handed to the adapter (input path, country filter, limit, signal).
+	 */
 	adapterOptions: AdapterOptions
 
-	/** Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it. */
+	/**
+	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
+	 */
 	outputDir: string
 
-	/** Corpus version stamped onto every row. Locked together with the tokenizer version. */
+	/**
+	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
+	 */
 	corpusVersion: string
 
 	/**
@@ -79,7 +101,9 @@ export interface RunAdapterOptions {
 	progressEvery?: number
 }
 
-/** Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk. */
+/**
+ * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
+ */
 export interface AdapterRunManifest {
 	adapter_id: string
 	corpus_version: string
@@ -105,7 +129,7 @@ export interface AdapterRunManifest {
  */
 export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
 	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
-	const progressEvery = opts.progressEvery ?? 1_000
+	const progressEvery = opts.progressEvery ?? 1000
 
 	const adapterDir = join(outputDir, adapter.id)
 	await mkdir(adapterDir, { recursive: true })
@@ -153,11 +177,13 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 					if (yielded % progressEvery === 0) {
 						emitProgress()
 					}
+
 					continue
 				}
 
 				if (seen.size >= DEDUP_MAX_SIZE) {
 					dedupExhausted = true
+
 					process.stderr.write(
 						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
 					)
@@ -169,6 +195,7 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 			const line = `${JSON.stringify(stamped)}\n`
 			hasher.update(line)
 			bytes += Buffer.byteLength(line, "utf8")
+
 			written++
 
 			if (!stream.write(line)) {
@@ -223,6 +250,7 @@ export async function runAllAdapters(
 
 	for (const adapter of registry.list()) {
 		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
+
 		out.push(
 			await runAdapter({
 				...common,
@@ -261,23 +289,29 @@ function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
 	}
 }
 
-/** Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. */
+/**
+ * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream.
+ */
 function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
 	return new Promise((resolve, reject) => {
 		const onEvent = (): void => {
 			emitter.off("error", onError)
 			resolve()
 		}
+
 		const onError = (err: Error): void => {
 			emitter.off(event, onEvent)
 			reject(err)
 		}
+
 		emitter.once(event, onEvent)
 		emitter.once("error", onError)
 	})
 }
 
-/** Convenience: ensure the parent directory of `filePath` exists. */
+/**
+ * Convenience: ensure the parent directory of `filePath` exists.
+ */
 export async function ensureParentDir(filePath: string): Promise<void> {
 	await mkdir(dirname(filePath), { recursive: true })
 }

package/src/shard-recipes/anchor-absorption.ts

@@ -21,11 +21,15 @@ import { alignRow } from "../align.ts"
 import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
 import { makeLcg, type ShardRecipe } from "./scaffold.ts"
 
-// The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through
-// the data-root helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
+/**
+ * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
+ * helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
+ */
 const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
 
-/** The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list). */
+/**
+ * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
+ */
 function loadRealUsZips(path: string): string[] {
 	const d = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>
 	const zips: string[] = []
@@ -39,6 +43,10 @@ function loadRealUsZips(path: string): string[] {
 	return zips
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const anchorAbsorptionRecipe: ShardRecipe = {
 	name: "anchor-absorption",
 	description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
@@ -47,8 +55,9 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 		// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
 		const random = makeLcg(opts.seed)
 		const source = opts.sourceName ?? "synth-anchor-absorption"
-		const count = opts.count ?? 50000
+		const count = opts.count ?? 50_000
 		const realZips = loadRealUsZips(ANCHOR_LOOKUP)
+
 		console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`)
 
 		let written = 0
@@ -57,7 +66,9 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 
 		for (let i = 0; i < count; i++) {
 			const synth = synthesizeAnchorAbsorptionRow({ random, realZips })
-			const country = synth.locale.split("-")[1] // "en-US" -> "US", "de-DE" -> "DE"
+			const country = synth.locale.split("-")[1]
+
+			// "en-US" -> "US", "de-DE" -> "DE"
 			const canonical = {
 				raw: synth.raw,
 				components: synth.components,
@@ -66,18 +77,23 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 				source,
 				source_id: stableSourceID(source, `${i}` as unknown as Parameters<typeof stableSourceID>[1]),
 			}
+
 			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
 			if (aligned.kind !== "labeled") {
 				quarantined++
+
 				continue
 			}
+
 			write(
 				JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n"
 			)
+
 			written++
 			byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1
 		}
+
 		console.error(`\nwrote ${written} rows (${quarantined} quarantined)`)
 		console.error("  by slice:", JSON.stringify(byTemplate))
 

package/src/shard-recipes/boundary-stress.ts

@@ -16,11 +16,13 @@ import { alignRow } from "../align.ts"
 import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.ts"
 import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
-// Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality
-// drop on bare "City, STATE" rows WITHOUT becoming a locality-first majority), and
-// house-number-before:after = 7:3 (FR's dominant order is number-BEFORE; 30% after breaks the order-bias
-// shortcut without risking FR hn-before accuracy). The three original non-number shapes keep the bulk.
-// Weights sum to 1.0. Key order is load-bearing — it drives the cumulative thresholds below.
+/**
+ * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
+ * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
+ * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
+ * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is load-bearing — it drives the cumulative
+ * thresholds below.
+ */
 const WEIGHTS: Record<BoundaryStressTemplate, number> = {
 	"street-eats-affix": 0.22,
 	"comma-less-city-state": 0.22,
@@ -29,6 +31,7 @@ const WEIGHTS: Record<BoundaryStressTemplate, number> = {
 	"house-number-before-street": 0.189,
 	"house-number-after-street": 0.081,
 }
+
 const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
 	let acc = 0
 
@@ -36,14 +39,19 @@ const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
 		([t, w]) => [t, (acc += w)] as [BoundaryStressTemplate, number]
 	)
 })()
+
 function pickTemplate(r: () => number): BoundaryStressTemplate {
 	const x = r()
 
 	for (const [t, c] of CUM) if (x <= c) return t
 
-	return CUM[CUM.length - 1]![0]
+	return CUM.at(-1)![0]
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const boundaryStressRecipe: ShardRecipe = {
 	name: "boundary-stress",
 	description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
@@ -51,7 +59,7 @@ export const boundaryStressRecipe: ShardRecipe = {
 	async run(opts, write) {
 		// Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
 		const random = makeMulberry32(opts.seed)
-		const count = opts.count ?? 20000
+		const count = opts.count ?? 20_000
 		let emitted = 0
 		let skipped = 0
 
@@ -59,6 +67,7 @@ export const boundaryStressRecipe: ShardRecipe = {
 			const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) })
 			const country = row.locale.split("-")[1] ?? "US"
 			const source_id = shardSourceID("synth-boundary-stress", { ...row.components, v: String(i) })
+
 			const canonical = {
 				raw: row.raw,
 				components: row.components,
@@ -69,16 +78,20 @@ export const boundaryStressRecipe: ShardRecipe = {
 				corpus_version: "0.6.0",
 				license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
 			}
+
 			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
 			if (aligned.kind !== "labeled") {
 				skipped++
+
 				continue
 			}
+
 			// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
 			write(
 				JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n"
 			)
+
 			emitted++
 		}