@mailwoman/corpus 8.1.0 → 8.3.0

package/src/codex/us-fips-state.ts

@@ -13,7 +13,9 @@
  *   join.
  */
 
-/** Per-state record: two-letter postal abbreviation + full canonical display name. */
+/**
+ * Per-state record: two-letter postal abbreviation + full canonical display name.
+ */
 export interface USStateInfo {
 	abbreviation: string
 	name: string
@@ -83,7 +85,9 @@ export const US_FIPS_STATE: Readonly<Record<string, USStateInfo>> = Object.freez
 	"78": { abbreviation: "VI", name: "Virgin Islands" },
 })
 
-/** Lookup helper. Returns null when the FIPS code isn't recognized. */
+/**
+ * Lookup helper. Returns null when the FIPS code isn't recognized.
+ */
 export function lookupFipsState(statefp: string | null | undefined): USStateInfo | null {
 	if (!statefp) return null
 

package/src/golden.ts

@@ -28,7 +28,9 @@ import { reconcileComponents } from "./format.ts"
 
 const TAG_SET = new Set<string>(COMPONENT_TAGS as readonly string[])
 
-/** One entry in a golden `.jsonl` file. */
+/**
+ * One entry in a golden `.jsonl` file.
+ */
 export interface GoldenEntry {
 	raw: string
 	components: Partial<Record<ComponentTag, string>>
@@ -37,25 +39,31 @@ export interface GoldenEntry {
 	notes?: string
 }
 
-/** Per-entry validation failure. */
+/**
+ * Per-entry validation failure.
+ */
 export interface GoldenIssue {
 	file: string
 	line: number
 	reason: string
 }
 
-/** Aggregate report from `validateGoldenDir`. */
+/**
+ * Aggregate report from `validateGoldenDir`.
+ */
 export interface GoldenReport {
 	entries: number
 	files: number
 	issues: GoldenIssue[]
 }
 
-/** Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations. */
+/**
+ * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
+ */
 export function parseGoldenLine(line: string): GoldenEntry {
 	const obj = JSON.parse(line) as Partial<GoldenEntry> & Record<string, unknown>
 
-	if (typeof obj.raw !== "string" || obj.raw.length === 0) {
+	if (typeof obj.raw !== "string" || !obj.raw.length) {
 		throw new Error("missing/empty raw")
 	}
 
@@ -66,12 +74,13 @@ export function parseGoldenLine(line: string): GoldenEntry {
 	if (obj.source !== "golden") {
 		throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`)
 	}
+
 	const components = (obj.components ?? {}) as Record<string, unknown>
 
 	for (const [k, v] of Object.entries(components)) {
 		if (!TAG_SET.has(k)) throw new Error(`unknown ComponentTag: ${k}`)
 
-		if (typeof v !== "string" || v.length === 0) {
+		if (typeof v !== "string" || !v.length) {
 			throw new Error(`components.${k} must be a non-empty string`)
 		}
 	}
@@ -85,7 +94,9 @@ export function parseGoldenLine(line: string): GoldenEntry {
 	}
 }
 
-/** Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent). */
+/**
+ * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
+ */
 export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 	const reconciled = reconcileComponents(entry.components, entry.raw)
 	const missing: ComponentTag[] = []
@@ -99,7 +110,9 @@ export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 	return missing
 }
 
-/** Validate one `.jsonl` file end-to-end, returning a list of issues. */
+/**
+ * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ */
 export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
 	const text = await readFile(path, "utf8")
 	const lines = text.split("\n")
@@ -114,24 +127,26 @@ export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
 			const entry = parseGoldenLine(line)
 			const unreachable = unreachableComponents(entry)
 
-			if (unreachable.length > 0) {
+			if (unreachable.length) {
 				issues.push({
 					file: path,
 					line: i + 1,
 					reason: `components not reachable in raw: ${unreachable.join(", ")}`,
 				})
 			}
-		} catch (err) {
-			issues.push({ file: path, line: i + 1, reason: (err as Error).message })
+		} catch (error) {
+			issues.push({ file: path, line: i + 1, reason: (error as Error).message })
 		}
 	}
 
 	return issues
 }
 
-/** Validate every `.jsonl` in a golden directory. */
+/**
+ * Validate every `.jsonl` in a golden directory.
+ */
 export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
-	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").sort()
+	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").toSorted()
 	const issues: GoldenIssue[] = []
 	let entries = 0
 

package/src/license.ts

@@ -37,10 +37,12 @@ export function compileLicenseExcludes(spec: string): RegExp[] {
 		.split(",")
 		.map((s) => s.trim())
 		.filter(Boolean)
-		.map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
+		.map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
 }
 
-/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
+/**
+ * True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
+ */
 export function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean {
 	const l = license ?? ""
 

package/src/parquet-wrapper/reader.ts

@@ -12,9 +12,11 @@ import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs"
 import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js"
 import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js"
 
-import { type ParquetRecordLike, ParquetSchema } from "./schema.ts"
+import type { ParquetSchema, ParquetRecordLike } from "./schema.ts"
 
-/** A typed Parquet reader, wrapping the base Parquet reader. */
+/**
+ * A typed Parquet reader, wrapping the base Parquet reader.
+ */
 export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
 	declare schema: ParquetSchema<T>
 
@@ -47,9 +49,9 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
 			const metadata = await envelopeReader.readFooter()
 
 			return new ParquetReader<T>(metadata, envelopeReader, opts)
-		} catch (err) {
+		} catch (error) {
 			await envelopeReader.close()
-			throw err
+			throw error
 		}
 	}
 

package/src/parquet-wrapper/schema.ts

@@ -21,22 +21,26 @@ import type { createSBBFParams as BloomFilterCreation } from "@dsnp/parquetjs/di
 import type { FieldDefinition } from "@dsnp/parquetjs/dist/lib/declare.js"
 import { LRUCache } from "lru-cache"
 
-/** A Parquet record-like object, i.e. a record with string keys and JSON-serializable values. */
-export type ParquetRecordLike = {
-	[key: string]: unknown | undefined
-}
+/**
+ * A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
+ */
+export type ParquetRecordLike = Record<string, unknown | undefined>
 
-/** Typed Parquet schema definition. */
-export type ParquetSchemaDefinition<T = ParquetRecordLike> = {
-	[field in Extract<keyof T, string>]: FieldDefinition
-}
+/**
+ * Typed Parquet schema definition.
+ */
+export type ParquetSchemaDefinition<T = ParquetRecordLike> = Record<Extract<keyof T, string>, FieldDefinition>
 
-/** Typed Parquet schema. */
+/**
+ * Typed Parquet schema.
+ */
 export class ParquetSchema<T> extends BaseParquetSchema {
 	declare schema: ParquetSchemaDefinition<T>
 }
 
-/** Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns. */
+/**
+ * Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns.
+ */
 export function createBloomFilters<T>(
 	parquetSchemaDef: ParquetSchemaDefinition<T>,
 	columns: Extract<keyof T, string>[]

package/src/parquet-wrapper/writer.ts

@@ -51,7 +51,9 @@ export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWrite
 		return new ParquetWriter<T>(schema, envelopeWriter, opts)
 	}
 
-	/** Convenience method to create a new buffered parquet writer that writes to the specified file. */
+	/**
+	 * Convenience method to create a new buffered parquet writer that writes to the specified file.
+	 */
 	static override async openFile<T extends ParquetRecordLike>(
 		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
 		sourcePath: string | Buffer | URL,
@@ -67,17 +69,23 @@ export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWrite
 	}
 
 	// @note This fixes invalid Markdown in the base class JSDoc.
-	/** Set a metadata key-value pair on the writer. */
+	/**
+	 * Set a metadata key-value pair on the writer.
+	 */
 	public override setMetadata(key: string, value: string): void {
 		return super.setMetadata(key, value)
 	}
 
-	/** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
+	/**
+	 * Append a row to the buffer. If the buffer is full, the data will be written to disk.
+	 */
 	public override async appendRow(row: T): Promise<void> {
 		return super.appendRow(row)
 	}
 
-	/** Flush all buffered data to disk, close the file, and release resources. */
+	/**
+	 * Flush all buffered data to disk, close the file, and release resources.
+	 */
 	public override async close(): Promise<void> {
 		await this.#flushing
 

package/src/parquet.ts

@@ -46,10 +46,14 @@ import { ParquetWriter, type ParquetSchemaDefinition } from "./parquet-wrapper/i
 import type { SplitName } from "./split.ts"
 import type { LabeledRow } from "./types.ts"
 
-/** Row groups flush at this many rows (parquetjs internal cadence within a shard). */
+/**
+ * Row groups flush at this many rows (parquetjs internal cadence within a shard).
+ */
 export const ROW_GROUP_SIZE = 50_000
 
-/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
+/**
+ * Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0.
+ */
 export const SHARD_COMPRESSION = "SNAPPY" as const
 
 /**
@@ -74,7 +78,9 @@ export interface ParquetRow {
 	[key: string]: unknown
 }
 
-/** Column names emitted into every shard. Matches `ParquetRow`. */
+/**
+ * Column names emitted into every shard. Matches `ParquetRow`.
+ */
 export const PARQUET_COLUMNS = [
 	"raw",
 	"tokens",
@@ -92,6 +98,10 @@ export const PARQUET_COLUMNS = [
 	"synth_base_id",
 ] as const
 
+/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
+   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
+   and the rule has no way to tell the two apart. */
+
 /**
  * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
  * tokens/labels arrays. Compression is per-column SNAPPY.
@@ -116,7 +126,9 @@ export const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow> = {
 	synth_base_id: { type: "UTF8", compression: SHARD_COMPRESSION, optional: true },
 }
 
-/** Per-shard metadata captured in `MANIFEST.json`. */
+/**
+ * Per-shard metadata captured in `MANIFEST.json`.
+ */
 export interface ShardDescriptor {
 	split: SplitName
 	path: string
@@ -140,13 +152,19 @@ export interface ShardManifest {
 }
 
 export interface WriteShardsOptions {
-	/** Root output directory; corpus version dir is created beneath. */
+	/**
+	 * Root output directory; corpus version dir is created beneath.
+	 */
 	outputDir: string
 
-	/** Corpus version stamped onto rows + into the output directory name. */
+	/**
+	 * Corpus version stamped onto rows + into the output directory name.
+	 */
 	corpusVersion: string
 
-	/** Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan. */
+	/**
+	 * Max rows per `.parquet` shard. Default 1_000_000 per the Phase 1 plan.
+	 */
 	rowsPerShard?: number
 }
 
@@ -274,9 +292,11 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			const splitDir = join(corpusDir, split)
 			await mkdir(splitDir, { recursive: true })
 			path = join(splitDir, `part-${String(shardIndex).padStart(4, "0")}.parquet`)
+
 			writer = await ParquetWriter.openFile<ParquetRow>(LABELED_ROW_SCHEMA, path, {
 				rowGroupSize: ROW_GROUP_SIZE,
 			})
+
 			writer.setMetadata("mailwoman.corpus_version", opts.corpusVersion)
 			writer.setMetadata("mailwoman.split", split)
 			writer.setMetadata("mailwoman.shard_index", String(shardIndex))
@@ -292,6 +312,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			if (shardRows > 0) {
 				const fileStat = await stat(path)
 				const sha256 = await hashFile(path)
+
 				shards.push({
 					split,
 					path,
@@ -304,6 +325,7 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 					last_source_id: lastSourceID,
 				})
 			}
+
 			writer = null
 		}
 
@@ -311,19 +333,25 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 			if (!writer) {
 				await openShard()
 			}
+
 			const pq = rowToParquet(row)
 			await writer!.appendRow(appendShape(pq) as unknown as ParquetRow)
 
 			if (shardRows === 0) {
 				firstSourceID = row.source_id
 			}
+
 			lastSourceID = row.source_id
+
 			shardRows++
+
 			counts[split]++
+
 			totalRows++
 
 			if (shardRows >= rowsPerShard) {
 				await closeShard()
+
 				shardIndex++
 			}
 		}
@@ -342,12 +370,15 @@ export async function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptio
 		counts,
 		total_rows: totalRows,
 	}
+
 	await writeFile(join(corpusDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
 
 	return manifest
 }
 
-/** Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput. */
+/**
+ * Single-pass SHA-256 over the file at `path`. Cheap relative to Parquet write throughput.
+ */
 async function hashFile(path: string): Promise<string> {
 	const hash = createHash("sha256")
 	const stream = createReadStream(path)

package/src/runner.ts

@@ -34,36 +34,58 @@ import { dirname, join } from "node:path"
 import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
 
-/** Snapshot of the runner's state, emitted on every progress tick. */
+/**
+ * Snapshot of the runner's state, emitted on every progress tick.
+ */
 export interface RunnerProgress {
-	/** Adapter being driven. */
+	/**
+	 * Adapter being driven.
+	 */
 	adapterID: string
 
-	/** Total rows the adapter has yielded (before dedup). */
+	/**
+	 * Total rows the adapter has yielded (before dedup).
+	 */
 	yielded: number
 
-	/** Rows actually written to JSONL (after dedup). */
+	/**
+	 * Rows actually written to JSONL (after dedup).
+	 */
 	written: number
 
-	/** Bytes written to JSONL so far. */
+	/**
+	 * Bytes written to JSONL so far.
+	 */
 	bytes: number
 
-	/** Wall-clock milliseconds since the run started. */
+	/**
+	 * Wall-clock milliseconds since the run started.
+	 */
 	elapsed_ms: number
 }
 
-/** Per-invocation options for `runAdapter`. */
+/**
+ * Per-invocation options for `runAdapter`.
+ */
 export interface RunAdapterOptions {
-	/** Adapter to drive. */
+	/**
+	 * Adapter to drive.
+	 */
 	adapter: CorpusAdapter
 
-	/** Options handed to the adapter (input path, country filter, limit, signal). */
+	/**
+	 * Options handed to the adapter (input path, country filter, limit, signal).
+	 */
 	adapterOptions: AdapterOptions
 
-	/** Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it. */
+	/**
+	 * Root output directory; the runner creates `<outputDir>/<adapter.id>/` under it.
+	 */
 	outputDir: string
 
-	/** Corpus version stamped onto every row. Locked together with the tokenizer version. */
+	/**
+	 * Corpus version stamped onto every row. Locked together with the tokenizer version.
+	 */
 	corpusVersion: string
 
 	/**
@@ -79,7 +101,9 @@ export interface RunAdapterOptions {
 	progressEvery?: number
 }
 
-/** Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk. */
+/**
+ * Return value of `runAdapter`: the same shape as `MANIFEST.json` on disk.
+ */
 export interface AdapterRunManifest {
 	adapter_id: string
 	corpus_version: string
@@ -105,7 +129,7 @@ export interface AdapterRunManifest {
  */
 export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest> {
 	const { adapter, adapterOptions, outputDir, corpusVersion } = opts
-	const progressEvery = opts.progressEvery ?? 1_000
+	const progressEvery = opts.progressEvery ?? 1000
 
 	const adapterDir = join(outputDir, adapter.id)
 	await mkdir(adapterDir, { recursive: true })
@@ -153,11 +177,13 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 					if (yielded % progressEvery === 0) {
 						emitProgress()
 					}
+
 					continue
 				}
 
 				if (seen.size >= DEDUP_MAX_SIZE) {
 					dedupExhausted = true
+
 					process.stderr.write(
 						`  runner: dedup set full at ${DEDUP_MAX_SIZE.toLocaleString()} — skipping dedup for remaining rows\n`
 					)
@@ -169,6 +195,7 @@ export async function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunMan
 			const line = `${JSON.stringify(stamped)}\n`
 			hasher.update(line)
 			bytes += Buffer.byteLength(line, "utf8")
+
 			written++
 
 			if (!stream.write(line)) {
@@ -223,6 +250,7 @@ export async function runAllAdapters(
 
 	for (const adapter of registry.list()) {
 		const adapterOptions = common.adapterOptionsFor?.(adapter) ?? common.adapterOptions
+
 		out.push(
 			await runAdapter({
 				...common,
@@ -261,23 +289,29 @@ function assertEmittedRow(adapter: CorpusAdapter, row: CanonicalRow): void {
 	}
 }
 
-/** Promise-ify a single event emission. Used to await `drain` / `close` on the write stream. */
+/**
+ * Promise-ify a single event emission. Used to await `drain` / `close` on the write stream.
+ */
 function once(emitter: WriteStream, event: "drain" | "close"): Promise<void> {
 	return new Promise((resolve, reject) => {
 		const onEvent = (): void => {
 			emitter.off("error", onError)
 			resolve()
 		}
+
 		const onError = (err: Error): void => {
 			emitter.off(event, onEvent)
 			reject(err)
 		}
+
 		emitter.once(event, onEvent)
 		emitter.once("error", onError)
 	})
 }
 
-/** Convenience: ensure the parent directory of `filePath` exists. */
+/**
+ * Convenience: ensure the parent directory of `filePath` exists.
+ */
 export async function ensureParentDir(filePath: string): Promise<void> {
 	await mkdir(dirname(filePath), { recursive: true })
 }

package/src/shard-recipes/anchor-absorption.ts

@@ -21,11 +21,15 @@ import { alignRow } from "../align.ts"
 import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
 import { makeLcg, type ShardRecipe } from "./scaffold.ts"
 
-// The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through
-// the data-root helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
+/**
+ * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
+ * helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
+ */
 const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
 
-/** The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list). */
+/**
+ * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
+ */
 function loadRealUsZips(path: string): string[] {
 	const d = JSON.parse(readFileSync(path, "utf8")) as Record<string, unknown>
 	const zips: string[] = []
@@ -39,6 +43,10 @@ function loadRealUsZips(path: string): string[] {
 	return zips
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const anchorAbsorptionRecipe: ShardRecipe = {
 	name: "anchor-absorption",
 	description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
@@ -47,8 +55,9 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 		// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
 		const random = makeLcg(opts.seed)
 		const source = opts.sourceName ?? "synth-anchor-absorption"
-		const count = opts.count ?? 50000
+		const count = opts.count ?? 50_000
 		const realZips = loadRealUsZips(ANCHOR_LOOKUP)
+
 		console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`)
 
 		let written = 0
@@ -57,7 +66,9 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 
 		for (let i = 0; i < count; i++) {
 			const synth = synthesizeAnchorAbsorptionRow({ random, realZips })
-			const country = synth.locale.split("-")[1] // "en-US" -> "US", "de-DE" -> "DE"
+			const country = synth.locale.split("-")[1]
+
+			// "en-US" -> "US", "de-DE" -> "DE"
 			const canonical = {
 				raw: synth.raw,
 				components: synth.components,
@@ -66,18 +77,23 @@ export const anchorAbsorptionRecipe: ShardRecipe = {
 				source,
 				source_id: stableSourceID(source, `${i}` as unknown as Parameters<typeof stableSourceID>[1]),
 			}
+
 			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
 			if (aligned.kind !== "labeled") {
 				quarantined++
+
 				continue
 			}
+
 			write(
 				JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n"
 			)
+
 			written++
 			byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1
 		}
+
 		console.error(`\nwrote ${written} rows (${quarantined} quarantined)`)
 		console.error("  by slice:", JSON.stringify(byTemplate))