@mailwoman/corpus 8.0.0 → 8.2.0

package/src/adapters/wof-postalcode-json/adapter.ts

@@ -136,6 +136,10 @@ export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string
 	return slots
 }
 
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
 export const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode"
 
 export function createWOFPostalcodeAdapter(): CorpusAdapter {
@@ -163,7 +167,7 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
 			const ancestry = buildAncestryIndex(byID)
 
 			// Pass 2: emit postcode rows only, sorted by id for determinism.
-			const ids = [...byID.keys()].sort((a, b) => a - b)
+			const ids = [...byID.keys()].toSorted((a, b) => a - b)
 			let emitted = 0
 
 			for (const id of ids) {
@@ -186,7 +190,7 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
 						if (!raw) continue
 						const aligned = reconcileComponents(variant.components, raw)
 
-						if (Object.keys(aligned).length === 0) continue
+						if (!Object.keys(aligned).length) continue
 
 						yield {
 							raw,
@@ -198,6 +202,7 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
 							corpus_version: "",
 							license: "CC0-1.0",
 						}
+
 						emitted++
 					}
 				}
@@ -206,4 +211,7 @@ export function createWOFPostalcodeAdapter(): CorpusAdapter {
 	}
 }
 
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
 export const wofPostalcodeAdapter = createWOFPostalcodeAdapter()

package/src/align.ts

@@ -39,9 +39,13 @@ import { distance as levenshteinDistance } from "fastest-levenshtein"
 import { whitespaceTokenizer, type TokenSpan, type Tokenizer } from "./tokenize.ts"
 import type { CanonicalRow, LabeledRow, QuarantinedRow } from "./types.ts"
 
-/** Options for `alignRow`. */
+/**
+ * Options for `alignRow`.
+ */
 export interface AlignOptions {
-	/** Tokenizer to use. Defaults to `whitespaceTokenizer()`. */
+	/**
+	 * Tokenizer to use. Defaults to `whitespaceTokenizer()`.
+	 */
 	tokenizer?: Tokenizer
 
 	/**
@@ -60,7 +64,9 @@ export interface AlignOptions {
 	caseInsensitive?: boolean
 }
 
-/** Either a successful labeled row or a quarantined one. */
+/**
+ * Either a successful labeled row or a quarantined one.
+ */
 export type AlignmentResult = { kind: "labeled"; row: LabeledRow } | { kind: "quarantined"; row: QuarantinedRow }
 
 /**
@@ -73,7 +79,9 @@ export interface ComponentSpan {
 	end: number
 }
 
-/** Align a single row. */
+/**
+ * Align a single row.
+ */
 export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentResult {
 	const tokenizer = opts.tokenizer ?? whitespaceTokenizer()
 	const maxEditDistance = opts.maxEditDistance ?? 2
@@ -107,7 +115,7 @@ export function alignRow(row: CanonicalRow, opts: AlignOptions = {}): AlignmentR
 	// component owns ("Alaska Regional Dr, Alaska" — region "Alaska" stealing the street's first
 	// word quarantined the street; pilot2's residual class). Emit order is unaffected — spans are
 	// re-sorted by start below.
-	const entries = (Object.entries(components) as Array<[ComponentTag, string | undefined]>).sort(
+	const entries = (Object.entries(components) as Array<[ComponentTag, string | undefined]>).toSorted(
 		(a, b) => (b[1]?.length ?? 0) - (a[1]?.length ?? 0)
 	)
 
@@ -220,7 +228,7 @@ function locateSpan(args: {
 }): { start: number; end: number } | undefined {
 	const { haystack, needle, claimed, maxEditDistance } = args
 
-	if (needle.length === 0) return undefined
+	if (!needle.length) return undefined
 
 	// Pass 1: verbatim substring. Word-boundary-aligned matches are PREFERRED over intra-word ones
 	// — leftmost-substring alone let a short value claim the inside of an earlier word (region "AK"
@@ -234,13 +242,14 @@ function locateSpan(args: {
 	while (true) {
 		const idx = haystack.indexOf(needle, from)
 
-		if (idx < 0) break
+		if (idx === -1) break
 		const end = idx + needle.length
 
 		if (!overlapsClaimed(idx, end, claimed)) {
 			if (isBoundaryAligned(haystack, idx, end)) return { start: idx, end }
 			intraWord ??= { start: idx, end }
 		}
+
 		from = idx + 1
 	}
 
@@ -267,7 +276,9 @@ function locateSpan(args: {
 
 const WORD_CHAR = /[\p{L}\p{N}]/u
 
-/** Both needle edges sit on word boundaries of the haystack (string edges count as boundaries). */
+/**
+ * Both needle edges sit on word boundaries of the haystack (string edges count as boundaries).
+ */
 function isBoundaryAligned(haystack: string, start: number, end: number): boolean {
 	const before = start === 0 || !WORD_CHAR.test(haystack[start - 1]!)
 	const after = end === haystack.length || !WORD_CHAR.test(haystack[end]!)
@@ -307,9 +318,11 @@ function labelTokens(tokens: readonly TokenSpan[], spans: readonly ComponentSpan
 				} else {
 					assigned = `I-${s.tag}` as BIOLabel
 				}
+
 				break
 			}
 		}
+
 		out.push(assigned)
 	}
 

package/src/build.ts

@@ -69,15 +69,23 @@ import {
 import { defaultAugmentationsForCountry, synthesizeRow } from "./synthesize.ts"
 import type { AdapterOptions, CanonicalRow, CorpusAdapter, LabeledRow } from "./types.ts"
 
-/** Stage tags surfaced to `onProgress`. */
+/**
+ * Stage tags surfaced to `onProgress`.
+ */
 export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest"
 
-/** Per-invocation options for `buildCorpus`. */
+/**
+ * Per-invocation options for `buildCorpus`.
+ */
 export interface BuildCorpusOptions {
-	/** Root output directory. All build artifacts land beneath it. */
+	/**
+	 * Root output directory. All build artifacts land beneath it.
+	 */
 	outputDir: string
 
-	/** Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name. */
+	/**
+	 * Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
+	 */
 	corpusVersion: string
 
 	/**
@@ -92,13 +100,19 @@ export interface BuildCorpusOptions {
 	 */
 	adapterInputs: Record<string, AdapterOptions>
 
-	/** Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests. */
+	/**
+	 * Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
+	 */
 	synthesize?: boolean
 
-	/** Forwarded to `writeShards`. Default 1_000_000. */
+	/**
+	 * Forwarded to `writeShards`. Default 1_000_000.
+	 */
 	rowsPerShard?: number
 
-	/** Progress hook. Errors thrown abort the build. */
+	/**
+	 * Progress hook. Errors thrown abort the build.
+	 */
 	onProgress?: (stage: BuildStage, message: string) => void
 
 	/**
@@ -110,7 +124,9 @@ export interface BuildCorpusOptions {
 	excludeLicenses?: readonly RegExp[]
 }
 
-/** Top-level manifest tying every stage together. */
+/**
+ * Top-level manifest tying every stage together.
+ */
 export interface BuildCorpusManifest {
 	corpus_version: string
 	built_at: string
@@ -155,8 +171,10 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 		if (!adapterOptions) {
 			skipped.push(adapter.id)
 			opts.onProgress?.("adapter-run", `skipped ${adapter.id} (no input configured)`)
+
 			continue
 		}
+
 		// Opt-in resume (MAILWOMAN_RESUME=1): if a complete per-adapter canonical.jsonl + MANIFEST.json
 		// already exist, reuse them instead of re-emitting. The MANIFEST is written only after the
 		// canonical is fully flushed, so its presence guarantees completeness; row order is identical,
@@ -173,15 +191,19 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 			const cached = JSON.parse(readFileSync(cachedManifest, "utf8")) as AdapterRunManifest
 			opts.onProgress?.("adapter-run", `resumed ${adapter.id} (reused ${cached.yielded} canonical rows)`)
 			adapterRuns.push(cached)
+
 			continue
 		}
+
 		opts.onProgress?.("adapter-run", `running ${adapter.id}`)
+
 		const m = await runAdapter({
 			adapter,
 			adapterOptions,
 			outputDir: intermediateDir,
 			corpusVersion: opts.corpusVersion,
 		})
+
 		adapterRuns.push(m)
 	}
 
@@ -195,11 +217,13 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 		val: join(intermediateDir, "labeled-val.jsonl"),
 		test: join(intermediateDir, "labeled-test.jsonl"),
 	}
+
 	const labeledStreams: Record<SplitName, WriteStream> = {
 		train: createWriteStream(labeledPaths.train, { encoding: "utf8" }),
 		val: createWriteStream(labeledPaths.val, { encoding: "utf8" }),
 		test: createWriteStream(labeledPaths.test, { encoding: "utf8" }),
 	}
+
 	const quarantinePath = join(intermediateDir, "quarantine.jsonl")
 	const quarantineStream = createWriteStream(quarantinePath, { encoding: "utf8" })
 
@@ -230,8 +254,10 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 			// set reflects what the corpus actually CONTAINED, and `excluded_by_license` what was cut.
 			if (licenseExcluded(row.license, excludeLicenses)) {
 				excludedByLicense++
+
 				continue
 			}
+
 			const fanned: CanonicalRow[] = [row]
 
 			if (synthesize) {
@@ -245,23 +271,28 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 
 				try {
 					result = alignRow(r)
-				} catch (err) {
+				} catch (error) {
 					// Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
 					// alignRow's targeted paths normalize/quarantine known issues with specific reasons;
 					// this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
 					// shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
-					writeQuarantine(r, `align-threw:${(err as Error).message.slice(0, 160)}`)
+					writeQuarantine(r, `align-threw:${(error as Error).message.slice(0, 160)}`)
+
 					quarantined++
+
 					continue
 				}
 
 				if (result.kind === "labeled") {
 					const split = splitForRow(result.row, holdouts)
 					labeledStreams[split].write(`${JSON.stringify(result.row)}\n`)
+
 					counts[split]++
+
 					aligned++
 				} else {
 					writeQuarantine(r, result.row.reason)
+
 					quarantined++
 				}
 			}
@@ -271,6 +302,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 	for (const s of Object.values(labeledStreams)) {
 		s.end()
 	}
+
 	quarantineStream.end()
 	await Promise.all([...Object.values(labeledStreams).map(streamEnd), streamEnd(quarantineStream)])
 
@@ -279,6 +311,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 	// manifests with disk spill for splits that exceed in-memory thresholds.
 	opts.onProgress?.("split", `splitting ${aligned} aligned rows`)
 	const splitsDir = join(opts.outputDir, "splits")
+
 	const splitCounts = await writeSplitManifestsFromLabeledFiles({
 		labeledPaths,
 		outputDir: splitsDir,
@@ -290,6 +323,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 	// 5. Parquet shards — per-split labeled JSONL streams in, sharded `.parquet` out. The prior
 	// `splitFor(source_id)` callback (and the `Map<source_id, SplitName>` behind it) is gone.
 	opts.onProgress?.("shard", "writing parquet shards")
+
 	const shardManifest = await writeShards(
 		{
 			train: streamJSONL<LabeledRow>(labeledPaths.train),
@@ -305,7 +339,8 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 
 	// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
 	// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
-	const licenseSummary = [...licenseCounts.entries()].sort((a, b) => b[1] - a[1])
+	const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1])
+
 	opts.onProgress?.(
 		"manifest",
 		`license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
@@ -316,6 +351,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 
 	// 6. Top-level manifest.
 	opts.onProgress?.("manifest", "writing top-level MANIFEST.json")
+
 	const manifest: BuildCorpusManifest = {
 		corpus_version: opts.corpusVersion,
 		built_at,
@@ -328,6 +364,7 @@ export async function buildCorpus(opts: BuildCorpusOptions): Promise<BuildCorpus
 		licenses: Object.fromEntries(licenseSummary),
 		excluded_by_license: excludedByLicense,
 	}
+
 	await writeFile(join(opts.outputDir, "MANIFEST.json"), `${JSON.stringify(manifest, null, 2)}\n`, "utf8")
 
 	return manifest

package/src/codex/us-fips-state.ts

@@ -13,7 +13,9 @@
  *   join.
  */
 
-/** Per-state record: two-letter postal abbreviation + full canonical display name. */
+/**
+ * Per-state record: two-letter postal abbreviation + full canonical display name.
+ */
 export interface USStateInfo {
 	abbreviation: string
 	name: string
@@ -83,7 +85,9 @@ export const US_FIPS_STATE: Readonly<Record<string, USStateInfo>> = Object.freez
 	"78": { abbreviation: "VI", name: "Virgin Islands" },
 })
 
-/** Lookup helper. Returns null when the FIPS code isn't recognized. */
+/**
+ * Lookup helper. Returns null when the FIPS code isn't recognized.
+ */
 export function lookupFipsState(statefp: string | null | undefined): USStateInfo | null {
 	if (!statefp) return null
 

package/src/golden.ts

@@ -28,7 +28,9 @@ import { reconcileComponents } from "./format.ts"
 
 const TAG_SET = new Set<string>(COMPONENT_TAGS as readonly string[])
 
-/** One entry in a golden `.jsonl` file. */
+/**
+ * One entry in a golden `.jsonl` file.
+ */
 export interface GoldenEntry {
 	raw: string
 	components: Partial<Record<ComponentTag, string>>
@@ -37,25 +39,31 @@ export interface GoldenEntry {
 	notes?: string
 }
 
-/** Per-entry validation failure. */
+/**
+ * Per-entry validation failure.
+ */
 export interface GoldenIssue {
 	file: string
 	line: number
 	reason: string
 }
 
-/** Aggregate report from `validateGoldenDir`. */
+/**
+ * Aggregate report from `validateGoldenDir`.
+ */
 export interface GoldenReport {
 	entries: number
 	files: number
 	issues: GoldenIssue[]
 }
 
-/** Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations. */
+/**
+ * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
+ */
 export function parseGoldenLine(line: string): GoldenEntry {
 	const obj = JSON.parse(line) as Partial<GoldenEntry> & Record<string, unknown>
 
-	if (typeof obj.raw !== "string" || obj.raw.length === 0) {
+	if (typeof obj.raw !== "string" || !obj.raw.length) {
 		throw new Error("missing/empty raw")
 	}
 
@@ -66,12 +74,13 @@ export function parseGoldenLine(line: string): GoldenEntry {
 	if (obj.source !== "golden") {
 		throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`)
 	}
+
 	const components = (obj.components ?? {}) as Record<string, unknown>
 
 	for (const [k, v] of Object.entries(components)) {
 		if (!TAG_SET.has(k)) throw new Error(`unknown ComponentTag: ${k}`)
 
-		if (typeof v !== "string" || v.length === 0) {
+		if (typeof v !== "string" || !v.length) {
 			throw new Error(`components.${k} must be a non-empty string`)
 		}
 	}
@@ -85,7 +94,9 @@ export function parseGoldenLine(line: string): GoldenEntry {
 	}
 }
 
-/** Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent). */
+/**
+ * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
+ */
 export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 	const reconciled = reconcileComponents(entry.components, entry.raw)
 	const missing: ComponentTag[] = []
@@ -99,7 +110,9 @@ export function unreachableComponents(entry: GoldenEntry): ComponentTag[] {
 	return missing
 }
 
-/** Validate one `.jsonl` file end-to-end, returning a list of issues. */
+/**
+ * Validate one `.jsonl` file end-to-end, returning a list of issues.
+ */
 export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
 	const text = await readFile(path, "utf8")
 	const lines = text.split("\n")
@@ -114,24 +127,26 @@ export async function validateGoldenFile(path: string): Promise<GoldenIssue[]> {
 			const entry = parseGoldenLine(line)
 			const unreachable = unreachableComponents(entry)
 
-			if (unreachable.length > 0) {
+			if (unreachable.length) {
 				issues.push({
 					file: path,
 					line: i + 1,
 					reason: `components not reachable in raw: ${unreachable.join(", ")}`,
 				})
 			}
-		} catch (err) {
-			issues.push({ file: path, line: i + 1, reason: (err as Error).message })
+		} catch (error) {
+			issues.push({ file: path, line: i + 1, reason: (error as Error).message })
 		}
 	}
 
 	return issues
 }
 
-/** Validate every `.jsonl` in a golden directory. */
+/**
+ * Validate every `.jsonl` in a golden directory.
+ */
 export async function validateGoldenDir(dir: string): Promise<GoldenReport> {
-	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").sort()
+	const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").toSorted()
 	const issues: GoldenIssue[] = []
 	let entries = 0
 

package/src/license.ts

@@ -37,10 +37,12 @@ export function compileLicenseExcludes(spec: string): RegExp[] {
 		.split(",")
 		.map((s) => s.trim())
 		.filter(Boolean)
-		.map((s) => new RegExp("^" + s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
+		.map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"))
 }
 
-/** True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded). */
+/**
+ * True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
+ */
 export function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean {
 	const l = license ?? ""
 

package/src/parquet-wrapper/reader.ts

@@ -12,9 +12,11 @@ import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs"
 import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js"
 import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js"
 
-import { type ParquetRecordLike, ParquetSchema } from "./schema.ts"
+import type { ParquetSchema, ParquetRecordLike } from "./schema.ts"
 
-/** A typed Parquet reader, wrapping the base Parquet reader. */
+/**
+ * A typed Parquet reader, wrapping the base Parquet reader.
+ */
 export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
 	declare schema: ParquetSchema<T>
 
@@ -47,9 +49,9 @@ export class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReade
 			const metadata = await envelopeReader.readFooter()
 
 			return new ParquetReader<T>(metadata, envelopeReader, opts)
-		} catch (err) {
+		} catch (error) {
 			await envelopeReader.close()
-			throw err
+			throw error
 		}
 	}
 

package/src/parquet-wrapper/schema.ts

@@ -21,22 +21,26 @@ import type { createSBBFParams as BloomFilterCreation } from "@dsnp/parquetjs/di
 import type { FieldDefinition } from "@dsnp/parquetjs/dist/lib/declare.js"
 import { LRUCache } from "lru-cache"
 
-/** A Parquet record-like object, i.e. a record with string keys and JSON-serializable values. */
-export type ParquetRecordLike = {
-	[key: string]: unknown | undefined
-}
+/**
+ * A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
+ */
+export type ParquetRecordLike = Record<string, unknown | undefined>
 
-/** Typed Parquet schema definition. */
-export type ParquetSchemaDefinition<T = ParquetRecordLike> = {
-	[field in Extract<keyof T, string>]: FieldDefinition
-}
+/**
+ * Typed Parquet schema definition.
+ */
+export type ParquetSchemaDefinition<T = ParquetRecordLike> = Record<Extract<keyof T, string>, FieldDefinition>
 
-/** Typed Parquet schema. */
+/**
+ * Typed Parquet schema.
+ */
 export class ParquetSchema<T> extends BaseParquetSchema {
 	declare schema: ParquetSchemaDefinition<T>
 }
 
-/** Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns. */
+/**
+ * Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns.
+ */
 export function createBloomFilters<T>(
 	parquetSchemaDef: ParquetSchemaDefinition<T>,
 	columns: Extract<keyof T, string>[]

package/src/parquet-wrapper/writer.ts

@@ -51,7 +51,9 @@ export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWrite
 		return new ParquetWriter<T>(schema, envelopeWriter, opts)
 	}
 
-	/** Convenience method to create a new buffered parquet writer that writes to the specified file. */
+	/**
+	 * Convenience method to create a new buffered parquet writer that writes to the specified file.
+	 */
 	static override async openFile<T extends ParquetRecordLike>(
 		schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>,
 		sourcePath: string | Buffer | URL,
@@ -67,17 +69,23 @@ export class ParquetWriter<T extends ParquetRecordLike> extends BaseParquetWrite
 	}
 
 	// @note This fixes invalid Markdown in the base class JSDoc.
-	/** Set a metadata key-value pair on the writer. */
+	/**
+	 * Set a metadata key-value pair on the writer.
+	 */
 	public override setMetadata(key: string, value: string): void {
 		return super.setMetadata(key, value)
 	}
 
-	/** Append a row to the buffer. If the buffer is full, the data will be written to disk. */
+	/**
+	 * Append a row to the buffer. If the buffer is full, the data will be written to disk.
+	 */
 	public override async appendRow(row: T): Promise<void> {
 		return super.appendRow(row)
 	}
 
-	/** Flush all buffered data to disk, close the file, and release resources. */
+	/**
+	 * Flush all buffered data to disk, close the file, and release resources.
+	 */
 	public override async close(): Promise<void> {
 		await this.#flushing