@mailwoman/corpus 8.0.0 → 8.2.0

package/src/tools/golden-promote.ts

@@ -38,6 +38,21 @@ import { readJSONL, sha256File, writeJSONL } from "@mailwoman/core/utils"
 
 // ── Types ──────────────────────────────────────────────────────────────────
 
+/**
+ * Components a candidate needs before it is rich enough to be worth a golden-set slot.
+ */
+const MIN_GOLDEN_COMPONENTS = 5
+
+/**
+ * Components an existing entry must still carry to survive promotion.
+ */
+const MIN_PROMOTABLE_COMPONENTS = 4
+
+/**
+ * Quote-delimited segments above which the raw line is malformed rather than merely quoted.
+ */
+const MAX_QUOTE_SEGMENTS = 3
+
 interface GoldenEntry {
 	raw: string
 	components: Record<string, string>
@@ -58,24 +73,36 @@ export interface PromoteStats {
 }
 
 export interface PromoteGoldenOptions {
-	/** Candidates JSONL (required). */
+	/**
+	 * Candidates JSONL (required).
+	 */
 	input: string
-	/** Target golden version dir (required, e.g. `v0.1.1`). */
+	/**
+	 * Target golden version dir (required, e.g. `v0.1.1`).
+	 */
 	bumpTo: string
-	/** Previous version to forward-copy + dedup against. Default `v0.1.0`. */
+	/**
+	 * Previous version to forward-copy + dedup against. Default `v0.1.0`.
+	 */
 	prior?: string
-	/** Golden dir root. Default `data/eval/golden`. */
+	/**
+	 * Golden dir root. Default `data/eval/golden`.
+	 */
 	goldenRoot?: string
-	/** Skip the human-typed-likelihood filters (keep everything that passed expand-golden's validator). */
+	/**
+	 * Skip the human-typed-likelihood filters (keep everything that passed expand-golden's validator).
+	 */
 	noFilters?: boolean
-	/** Report what would be written but don't touch disk. */
+	/**
+	 * Report what would be written but don't touch disk.
+	 */
 	dryRun?: boolean
 }
 
 // ── Filters ───────────────────────────────────────────────────────────────
 
 function normalize(s: string): string {
-	return s.toLowerCase().replace(/\s+/g, " ").trim()
+	return s.toLowerCase().replaceAll(/\s+/g, " ").trim()
 }
 
 /**
@@ -85,7 +112,7 @@ function normalize(s: string): string {
 function isComponentsGlued(entry: GoldenEntry): boolean {
 	const componentCount = Object.keys(entry.components).length
 
-	if (componentCount < 5) return false
+	if (componentCount < MIN_GOLDEN_COMPONENTS) return false
 	const separators = (entry.raw.match(/[,\n;]/g) ?? []).length
 
 	return separators < 2
@@ -97,7 +124,7 @@ function isComponentsGlued(entry: GoldenEntry): boolean {
  * often precedes locality there).
  */
 function isPostcodeBadlyLeading(entry: GoldenEntry): boolean {
-	if (Object.keys(entry.components).length < 4) return false
+	if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS) return false
 
 	if (entry.country === "FR" || entry.country === "France") return false
 	const postcode = entry.components.postcode
@@ -118,13 +145,13 @@ function isSuspicious(entry: GoldenEntry): boolean {
 	const raw = entry.raw
 
 	// eslint-disable-next-line no-control-regex
-	if (/[\x00-\x1f\x7f]/.test(raw)) return true
+	if (/[\u0000-\u001F\u007F]/.test(raw)) return true
 	const openBrackets = (raw.match(/[[({<]/g) ?? []).length
 	const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length
 
 	if (openBrackets !== closeBrackets) return true
 
-	if (raw.split('"').length > 3) return true
+	if (raw.split('"').length > MAX_QUOTE_SEGMENTS) return true
 
 	// too many quote marks
 	return false
@@ -132,7 +159,9 @@ function isSuspicious(entry: GoldenEntry): boolean {
 
 // ── IO ─────────────────────────────────────────────────────────────────────
 
-/** Read a JSONL, tolerating a missing file (returns `[]`) — prior golden dirs may lack a bucket. */
+/**
+ * Read a JSONL, tolerating a missing file (returns `[]`) — prior golden dirs may lack a bucket.
+ */
 function readJSONLIfPresent<T>(path: string): T[] {
 	if (!existsSync(path)) return []
 
@@ -169,6 +198,7 @@ export async function promoteGolden(
 			for (const e of entries) {
 				seenNormalized.add(normalize(e.raw))
 			}
+
 			report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`)
 		}
 	} else {
@@ -182,6 +212,7 @@ export async function promoteGolden(
 		kept: 0,
 		perCountry: {},
 	}
+
 	const accepted: GoldenEntry[] = []
 	const seenInBatch = new Set<string>()
 
@@ -191,34 +222,40 @@ export async function promoteGolden(
 		// Dedup pass 1: against prior versioned golden
 		if (seenNormalized.has(norm)) {
 			stats.filteredOut.forwardDup++
+
 			continue
 		}
 
 		// Dedup pass 2: against this batch
 		if (seenInBatch.has(norm)) {
 			stats.filteredOut.duplicate++
+
 			continue
 		}
 
 		if (applyFilters) {
 			if (isComponentsGlued(cand)) {
 				stats.filteredOut.glued++
+
 				continue
 			}
 
 			if (isPostcodeBadlyLeading(cand)) {
 				stats.filteredOut.postcodeLeading++
+
 				continue
 			}
 
 			if (isSuspicious(cand)) {
 				stats.filteredOut.suspicious++
+
 				continue
 			}
 		}
 
 		accepted.push(cand)
 		seenInBatch.add(norm)
+
 		stats.kept++
 		const country = cand.country || "OTHER"
 		stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1
@@ -238,6 +275,7 @@ export async function promoteGolden(
 		if (!buckets.has(key)) {
 			buckets.set(key, [])
 		}
+
 		buckets.get(key)!.push(cand)
 	}
 
@@ -249,6 +287,7 @@ export async function promoteGolden(
 	for (const [key, entries] of buckets) {
 		report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`)
 	}
+
 	report?.(`=== filter stats ===`)
 	report?.(`candidates in:        ${stats.candidatesIn}`)
 	report?.(`  filtered (glued):           ${stats.filteredOut.glued}`)
@@ -259,7 +298,7 @@ export async function promoteGolden(
 	report?.(`  kept:                       ${stats.kept}`)
 	report?.(`per-country (kept):`)
 
-	for (const [c, n] of Object.entries(stats.perCountry).sort((a, b) => b[1] - a[1])) {
+	for (const [c, n] of Object.entries(stats.perCountry).toSorted((a, b) => b[1] - a[1])) {
 		report?.(`  ${c}: ${n}`)
 	}
 
@@ -270,6 +309,7 @@ export async function promoteGolden(
 	}
 
 	mkdirSync(outDir, { recursive: true })
+
 	const manifest: {
 		promoted_at: string
 		from: string

package/src/tools/ingest-csv.ts

@@ -31,9 +31,7 @@ import type { SQLInputValue } from "node:sqlite"
 
 import { TextSpliterator } from "spliterator"
 
-// ---------------------------------------------------------------------------
-// Core: quote-aware CSV field splitting
-// ---------------------------------------------------------------------------
+//#region Core: quote-aware CSV field splitting
 
 const COMMA = 44
 const DOUBLE_QUOTE = 34
@@ -59,6 +57,7 @@ function splitCSVLine(line: string, separator: number = COMMA): string[] {
 			start = i + 1
 		}
 	}
+
 	fields.push(line.slice(start))
 
 	return fields
@@ -68,24 +67,24 @@ function stripQuotes(field: string): string {
 	const trimmed = field.trim()
 
 	if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
-		return trimmed.slice(1, -1).replace(/""/g, '"')
+		return trimmed.slice(1, -1).replaceAll('""', '"')
 	}
 
 	return trimmed
 }
 
-// ---------------------------------------------------------------------------
-// Column name normalization
-// ---------------------------------------------------------------------------
+//#endregion
+
+//#region Column name normalization
 
 function normalizeColumnName(raw: string): string {
 	return (
 		raw
 			.trim()
-			.replace(/^"+|"+$/g, "")
+			.replaceAll(/^"+|"+$/g, "")
 			.toLowerCase()
-			.replace(/[^a-z0-9]+/g, "_")
-			.replace(/^_|_$/g, "") || "unnamed"
+			.replaceAll(/[^a-z0-9]+/g, "_")
+			.replaceAll(/^_|_$/g, "") || "unnamed"
 	)
 }
 
@@ -100,9 +99,9 @@ function dedupColumns(names: string[]): string[] {
 	})
 }
 
-// ---------------------------------------------------------------------------
-// Type inference
-// ---------------------------------------------------------------------------
+//#endregion
+
+//#region Type inference
 
 type SQLiteColType = "INTEGER" | "REAL" | "TEXT"
 
@@ -116,7 +115,7 @@ function normalizeField(raw: string): string | null {
 	let s = raw.trim()
 
 	if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
-		s = s.slice(1, -1).replace(/""/g, '"')
+		s = s.slice(1, -1).replaceAll('""', '"')
 	}
 
 	// Normalize common null-like values
@@ -136,6 +135,7 @@ function inferColumnType(samples: (string | null)[]): ColumnInfo {
 	for (const s of samples) {
 		if (s === null) {
 			nullCount++
+
 			continue
 		}
 
@@ -154,9 +154,9 @@ function inferColumnType(samples: (string | null)[]): ColumnInfo {
 	return { name: "", type, nullable: nullCount / total >= 0.5 }
 }
 
-// ---------------------------------------------------------------------------
-// Main: read CSV, infer schema, produce SQL
-// ---------------------------------------------------------------------------
+//#endregion
+
+//#region Main: read CSV, infer schema, produce SQL
 
 interface IngestOptions {
 	inputPath: string
@@ -192,6 +192,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 		if (!headerLine && opts.hasHeader) {
 			headerLine = line
+
 			continue
 		}
 
@@ -226,6 +227,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 			return normalizeField(raw)
 		})
+
 		const info = inferColumnType(samples)
 		info.name = name
 
@@ -263,6 +265,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	// Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
 	// NULL normalization and type coercion.
 	const csvBasename = basename(opts.inputPath)
+
 	const importSQL = [
 		`CREATE TEMP TABLE "${opts.tableName}_source" (${colDefs});`,
 		`.mode csv`,
@@ -291,15 +294,16 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 			for (const row of batch) {
 				insertStmt.run(...row)
 			}
+
 			db.exec("COMMIT")
-		} catch (err) {
+		} catch (error) {
 			db.exec("ROLLBACK")
-			throw err
+			throw error
 		}
 	}
 
 	const batch: SQLInputValue[][] = []
-	const BATCH_SIZE = 10000
+	const BATCH_SIZE = 10_000
 
 	for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
 		lineNum++
@@ -308,19 +312,21 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 		if (opts.hasHeader && !headerSkipped) {
 			headerSkipped = true
+
 			continue
 		}
 
 		const fields = splitCSVLine(line, sep).map(stripQuotes)
+
 		const values = fields.map((f, i) => {
 			const v = normalizeField(f)
 
 			if (v === null) return null
 			const col = columns[i]
 
-			if (col?.type === "INTEGER" && /^-?\d+$/.test(v)) return parseInt(v, 10)
+			if (col?.type === "INTEGER" && /^-?\d+$/.test(v)) return Number.parseInt(v, 10)
 
-			if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v)) return parseFloat(v)
+			if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v)) return Number.parseFloat(v)
 
 			return v
 		})
@@ -329,6 +335,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		while (values.length < columns.length) {
 			values.push(null)
 		}
+
 		values.length = columns.length
 
 		batch.push(values)
@@ -338,14 +345,14 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 			imported += batch.length
 			batch.length = 0
 
-			if (imported % 100000 === 0) {
+			if (imported % 100_000 === 0) {
 				process.stderr.write(`  ${(imported / 1_000_000).toFixed(1)}M rows...\n`)
 			}
 		}
 	}
 
 	// Flush remaining
-	if (batch.length > 0) {
+	if (batch.length) {
 		doInsert()
 		imported += batch.length
 	}
@@ -357,6 +364,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 	if (firstTextCol) {
 		process.stderr.write(`Building index on "${firstTextCol.name}"...\n`)
+
 		db.exec(
 			`CREATE INDEX IF NOT EXISTS idx_${opts.tableName}_${firstTextCol.name} ON "${opts.tableName}"("${firstTextCol.name}");`
 		)
@@ -367,6 +375,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	// Write MANIFEST
 	const fileSize = (await import("node:fs/promises")).stat
 	const stat = await (await import("node:fs/promises")).stat(opts.outputPath)
+
 	const manifest = {
 		ingested_at: new Date().toISOString(),
 		source_csv: basename(opts.inputPath),
@@ -375,6 +384,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		row_count: imported,
 		db_bytes: stat.size,
 	}
+
 	const manifestPath = opts.outputPath.replace(/\.db$/, ".manifest.json")
 	writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n")
 
@@ -383,7 +393,9 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	)
 }
 
-/** Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted. */
+/**
+ * Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted.
+ */
 export interface IngestCSVOptions {
 	input: string
 	table?: string
@@ -405,11 +417,12 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	if (!existsSync(options.input)) {
 		throw new Error(`File not found: ${options.input}`)
 	}
+
 	const csvName = basename(options.input, extname(options.input))
 
 	await runIngest({
 		inputPath: options.input,
-		tableName: options.table ?? csvName.replace(/[^a-zA-Z0-9_]/g, "_"),
+		tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
 		outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
 		sampleSize: options.sample ?? 100,
 		separator: options.separator ?? ",",
@@ -418,3 +431,5 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 		dryRun: options.dryRun ?? false,
 	})
 }
+
+//#endregion

package/src/tools/jsonl-to-parquet.ts

@@ -82,17 +82,27 @@ const COLUMN_TYPES: Record<(typeof REQUIRED_COLUMNS)[number], string> = {
 	synth_base_id: "VARCHAR",
 }
 
-/** Options for {@linkcode jsonlToParquet}. */
+/**
+ * Options for {@linkcode jsonlToParquet}.
+ */
 export interface JSONLToParquetOptions {
-	/** The labeled-row JSONL to convert. */
+	/**
+	 * The labeled-row JSONL to convert.
+	 */
 	input: string
-	/** The parquet shard to write. */
+	/**
+	 * The parquet shard to write.
+	 */
 	output: string
-	/** Parquet row-group size. Default 50000. */
+	/**
+	 * Parquet row-group size. Default 50000.
+	 */
 	rowGroupSize?: number
 }
 
-/** Summary returned by {@linkcode jsonlToParquet}. */
+/**
+ * Summary returned by {@linkcode jsonlToParquet}.
+ */
 export interface JSONLToParquetSummary {
 	read: number
 	written: number
@@ -115,6 +125,7 @@ function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
 				"must carry span_starts/span_ends/span_tags; re-emit this shard through alignRow."
 		)
 	}
+
 	const n = (row.span_starts as unknown[]).length
 
 	if ((row.span_ends as unknown[]).length !== n || (row.span_tags as unknown[]).length !== n) {
@@ -126,17 +137,21 @@ function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
 	}
 }
 
-/** Escape a path for single-quoted SQL string literals. */
+/**
+ * Escape a path for single-quoted SQL string literals.
+ */
 function sqlString(value: string): string {
-	return value.replace(/'/g, "''")
+	return value.replaceAll("'", "''")
 }
 
-/** Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard. */
+/**
+ * Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
+ */
 export async function jsonlToParquet(
 	options: JSONLToParquetOptions,
 	report?: (line: string) => void
 ): Promise<JSONLToParquetSummary> {
-	const rowGroupSize = options.rowGroupSize ?? 50000
+	const rowGroupSize = options.rowGroupSize ?? 50_000
 
 	if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
 		throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`)
@@ -166,9 +181,13 @@ export async function jsonlToParquet(
 			// Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
 			// map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
 			stage.write(line + "\n")
+
 			rows++
 		}
-		await new Promise<void>((resolve, reject) => stage.end((err?: Error | null) => (err ? reject(err) : resolve())))
+
+		await new Promise<void>((resolve, reject) => {
+			stage.end((err?: Error | null) => (err ? reject(err) : resolve()))
+		})
 
 		report?.(`Read ${rows} rows from ${options.input}`)
 
@@ -182,6 +201,7 @@ export async function jsonlToParquet(
 		// Row order is load-bearing: the overlay-manifest assembler records first/last source_id from
 		// shard order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
 		await db.run("SET preserve_insertion_order=true")
+
 		await db.run(
 			`COPY (SELECT ${selectList} FROM read_json('${sqlString(stagePath)}', ` +
 				`columns = ${columnsLiteral}, format = 'newline_delimited')) ` +