@mailwoman/corpus 9.2.0 → 9.4.0

package/lib/tools/index.ts

@@ -0,0 +1,28 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus operator tools — the `run()`-style modules behind `mailwoman corpus …` commands. No argv,
+ *   no `process.exit`: commands own parsing, rendering, and exit codes.
+ */
+
+export * from "#tools/align-slice"
+export * from "#tools/audit"
+export * from "#tools/corpus-stats"
+export * from "#tools/ingest-csv"
+export * from "#tools/overlay-manifest"
+export * from "#tools/fetch/download"
+export * from "#tools/fetch/index"
+export * from "#tools/golden-expand"
+export * from "#tools/golden-promote"
+export * from "#tools/golden-relabel-street"
+export * from "#tools/jsonl-to-parquet"
+export * from "#tools/lint-slice"
+export * from "#tools/lint-slice-vocab"
+export * from "#tools/slice-kryptonite"
+export * from "#tools/overture-subvenue"
+export * from "#tools/postcode-triples"
+export * from "#tools/sub-venue-lexicon"
+export * from "#tools/sub-venue-promotions"
+export * from "#tools/slice-translit"

package/lib/tools/ingest-csv.ts

@@ -0,0 +1,378 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ *
+ *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
+ *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus ingest-csv \
+ *   --input /data/corpus/sources/usgov-nppes/npidata_pfile.csv \
+ *   --table nppes_providers \
+ *   --output /data/corpus/sources/usgov-nppes/nppes.db
+ * ```
+ *
+ *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
+ *   derived from input filename) --output <path> SQLite database path (default: input dir /
+ *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
+ *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
+ *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
+ *   CREATE TABLE, but don't import
+ */
+
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat, pathExists } from "@mailwoman/core/fs/readers"
+import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import type { SQLInputValue } from "@mailwoman/sqlite/client"
+import type { Database } from "@mailwoman/sqlite/database-schema"
+import { basename, dirname, extname, join } from "path-ts"
+import { CSVSpliterator } from "spliterator"
+
+//#region Column name normalization
+
+/**
+ * @deprecated belongs in core, if really needed. spliterator should handle this.
+ */
+function normalizeColumnName(raw: string): string {
+	return (
+		raw
+			.trim()
+			.replaceAll(/^"+|"+$/g, "")
+			.toLowerCase()
+			.replaceAll(/[^a-z0-9]+/g, "_")
+			.replaceAll(/^_|_$/g, "") || "unnamed"
+	)
+}
+
+function dedupColumns(names: string[]): string[] {
+	const seen = new Map<string, number>()
+
+	return names.map((name) => {
+		const count = seen.get(name) ?? 0
+		seen.set(name, count + 1)
+
+		return count === 0 ? name : `${name}_${count + 1}`
+	})
+}
+
+//#endregion
+
+//#region Type inference
+
+type SQLiteColType = "INTEGER" | "REAL" | "TEXT"
+
+interface ColumnInfo {
+	name: string
+	type: SQLiteColType
+	nullable: boolean
+}
+
+function normalizeField(raw: string): string | null {
+	let s = raw.trim()
+
+	if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
+		s = s.slice(1, -1).replaceAll('""', '"')
+	}
+
+	// Normalize common null-like values
+	if (!s || s === "null" || s === "NULL" || s === "N/A" || s === "n/a" || s === "-" || s === "<UNAVAIL>") {
+		return null
+	}
+
+	return s
+}
+
+function inferColumnType(samples: (string | null)[]): ColumnInfo {
+	let nullCount = 0
+	let intCount = 0
+	let realCount = 0
+	let textCount = 0
+
+	for (const s of samples) {
+		if (s === null) {
+			nullCount++
+
+			continue
+		}
+
+		if (/^-?\d+$/.test(s)) {
+			intCount++
+		} else if (/^-?\d+\.?\d+$/.test(s)) {
+			realCount++
+		} else {
+			textCount++
+		}
+	}
+
+	const total = samples.length
+	const type: SQLiteColType = realCount / total >= 0.5 ? "REAL" : intCount / total >= 0.5 ? "INTEGER" : "TEXT"
+
+	return { name: "", type, nullable: nullCount / total >= 0.5 }
+}
+
+//#endregion
+
+//#region Main: read CSV, infer schema, produce SQL
+
+interface IngestOptions {
+	inputPath: string
+	tableName: string
+	outputPath: string
+	sampleSize: number
+	separator: string
+	skipLines: number
+	hasHeader: boolean
+	dryRun: boolean
+}
+
+async function runIngest(opts: IngestOptions): Promise<void> {
+	// `header: false` with `drop` is what expresses `--skip N`: the spliterator's own header handling consumes the
+	// FIRST row as the header, and `drop` counts from the row after it, so a preamble before the header has to be
+	// dropped here and the header row taken by hand. Quote handling is end-to-end (quoted delimiters, doubled quotes);
+	// `skipEmpty` drops blank lines that readline would have turned into all-null rows. The early `break` closes the
+	// file descriptor.
+	const rows = (): AsyncIterable<string[]> =>
+		CSVSpliterator.fromAsync<string[]>(opts.inputPath, {
+			mode: "array",
+			header: false,
+			columnDelimiter: opts.separator,
+			drop: opts.skipLines,
+			enableQuoteHandling: true,
+		})
+
+	// --- Pass 1: read header + sample rows for type inference ---
+	process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`)
+
+	let headerRow: string[] | null = null
+	const sampleRows: string[][] = []
+
+	for await (const row of rows()) {
+		if (!headerRow && opts.hasHeader) {
+			headerRow = row
+
+			continue
+		}
+
+		if (sampleRows.length < opts.sampleSize) {
+			sampleRows.push(row)
+		} else {
+			break
+		}
+	}
+
+	if (!headerRow && opts.hasHeader) {
+		throw new Error("No header line found in CSV")
+	}
+
+	// --- Determine column names ---
+	let rawHeaders: string[]
+
+	if (opts.hasHeader && headerRow) {
+		rawHeaders = headerRow
+	} else {
+		// Auto-generate column names: col_0, col_1, ...
+		const numCols = sampleRows[0]?.length ?? 0
+		rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`)
+	}
+
+	const colNames = dedupColumns(rawHeaders.map(normalizeColumnName))
+
+	// --- Infer column types ---
+	const columns: ColumnInfo[] = colNames.map((name, i) => {
+		const samples = sampleRows.map((row) => {
+			const raw = row[i] ?? ""
+
+			return normalizeField(raw)
+		})
+
+		const info = inferColumnType(samples)
+		info.name = name
+
+		return info
+	})
+
+	// --- Generate SQL ---
+	const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ")
+	// Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
+	// so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+	const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`
+
+	const tempCols = columns.map((c) => `"${c.name}"`).join(", ")
+
+	process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`)
+	process.stderr.write(`${createTableSQL}\n\n`)
+
+	if (opts.dryRun) {
+		process.stderr.write("--dry-run: stopping before import\n")
+
+		return
+	}
+
+	// --- Create database + import ---
+	const { DatabaseClient } = await import("@mailwoman/sqlite/client")
+	await makeDirectories(dirname(opts.outputPath))
+
+	// `Database` — the EMPTY schema — deliberately, not by default: `createTableSQL` is built from the columns and
+	// types inferred from the CSV at runtime, so there is no table this file could name at compile time. Every write
+	// below goes through `exec`/`prepare` for the same reason.
+	using db = new DatabaseClient<Database>(opts.outputPath)
+	db.exec("PRAGMA journal_mode = OFF") // faster for bulk import
+	db.exec("PRAGMA synchronous = OFF")
+
+	db.exec(createTableSQL)
+
+	// Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
+	// NULL normalization and type coercion.
+	// better-sqlite3 doesn't support .import natively, so we use a different approach:
+	// Read the CSV line-by-line and INSERT in a transaction.
+	process.stderr.write(`Importing rows...\n`)
+
+	const insertStmt = db.prepare(
+		`INSERT INTO "${opts.tableName}" (${tempCols}) VALUES (${columns.map(() => "?").join(", ")})`
+	)
+
+	let imported = 0
+	let headerSkipped = false
+
+	// node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+	const doInsert = () => {
+		db.exec("BEGIN")
+
+		try {
+			for (const row of batch) {
+				insertStmt.run(...row)
+			}
+
+			db.exec("COMMIT")
+		} catch (error) {
+			db.exec("ROLLBACK")
+			throw error
+		}
+	}
+
+	const batch: SQLInputValue[][] = []
+	const BATCH_SIZE = 10_000
+
+	for await (const fields of rows()) {
+		if (opts.hasHeader && !headerSkipped) {
+			headerSkipped = true
+
+			continue
+		}
+
+		const values = fields.map((f, i) => {
+			const v = normalizeField(f)
+
+			if (v === null) return null
+			const col = columns[i]
+
+			if (col?.type === "INTEGER" && /^-?\d+$/.test(v)) return Number.parseInt(v, 10)
+
+			if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v)) return Number.parseFloat(v)
+
+			return v
+		})
+
+		// Pad or truncate to column count
+		while (values.length < columns.length) {
+			values.push(null)
+		}
+
+		values.length = columns.length
+
+		batch.push(values)
+
+		if (batch.length >= BATCH_SIZE) {
+			doInsert()
+			imported += batch.length
+			batch.length = 0
+
+			if (imported % 100_000 === 0) {
+				process.stderr.write(`  ${(imported / 1_000_000).toFixed(1)}M rows...\n`)
+			}
+		}
+	}
+
+	// Flush remaining
+	if (batch.length) {
+		doInsert()
+		imported += batch.length
+	}
+
+	process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`)
+
+	// Build a basic index on the first TEXT column (likely the primary key)
+	const firstTextCol = columns.find((c) => c.type === "TEXT")
+
+	if (firstTextCol) {
+		process.stderr.write(`Building index on "${firstTextCol.name}"...\n`)
+
+		db.exec(
+			`CREATE INDEX IF NOT EXISTS idx_${opts.tableName}_${firstTextCol.name} ON "${opts.tableName}"("${firstTextCol.name}");`
+		)
+	}
+
+	const stat = await tryStat(opts.outputPath)
+
+	const manifest = {
+		ingested_at: new Date().toISOString(),
+		source_csv: basename(opts.inputPath),
+		table_name: opts.tableName,
+		columns: columns.map((c) => ({ name: c.name, type: c.type, nullable: c.nullable })),
+		row_count: imported,
+		db_bytes: stat!.size,
+	}
+
+	const manifestPath = opts.outputPath.replace(/\.db$/, ".manifest.json")
+	await writeLocalJSONFile(manifest, manifestPath)
+
+	process.stderr.write(
+		`Done. ${imported.toLocaleString()} rows → ${opts.outputPath} (${ByteFormatter.formatIEC(stat!.size)})\n`
+	)
+}
+
+/**
+ * Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted.
+ */
+export interface IngestCSVOptions {
+	input: string
+	table?: string
+	output?: string
+	sample?: number
+	separator?: string
+	skip?: number
+	noHeader?: boolean
+	dryRun?: boolean
+}
+
+/**
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
+ * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
+ * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
+ * it.
+ */
+export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
+	if (!(await pathExists(options.input))) {
+		throw new Error(`File not found: ${options.input}`)
+	}
+
+	const csvName = basename(options.input, extname(options.input))
+
+	await runIngest({
+		inputPath: options.input,
+		tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+		outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+		sampleSize: options.sample ?? 100,
+		separator: options.separator ?? ",",
+		skipLines: options.skip ?? 0,
+		hasHeader: !options.noHeader,
+		dryRun: options.dryRun ?? false,
+	})
+}
+
+//#endregion

package/lib/tools/jsonl-to-parquet.ts

@@ -0,0 +1,207 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ *
+ *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
+ *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
+ *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
+ *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
+ *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
+ *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
+ *   Verified field-for-field against the PyArrow original (same column order, same logical types,
+ *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
+ *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
+ *   which is blind to physical layout. INT32 matches the corpus's native TS writer
+ *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
+ *   rides alongside.
+ *
+ *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
+ *   source_id, corpus_version, license, synth_method, synth_base_id.
+ *
+ *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
+ *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
+ *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
+ *   number, instead.
+ *
+ *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
+ *   /tmp/part-po-box.parquet
+ */
+
+import { openWriteStream } from "@mailwoman/core/fs/streams"
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
+import { parseJSONStrict } from "@mailwoman/core/json"
+import { join } from "path-ts"
+import { TextSpliterator } from "spliterator"
+
+import { connectDuckDB, escapeSQLString } from "#utils/parquet"
+
+const REQUIRED_COLUMNS = [
+	"raw",
+	"tokens",
+	"labels",
+	"span_starts",
+	"span_ends",
+	"span_tags",
+	"country",
+	"locale",
+	"source",
+	"source_id",
+	"corpus_version",
+	"license",
+	"synth_method",
+	"synth_base_id",
+] as const
+
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"] as const
+
+/**
+ * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ *
+ * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
+ * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
+ * round-trips as a plain integer where INT64 would surface as bigint.
+ */
+const COLUMN_TYPES: Record<(typeof REQUIRED_COLUMNS)[number], string> = {
+	raw: "VARCHAR",
+	tokens: "VARCHAR[]",
+	labels: "VARCHAR[]",
+	span_starts: "INTEGER[]",
+	span_ends: "INTEGER[]",
+	span_tags: "VARCHAR[]",
+	country: "VARCHAR",
+	locale: "VARCHAR",
+	source: "VARCHAR",
+	source_id: "VARCHAR",
+	corpus_version: "VARCHAR",
+	license: "VARCHAR",
+	synth_method: "VARCHAR",
+	synth_base_id: "VARCHAR",
+}
+
+/**
+ * Options for {@linkcode jsonlToParquet}.
+ */
+export interface JSONLToParquetOptions {
+	/**
+	 * The labeled-row JSONL to convert.
+	 */
+	input: string
+	/**
+	 * The parquet slice to write.
+	 */
+	output: string
+	/**
+	 * Parquet row-group size. Default 50000.
+	 */
+	rowGroupSize?: number
+}
+
+/**
+ * Summary returned by {@linkcode jsonlToParquet}.
+ */
+export interface JSONLToParquetSummary {
+	read: number
+	written: number
+	outPath: string
+}
+
+/**
+ * Enforce the #519 span contract per row: all three present, parallel lengths.
+ *
+ * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ */
+function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
+	const present = SPAN_COLUMNS.filter((c) => row[c] != null)
+
+	if (present.length !== SPAN_COLUMNS.length) {
+		const missing = SPAN_COLUMNS.filter((c) => row[c] == null)
+		throw new Error(
+			`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
+				`missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
+				"must carry span_starts/span_ends/span_tags; re-emit this slice through alignRow."
+		)
+	}
+
+	const n = (row.span_starts as unknown[]).length
+
+	if ((row.span_ends as unknown[]).length !== n || (row.span_tags as unknown[]).length !== n) {
+		throw new Error(
+			`line ${lineNo}: span triple arrays are not parallel — ` +
+				`starts=${(row.span_starts as unknown[]).length} ends=${(row.span_ends as unknown[]).length} ` +
+				`tags=${(row.span_tags as unknown[]).length} (source_id=${JSON.stringify(row.source_id ?? null)})`
+		)
+	}
+}
+
+/**
+ * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ */
+export async function jsonlToParquet(
+	options: JSONLToParquetOptions,
+	report?: (line: string) => void
+): Promise<JSONLToParquetSummary> {
+	const rowGroupSize = options.rowGroupSize ?? 50_000
+
+	if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
+		throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`)
+	}
+
+	// Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
+	// memory O(1) on the Node side (the Python original buffered every column into memory first). The
+	// staging directory owns the write stream, so it is closed before the directory is removed — and a
+	// mid-stream span-triple failure leaves no orphan.
+	await using staging = await temporaryDirectory("mw-jsonl-to-parquet-")
+	const stagePath = join(staging.path, "rows.ndjson")
+	const stage = staging.use(openWriteStream(stagePath, { encoding: "utf8" }))
+
+	let rows = 0
+	let lineNo = 0
+
+	// TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
+	// DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
+	// defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
+	// same as readline's crlfDelay:Infinity did.
+	for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+		lineNo++
+		const line = rawLine.trim()
+
+		if (!line) continue
+		const row = parseJSONStrict<Record<string, unknown>>(line)
+		assertSpanTriple(row, lineNo)
+		// Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
+		// map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+		stage.write(line + "\n")
+
+		rows++
+	}
+
+	await new Promise<void>((resolve, reject) => {
+		stage.end((err?: Error | null) => (err ? reject(err) : resolve()))
+	})
+
+	report?.(`Read ${rows} rows from ${options.input}`)
+
+	const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}"
+	const selectList = REQUIRED_COLUMNS.join(", ")
+
+	const db = await connectDuckDB()
+	// Row order is required: the overlay-manifest assembler records first/last source_id from
+	// slice order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+	await db.run("SET preserve_insertion_order=true")
+
+	await db.run(
+		`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+			`columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
+			`TO '${escapeSQLString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`
+	)
+
+	const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(options.output)}')`)
+	const written = Number(counted.getRowObjects()[0]!.n)
+	report?.(`Wrote ${written} rows to ${options.output}`)
+
+	return { read: rows, written, outPath: options.output }
+}