@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/tools/ingest-csv.ts

@@ -3,12 +3,13 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ * CSV → SQLite ingest tool.
  *
- *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
- *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ * The tool reads a CSV and samples rows to infer column types.
+ * It creates a SQLite table and imports all rows.
+ * It handles quoted fields, null-like values and duplicate column names.
  *
- *   ## Usage
+ * ## Usage
  *
  *   ```sh
  *   mailwoman corpus ingest-csv \
@@ -17,12 +18,15 @@
  *   --output /data/corpus/sources/usgov-nppes/nppes.db
  * ```
  *
- *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
- *   derived from input filename) --output <path> SQLite database path (default: input dir /
- *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
- *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
- *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
- *   CREATE TABLE, but don't import
+ * Options:
+ * --input <path> Required CSV path
+ * --table <name> Table name (default: from filename)
+ * --output <path> DB path (default: input dir/<table>.db)
+ * --sample <n> Rows used for type inference (default: 100)
+ * --separator <char> Field delimiter (default: ,)
+ * --skip <n> Lines to skip before header (default: 0)
+ * --no-header Treat CSV as headerless (col_0, col_1, ...)
+ * --dry-run Print inferred schema, skip import
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
@@ -30,13 +34,14 @@ import { tryStat, pathExists } from "@mailwoman/core/fs/readers"
 import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import type { SQLInputValue } from "@mailwoman/sqlite/client"
 import type { Database } from "@mailwoman/sqlite/database-schema"
-import { basename, dirname, extname, join } from "path-ts"
+import { basename, dirname, extname, PathBuilder } from "path-ts"
 import { CSVSpliterator } from "spliterator"
 
-//#region Column name normalization
+// #region Column name normalization
 
 /**
- * @deprecated belongs in core, if really needed. spliterator should handle this.
+ * @deprecated This helper may belong in core.
+ * Keep it here until then.
  */
 function normalizeColumnName(raw: string): string {
 	return (
@@ -60,9 +65,9 @@ function dedupColumns(names: string[]): string[] {
 	})
 }
 
-//#endregion
+// #endregion
 
-//#region Type inference
+// #region Type inference
 
 type SQLiteColType = "INTEGER" | "REAL" | "TEXT"
 
@@ -115,9 +120,9 @@ function inferColumnType(samples: (string | null)[]): ColumnInfo {
 	return { name: "", type, nullable: nullCount / total >= 0.5 }
 }
 
-//#endregion
+// #endregion
 
-//#region Main: read CSV, infer schema, produce SQL
+// #region Main: read CSV, infer schema, produce SQL
 
 interface IngestOptions {
 	inputPath: string
@@ -131,11 +136,8 @@ interface IngestOptions {
 }
 
 async function runIngest(opts: IngestOptions): Promise<void> {
-	// `header: false` with `drop` is what expresses `--skip N`: the spliterator's own header handling consumes the
-	// FIRST row as the header, and `drop` counts from the row after it, so a preamble before the header has to be
-	// dropped here and the header row taken by hand. Quote handling is end-to-end (quoted delimiters, doubled quotes);
-	// `skipEmpty` drops blank lines that readline would have turned into all-null rows. The early `break` closes the
-	// file descriptor.
+	// Keep parser header handling off so `--skip` works predictably.
+	// We read the header row ourselves.
 	const rows = (): AsyncIterable<string[]> =>
 		CSVSpliterator.fromAsync<string[]>(opts.inputPath, {
 			header: false,
@@ -143,7 +145,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 			drop: opts.skipLines,
 		})
 
-	// Read the header and sample rows before inferring the SQLite schema.
+	// Read header + sample rows to infer schema.
 	process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`)
 
 	let headerRow: string[] | null = null
@@ -167,20 +169,20 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		throw new Error("No header line found in CSV")
 	}
 
-	// Normalize and deduplicate the input column names.
+	// Normalize and dedupe column names.
 	let rawHeaders: string[]
 
 	if (opts.hasHeader && headerRow) {
 		rawHeaders = headerRow
 	} else {
-		// Auto-generate column names: col_0, col_1, ...
+		// No header: generate col_0, col_1, ...
 		const numCols = sampleRows[0]?.length ?? 0
 		rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`)
 	}
 
 	const colNames = dedupColumns(rawHeaders.map(normalizeColumnName))
 
-	// Infer each column's SQLite type from its sample values.
+	// Infer each column type from samples.
 	const columns: ColumnInfo[] = colNames.map((name, i) => {
 		const samples = sampleRows.map((row) => {
 			const raw = row[i] ?? ""
@@ -194,10 +196,9 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		return info
 	})
 
-	// Generate the table definition and insert statement from the inferred schema.
+	// Build CREATE TABLE and INSERT SQL.
 	const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ")
-	// Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
-	// so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+	// Raw SQL is intentional: columns and types are dynamic at runtime.
 	const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`
 
 	const tempCols = columns.map((c) => `"${c.name}"`).join(", ")
@@ -211,23 +212,18 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		return
 	}
 
-	// Create the database, then import every input row.
+	// Create database and import rows.
 	const { DatabaseClient } = await import("@mailwoman/sqlite/client")
 	await makeDirectories(dirname(opts.outputPath))
 
-	// `Database` — the EMPTY schema — deliberately, not by default: `createTableSQL` is built from the columns and
-	// types inferred from the CSV at runtime, so there is no table this file could name at compile time. Every write
-	// below goes through `exec`/`prepare` for the same reason.
+	// Empty schema type is intentional: table shape is runtime-generated.
 	using db = new DatabaseClient<Database>(opts.outputPath)
 	db.exec("PRAGMA journal_mode = OFF") // faster for bulk import
 	db.exec("PRAGMA synchronous = OFF")
 
 	db.exec(createTableSQL)
 
-	// Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
-	// NULL normalization and type coercion.
-	// better-sqlite3 doesn't support .import natively, so we use a different approach:
-	// Read the CSV line-by-line and INSERT in a transaction.
+	// Insert line-by-line in batches (no native .import here).
 	process.stderr.write(`Importing rows...\n`)
 
 	const insertStmt = db.prepare(
@@ -237,7 +233,8 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	let imported = 0
 	let headerSkipped = false
 
-	// node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+	// node:sqlite has no transaction helper.
+	// The code uses BEGIN/COMMIT.
 	const doInsert = () => {
 		db.exec("BEGIN")
 
@@ -276,7 +273,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 			return v
 		})
 
-		// Pad or truncate to column count
+		// Pad or trim to expected column count.
 		while (values.length < columns.length) {
 			values.push(null)
 		}
@@ -296,7 +293,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		}
 	}
 
-	// Flush remaining
+	// Flush remaining rows.
 	if (batch.length) {
 		doInsert()
 		imported += batch.length
@@ -304,7 +301,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 	process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`)
 
-	// Build a basic index on the first TEXT column (likely the primary key)
+	// Create a simple index on the first text column.
 	const firstTextCol = columns.find((c) => c.type === "TEXT")
 
 	if (firstTextCol) {
@@ -335,7 +332,8 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 }
 
 /**
- * Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted.
+ * Options for {@linkcode ingestCSV}.
+ * `table` and `output` default from `input`.
  */
 export interface IngestCSVOptions {
 	input: string
@@ -349,10 +347,9 @@ export interface IngestCSVOptions {
 }
 
 /**
- * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
- * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
- * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
- * it.
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows.
+ *
+ * @throws When `input` does not exist.
  */
 export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	if (!(await pathExists(options.input))) {
@@ -364,7 +361,7 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	await runIngest({
 		inputPath: options.input,
 		tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-		outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+		outputPath: options.output ?? PathBuilder.from(options.input).dirname()(`${csvName}.db`).toString(),
 		sampleSize: options.sample ?? 100,
 		separator: options.separator ?? ",",
 		skipLines: options.skip ?? 0,
@@ -373,4 +370,4 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	})
 }
 
-//#endregion
+// #endregion

package/lib/tools/jsonl-to-parquet.ts

@@ -3,99 +3,59 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ * Convert a jsonl of LabeledRow objects to a Parquet file matching the schema the corpus writes.
  *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
- *   rides alongside.
+ * DuckDB reproduces the PyArrow file's logical schema and column order. A PyArrow reader therefore sees an identical table.
+ * The trainer reads Parquet by column name, so physical layout does not affect it.
  *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
+ * Schema: `PARQUET_COLUMNS` from `#parquet/schema`, the same list the native writer uses.
  *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
+ * Every row must contain the span triple. A row without it came from a producer that has not migrated.
+ * The writer would silently drop the character-offset labels if it wrote that row. Fail loudly and report the row number.
  */
 
+import { delimitedSource } from "@mailwoman/core/fs/delimited"
 import { openWriteStream } from "@mailwoman/core/fs/streams"
 import { temporaryDirectory } from "@mailwoman/core/fs/temporary"
-import { parseJSONStrict } from "@mailwoman/core/json"
-import { join } from "path-ts"
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
 import { TextSpliterator } from "spliterator"
 
-import { connectDuckDB, escapeSQLString } from "#utils/parquet"
-
-const REQUIRED_COLUMNS = [
-	"raw",
-	"tokens",
-	"labels",
-	"span_starts",
-	"span_ends",
-	"span_tags",
-	"country",
-	"locale",
-	"source",
-	"source_id",
-	"corpus_version",
-	"license",
-	"synth_method",
-	"synth_base_id",
-] as const
-
-const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"] as const
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
+import { PARQUET_COLUMNS, PARQUET_COLUMN_TYPES } from "#parquet/schema"
 
 /**
- * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ * The columns this converter writes and the DuckDB type for each column.
+ *
+ * Both come from `#parquet/schema`, the one definition the native writer,
+ * the reader and the manifest share.
  *
- * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
- * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
- * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
- * round-trips as a plain integer where INT64 would surface as bigint.
+ * The span offsets are INT32: parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping).
+ * `raw` is a short address string, so INT32 round-trips as a plain integer
+ * where INT64 would surface as bigint.
  */
-const COLUMN_TYPES: Record<(typeof REQUIRED_COLUMNS)[number], string> = {
-	raw: "VARCHAR",
-	tokens: "VARCHAR[]",
-	labels: "VARCHAR[]",
-	span_starts: "INTEGER[]",
-	span_ends: "INTEGER[]",
-	span_tags: "VARCHAR[]",
-	country: "VARCHAR",
-	locale: "VARCHAR",
-	source: "VARCHAR",
-	source_id: "VARCHAR",
-	corpus_version: "VARCHAR",
-	license: "VARCHAR",
-	synth_method: "VARCHAR",
-	synth_base_id: "VARCHAR",
-}
+const REQUIRED_COLUMNS = PARQUET_COLUMNS
+const COLUMN_TYPES = PARQUET_COLUMN_TYPES
+
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"] as const
 
 /**
  * Options for {@linkcode jsonlToParquet}.
  */
 export interface JSONLToParquetOptions {
 	/**
-	 * The labeled-row JSONL to convert.
+	 * The labeled-row jsonl to convert.
 	 */
-	input: string
+	input: PathBuilderLike
 	/**
-	 * The parquet slice to write.
+	 * The parquet file to write.
 	 */
-	output: string
+	output: PathBuilderLike
 	/**
-	 * Parquet row-group size. Default 50000.
+	 * Parquet row-group size.
+	 *
+	 * Default 50000.
 	 */
 	rowGroupSize?: number
 }
@@ -110,9 +70,9 @@ export interface JSONLToParquetSummary {
 }
 
 /**
- * Enforce the #519 span contract per row: all three present, parallel lengths.
+ * Enforce the span interface per row: all three present, parallel lengths.
  *
- * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ * A row with span_starts but no span_tags is a corrupt row, never a silent fallback.
  */
 function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
 	const present = SPAN_COLUMNS.filter((c) => row[c] != null)
@@ -121,8 +81,8 @@ function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
 		const missing = SPAN_COLUMNS.filter((c) => row[c] == null)
 		throw new Error(
 			`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
-				`missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
-				"must carry span_starts/span_ends/span_tags; re-emit this slice through alignRow."
+				`missing ${stringifyJSON(missing)} (source_id=${stringifyJSON(row.source_id ?? null)}). Every parquet-bound row ` +
+				"must carry span_starts/span_ends/span_tags; re-emit this file through alignRow."
 		)
 	}
 
@@ -132,13 +92,13 @@ function assertSpanTriple(row: Record<string, unknown>, lineNo: number): void {
 		throw new Error(
 			`line ${lineNo}: span triple arrays are not parallel — ` +
 				`starts=${(row.span_starts as unknown[]).length} ends=${(row.span_ends as unknown[]).length} ` +
-				`tags=${(row.span_tags as unknown[]).length} (source_id=${JSON.stringify(row.source_id ?? null)})`
+				`tags=${(row.span_tags as unknown[]).length} (source_id=${stringifyJSON(row.source_id ?? null)})`
 		)
 	}
 }
 
 /**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ * Convert a labeled-row jsonl to a v0.5.0-schema Parquet file.
  */
 export async function jsonlToParquet(
 	options: JSONLToParquetOptions,
@@ -147,33 +107,32 @@ export async function jsonlToParquet(
 	const rowGroupSize = options.rowGroupSize ?? 50_000
 
 	if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
-		throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`)
+		throw new Error(`rowGroupSize must be a positive integer (got ${stringifyJSON(rowGroupSize)})`)
 	}
 
-	// Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
-	// memory O(1) on the Node side (the Python original buffered every column into memory first). The
-	// staging directory owns the write stream, so it is closed before the directory is removed — and a
-	// mid-stream span-triple failure leaves no orphan.
+	// The Node-side writer uses O(1) memory by streaming rows.
+	// The staging directory owns the write stream, so it is closed before the directory is removed.
+	// A mid-stream span-triple failure leaves no orphan.
 	await using staging = await temporaryDirectory("mw-jsonl-to-parquet-")
-	const stagePath = join(staging.path, "rows.ndjson")
+	const stagePath = staging.path("rows.ndjson")
 	const stage = staging.use(openWriteStream(stagePath, { encoding: "utf8" }))
 
 	let rows = 0
 	let lineNo = 0
 
-	// TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
-	// DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
-	// defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
-	// same as readline's crlfDelay:Infinity did.
-	for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+	// TextSpliterator rather than JSONSpliterator: the staging write below streams
+	// the RAW line bytes to DuckDB verbatim (the parse here only validates),
+	// so a re-serialized JSONSpliterator row would defeat the point.
+	// The reader trims each line (a trailing \r included); `skipEmpty: false` keeps
+	// blank lines so `lineNo` matches the file.
+	for await (const line of TextSpliterator.fromAsync(delimitedSource(options.input), { skipEmpty: false })) {
 		lineNo++
-		const line = rawLine.trim()
 
 		if (!line) continue
 		const row = parseJSONStrict<Record<string, unknown>>(line)
 		assertSpanTriple(row, lineNo)
-		// Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
-		// map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+		// DuckDB's `read_json` projects to the explicit `columns` map below
+		// (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
 		stage.write(line + "\n")
 
 		rows++
@@ -188,20 +147,22 @@ export async function jsonlToParquet(
 	const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}"
 	const selectList = REQUIRED_COLUMNS.join(", ")
 
-	const db = await connectDuckDB()
-	// Row order is required: the overlay-manifest assembler records first/last source_id from
-	// slice order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+	const output = options.output.toString()
+	using db = await openDuckDB()
+
+	// Row order is required: the overlay-manifest assembler records first/last source_id from file order.
+	// `preserve_insertion_order` (DuckDB default) keeps output order = input order.
 	await db.run("SET preserve_insertion_order=true")
 
 	await db.run(
-		`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+		`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath.toString())}', ` +
 			`columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-			`TO '${escapeSQLString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`
+			`TO '${escapeSQLString(output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`
 	)
 
-	const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(options.output)}')`)
+	const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(output)}')`)
 	const written = Number(counted.getRowObjects()[0]!.n)
-	report?.(`Wrote ${written} rows to ${options.output}`)
+	report?.(`Wrote ${written} rows to ${output}`)
 
-	return { read: rows, written, outPath: options.output }
+	return { read: rows, written, outPath: output }
 }