@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/tools/golden/relabel-street/row.ts

@@ -0,0 +1,283 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Splits the street span of US golden rows into `street_prefix`, `street` and `street_suffix` with the USPS Pub-28 table.
+ */
+
+import {
+	isStreetDirectionalToken,
+	matchTrailingSuffix,
+	type USStreetSuffix,
+	NAME_PRONE_US_SUFFIXES,
+} from "@mailwoman/codex/us"
+import { stringifyJSON } from "@mailwoman/core/json"
+import { escapeRegExp } from "@mailwoman/core/strings/regexp"
+
+/**
+ * The golden row fields that the relabeler reads.
+ * Other fields are copied unchanged.
+ */
+export interface GoldenStreetRow {
+	raw: string
+	components: Record<string, string>
+	country?: string
+	source?: string
+	notes?: string
+	[key: string]: unknown
+}
+
+/**
+ * The relabel outcome class for one row.
+ */
+export type GoldenRelabelClass =
+	| "split-suffix"
+	| "split-suffix-postdirectional"
+	| "split-prefix-only"
+	| "already-split"
+	| "single-token"
+	| "suffix-only-street"
+	| "postdirectional-tail-only"
+	| "no-suffix-match"
+	| "no-street"
+	| "not-us"
+	| "untrimmed-street"
+
+/**
+ * A review note attached to a changed row.
+ * The relabel decision ignores it.
+ */
+export interface GoldenRelabelFlag {
+	kind: "name-prone-suffix" | "venue-context" | "remainder-is-affix"
+	detail: string
+}
+
+/**
+ * The relabel result for one row.
+ */
+export interface GoldenRelabelResult {
+	/**
+	 * The row to write.
+	 * This is the input object itself when no field changed.
+	 */
+	row: GoldenStreetRow
+	changed: boolean
+	rowClass: GoldenRelabelClass
+	flags: GoldenRelabelFlag[]
+	prefixSplit: boolean
+	/**
+	 * The original street span of a changed row, for review.
+	 */
+	beforeStreet?: string
+}
+
+interface TailSplit {
+	head: string
+	gap: string
+	tail: string
+}
+
+/**
+ * Splits `s` at its last whitespace run, or returns `null` for an empty, untrimmed, or single-token string.
+ */
+function splitLastWord(s: string): TailSplit | null {
+	if (s !== s.trim() || !s) return null
+	const match = /^(.*\S)(\s+)(\S+)$/.exec(s)
+
+	if (!match) return null
+
+	return { head: match[1]!, gap: match[2]!, tail: match[3]! }
+}
+
+/**
+ * Splits `s` at its first whitespace run, with the same `null` cases as {@link splitLastWord}.
+ */
+function splitFirstWord(s: string): TailSplit | null {
+	if (s !== s.trim() || !s) return null
+	const match = /^(\S+)(\s+)(\S.*)$/.exec(s)
+
+	if (!match) return null
+
+	return { head: match[1]!, gap: match[2]!, tail: match[3]! }
+}
+
+/**
+ * Rebuilds `components` with the street spans in place of `street`, so the keys stay in address order.
+ */
+function withStreetSpans(
+	components: Record<string, string>,
+	spans: { prefix?: string; street: string; suffix?: string }
+): Record<string, string> {
+	const out: Record<string, string> = {}
+
+	for (const [key, value] of Object.entries(components)) {
+		if (key === "street") {
+			if (spans.prefix) {
+				out.street_prefix = spans.prefix
+			}
+
+			out.street = spans.street
+
+			if (spans.suffix) {
+				out.street_suffix = spans.suffix
+			}
+
+			continue
+		}
+
+		if (key === "street_prefix" || key === "street_suffix") continue
+		out[key] = value
+	}
+
+	return out
+}
+
+/**
+ * Options for {@linkcode relabelGoldenStreetRow}.
+ */
+export interface RelabelStreetRowOptions {
+	/**
+	 * Whether to also move a leading directional into `street_prefix`.
+	 *
+	 * Defaults to `true`; without it `N Desmet Avenue` grades as `street: "N Desmet"`
+	 * against a model that emits a prefix.
+	 */
+	splitPrefix?: boolean
+}
+
+/**
+ * Splits the street span of one US golden row without mutating its argument,
+ * returning it when no field changes.
+ *
+ * @throws When the split spans do not rebuild the original street byte for byte.
+ */
+export function relabelGoldenStreetRow(
+	row: GoldenStreetRow,
+	options: RelabelStreetRowOptions = {}
+): GoldenRelabelResult {
+	const splitPrefix = options.splitPrefix ?? true
+
+	const keep = (rowClass: GoldenRelabelClass): GoldenRelabelResult => ({
+		row,
+		changed: false,
+		rowClass,
+		flags: [],
+		prefixSplit: false,
+	})
+
+	if ((row.country ?? "").toUpperCase() !== "US") return keep("not-us")
+	const street = row.components.street
+
+	if (!street) return keep("no-street")
+
+	if (row.components.street_suffix) return keep("already-split")
+
+	if (street !== street.trim()) return keep("untrimmed-street")
+
+	const flags: GoldenRelabelFlag[] = []
+	let rowClass: GoldenRelabelClass
+	let name = street
+	let suffix: string | undefined
+	let suffixGap = ""
+	let canonical: USStreetSuffix | undefined
+
+	const split = splitLastWord(street)
+
+	if (!split) {
+		rowClass = matchTrailingSuffix(street) ? "suffix-only-street" : "single-token"
+	} else if (isStreetDirectionalToken(split.tail)) {
+		// A street type and a post-directional, as in "Pennsylvania Avenue NW",
+		// form one suffix span because no post-directional tag exists.
+		const inner = splitLastWord(split.head)
+		const typeMatch = inner ? matchTrailingSuffix(inner.tail) : null
+
+		if (!inner || !typeMatch) {
+			rowClass = "postdirectional-tail-only"
+		} else {
+			rowClass = "split-suffix-postdirectional"
+			name = inner.head
+			suffix = `${inner.tail}${split.gap}${split.tail}`
+			suffixGap = inner.gap
+			canonical = typeMatch.canonical
+		}
+	} else {
+		const typeMatch = matchTrailingSuffix(street)
+
+		if (!typeMatch) {
+			rowClass = "no-suffix-match"
+		} else {
+			rowClass = "split-suffix"
+			name = split.head
+			suffix = split.tail
+			suffixGap = split.gap
+			canonical = typeMatch.canonical
+		}
+	}
+
+	let prefix: string | undefined
+	let prefixGap = ""
+
+	if (splitPrefix && !row.components.street_prefix) {
+		const lead = splitFirstWord(name)
+
+		if (lead && isStreetDirectionalToken(lead.head)) {
+			prefix = lead.head
+			prefixGap = lead.gap
+			name = lead.tail
+		}
+	}
+
+	if (!suffix && !prefix) return keep(rowClass)
+
+	if (!suffix) {
+		rowClass = "split-prefix-only"
+	}
+
+	const rebuilt = `${prefix ? prefix + prefixGap : ""}${name}${suffix ? suffixGap + suffix : ""}`
+
+	if (rebuilt !== street) {
+		throw new Error(`golden-relabel: span reconstruction failed for ${stringifyJSON(street)}`)
+	}
+
+	if (canonical && NAME_PRONE_US_SUFFIXES.has(canonical)) {
+		flags.push({
+			kind: "name-prone-suffix",
+			detail: `${canonical} heads proper names as often as street types`,
+		})
+	}
+
+	const venue = row.components.venue
+
+	if (suffix && venue && new RegExp(`(^|\\W)${escapeRegExp(suffix)}(\\W|$)`, "i").test(venue)) {
+		flags.push({
+			kind: "venue-context",
+			detail: `venue ${stringifyJSON(venue)} also carries ${stringifyJSON(suffix)}`,
+		})
+	}
+
+	// Only a bare directional remainder is flagged, as in "East Rd"; a Pub-28 suffix
+	// word like "Valley Dr" is an ordinary street.
+	if (isStreetDirectionalToken(name)) {
+		flags.push({
+			kind: "remainder-is-affix",
+			detail: `street would become the bare directional ${stringifyJSON(name)}`,
+		})
+	}
+
+	return {
+		row: {
+			...row,
+			components: withStreetSpans(row.components, {
+				...(prefix ? { prefix } : row.components.street_prefix ? { prefix: row.components.street_prefix } : {}),
+				street: name,
+				...(suffix ? { suffix } : {}),
+			}),
+		},
+		changed: true,
+		rowClass,
+		flags,
+		prefixSplit: Boolean(prefix),
+		beforeStreet: street,
+	}
+}

package/lib/tools/golden/relabel-street.ts

@@ -0,0 +1,9 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Public entry point for golden street relabeling.
+ */
+
+export * from "#tools/golden/relabel-street/directory"
+export * from "#tools/golden/relabel-street/row"

package/lib/tools/ingest-csv.ts

@@ -3,12 +3,13 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ * CSV → SQLite ingest tool.
  *
- *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
- *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ * The tool reads a CSV and samples rows to infer column types.
+ * It creates a SQLite table and imports all rows.
+ * It handles quoted fields, null-like values and duplicate column names.
  *
- *   ## Usage
+ * ## Usage
  *
  *   ```sh
  *   mailwoman corpus ingest-csv \
@@ -17,12 +18,15 @@
  *   --output /data/corpus/sources/usgov-nppes/nppes.db
  * ```
  *
- *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
- *   derived from input filename) --output <path> SQLite database path (default: input dir /
- *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
- *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
- *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
- *   CREATE TABLE, but don't import
+ * Options:
+ * --input <path> Required CSV path
+ * --table <name> Table name (default: from filename)
+ * --output <path> DB path (default: input dir/<table>.db)
+ * --sample <n> Rows used for type inference (default: 100)
+ * --separator <char> Field delimiter (default: ,)
+ * --skip <n> Lines to skip before header (default: 0)
+ * --no-header Treat CSV as headerless (col_0, col_1, ...)
+ * --dry-run Print inferred schema, skip import
  */
 
 import { ByteFormatter } from "@mailwoman/core/fs/formatters"
@@ -30,13 +34,14 @@ import { tryStat, pathExists } from "@mailwoman/core/fs/readers"
 import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers"
 import type { SQLInputValue } from "@mailwoman/sqlite/client"
 import type { Database } from "@mailwoman/sqlite/database-schema"
-import { basename, dirname, extname, join } from "path-ts"
+import { basename, dirname, extname, PathBuilder } from "path-ts"
 import { CSVSpliterator } from "spliterator"
 
-//#region Column name normalization
+// #region Column name normalization
 
 /**
- * @deprecated belongs in core, if really needed. spliterator should handle this.
+ * @deprecated This helper may belong in core.
+ * Keep it here until then.
  */
 function normalizeColumnName(raw: string): string {
 	return (
@@ -60,9 +65,9 @@ function dedupColumns(names: string[]): string[] {
 	})
 }
 
-//#endregion
+// #endregion
 
-//#region Type inference
+// #region Type inference
 
 type SQLiteColType = "INTEGER" | "REAL" | "TEXT"
 
@@ -115,9 +120,9 @@ function inferColumnType(samples: (string | null)[]): ColumnInfo {
 	return { name: "", type, nullable: nullCount / total >= 0.5 }
 }
 
-//#endregion
+// #endregion
 
-//#region Main: read CSV, infer schema, produce SQL
+// #region Main: read CSV, infer schema, produce SQL
 
 interface IngestOptions {
 	inputPath: string
@@ -131,21 +136,16 @@ interface IngestOptions {
 }
 
 async function runIngest(opts: IngestOptions): Promise<void> {
-	// `header: false` with `drop` is what expresses `--skip N`: the spliterator's own header handling consumes the
-	// FIRST row as the header, and `drop` counts from the row after it, so a preamble before the header has to be
-	// dropped here and the header row taken by hand. Quote handling is end-to-end (quoted delimiters, doubled quotes);
-	// `skipEmpty` drops blank lines that readline would have turned into all-null rows. The early `break` closes the
-	// file descriptor.
+	// Keep parser header handling off so `--skip` works predictably.
+	// We read the header row ourselves.
 	const rows = (): AsyncIterable<string[]> =>
 		CSVSpliterator.fromAsync<string[]>(opts.inputPath, {
-			mode: "array",
 			header: false,
 			columnDelimiter: opts.separator,
 			drop: opts.skipLines,
-			enableQuoteHandling: true,
 		})
 
-	// --- Pass 1: read header + sample rows for type inference ---
+	// Read header + sample rows to infer schema.
 	process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`)
 
 	let headerRow: string[] | null = null
@@ -169,20 +169,20 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		throw new Error("No header line found in CSV")
 	}
 
-	// --- Determine column names ---
+	// Normalize and dedupe column names.
 	let rawHeaders: string[]
 
 	if (opts.hasHeader && headerRow) {
 		rawHeaders = headerRow
 	} else {
-		// Auto-generate column names: col_0, col_1, ...
+		// No header: generate col_0, col_1, ...
 		const numCols = sampleRows[0]?.length ?? 0
 		rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`)
 	}
 
 	const colNames = dedupColumns(rawHeaders.map(normalizeColumnName))
 
-	// --- Infer column types ---
+	// Infer each column type from samples.
 	const columns: ColumnInfo[] = colNames.map((name, i) => {
 		const samples = sampleRows.map((row) => {
 			const raw = row[i] ?? ""
@@ -196,10 +196,9 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		return info
 	})
 
-	// --- Generate SQL ---
+	// Build CREATE TABLE and INSERT SQL.
 	const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ")
-	// Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
-	// so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+	// Raw SQL is intentional: columns and types are dynamic at runtime.
 	const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`
 
 	const tempCols = columns.map((c) => `"${c.name}"`).join(", ")
@@ -213,23 +212,18 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		return
 	}
 
-	// --- Create database + import ---
+	// Create database and import rows.
 	const { DatabaseClient } = await import("@mailwoman/sqlite/client")
 	await makeDirectories(dirname(opts.outputPath))
 
-	// `Database` — the EMPTY schema — deliberately, not by default: `createTableSQL` is built from the columns and
-	// types inferred from the CSV at runtime, so there is no table this file could name at compile time. Every write
-	// below goes through `exec`/`prepare` for the same reason.
+	// Empty schema type is intentional: table shape is runtime-generated.
 	using db = new DatabaseClient<Database>(opts.outputPath)
 	db.exec("PRAGMA journal_mode = OFF") // faster for bulk import
 	db.exec("PRAGMA synchronous = OFF")
 
 	db.exec(createTableSQL)
 
-	// Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
-	// NULL normalization and type coercion.
-	// better-sqlite3 doesn't support .import natively, so we use a different approach:
-	// Read the CSV line-by-line and INSERT in a transaction.
+	// Insert line-by-line in batches (no native .import here).
 	process.stderr.write(`Importing rows...\n`)
 
 	const insertStmt = db.prepare(
@@ -239,7 +233,8 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 	let imported = 0
 	let headerSkipped = false
 
-	// node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+	// node:sqlite has no transaction helper.
+	// The code uses BEGIN/COMMIT.
 	const doInsert = () => {
 		db.exec("BEGIN")
 
@@ -278,7 +273,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 			return v
 		})
 
-		// Pad or truncate to column count
+		// Pad or trim to expected column count.
 		while (values.length < columns.length) {
 			values.push(null)
 		}
@@ -298,7 +293,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 		}
 	}
 
-	// Flush remaining
+	// Flush remaining rows.
 	if (batch.length) {
 		doInsert()
 		imported += batch.length
@@ -306,7 +301,7 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 
 	process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`)
 
-	// Build a basic index on the first TEXT column (likely the primary key)
+	// Create a simple index on the first text column.
 	const firstTextCol = columns.find((c) => c.type === "TEXT")
 
 	if (firstTextCol) {
@@ -337,7 +332,8 @@ async function runIngest(opts: IngestOptions): Promise<void> {
 }
 
 /**
- * Flag-shaped options for {@linkcode ingestCSV} — `table`/`output` derive from `input` when omitted.
+ * Options for {@linkcode ingestCSV}.
+ * `table` and `output` default from `input`.
  */
 export interface IngestCSVOptions {
 	input: string
@@ -351,10 +347,9 @@ export interface IngestCSVOptions {
 }
 
 /**
- * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
- * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
- * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
- * it.
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows.
+ *
+ * @throws When `input` does not exist.
  */
 export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	if (!(await pathExists(options.input))) {
@@ -366,7 +361,7 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	await runIngest({
 		inputPath: options.input,
 		tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-		outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+		outputPath: options.output ?? PathBuilder.from(options.input).dirname()(`${csvName}.db`).toString(),
 		sampleSize: options.sample ?? 100,
 		separator: options.separator ?? ",",
 		skipLines: options.skip ?? 0,
@@ -375,4 +370,4 @@ export async function ingestCSV(options: IngestCSVOptions): Promise<void> {
 	})
 }
 
-//#endregion
+// #endregion