@mailwoman/corpus 9.2.0 → 9.4.0

package/out/src/tools/ingest-csv.js

@@ -1,296 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
- *
- *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
- *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus ingest-csv \
- *   --input /data/corpus/sources/usgov-nppes/npidata_pfile.csv \
- *   --table nppes_providers \
- *   --output /data/corpus/sources/usgov-nppes/nppes.db
- * ```
- *
- *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
- *   derived from input filename) --output <path> SQLite database path (default: input dir /
- *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
- *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
- *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
- *   CREATE TABLE, but don't import
- */
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import { basename, dirname, extname, join } from "node:path";
-import { TextSpliterator } from "spliterator";
-//#region Core: quote-aware CSV field splitting
-const COMMA = 44;
-const DOUBLE_QUOTE = 34;
-// These stay hand-rolled on purpose — spliterator ≥ 3.2.0's CSVSpliterator does quote handling
-// correctly now, but it can't express this tool's CLI contract: `--skip N` drops N preamble lines
-// BEFORE the header (CSVSpliterator consumes row 1 as the header immediately, with no skip-first-N
-// hook), `--no-header` auto-generates `col_0…col_N` names from the first data row's width, and the
-// separator is any single caller-supplied byte. If a future edit removes those knobs, revisit;
-// until then, don't "finish the job" by swapping in CSVSpliterator.
-function splitCSVLine(line, separator = COMMA) {
-    const fields = [];
-    let start = 0;
-    let inQuotes = false;
-    for (let i = 0; i < line.length; i++) {
-        const ch = line.charCodeAt(i);
-        if (ch === DOUBLE_QUOTE) {
-            inQuotes = !inQuotes;
-        }
-        else if (ch === separator && !inQuotes) {
-            fields.push(line.slice(start, i));
-            start = i + 1;
-        }
-    }
-    fields.push(line.slice(start));
-    return fields;
-}
-function stripQuotes(field) {
-    const trimmed = field.trim();
-    if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
-        return trimmed.slice(1, -1).replaceAll('""', '"');
-    }
-    return trimmed;
-}
-//#endregion
-//#region Column name normalization
-function normalizeColumnName(raw) {
-    return (raw
-        .trim()
-        .replaceAll(/^"+|"+$/g, "")
-        .toLowerCase()
-        .replaceAll(/[^a-z0-9]+/g, "_")
-        .replaceAll(/^_|_$/g, "") || "unnamed");
-}
-function dedupColumns(names) {
-    const seen = new Map();
-    return names.map((name) => {
-        const count = seen.get(name) ?? 0;
-        seen.set(name, count + 1);
-        return count === 0 ? name : `${name}_${count + 1}`;
-    });
-}
-function normalizeField(raw) {
-    let s = raw.trim();
-    if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
-        s = s.slice(1, -1).replaceAll('""', '"');
-    }
-    // Normalize common null-like values
-    if (!s || s === "null" || s === "NULL" || s === "N/A" || s === "n/a" || s === "-" || s === "<UNAVAIL>") {
-        return null;
-    }
-    return s;
-}
-function inferColumnType(samples) {
-    let nullCount = 0;
-    let intCount = 0;
-    let realCount = 0;
-    let textCount = 0;
-    for (const s of samples) {
-        if (s === null) {
-            nullCount++;
-            continue;
-        }
-        if (/^-?\d+$/.test(s)) {
-            intCount++;
-        }
-        else if (/^-?\d+\.?\d+$/.test(s)) {
-            realCount++;
-        }
-        else {
-            textCount++;
-        }
-    }
-    const total = samples.length;
-    const type = realCount / total >= 0.5 ? "REAL" : intCount / total >= 0.5 ? "INTEGER" : "TEXT";
-    return { name: "", type, nullable: nullCount / total >= 0.5 };
-}
-async function runIngest(opts) {
-    const sep = opts.separator.charCodeAt(0);
-    // --- Pass 1: read header + sample rows for type inference ---
-    process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`);
-    // CRLF-safe by construction: every field below flows through stripQuotes/normalizeField, both of
-    // which `.trim()`, so a trailing CR on the last column of a CRLF file is stripped. TextSpliterator's
-    // default skipEmpty matches readline row-for-row on files with a trailing newline (the common case);
-    // it drops interior blank lines that readline would have turned into all-null rows. The early `break`
-    // closes the file descriptor.
-    let headerLine = null;
-    const sampleRows = [];
-    let lineNum = 0;
-    for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
-        lineNum++;
-        // Skip lines before header
-        if (lineNum <= opts.skipLines)
-            continue;
-        if (!headerLine && opts.hasHeader) {
-            headerLine = line;
-            continue;
-        }
-        if (sampleRows.length < opts.sampleSize) {
-            sampleRows.push(splitCSVLine(line, sep).map(stripQuotes));
-        }
-        else {
-            break;
-        }
-    }
-    if (!headerLine && opts.hasHeader) {
-        throw new Error("No header line found in CSV");
-    }
-    // --- Determine column names ---
-    let rawHeaders;
-    if (opts.hasHeader && headerLine) {
-        rawHeaders = splitCSVLine(headerLine, sep).map(stripQuotes);
-    }
-    else {
-        // Auto-generate column names: col_0, col_1, ...
-        const numCols = sampleRows[0]?.length ?? 0;
-        rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`);
-    }
-    const colNames = dedupColumns(rawHeaders.map(normalizeColumnName));
-    // --- Infer column types ---
-    const columns = colNames.map((name, i) => {
-        const samples = sampleRows.map((row) => {
-            const raw = row[i] ?? "";
-            return normalizeField(raw);
-        });
-        const info = inferColumnType(samples);
-        info.name = name;
-        return info;
-    });
-    // --- Generate SQL ---
-    const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ");
-    // Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
-    // so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
-    const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`;
-    const tempCols = columns.map((c) => `"${c.name}"`).join(", ");
-    process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`);
-    process.stderr.write(`${createTableSQL}\n\n`);
-    if (opts.dryRun) {
-        process.stderr.write("--dry-run: stopping before import\n");
-        return;
-    }
-    // --- Create database + import ---
-    const { DatabaseSync } = await import("node:sqlite");
-    mkdirSync(dirname(opts.outputPath), { recursive: true });
-    const db = new DatabaseSync(opts.outputPath);
-    db.exec("PRAGMA journal_mode = OFF"); // faster for bulk import
-    db.exec("PRAGMA synchronous = OFF");
-    db.exec(createTableSQL);
-    // Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
-    // NULL normalization and type coercion.
-    // better-sqlite3 doesn't support .import natively, so we use a different approach:
-    // Read the CSV line-by-line and INSERT in a transaction.
-    process.stderr.write(`Importing rows...\n`);
-    const insertStmt = db.prepare(`INSERT INTO "${opts.tableName}" (${tempCols}) VALUES (${columns.map(() => "?").join(", ")})`);
-    let imported = 0;
-    let headerSkipped = false;
-    // node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
-    const doInsert = () => {
-        db.exec("BEGIN");
-        try {
-            for (const row of batch) {
-                insertStmt.run(...row);
-            }
-            db.exec("COMMIT");
-        }
-        catch (error) {
-            db.exec("ROLLBACK");
-            throw error;
-        }
-    };
-    const batch = [];
-    const BATCH_SIZE = 10_000;
-    for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
-        lineNum++;
-        if (lineNum <= opts.skipLines)
-            continue;
-        if (opts.hasHeader && !headerSkipped) {
-            headerSkipped = true;
-            continue;
-        }
-        const fields = splitCSVLine(line, sep).map(stripQuotes);
-        const values = fields.map((f, i) => {
-            const v = normalizeField(f);
-            if (v === null)
-                return null;
-            const col = columns[i];
-            if (col?.type === "INTEGER" && /^-?\d+$/.test(v))
-                return Number.parseInt(v, 10);
-            if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v))
-                return Number.parseFloat(v);
-            return v;
-        });
-        // Pad or truncate to column count
-        while (values.length < columns.length) {
-            values.push(null);
-        }
-        values.length = columns.length;
-        batch.push(values);
-        if (batch.length >= BATCH_SIZE) {
-            doInsert();
-            imported += batch.length;
-            batch.length = 0;
-            if (imported % 100_000 === 0) {
-                process.stderr.write(`  ${(imported / 1_000_000).toFixed(1)}M rows...\n`);
-            }
-        }
-    }
-    // Flush remaining
-    if (batch.length) {
-        doInsert();
-        imported += batch.length;
-    }
-    process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`);
-    // Build a basic index on the first TEXT column (likely the primary key)
-    const firstTextCol = columns.find((c) => c.type === "TEXT");
-    if (firstTextCol) {
-        process.stderr.write(`Building index on "${firstTextCol.name}"...\n`);
-        db.exec(`CREATE INDEX IF NOT EXISTS idx_${opts.tableName}_${firstTextCol.name} ON "${opts.tableName}"("${firstTextCol.name}");`);
-    }
-    db.close();
-    // Write MANIFEST
-    const stat = await (await import("node:fs/promises")).stat(opts.outputPath);
-    const manifest = {
-        ingested_at: new Date().toISOString(),
-        source_csv: basename(opts.inputPath),
-        table_name: opts.tableName,
-        columns: columns.map((c) => ({ name: c.name, type: c.type, nullable: c.nullable })),
-        row_count: imported,
-        db_bytes: stat.size,
-    };
-    const manifestPath = opts.outputPath.replace(/\.db$/, ".manifest.json");
-    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
-    process.stderr.write(`Done. ${imported.toLocaleString()} rows → ${opts.outputPath} (${(stat.size / 1024 / 1024).toFixed(0)} MB)\n`);
-}
-/**
- * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
- * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
- * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
- * it.
- */
-export async function ingestCSV(options) {
-    if (!existsSync(options.input)) {
-        throw new Error(`File not found: ${options.input}`);
-    }
-    const csvName = basename(options.input, extname(options.input));
-    await runIngest({
-        inputPath: options.input,
-        tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-        outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
-        sampleSize: options.sample ?? 100,
-        separator: options.separator ?? ",",
-        skipLines: options.skip ?? 0,
-        hasHeader: !options.noHeader,
-        dryRun: options.dryRun ?? false,
-    });
-}
-//#endregion
-//# sourceMappingURL=ingest-csv.js.map

package/out/src/tools/ingest-csv.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../../src/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,+CAA+C;AAE/C,MAAM,KAAK,GAAG,EAAE,CAAA;AAChB,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB,+FAA+F;AAC/F,kGAAkG;AAClG,mGAAmG;AACnG,mGAAmG;AACnG,+FAA+F;AAC/F,oEAAoE;AACpE,SAAS,YAAY,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC5D,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAE7B,IAAI,EAAE,KAAK,YAAY,EAAE,CAAC;YACzB,QAAQ,GAAG,CAAC,QAAQ,CAAA;QACrB,CAAC;aAAM,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;YACjC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IAE9B,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAE5B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,YAAY;AAEZ,mCAAmC;AAEnC,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACvC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAExC,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,iGAAiG;IACjG,qGAAqG;IACrG,qGAAqG;IACrG,sGAAsG;IACtG,8BAA8B;IAC9B,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,2BAA2B;QAC3B,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,UAAU,GAAG,IAAI,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,UAAU,EAAE,CAAC;QAClC,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5D,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,6BAA6B;IAC7B,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,uBAAuB;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IACpD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,MAAM,CAAA;IAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAEvD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAE/E,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAEhF,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QAED,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QAErE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAA;IAEV,iBAAiB;IACjB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE3E,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;KACnB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IAErE,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAC7G,CAAA;AACF,CAAC;AAgBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,GAAG,CAAC;QACrE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC;AAED,YAAY"}

package/out/src/tools/jsonl-to-parquet.d.ts

@@ -1,61 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema.
- *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay
- *   rides alongside.
- *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
- *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
- */
-/**
- * Options for {@linkcode jsonlToParquet}.
- */
-export interface JSONLToParquetOptions {
-    /**
-     * The labeled-row JSONL to convert.
-     */
-    input: string;
-    /**
-     * The parquet shard to write.
-     */
-    output: string;
-    /**
-     * Parquet row-group size. Default 50000.
-     */
-    rowGroupSize?: number;
-}
-/**
- * Summary returned by {@linkcode jsonlToParquet}.
- */
-export interface JSONLToParquetSummary {
-    read: number;
-    written: number;
-    outPath: string;
-}
-/**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
- */
-export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
-//# sourceMappingURL=jsonl-to-parquet.d.ts.map

package/out/src/tools/jsonl-to-parquet.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAuDH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AAqCD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CAmEhC"}

package/out/src/tools/jsonl-to-parquet.js

@@ -1,164 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema.
- *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay
- *   rides alongside.
- *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
- *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
- */
-import { randomUUID } from "node:crypto";
-import { createWriteStream } from "node:fs";
-import { unlink } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { parseJSONStrict } from "@mailwoman/core/objects";
-import { TextSpliterator } from "spliterator";
-const REQUIRED_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
-/**
- * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
- *
- * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
- * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
- * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
- * round-trips as a plain integer where INT64 would surface as bigint.
- */
-const COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
-/**
- * Enforce the #519 span contract per row: all three present, parallel lengths.
- *
- * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
- */
-function assertSpanTriple(row, lineNo) {
-    const present = SPAN_COLUMNS.filter((c) => row[c] != null);
-    if (present.length !== SPAN_COLUMNS.length) {
-        const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
-        throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
-            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
-            "must carry span_starts/span_ends/span_tags; re-emit this shard through alignRow.");
-    }
-    const n = row.span_starts.length;
-    if (row.span_ends.length !== n || row.span_tags.length !== n) {
-        throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
-            `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
-            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
-    }
-}
-/**
- * Escape a path for single-quoted SQL string literals.
- */
-function sqlString(value) {
-    return value.replaceAll("'", "''");
-}
-/**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
- */
-export async function jsonlToParquet(options, report) {
-    const rowGroupSize = options.rowGroupSize ?? 50_000;
-    if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
-        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
-    }
-    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
-    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
-    // `finally` covers the validation pass too, so a mid-stream span-triple failure leaves no orphan.
-    const stagePath = join(tmpdir(), `mw-jsonl-to-parquet-${randomUUID()}.ndjson`);
-    const stage = createWriteStream(stagePath, { encoding: "utf8" });
-    try {
-        let rows = 0;
-        let lineNo = 0;
-        // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
-        // DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
-        // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
-        // same as readline's crlfDelay:Infinity did.
-        for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
-            lineNo++;
-            const line = rawLine.trim();
-            if (!line)
-                continue;
-            const row = parseJSONStrict(line);
-            assertSpanTriple(row, lineNo);
-            // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
-            // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
-            stage.write(line + "\n");
-            rows++;
-        }
-        await new Promise((resolve, reject) => {
-            stage.end((err) => (err ? reject(err) : resolve()));
-        });
-        report?.(`Read ${rows} rows from ${options.input}`);
-        const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
-        const selectList = REQUIRED_COLUMNS.join(", ");
-        // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
-        const { DuckDBInstance } = await import("@duckdb/node-api");
-        const instance = await DuckDBInstance.create();
-        const db = await instance.connect();
-        // Row order is load-bearing: the overlay-manifest assembler records first/last source_id from
-        // shard order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
-        await db.run("SET preserve_insertion_order=true");
-        await db.run(`COPY (SELECT ${selectList} FROM read_json('${sqlString(stagePath)}', ` +
-            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-            `TO '${sqlString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
-        const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${sqlString(options.output)}')`);
-        const written = Number(counted.getRowObjects()[0].n);
-        report?.(`Wrote ${written} rows to ${options.output}`);
-        return { read: rows, written, outPath: options.output };
-    }
-    finally {
-        stage.destroy();
-        await unlink(stagePath).catch(() => { });
-    }
-}
-//# sourceMappingURL=jsonl-to-parquet.js.map

package/out/src/tools/jsonl-to-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,kGAAkG;IAClG,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,UAAU,EAAE,SAAS,CAAC,CAAA;IAC9E,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhE,IAAI,CAAC;QACJ,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,8FAA8F;QAC9F,gGAAgG;QAChG,6FAA6F;QAC7F,6CAA6C;QAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAA;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;YAE3B,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;YAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC7B,6FAA6F;YAC7F,yFAAyF;YACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;YAExB,IAAI,EAAE,CAAA;QACP,CAAC;QAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QACnE,CAAC,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;QACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE9C,gFAAgF;QAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;QAC9C,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QACnC,8FAA8F;QAC9F,6FAA6F;QAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,SAAS,CAAC,SAAS,CAAC,KAAK;YACtE,aAAa,cAAc,mCAAmC;YAC9D,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACzG,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAChH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,KAAK,CAAC,OAAO,EAAE,CAAA;QACf,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACxC,CAAC;AACF,CAAC"}

package/out/src/tools/lint-shard-vocab.d.ts

@@ -1,90 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
- *
- *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
- *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
- */
-/**
- * Options for {@linkcode lintShardVocab}.
- */
-export interface LintShardVocabOptions {
-    /**
-     * The shard parquet to lint.
-     */
-    shard: string;
-    /**
-     * Base corpus version. Default `v0.5.0`.
-     */
-    baseVersion?: string;
-    /**
-     * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
-     */
-    baseRoot?: string;
-    /**
-     * Base-majority confidence floor for a contradiction. Default 0.7.
-     */
-    threshold?: number;
-    /**
-     * Minimum base support to judge a token. Default 50.
-     */
-    minCount?: number;
-    /**
-     * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
-     */
-    fraction?: number;
-}
-/**
- * One contradiction row: token, shard tag, base tag, base fraction, base total.
- */
-export type ShardVocabRow = [token: string, shardTag: string, baseTag: string, baseFrac: number, baseTotal: number];
-/**
- * Findings summary returned by {@linkcode lintShardVocab}.
- */
-export interface LintShardVocabSummary {
-    /**
-     * Real contradictions — the command exits 1 when nonzero.
-     */
-    errors: number;
-    /**
-     * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
-     */
-    warnings: number;
-    findings: {
-        contradictions: ShardVocabRow[];
-        affixSplits: ShardVocabRow[];
-    };
-}
-/**
- * Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
- */
-export declare function lintShardVocab(options: LintShardVocabOptions): Promise<LintShardVocabSummary>;
-//# sourceMappingURL=lint-shard-vocab.d.ts.map

package/out/src/tools/lint-shard-vocab.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard-vocab.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA8KH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAmJnG"}