@mailwoman/corpus 10.0.0 → 10.2.0

package/out/tools/ingest-csv.js

@@ -3,12 +3,13 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   CSV → SQLite ingestion utility. Ported from isp-nexus's `sdk/data/csv.ts`.
+ * CSV → SQLite ingest tool.
  *
- *   Reads a CSV file, infers column types from a sample of rows, creates a SQLite table, and imports
- *   the data. Handles quoted fields, NULL normalization, and duplicate column name disambiguation.
+ * The tool reads a CSV and samples rows to infer column types.
+ * It creates a SQLite table and imports all rows.
+ * It handles quoted fields, null-like values and duplicate column names.
  *
- *   ## Usage
+ * ## Usage
  *
  *   ```sh
  *   mailwoman corpus ingest-csv \
@@ -17,21 +18,25 @@
  *   --output /data/corpus/sources/usgov-nppes/nppes.db
  * ```
  *
- *   Options: --input <path> CSV file to ingest (required) --table <name> SQLite table name (default:
- *   derived from input filename) --output <path> SQLite database path (default: input dir /
- *   table.db) --sample <n> Rows to sample for type inference (default: 100) --separator <char>
- *   Field separator (default: ,) --skip <n> Lines to skip before header (default: 0) --no-header
- *   CSV has no header row — columns will be col_0, col_1, etc. --dry-run Infer schema and print
- *   CREATE TABLE, but don't import
+ * Options:
+ * --input <path> Required CSV path
+ * --table <name> Table name (default: from filename)
+ * --output <path> DB path (default: input dir/<table>.db)
+ * --sample <n> Rows used for type inference (default: 100)
+ * --separator <char> Field delimiter (default: ,)
+ * --skip <n> Lines to skip before header (default: 0)
+ * --no-header Treat CSV as headerless (col_0, col_1, ...)
+ * --dry-run Print inferred schema, skip import
  */
 import { ByteFormatter } from "@mailwoman/core/fs/formatters";
-import { tryStat, pathExists } from "@mailwoman/core/fs/readers";
+import { tryStat, pathExists } from "@mailwoman/core/fs/readers/stat";
 import { makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
-import { basename, dirname, extname, join } from "path-ts";
+import { basename, dirname, extname, PathBuilder } from "path-ts";
 import { CSVSpliterator } from "spliterator";
-//#region Column name normalization
+// #region Column name normalization
 /**
- * @deprecated belongs in core, if really needed. spliterator should handle this.
+ * @deprecated This helper may belong in core.
+ * Keep it here until then.
  */
 function normalizeColumnName(raw) {
     return (raw
@@ -66,7 +71,7 @@ function inferColumnType(samples) {
     let realCount = 0;
     let textCount = 0;
     for (const s of samples) {
-        if (s === null) {
+        if (!s) {
             nullCount++;
             continue;
         }
@@ -85,17 +90,14 @@ function inferColumnType(samples) {
     return { name: "", type, nullable: nullCount / total >= 0.5 };
 }
 async function runIngest(opts) {
-    // `header: false` with `drop` is what expresses `--skip N`: the spliterator's own header handling consumes the
-    // FIRST row as the header, and `drop` counts from the row after it, so a preamble before the header has to be
-    // dropped here and the header row taken by hand. Quote handling is end-to-end (quoted delimiters, doubled quotes);
-    // `skipEmpty` drops blank lines that readline would have turned into all-null rows. The early `break` closes the
-    // file descriptor.
+    // Keep parser header handling off so `--skip` works predictably.
+    // We read the header row ourselves.
     const rows = () => CSVSpliterator.fromAsync(opts.inputPath, {
         header: false,
         columnDelimiter: opts.separator,
         drop: opts.skipLines,
     });
-    // Read the header and sample rows before inferring the SQLite schema.
+    // Read header + sample rows to infer schema.
     process.stderr.write(`Reading ${opts.inputPath} for schema inference...\n`);
     let headerRow = null;
     const sampleRows = [];
@@ -114,18 +116,18 @@ async function runIngest(opts) {
     if (!headerRow && opts.hasHeader) {
         throw new Error("No header line found in CSV");
     }
-    // Normalize and deduplicate the input column names.
+    // Normalize and dedupe column names.
     let rawHeaders;
     if (opts.hasHeader && headerRow) {
         rawHeaders = headerRow;
     }
     else {
-        // Auto-generate column names: col_0, col_1, ...
+        // No header: generate col_0, col_1, ...
         const numCols = sampleRows[0]?.length ?? 0;
         rawHeaders = Array.from({ length: numCols }, (_, i) => `col_${i}`);
     }
     const colNames = dedupColumns(rawHeaders.map(normalizeColumnName));
-    // Infer each column's SQLite type from its sample values.
+    // Infer each column type from samples.
     const columns = colNames.map((name, i) => {
         const samples = sampleRows.map((row) => {
             const raw = row[i] ?? "";
@@ -135,10 +137,9 @@ async function runIngest(opts) {
         info.name = name;
         return info;
     });
-    // Generate the table definition and insert statement from the inferred schema.
+    // Build CREATE TABLE and INSERT SQL.
     const colDefs = columns.map((c) => `"${c.name}" ${c.type}`).join(",\n  ");
-    // Raw DDL by design: the column set + types are INFERRED from the CSV at runtime (colDefs above),
-    // so a Kysely builder loop would just wrap the same dynamic strings with ceremony and no type safety.
+    // Raw SQL is intentional: columns and types are dynamic at runtime.
     const createTableSQL = `CREATE TABLE IF NOT EXISTS "${opts.tableName}" (\n  ${colDefs}\n);`;
     const tempCols = columns.map((c) => `"${c.name}"`).join(", ");
     process.stderr.write(`\nSchema inferred from ${sampleRows.length} sample rows:\n`);
@@ -147,25 +148,21 @@ async function runIngest(opts) {
         process.stderr.write("--dry-run: stopping before import\n");
         return;
     }
-    // Create the database, then import every input row.
+    // Create database and import rows.
     const { DatabaseClient } = await import("@mailwoman/sqlite/client");
     await makeDirectories(dirname(opts.outputPath));
-    // `Database` — the EMPTY schema — deliberately, not by default: `createTableSQL` is built from the columns and
-    // types inferred from the CSV at runtime, so there is no table this file could name at compile time. Every write
-    // below goes through `exec`/`prepare` for the same reason.
+    // Empty schema type is intentional: table shape is runtime-generated.
     using db = new DatabaseClient(opts.outputPath);
     db.exec("PRAGMA journal_mode = OFF"); // faster for bulk import
     db.exec("PRAGMA synchronous = OFF");
     db.exec(createTableSQL);
-    // Use the .import approach via a temp table, then INSERT INTO ... SELECT to handle
-    // NULL normalization and type coercion.
-    // better-sqlite3 doesn't support .import natively, so we use a different approach:
-    // Read the CSV line-by-line and INSERT in a transaction.
+    // Insert line-by-line in batches (no native .import here).
     process.stderr.write(`Importing rows...\n`);
     const insertStmt = db.prepare(`INSERT INTO "${opts.tableName}" (${tempCols}) VALUES (${columns.map(() => "?").join(", ")})`);
     let imported = 0;
     let headerSkipped = false;
-    // node:sqlite has no `db.transaction(fn)` wrapper; use raw BEGIN/COMMIT around the batch.
+    // node:sqlite has no transaction helper.
+    // The code uses BEGIN/COMMIT.
     const doInsert = () => {
         db.exec("BEGIN");
         try {
@@ -188,7 +185,7 @@ async function runIngest(opts) {
         }
         const values = fields.map((f, i) => {
             const v = normalizeField(f);
-            if (v === null)
+            if (!v)
                 return null;
             const col = columns[i];
             if (col?.type === "INTEGER" && /^-?\d+$/.test(v))
@@ -197,7 +194,7 @@ async function runIngest(opts) {
                 return Number.parseFloat(v);
             return v;
         });
-        // Pad or truncate to column count
+        // Pad or trim to expected column count.
         while (values.length < columns.length) {
             values.push(null);
         }
@@ -212,13 +209,13 @@ async function runIngest(opts) {
             }
         }
     }
-    // Flush remaining
+    // Flush remaining rows.
     if (batch.length) {
         doInsert();
         imported += batch.length;
     }
     process.stderr.write(`  Imported ${imported.toLocaleString()} rows into "${opts.tableName}"\n`);
-    // Build a basic index on the first TEXT column (likely the primary key)
+    // Create a simple index on the first text column.
     const firstTextCol = columns.find((c) => c.type === "TEXT");
     if (firstTextCol) {
         process.stderr.write(`Building index on "${firstTextCol.name}"...\n`);
@@ -238,10 +235,9 @@ async function runIngest(opts) {
     process.stderr.write(`Done. ${imported.toLocaleString()} rows → ${opts.outputPath} (${ByteFormatter.formatIEC(stat.size)})\n`);
 }
 /**
- * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows. Throws when `input` is
- * missing. NOTE(phase1): progress narration still writes stderr directly — this predates the report-callback contract
- * and the write sites are deep in the type-inference helpers; thread a report param if a caller ever needs to capture
- * it.
+ * Ingest a CSV into SQLite: infer column types from a sample, create the table, import the rows.
+ *
+ * @throws When `input` does not exist.
  */
 export async function ingestCSV(options) {
     if (!(await pathExists(options.input))) {
@@ -251,7 +247,7 @@ export async function ingestCSV(options) {
     await runIngest({
         inputPath: options.input,
         tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-        outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
+        outputPath: options.output ?? PathBuilder.from(options.input).dirname()(`${csvName}.db`).toString(),
         sampleSize: options.sample ?? 100,
         separator: options.separator ?? ",",
         skipLines: options.skip ?? 0,
@@ -259,5 +255,5 @@ export async function ingestCSV(options) {
         dryRun: options.dryRun ?? false,
     });
 }
-//#endregion
+// #endregion
 //# sourceMappingURL=ingest-csv.js.map

package/out/tools/ingest-csv.js.map

@@ -1 +1 @@
-{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../lib/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAGhF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,mCAAmC;AAEnC;;GAEG;AACH,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACvC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,+GAA+G;IAC/G,8GAA8G;IAC9G,mHAAmH;IACnH,iHAAiH;IACjH,mBAAmB;IACnB,MAAM,IAAI,GAAG,GAA4B,EAAE,CAC1C,cAAc,CAAC,SAAS,CAAW,IAAI,CAAC,SAAS,EAAE;QAClD,MAAM,EAAE,KAAK;QACb,eAAe,EAAE,IAAI,CAAC,SAAS;QAC/B,IAAI,EAAE,IAAI,CAAC,SAAS;KACpB,CAAC,CAAA;IAEH,sEAAsE;IACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,IAAI,SAAS,GAAoB,IAAI,CAAA;IACrC,MAAM,UAAU,GAAe,EAAE,CAAA;IAEjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,SAAS,GAAG,GAAG,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrB,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,oDAAoD;IACpD,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;QACjC,UAAU,GAAG,SAAS,CAAA;IACvB,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,0DAA0D;IAC1D,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,+EAA+E;IAC/E,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,oDAAoD;IACpD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAA;IACnE,MAAM,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAA;IAE/C,+GAA+G;IAC/G,iHAAiH;IACjH,2DAA2D;IAC3D,MAAM,EAAE,GAAG,IAAI,cAAc,CAAW,IAAI,CAAC,UAAU,CAAC,CAAA;IACxD,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,MAAM,CAAA;IAEzB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAE/E,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAEhF,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QAED,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QAErE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE3C,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAK,CAAC,IAAI;KACpB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAEhD,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,SAAS,CAAC,IAAK,CAAC,IAAI,CAAC,KAAK,CACzG,CAAA;AACF,CAAC;AAgBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,GAAG,CAAC;QACrE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC;AAED,YAAY"}
+{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../lib/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AACrE,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAGhF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACjE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,oCAAoC;AAEpC;;;GAGG;AACH,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACvC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,EAAE,CAAC;YACR,SAAS,EAAE,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,iEAAiE;IACjE,oCAAoC;IACpC,MAAM,IAAI,GAAG,GAA4B,EAAE,CAC1C,cAAc,CAAC,SAAS,CAAW,IAAI,CAAC,SAAS,EAAE;QAClD,MAAM,EAAE,KAAK;QACb,eAAe,EAAE,IAAI,CAAC,SAAS;QAC/B,IAAI,EAAE,IAAI,CAAC,SAAS;KACpB,CAAC,CAAA;IAEH,6CAA6C;IAC7C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,IAAI,SAAS,GAAoB,IAAI,CAAA;IACrC,MAAM,UAAU,GAAe,EAAE,CAAA;IAEjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,SAAS,GAAG,GAAG,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACrB,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,qCAAqC;IACrC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;QACjC,UAAU,GAAG,SAAS,CAAA;IACvB,CAAC;SAAM,CAAC;QACP,wCAAwC;QACxC,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,uCAAuC;IACvC,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,qCAAqC;IACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,oEAAoE;IACpE,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAA;IACnE,MAAM,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAA;IAE/C,sEAAsE;IACtE,MAAM,EAAE,GAAG,IAAI,cAAc,CAAW,IAAI,CAAC,UAAU,CAAC,CAAA;IACxD,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,2DAA2D;IAC3D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,yCAAyC;IACzC,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,MAAM,CAAA;IAEzB,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,CAAC;gBAAE,OAAO,IAAI,CAAA;YACnB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAE/E,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAEhF,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,wCAAwC;QACxC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QAED,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,wBAAwB;IACxB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,kDAAkD;IAClD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QAErE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE3C,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAK,CAAC,IAAI;KACpB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAEhD,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,aAAa,CAAC,SAAS,CAAC,IAAK,CAAC,IAAI,CAAC,KAAK,CACzG,CAAA;AACF,CAAC;AAiBD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,GAAG,CAAC;QACrE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,KAAK,CAAC,CAAC,QAAQ,EAAE;QACnG,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC;AAED,aAAa"}

package/out/tools/jsonl-to-parquet.d.ts

@@ -3,46 +3,33 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ * Convert a jsonl of LabeledRow objects to a Parquet file matching the schema the corpus writes.
  *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
- *   rides alongside.
+ * DuckDB reproduces the PyArrow file's logical schema and column order. A PyArrow reader therefore sees an identical table.
+ * The trainer reads Parquet by column name, so physical layout does not affect it.
  *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
+ * Schema: `PARQUET_COLUMNS` from `#parquet/schema`, the same list the native writer uses.
  *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
+ * Every row must contain the span triple. A row without it came from a producer that has not migrated.
+ * The writer would silently drop the character-offset labels if it wrote that row. Fail loudly and report the row number.
  */
+import type { PathBuilderLike } from "path-ts";
 /**
  * Options for {@linkcode jsonlToParquet}.
  */
 export interface JSONLToParquetOptions {
     /**
-     * The labeled-row JSONL to convert.
+     * The labeled-row jsonl to convert.
      */
-    input: string;
+    input: PathBuilderLike;
     /**
-     * The parquet slice to write.
+     * The parquet file to write.
      */
-    output: string;
+    output: PathBuilderLike;
     /**
-     * Parquet row-group size. Default 50000.
+     * Parquet row-group size.
+     *
+     * @defaultValue `50000`
      */
     rowGroupSize?: number;
 }
@@ -55,7 +42,7 @@ export interface JSONLToParquetSummary {
     outPath: string;
 }
 /**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ * Convert a labeled-row jsonl to a v0.5.0-schema Parquet file.
  */
 export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
 //# sourceMappingURL=jsonl-to-parquet.d.ts.map

package/out/tools/jsonl-to-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA8BD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA6DhC"}
+{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAsB9C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,eAAe,CAAA;IACtB;;OAEG;IACH,MAAM,EAAE,eAAe,CAAA;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA8BD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA8DhC"}

package/out/tools/jsonl-to-parquet.js

@@ -3,129 +3,86 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ * Convert a jsonl of LabeledRow objects to a Parquet file matching the schema the corpus writes.
  *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
- *   rides alongside.
+ * DuckDB reproduces the PyArrow file's logical schema and column order. A PyArrow reader therefore sees an identical table.
+ * The trainer reads Parquet by column name, so physical layout does not affect it.
  *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
+ * Schema: `PARQUET_COLUMNS` from `#parquet/schema`, the same list the native writer uses.
  *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
+ * Every row must contain the span triple. A row without it came from a producer that has not migrated.
+ * The writer would silently drop the character-offset labels if it wrote that row. Fail loudly and report the row number.
  */
+import { delimitedSource } from "@mailwoman/core/fs/delimited";
 import { openWriteStream } from "@mailwoman/core/fs/streams";
 import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
-import { parseJSONStrict } from "@mailwoman/core/json";
-import { join } from "path-ts";
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json";
 import { TextSpliterator } from "spliterator";
-import { connectDuckDB, escapeSQLString } from "#utils/parquet";
-const REQUIRED_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb";
+import { PARQUET_COLUMNS, PARQUET_COLUMN_TYPES } from "#parquet/schema";
 /**
- * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ * The columns this converter writes and the DuckDB type for each column.
  *
- * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
- * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
- * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
- * round-trips as a plain integer where INT64 would surface as bigint.
+ * Both come from `#parquet/schema`, the one definition the native writer,
+ * the reader and the manifest share.
+ *
+ * The span offsets are INT32: parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping).
+ * `raw` is a short address string, so INT32 round-trips as a plain integer
+ * where INT64 would surface as bigint.
  */
-const COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
+const REQUIRED_COLUMNS = PARQUET_COLUMNS;
+const COLUMN_TYPES = PARQUET_COLUMN_TYPES;
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
 /**
- * Enforce the #519 span contract per row: all three present, parallel lengths.
+ * Enforce the span interface per row: all three present, parallel lengths.
  *
- * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ * A row with span_starts but no span_tags is a corrupt row, never a silent fallback.
  */
 function assertSpanTriple(row, lineNo) {
     const present = SPAN_COLUMNS.filter((c) => row[c] != null);
     if (present.length !== SPAN_COLUMNS.length) {
         const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
         throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
-            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
-            "must carry span_starts/span_ends/span_tags; re-emit this slice through alignRow.");
+            `missing ${stringifyJSON(missing)} (source_id=${stringifyJSON(row.source_id ?? null)}). Every parquet-bound row ` +
+            "must carry span_starts/span_ends/span_tags; re-emit this file through alignRow.");
     }
     const n = row.span_starts.length;
     if (row.span_ends.length !== n || row.span_tags.length !== n) {
         throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
             `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
-            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
+            `tags=${row.span_tags.length} (source_id=${stringifyJSON(row.source_id ?? null)})`);
     }
 }
 /**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ * Convert a labeled-row jsonl to a v0.5.0-schema Parquet file.
  */
 export async function jsonlToParquet(options, report) {
     const rowGroupSize = options.rowGroupSize ?? 50_000;
     if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
-        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
+        throw new Error(`rowGroupSize must be a positive integer (got ${stringifyJSON(rowGroupSize)})`);
     }
-    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
-    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
-    // staging directory owns the write stream, so it is closed before the directory is removed — and a
-    // mid-stream span-triple failure leaves no orphan.
+    // The Node-side writer uses O(1) memory by streaming rows.
+    // The staging directory owns the write stream, so it is closed before the directory is removed.
+    // A mid-stream span-triple failure leaves no orphan.
     await using staging = await temporaryDirectory("mw-jsonl-to-parquet-");
-    const stagePath = join(staging.path, "rows.ndjson");
+    const stagePath = staging.path("rows.ndjson");
     const stage = staging.use(openWriteStream(stagePath, { encoding: "utf8" }));
     let rows = 0;
     let lineNo = 0;
-    // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
-    // DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
-    // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
-    // same as readline's crlfDelay:Infinity did.
-    for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+    // TextSpliterator rather than JSONSpliterator: the staging write below streams
+    // the RAW line bytes to DuckDB verbatim (the parse here only validates),
+    // so a re-serialized JSONSpliterator row would defeat the point.
+    // The reader trims each line (a trailing \r included); `skipEmpty: false` keeps
+    // blank lines so `lineNo` matches the file.
+    for await (const line of TextSpliterator.fromAsync(delimitedSource(options.input), { skipEmpty: false })) {
         lineNo++;
-        const line = rawLine.trim();
         if (!line)
             continue;
         const row = parseJSONStrict(line);
         assertSpanTriple(row, lineNo);
-        // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
-        // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+        // DuckDB's `read_json` projects to the explicit `columns` map below
+        // (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
         stage.write(line + "\n");
         rows++;
     }
@@ -135,16 +92,17 @@ export async function jsonlToParquet(options, report) {
     report?.(`Read ${rows} rows from ${options.input}`);
     const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
     const selectList = REQUIRED_COLUMNS.join(", ");
-    const db = await connectDuckDB();
-    // Row order is required: the overlay-manifest assembler records first/last source_id from
-    // slice order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+    const output = options.output.toString();
+    using db = await openDuckDB();
+    // Row order is required: the overlay-manifest assembler records first/last source_id from file order.
+    // `preserve_insertion_order` (DuckDB default) keeps output order = input order.
     await db.run("SET preserve_insertion_order=true");
-    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath.toString())}', ` +
         `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-        `TO '${escapeSQLString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
-    const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(options.output)}')`);
+        `TO '${escapeSQLString(output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
+    const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(output)}')`);
     const written = Number(counted.getRowObjects()[0].n);
-    report?.(`Wrote ${written} rows to ${options.output}`);
-    return { read: rows, written, outPath: options.output };
+    report?.(`Wrote ${written} rows to ${output}`);
+    return { read: rows, written, outPath: output };
 }
 //# sourceMappingURL=jsonl-to-parquet.js.map

package/out/tools/jsonl-to-parquet.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAE/D,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,mGAAmG;IACnG,mDAAmD;IACnD,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,8FAA8F;IAC9F,gGAAgG;IAChG,6FAA6F;IAC7F,6CAA6C;IAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,EAAE,CAAA;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,6FAA6F;QAC7F,yFAAyF;QACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;QAExB,IAAI,EAAE,CAAA;IACP,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9C,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,0FAA0F;IAC1F,6FAA6F;IAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;QAC5E,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CAC/G,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;AACxD,CAAC"}
+{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAErE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAEvE;;;;;;;;;;GAUG;AACH,MAAM,gBAAgB,GAAG,eAAe,CAAA;AACxC,MAAM,YAAY,GAAG,oBAAoB,CAAA;AAEzC,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AA+BvE;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,aAAa,CAAC,OAAO,CAAC,eAAe,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACjH,iFAAiF,CAClF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CAClG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IAChG,CAAC;IAED,2DAA2D;IAC3D,gGAAgG;IAChG,qDAAqD;IACrD,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,+EAA+E;IAC/E,yEAAyE;IACzE,iEAAiE;IACjE,gFAAgF;IAChF,4CAA4C;IAC5C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC1G,MAAM,EAAE,CAAA;QAER,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,oEAAoE;QACpE,+EAA+E;QAC/E,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;QAExB,IAAI,EAAE,CAAA;IACP,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IACxC,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,sGAAsG;IACtG,gFAAgF;IAChF,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK;QACvF,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACvG,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC9G,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC"}