kirograph 0.16.1 → 0.17.0

package/dist/data/lint.js

@@ -0,0 +1,109 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var lint_exports = {};
+__export(lint_exports, {
+  DataLinter: () => DataLinter
+});
+module.exports = __toCommonJS(lint_exports);
+var fs = __toESM(require("fs"));
+var path = __toESM(require("path"));
+var crypto = __toESM(require("crypto"));
+class DataLinter {
+  constructor(db, projectRoot) {
+    this.db = db;
+    this.projectRoot = projectRoot;
+  }
+  lint() {
+    const issues = [];
+    const datasets = this.db.all("SELECT * FROM data_datasets");
+    let healthy = 0;
+    for (const ds of datasets) {
+      const dsIssues = [];
+      const absPath = path.join(this.projectRoot, ds.file_path);
+      if (!fs.existsSync(absPath)) {
+        dsIssues.push({ severity: "error", dataset: ds.id, message: `Source file missing: ${ds.file_path}` });
+      } else {
+        try {
+          const content = fs.readFileSync(absPath);
+          const currentHash = crypto.createHash("sha256").update(content).digest("hex");
+          if (currentHash !== ds.content_hash) {
+            dsIssues.push({ severity: "warning", dataset: ds.id, message: `Content changed since last index (hash mismatch). Run kirograph data reindex.` });
+          }
+        } catch {
+          dsIssues.push({ severity: "warning", dataset: ds.id, message: `Cannot read source file: ${ds.file_path}` });
+        }
+      }
+      const tableName = `data_rows_${ds.id.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+      try {
+        const countRow = this.db.get(`SELECT COUNT(*) as cnt FROM "${tableName}"`);
+        const actualRows = countRow?.cnt ?? 0;
+        if (actualRows !== ds.row_count) {
+          dsIssues.push({ severity: "error", dataset: ds.id, message: `Row count mismatch: metadata says ${ds.row_count}, table has ${actualRows}` });
+        }
+      } catch {
+        dsIssues.push({ severity: "error", dataset: ds.id, message: `Row table "${tableName}" does not exist` });
+      }
+      const profileCount = this.db.get("SELECT COUNT(*) as cnt FROM data_columns WHERE dataset_id = ?", [ds.id])?.cnt ?? 0;
+      if (profileCount !== ds.column_count) {
+        dsIssues.push({ severity: "warning", dataset: ds.id, message: `Column count mismatch: metadata says ${ds.column_count}, profiles has ${profileCount}` });
+      }
+      if (dsIssues.length === 0) {
+        healthy++;
+      }
+      issues.push(...dsIssues);
+    }
+    const orphanRefs = this.db.all(`
+      SELECT r.dataset_id, r.qualified_name FROM data_code_refs r
+      LEFT JOIN data_datasets d ON r.dataset_id = d.id
+      WHERE d.id IS NULL
+    `);
+    if (orphanRefs.length > 0) {
+      issues.push({ severity: "warning", message: `${orphanRefs.length} orphan code ref(s) pointing to deleted datasets` });
+    }
+    try {
+      require.resolve("xlsx");
+    } catch {
+      const xlsxFiles = this.db.all(`SELECT file_path FROM data_datasets WHERE format IN ('xlsx', 'xls')`);
+      if (xlsxFiles.length === 0) {
+        issues.push({ severity: "info", message: `Optional dep 'xlsx' not installed. Excel files (.xlsx/.xls) will be skipped during indexing.` });
+      }
+    }
+    try {
+      require.resolve("parquetjs-lite");
+    } catch {
+      issues.push({ severity: "info", message: `Optional dep 'parquetjs-lite' not installed. Parquet files will be skipped during indexing.` });
+    }
+    return { issues, datasetsChecked: datasets.length, healthy };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DataLinter
+});
+//# sourceMappingURL=lint.js.map

package/dist/data/lint.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/data/lint.ts"],
+  "sourcesContent": ["/**\n * Data Lint\n *\n * Integrity checks for the data index:\n * - Row count mismatch (stored vs actual table rows)\n * - Schema consistency (columns in profile vs actual table)\n * - Stale datasets (source file deleted or changed)\n * - Missing optional deps (files skipped due to missing xlsx/parquetjs-lite)\n * - Orphan code refs (refs pointing to deleted datasets)\n */\n\nimport * as fs from 'fs';\nimport * as path from 'path';\nimport * as crypto from 'crypto';\n\nexport interface LintIssue {\n  severity: 'error' | 'warning' | 'info';\n  dataset?: string;\n  message: string;\n}\n\nexport interface LintResult {\n  issues: LintIssue[];\n  datasetsChecked: number;\n  healthy: number;\n}\n\nexport class DataLinter {\n  private readonly db: any;\n  private readonly projectRoot: string;\n\n  constructor(db: any, projectRoot: string) {\n    this.db = db;\n    this.projectRoot = projectRoot;\n  }\n\n  lint(): LintResult {\n    const issues: LintIssue[] = [];\n    const datasets = this.db.all('SELECT * FROM data_datasets') as any[];\n    let healthy = 0;\n\n    for (const ds of datasets) {\n      const dsIssues: LintIssue[] = [];\n\n      // Check source file exists\n      const absPath = path.join(this.projectRoot, ds.file_path);\n      if (!fs.existsSync(absPath)) {\n        dsIssues.push({ severity: 'error', dataset: ds.id, message: `Source file missing: ${ds.file_path}` });\n      } else {\n        // Check content hash (stale detection)\n        try {\n          const content = fs.readFileSync(absPath);\n          const currentHash = crypto.createHash('sha256').update(content).digest('hex');\n          if (currentHash !== ds.content_hash) {\n            dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Content changed since last index (hash mismatch). Run kirograph data reindex.` });\n          }\n        } catch {\n          dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Cannot read source file: ${ds.file_path}` });\n        }\n      }\n\n      // Check row count matches actual table\n      const tableName = `data_rows_${ds.id.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n      try {\n        const countRow = this.db.get(`SELECT COUNT(*) as cnt FROM \"${tableName}\"`);\n        const actualRows = countRow?.cnt ?? 0;\n        if (actualRows !== ds.row_count) {\n          dsIssues.push({ severity: 'error', dataset: ds.id, message: `Row count mismatch: metadata says ${ds.row_count}, table has ${actualRows}` });\n        }\n      } catch {\n        dsIssues.push({ severity: 'error', dataset: ds.id, message: `Row table \"${tableName}\" does not exist` });\n      }\n\n      // Check column count matches profiles\n      const profileCount = this.db.get('SELECT COUNT(*) as cnt FROM data_columns WHERE dataset_id = ?', [ds.id])?.cnt ?? 0;\n      if (profileCount !== ds.column_count) {\n        dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Column count mismatch: metadata says ${ds.column_count}, profiles has ${profileCount}` });\n      }\n\n      if (dsIssues.length === 0) {\n        healthy++;\n      }\n      issues.push(...dsIssues);\n    }\n\n    // Check for orphan code refs\n    const orphanRefs = this.db.all(`\n      SELECT r.dataset_id, r.qualified_name FROM data_code_refs r\n      LEFT JOIN data_datasets d ON r.dataset_id = d.id\n      WHERE d.id IS NULL\n    `) as any[];\n\n    if (orphanRefs.length > 0) {\n      issues.push({ severity: 'warning', message: `${orphanRefs.length} orphan code ref(s) pointing to deleted datasets` });\n    }\n\n    // Check for missing optional deps\n    try {\n      require.resolve('xlsx');\n    } catch {\n      // Check if any xlsx/xls files exist in include patterns\n      const xlsxFiles = this.db.all(`SELECT file_path FROM data_datasets WHERE format IN ('xlsx', 'xls')`) as any[];\n      if (xlsxFiles.length === 0) {\n        // Check if there are xlsx files on disk that were skipped\n        issues.push({ severity: 'info', message: `Optional dep 'xlsx' not installed. Excel files (.xlsx/.xls) will be skipped during indexing.` });\n      }\n    }\n\n    try {\n      require.resolve('parquetjs-lite');\n    } catch {\n      issues.push({ severity: 'info', message: `Optional dep 'parquetjs-lite' not installed. Parquet files will be skipped during indexing.` });\n    }\n\n    return { issues, datasetsChecked: datasets.length, healthy };\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,SAAoB;AACpB,WAAsB;AACtB,aAAwB;AAcjB,MAAM,WAAW;AAAA,EAItB,YAAY,IAAS,aAAqB;AACxC,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,OAAmB;AACjB,UAAM,SAAsB,CAAC;AAC7B,UAAM,WAAW,KAAK,GAAG,IAAI,6BAA6B;AAC1D,QAAI,UAAU;AAEd,eAAW,MAAM,UAAU;AACzB,YAAM,WAAwB,CAAC;AAG/B,YAAM,UAAU,KAAK,KAAK,KAAK,aAAa,GAAG,SAAS;AACxD,UAAI,CAAC,GAAG,WAAW,OAAO,GAAG;AAC3B,iBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,wBAAwB,GAAG,SAAS,GAAG,CAAC;AAAA,MACtG,OAAO;AAEL,YAAI;AACF,gBAAM,UAAU,GAAG,aAAa,OAAO;AACvC,gBAAM,cAAc,OAAO,WAAW,QAAQ,EAAE,OAAO,OAAO,EAAE,OAAO,KAAK;AAC5E,cAAI,gBAAgB,GAAG,cAAc;AACnC,qBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,gFAAgF,CAAC;AAAA,UACjJ;AAAA,QACF,QAAQ;AACN,mBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,4BAA4B,GAAG,SAAS,GAAG,CAAC;AAAA,QAC5G;AAAA,MACF;AAGA,YAAM,YAAY,aAAa,GAAG,GAAG,QAAQ,kBAAkB,GAAG,CAAC;AACnE,UAAI;AACF,cAAM,WAAW,KAAK,GAAG,IAAI,gCAAgC,SAAS,GAAG;AACzE,cAAM,aAAa,UAAU,OAAO;AACpC,YAAI,eAAe,GAAG,WAAW;AAC/B,mBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,qCAAqC,GAAG,SAAS,eAAe,UAAU,GAAG,CAAC;AAAA,QAC5I;AAAA,MACF,QAAQ;AACN,iBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,cAAc,SAAS,mBAAmB,CAAC;AAAA,MACzG;AAGA,YAAM,eAAe,KAAK,GAAG,IAAI,iEAAiE,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO;AACnH,UAAI,iBAAiB,GAAG,cAAc;AACpC,iBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,wCAAwC,GAAG,YAAY,kBAAkB,YAAY,GAAG,CAAC;AAAA,MACzJ;AAEA,UAAI,SAAS,WAAW,GAAG;AACzB;AAAA,MACF;AACA,aAAO,KAAK,GAAG,QAAQ;AAAA,IACzB;AAGA,UAAM,aAAa,KAAK,GAAG,IAAI;AAAA;AAAA;AAAA;AAAA,KAI9B;AAED,QAAI,WAAW,SAAS,GAAG;AACzB,aAAO,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,WAAW,MAAM,mDAAmD,CAAC;AAAA,IACtH;AAGA,QAAI;AACF,sBAAgB,MAAM;AAAA,IACxB,QAAQ;AAEN,YAAM,YAAY,KAAK,GAAG,IAAI,qEAAqE;AACnG,UAAI,UAAU,WAAW,GAAG;AAE1B,eAAO,KAAK,EAAE,UAAU,QAAQ,SAAS,+FAA+F,CAAC;AAAA,MAC3I;AAAA,IACF;AAEA,QAAI;AACF,sBAAgB,gBAAgB;AAAA,IAClC,QAAQ;AACN,aAAO,KAAK,EAAE,UAAU,QAAQ,SAAS,8FAA8F,CAAC;AAAA,IAC1I;AAEA,WAAO,EAAE,QAAQ,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,EAC7D;AACF;",
+  "names": []
+}

package/dist/data/parsers/csv.js

@@ -0,0 +1,132 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var csv_exports = {};
+__export(csv_exports, {
+  csvParser: () => csvParser,
+  tsvParser: () => tsvParser
+});
+module.exports = __toCommonJS(csv_exports);
+var fs = __toESM(require("fs"));
+var readline = __toESM(require("readline"));
+const BATCH_SIZE = 1e4;
+function parseCSVLine(line, delimiter) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += char;
+      }
+    } else {
+      if (char === '"') {
+        inQuotes = true;
+      } else if (char === delimiter) {
+        fields.push(current.trim());
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+  }
+  fields.push(current.trim());
+  return fields;
+}
+function inferValue(raw) {
+  if (raw === "" || raw.toLowerCase() === "null" || raw.toLowerCase() === "na" || raw.toLowerCase() === "n/a") return null;
+  if (raw.toLowerCase() === "true") return true;
+  if (raw.toLowerCase() === "false") return false;
+  if (/^-?\d+$/.test(raw)) {
+    const n = parseInt(raw, 10);
+    if (n >= Number.MIN_SAFE_INTEGER && n <= Number.MAX_SAFE_INTEGER) return n;
+  }
+  if (/^-?\d+\.\d+$/.test(raw) || /^-?\d+[eE][+-]?\d+$/.test(raw)) {
+    const f = parseFloat(raw);
+    if (!isNaN(f)) return f;
+  }
+  return raw;
+}
+async function parseFile(filePath, delimiter, opts) {
+  const stream = fs.createReadStream(filePath, { encoding: "utf8" });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+  let columns = [];
+  let totalRows = 0;
+  let batch = [];
+  let isHeader = true;
+  for await (const line of rl) {
+    if (line.trim() === "") continue;
+    if (isHeader) {
+      columns = parseCSVLine(line, delimiter);
+      isHeader = false;
+      continue;
+    }
+    if (totalRows >= opts.maxRows) break;
+    const values = parseCSVLine(line, delimiter);
+    const row = {};
+    for (let i = 0; i < columns.length; i++) {
+      row[columns[i]] = inferValue(values[i] ?? "");
+    }
+    batch.push(row);
+    totalRows++;
+    if (batch.length >= BATCH_SIZE) {
+      opts.onBatch(batch, columns);
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    opts.onBatch(batch, columns);
+  }
+  return { columns, totalRows };
+}
+const csvParser = {
+  name: "csv",
+  extensions: [".csv"],
+  isAvailable: () => true,
+  parse: (filePath, opts) => parseFile(filePath, ",", opts)
+};
+const tsvParser = {
+  name: "tsv",
+  extensions: [".tsv"],
+  isAvailable: () => true,
+  parse: (filePath, opts) => parseFile(filePath, "	", opts)
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  csvParser,
+  tsvParser
+});
+//# sourceMappingURL=csv.js.map

package/dist/data/parsers/csv.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/csv.ts"],
+  "sourcesContent": ["/**\n * CSV/TSV Streaming Parser\n *\n * Handles: .csv, .tsv\n * Parses line-by-line without loading the full file into memory.\n * Handles quoted fields, newlines within quotes, and various delimiters.\n */\n\nimport * as fs from 'fs';\nimport * as readline from 'readline';\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nconst BATCH_SIZE = 10_000;\n\nfunction parseCSVLine(line: string, delimiter: string): string[] {\n  const fields: string[] = [];\n  let current = '';\n  let inQuotes = false;\n\n  for (let i = 0; i < line.length; i++) {\n    const char = line[i];\n\n    if (inQuotes) {\n      if (char === '\"') {\n        if (i + 1 < line.length && line[i + 1] === '\"') {\n          current += '\"';\n          i++; // skip escaped quote\n        } else {\n          inQuotes = false;\n        }\n      } else {\n        current += char;\n      }\n    } else {\n      if (char === '\"') {\n        inQuotes = true;\n      } else if (char === delimiter) {\n        fields.push(current.trim());\n        current = '';\n      } else {\n        current += char;\n      }\n    }\n  }\n  fields.push(current.trim());\n  return fields;\n}\n\nfunction inferValue(raw: string): string | number | boolean | null {\n  if (raw === '' || raw.toLowerCase() === 'null' || raw.toLowerCase() === 'na' || raw.toLowerCase() === 'n/a') return null;\n  if (raw.toLowerCase() === 'true') return true;\n  if (raw.toLowerCase() === 'false') return false;\n\n  // Try integer\n  if (/^-?\\d+$/.test(raw)) {\n    const n = parseInt(raw, 10);\n    if (n >= Number.MIN_SAFE_INTEGER && n <= Number.MAX_SAFE_INTEGER) return n;\n  }\n\n  // Try float\n  if (/^-?\\d+\\.\\d+$/.test(raw) || /^-?\\d+[eE][+-]?\\d+$/.test(raw)) {\n    const f = parseFloat(raw);\n    if (!isNaN(f)) return f;\n  }\n\n  return raw;\n}\n\nasync function parseFile(\n  filePath: string,\n  delimiter: string,\n  opts: { maxRows: number; onBatch: (rows: ParsedRow[], columns: string[]) => void },\n): Promise<{ columns: string[]; totalRows: number }> {\n  const stream = fs.createReadStream(filePath, { encoding: 'utf8' });\n  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });\n\n  let columns: string[] = [];\n  let totalRows = 0;\n  let batch: ParsedRow[] = [];\n  let isHeader = true;\n\n  for await (const line of rl) {\n    if (line.trim() === '') continue;\n\n    if (isHeader) {\n      columns = parseCSVLine(line, delimiter);\n      isHeader = false;\n      continue;\n    }\n\n    if (totalRows >= opts.maxRows) break;\n\n    const values = parseCSVLine(line, delimiter);\n    const row: ParsedRow = {};\n    for (let i = 0; i < columns.length; i++) {\n      row[columns[i]] = inferValue(values[i] ?? '');\n    }\n\n    batch.push(row);\n    totalRows++;\n\n    if (batch.length >= BATCH_SIZE) {\n      opts.onBatch(batch, columns);\n      batch = [];\n    }\n  }\n\n  if (batch.length > 0) {\n    opts.onBatch(batch, columns);\n  }\n\n  return { columns, totalRows };\n}\n\nexport const csvParser: DataFormatParser = {\n  name: 'csv',\n  extensions: ['.csv'],\n  isAvailable: () => true,\n  parse: (filePath, opts) => parseFile(filePath, ',', opts),\n};\n\nexport const tsvParser: DataFormatParser = {\n  name: 'tsv',\n  extensions: ['.tsv'],\n  isAvailable: () => true,\n  parse: (filePath, opts) => parseFile(filePath, '\\t', opts),\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,SAAoB;AACpB,eAA0B;AAG1B,MAAM,aAAa;AAEnB,SAAS,aAAa,MAAc,WAA6B;AAC/D,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,OAAO,KAAK,CAAC;AAEnB,QAAI,UAAU;AACZ,UAAI,SAAS,KAAK;AAChB,YAAI,IAAI,IAAI,KAAK,UAAU,KAAK,IAAI,CAAC,MAAM,KAAK;AAC9C,qBAAW;AACX;AAAA,QACF,OAAO;AACL,qBAAW;AAAA,QACb;AAAA,MACF,OAAO;AACL,mBAAW;AAAA,MACb;AAAA,IACF,OAAO;AACL,UAAI,SAAS,KAAK;AAChB,mBAAW;AAAA,MACb,WAAW,SAAS,WAAW;AAC7B,eAAO,KAAK,QAAQ,KAAK,CAAC;AAC1B,kBAAU;AAAA,MACZ,OAAO;AACL,mBAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AACA,SAAO,KAAK,QAAQ,KAAK,CAAC;AAC1B,SAAO;AACT;AAEA,SAAS,WAAW,KAA+C;AACjE,MAAI,QAAQ,MAAM,IAAI,YAAY,MAAM,UAAU,IAAI,YAAY,MAAM,QAAQ,IAAI,YAAY,MAAM,MAAO,QAAO;AACpH,MAAI,IAAI,YAAY,MAAM,OAAQ,QAAO;AACzC,MAAI,IAAI,YAAY,MAAM,QAAS,QAAO;AAG1C,MAAI,UAAU,KAAK,GAAG,GAAG;AACvB,UAAM,IAAI,SAAS,KAAK,EAAE;AAC1B,QAAI,KAAK,OAAO,oBAAoB,KAAK,OAAO,iBAAkB,QAAO;AAAA,EAC3E;AAGA,MAAI,eAAe,KAAK,GAAG,KAAK,sBAAsB,KAAK,GAAG,GAAG;AAC/D,UAAM,IAAI,WAAW,GAAG;AACxB,QAAI,CAAC,MAAM,CAAC,EAAG,QAAO;AAAA,EACxB;AAEA,SAAO;AACT;AAEA,eAAe,UACb,UACA,WACA,MACmD;AACnD,QAAM,SAAS,GAAG,iBAAiB,UAAU,EAAE,UAAU,OAAO,CAAC;AACjE,QAAM,KAAK,SAAS,gBAAgB,EAAE,OAAO,QAAQ,WAAW,SAAS,CAAC;AAE1E,MAAI,UAAoB,CAAC;AACzB,MAAI,YAAY;AAChB,MAAI,QAAqB,CAAC;AAC1B,MAAI,WAAW;AAEf,mBAAiB,QAAQ,IAAI;AAC3B,QAAI,KAAK,KAAK,MAAM,GAAI;AAExB,QAAI,UAAU;AACZ,gBAAU,aAAa,MAAM,SAAS;AACtC,iBAAW;AACX;AAAA,IACF;AAEA,QAAI,aAAa,KAAK,QAAS;AAE/B,UAAM,SAAS,aAAa,MAAM,SAAS;AAC3C,UAAM,MAAiB,CAAC;AACxB,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAI,QAAQ,CAAC,CAAC,IAAI,WAAW,OAAO,CAAC,KAAK,EAAE;AAAA,IAC9C;AAEA,UAAM,KAAK,GAAG;AACd;AAEA,QAAI,MAAM,UAAU,YAAY;AAC9B,WAAK,QAAQ,OAAO,OAAO;AAC3B,cAAQ,CAAC;AAAA,IACX;AAAA,EACF;AAEA,MAAI,MAAM,SAAS,GAAG;AACpB,SAAK,QAAQ,OAAO,OAAO;AAAA,EAC7B;AAEA,SAAO,EAAE,SAAS,UAAU;AAC9B;AAEO,MAAM,YAA8B;AAAA,EACzC,MAAM;AAAA,EACN,YAAY,CAAC,MAAM;AAAA,EACnB,aAAa,MAAM;AAAA,EACnB,OAAO,CAAC,UAAU,SAAS,UAAU,UAAU,KAAK,IAAI;AAC1D;AAEO,MAAM,YAA8B;AAAA,EACzC,MAAM;AAAA,EACN,YAAY,CAAC,MAAM;AAAA,EACnB,aAAa,MAAM;AAAA,EACnB,OAAO,CAAC,UAAU,SAAS,UAAU,UAAU,KAAM,IAAI;AAC3D;",
+  "names": []
+}

package/dist/data/parsers/excel.js

@@ -0,0 +1,88 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var excel_exports = {};
+__export(excel_exports, {
+  excelParser: () => excelParser
+});
+module.exports = __toCommonJS(excel_exports);
+const BATCH_SIZE = 1e4;
+let xlsxModule = null;
+let xlsxChecked = false;
+function tryLoadXlsx() {
+  if (xlsxChecked) return xlsxModule !== null;
+  xlsxChecked = true;
+  try {
+    xlsxModule = require("xlsx");
+    return true;
+  } catch {
+    return false;
+  }
+}
+const excelParser = {
+  name: "xlsx",
+  extensions: [".xlsx", ".xls"],
+  isAvailable() {
+    return tryLoadXlsx();
+  },
+  async parse(filePath, opts) {
+    if (!tryLoadXlsx()) {
+      return { columns: [], totalRows: 0 };
+    }
+    const workbook = xlsxModule.readFile(filePath, { type: "file", cellDates: true });
+    const sheetName = workbook.SheetNames[0];
+    if (!sheetName) return { columns: [], totalRows: 0 };
+    const sheet = workbook.Sheets[sheetName];
+    const jsonData = xlsxModule.utils.sheet_to_json(sheet, { defval: null });
+    if (jsonData.length === 0) return { columns: [], totalRows: 0 };
+    const columns = Object.keys(jsonData[0]);
+    let totalRows = 0;
+    let batch = [];
+    for (const item of jsonData) {
+      if (totalRows >= opts.maxRows) break;
+      const row = {};
+      for (const col of columns) {
+        const val = item[col];
+        if (val === null || val === void 0) {
+          row[col] = null;
+        } else if (val instanceof Date) {
+          row[col] = val.toISOString();
+        } else if (typeof val === "object") {
+          row[col] = JSON.stringify(val);
+        } else {
+          row[col] = val;
+        }
+      }
+      batch.push(row);
+      totalRows++;
+      if (batch.length >= BATCH_SIZE) {
+        opts.onBatch(batch, columns);
+        batch = [];
+      }
+    }
+    if (batch.length > 0) {
+      opts.onBatch(batch, columns);
+    }
+    return { columns, totalRows };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  excelParser
+});
+//# sourceMappingURL=excel.js.map

package/dist/data/parsers/excel.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/excel.ts"],
+  "sourcesContent": ["/**\n * Excel Parser (Optional Dependency)\n *\n * Handles: .xlsx, .xls\n * Requires: xlsx package (optional dependency)\n * Gracefully returns isAvailable()=false if not installed.\n */\n\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nconst BATCH_SIZE = 10_000;\n\nlet xlsxModule: any = null;\nlet xlsxChecked = false;\n\nfunction tryLoadXlsx(): boolean {\n  if (xlsxChecked) return xlsxModule !== null;\n  xlsxChecked = true;\n  try {\n    xlsxModule = require('xlsx');\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nexport const excelParser: DataFormatParser = {\n  name: 'xlsx',\n  extensions: ['.xlsx', '.xls'],\n\n  isAvailable(): boolean {\n    return tryLoadXlsx();\n  },\n\n  async parse(filePath, opts) {\n    if (!tryLoadXlsx()) {\n      return { columns: [], totalRows: 0 };\n    }\n\n    const workbook = xlsxModule.readFile(filePath, { type: 'file', cellDates: true });\n    const sheetName = workbook.SheetNames[0];\n    if (!sheetName) return { columns: [], totalRows: 0 };\n\n    const sheet = workbook.Sheets[sheetName];\n    const jsonData = xlsxModule.utils.sheet_to_json(sheet, { defval: null }) as Record<string, any>[];\n\n    if (jsonData.length === 0) return { columns: [], totalRows: 0 };\n\n    const columns = Object.keys(jsonData[0]);\n    let totalRows = 0;\n    let batch: ParsedRow[] = [];\n\n    for (const item of jsonData) {\n      if (totalRows >= opts.maxRows) break;\n\n      const row: ParsedRow = {};\n      for (const col of columns) {\n        const val = item[col];\n        if (val === null || val === undefined) {\n          row[col] = null;\n        } else if (val instanceof Date) {\n          row[col] = val.toISOString();\n        } else if (typeof val === 'object') {\n          row[col] = JSON.stringify(val);\n        } else {\n          row[col] = val;\n        }\n      }\n\n      batch.push(row);\n      totalRows++;\n\n      if (batch.length >= BATCH_SIZE) {\n        opts.onBatch(batch, columns);\n        batch = [];\n      }\n    }\n\n    if (batch.length > 0) {\n      opts.onBatch(batch, columns);\n    }\n\n    return { columns, totalRows };\n  },\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAUA,MAAM,aAAa;AAEnB,IAAI,aAAkB;AACtB,IAAI,cAAc;AAElB,SAAS,cAAuB;AAC9B,MAAI,YAAa,QAAO,eAAe;AACvC,gBAAc;AACd,MAAI;AACF,iBAAa,QAAQ,MAAM;AAC3B,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAgC;AAAA,EAC3C,MAAM;AAAA,EACN,YAAY,CAAC,SAAS,MAAM;AAAA,EAE5B,cAAuB;AACrB,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAM,MAAM,UAAU,MAAM;AAC1B,QAAI,CAAC,YAAY,GAAG;AAClB,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAEA,UAAM,WAAW,WAAW,SAAS,UAAU,EAAE,MAAM,QAAQ,WAAW,KAAK,CAAC;AAChF,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,UAAW,QAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAEnD,UAAM,QAAQ,SAAS,OAAO,SAAS;AACvC,UAAM,WAAW,WAAW,MAAM,cAAc,OAAO,EAAE,QAAQ,KAAK,CAAC;AAEvE,QAAI,SAAS,WAAW,EAAG,QAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAE9D,UAAM,UAAU,OAAO,KAAK,SAAS,CAAC,CAAC;AACvC,QAAI,YAAY;AAChB,QAAI,QAAqB,CAAC;AAE1B,eAAW,QAAQ,UAAU;AAC3B,UAAI,aAAa,KAAK,QAAS;AAE/B,YAAM,MAAiB,CAAC;AACxB,iBAAW,OAAO,SAAS;AACzB,cAAM,MAAM,KAAK,GAAG;AACpB,YAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,cAAI,GAAG,IAAI;AAAA,QACb,WAAW,eAAe,MAAM;AAC9B,cAAI,GAAG,IAAI,IAAI,YAAY;AAAA,QAC7B,WAAW,OAAO,QAAQ,UAAU;AAClC,cAAI,GAAG,IAAI,KAAK,UAAU,GAAG;AAAA,QAC/B,OAAO;AACL,cAAI,GAAG,IAAI;AAAA,QACb;AAAA,MACF;AAEA,YAAM,KAAK,GAAG;AACd;AAEA,UAAI,MAAM,UAAU,YAAY;AAC9B,aAAK,QAAQ,OAAO,OAAO;AAC3B,gBAAQ,CAAC;AAAA,MACX;AAAA,IACF;AAEA,QAAI,MAAM,SAAS,GAAG;AACpB,WAAK,QAAQ,OAAO,OAAO;AAAA,IAC7B;AAEA,WAAO,EAAE,SAAS,UAAU;AAAA,EAC9B;AACF;",
+  "names": []
+}

package/dist/data/parsers/index.js

@@ -0,0 +1,79 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parsers_exports = {};
+__export(parsers_exports, {
+  getAvailableExtensions: () => getAvailableExtensions,
+  getDataParser: () => getDataParser,
+  getUnavailableExtensions: () => getUnavailableExtensions,
+  isSupportedDataFormat: () => isSupportedDataFormat
+});
+module.exports = __toCommonJS(parsers_exports);
+var import_csv = require("./csv");
+var import_jsonl = require("./jsonl");
+var import_json_array = require("./json-array");
+var import_excel = require("./excel");
+var import_parquet = require("./parquet");
+const allParsers = [
+  import_csv.csvParser,
+  import_csv.tsvParser,
+  import_jsonl.jsonlParser,
+  import_json_array.jsonArrayParser,
+  import_excel.excelParser,
+  import_parquet.parquetParser
+];
+const extensionMap = /* @__PURE__ */ new Map();
+for (const parser of allParsers) {
+  for (const ext of parser.extensions) {
+    extensionMap.set(ext.toLowerCase(), parser);
+  }
+}
+function getDataParser(filePath) {
+  const ext = filePath.slice(filePath.lastIndexOf(".")).toLowerCase();
+  const parser = extensionMap.get(ext) ?? null;
+  if (!parser) return null;
+  if (!parser.isAvailable()) return null;
+  return parser;
+}
+function isSupportedDataFormat(filePath) {
+  return getDataParser(filePath) !== null;
+}
+function getAvailableExtensions() {
+  const available = [];
+  for (const [ext, parser] of extensionMap) {
+    if (parser.isAvailable()) available.push(ext);
+  }
+  return available;
+}
+function getUnavailableExtensions() {
+  const unavailable = [];
+  for (const [ext, parser] of extensionMap) {
+    if (!parser.isAvailable()) {
+      unavailable.push({ ext, parser: parser.name });
+    }
+  }
+  return unavailable;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getAvailableExtensions,
+  getDataParser,
+  getUnavailableExtensions,
+  isSupportedDataFormat
+});
+//# sourceMappingURL=index.js.map

package/dist/data/parsers/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/index.ts"],
+  "sourcesContent": ["/**\n * Data Format Registry\n *\n * Maps file extensions to their appropriate parser.\n * Gracefully skips formats whose optional dependencies are not installed.\n */\n\nimport type { DataFormatParser } from '../types';\nimport { csvParser, tsvParser } from './csv';\nimport { jsonlParser } from './jsonl';\nimport { jsonArrayParser } from './json-array';\nimport { excelParser } from './excel';\nimport { parquetParser } from './parquet';\n\n// \u2500\u2500 Registry \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nconst allParsers: DataFormatParser[] = [\n  csvParser,\n  tsvParser,\n  jsonlParser,\n  jsonArrayParser,\n  excelParser,\n  parquetParser,\n];\n\n/** Extension \u2192 parser lookup */\nconst extensionMap = new Map<string, DataFormatParser>();\nfor (const parser of allParsers) {\n  for (const ext of parser.extensions) {\n    extensionMap.set(ext.toLowerCase(), parser);\n  }\n}\n\n// \u2500\u2500 Public API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n/**\n * Get the parser for a given file extension.\n * Returns null if no parser is registered or the optional dep is missing.\n */\nexport function getDataParser(filePath: string): DataFormatParser | null {\n  const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();\n  const parser = extensionMap.get(ext) ?? null;\n  if (!parser) return null;\n  if (!parser.isAvailable()) return null;\n  return parser;\n}\n\n/**\n * Check if a file extension is supported for data parsing.\n */\nexport function isSupportedDataFormat(filePath: string): boolean {\n  return getDataParser(filePath) !== null;\n}\n\n/**\n * Get all supported extensions (only those with available deps).\n */\nexport function getAvailableExtensions(): string[] {\n  const available: string[] = [];\n  for (const [ext, parser] of extensionMap) {\n    if (parser.isAvailable()) available.push(ext);\n  }\n  return available;\n}\n\n/**\n * Get extensions that are registered but have missing optional deps.\n */\nexport function getUnavailableExtensions(): Array<{ ext: string; parser: string }> {\n  const unavailable: Array<{ ext: string; parser: string }> = [];\n  for (const [ext, parser] of extensionMap) {\n    if (!parser.isAvailable()) {\n      unavailable.push({ ext, parser: parser.name });\n    }\n  }\n  return unavailable;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,iBAAqC;AACrC,mBAA4B;AAC5B,wBAAgC;AAChC,mBAA4B;AAC5B,qBAA8B;AAI9B,MAAM,aAAiC;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,eAAe,oBAAI,IAA8B;AACvD,WAAW,UAAU,YAAY;AAC/B,aAAW,OAAO,OAAO,YAAY;AACnC,iBAAa,IAAI,IAAI,YAAY,GAAG,MAAM;AAAA,EAC5C;AACF;AAQO,SAAS,cAAc,UAA2C;AACvE,QAAM,MAAM,SAAS,MAAM,SAAS,YAAY,GAAG,CAAC,EAAE,YAAY;AAClE,QAAM,SAAS,aAAa,IAAI,GAAG,KAAK;AACxC,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI,CAAC,OAAO,YAAY,EAAG,QAAO;AAClC,SAAO;AACT;AAKO,SAAS,sBAAsB,UAA2B;AAC/D,SAAO,cAAc,QAAQ,MAAM;AACrC;AAKO,SAAS,yBAAmC;AACjD,QAAM,YAAsB,CAAC;AAC7B,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,OAAO,YAAY,EAAG,WAAU,KAAK,GAAG;AAAA,EAC9C;AACA,SAAO;AACT;AAKO,SAAS,2BAAmE;AACjF,QAAM,cAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,CAAC,OAAO,YAAY,GAAG;AACzB,kBAAY,KAAK,EAAE,KAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC/C;AAAA,EACF;AACA,SAAO;AACT;",
+  "names": []
+}

package/dist/data/parsers/json-array.js

@@ -0,0 +1,89 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var json_array_exports = {};
+__export(json_array_exports, {
+  jsonArrayParser: () => jsonArrayParser
+});
+module.exports = __toCommonJS(json_array_exports);
+var fs = __toESM(require("fs"));
+const BATCH_SIZE = 1e4;
+const jsonArrayParser = {
+  name: "json",
+  extensions: [".json"],
+  isAvailable: () => true,
+  async parse(filePath, opts) {
+    const content = fs.readFileSync(filePath, "utf8");
+    let data;
+    try {
+      data = JSON.parse(content);
+    } catch {
+      return { columns: [], totalRows: 0 };
+    }
+    if (!Array.isArray(data) || data.length === 0) {
+      return { columns: [], totalRows: 0 };
+    }
+    if (typeof data[0] !== "object" || data[0] === null || Array.isArray(data[0])) {
+      return { columns: [], totalRows: 0 };
+    }
+    const columnSet = /* @__PURE__ */ new Set();
+    let totalRows = 0;
+    let batch = [];
+    for (const item of data) {
+      if (totalRows >= opts.maxRows) break;
+      if (typeof item !== "object" || item === null || Array.isArray(item)) continue;
+      const row = {};
+      for (const [key, value] of Object.entries(item)) {
+        columnSet.add(key);
+        if (value === null || value === void 0) {
+          row[key] = null;
+        } else if (typeof value === "object") {
+          row[key] = JSON.stringify(value);
+        } else {
+          row[key] = value;
+        }
+      }
+      batch.push(row);
+      totalRows++;
+      if (batch.length >= BATCH_SIZE) {
+        opts.onBatch(batch, [...columnSet]);
+        batch = [];
+      }
+    }
+    const columns = [...columnSet];
+    if (batch.length > 0) {
+      opts.onBatch(batch, columns);
+    }
+    return { columns, totalRows };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  jsonArrayParser
+});
+//# sourceMappingURL=json-array.js.map

package/dist/data/parsers/json-array.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/json-array.ts"],
+  "sourcesContent": ["/**\n * JSON Array Parser\n *\n * Handles: .json files that contain an array of objects\n * For files in data/ directories (not package.json, tsconfig.json, etc.)\n */\n\nimport * as fs from 'fs';\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nconst BATCH_SIZE = 10_000;\n\nexport const jsonArrayParser: DataFormatParser = {\n  name: 'json',\n  extensions: ['.json'],\n  isAvailable: () => true,\n\n  async parse(filePath, opts) {\n    const content = fs.readFileSync(filePath, 'utf8');\n    let data: unknown;\n\n    try {\n      data = JSON.parse(content);\n    } catch {\n      return { columns: [], totalRows: 0 };\n    }\n\n    // Must be an array of objects\n    if (!Array.isArray(data) || data.length === 0) {\n      return { columns: [], totalRows: 0 };\n    }\n\n    // Check first element is an object\n    if (typeof data[0] !== 'object' || data[0] === null || Array.isArray(data[0])) {\n      return { columns: [], totalRows: 0 };\n    }\n\n    const columnSet = new Set<string>();\n    let totalRows = 0;\n    let batch: ParsedRow[] = [];\n\n    for (const item of data) {\n      if (totalRows >= opts.maxRows) break;\n      if (typeof item !== 'object' || item === null || Array.isArray(item)) continue;\n\n      const row: ParsedRow = {};\n      for (const [key, value] of Object.entries(item)) {\n        columnSet.add(key);\n        if (value === null || value === undefined) {\n          row[key] = null;\n        } else if (typeof value === 'object') {\n          row[key] = JSON.stringify(value);\n        } else {\n          row[key] = value as string | number | boolean;\n        }\n      }\n\n      batch.push(row);\n      totalRows++;\n\n      if (batch.length >= BATCH_SIZE) {\n        opts.onBatch(batch, [...columnSet]);\n        batch = [];\n      }\n    }\n\n    const columns = [...columnSet];\n    if (batch.length > 0) {\n      opts.onBatch(batch, columns);\n    }\n\n    return { columns, totalRows };\n  },\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,SAAoB;AAGpB,MAAM,aAAa;AAEZ,MAAM,kBAAoC;AAAA,EAC/C,MAAM;AAAA,EACN,YAAY,CAAC,OAAO;AAAA,EACpB,aAAa,MAAM;AAAA,EAEnB,MAAM,MAAM,UAAU,MAAM;AAC1B,UAAM,UAAU,GAAG,aAAa,UAAU,MAAM;AAChD,QAAI;AAEJ,QAAI;AACF,aAAO,KAAK,MAAM,OAAO;AAAA,IAC3B,QAAQ;AACN,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAGA,QAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,WAAW,GAAG;AAC7C,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAGA,QAAI,OAAO,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,QAAQ,MAAM,QAAQ,KAAK,CAAC,CAAC,GAAG;AAC7E,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAEA,UAAM,YAAY,oBAAI,IAAY;AAClC,QAAI,YAAY;AAChB,QAAI,QAAqB,CAAC;AAE1B,eAAW,QAAQ,MAAM;AACvB,UAAI,aAAa,KAAK,QAAS;AAC/B,UAAI,OAAO,SAAS,YAAY,SAAS,QAAQ,MAAM,QAAQ,IAAI,EAAG;AAEtE,YAAM,MAAiB,CAAC;AACxB,iBAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,IAAI,GAAG;AAC/C,kBAAU,IAAI,GAAG;AACjB,YAAI,UAAU,QAAQ,UAAU,QAAW;AACzC,cAAI,GAAG,IAAI;AAAA,QACb,WAAW,OAAO,UAAU,UAAU;AACpC,cAAI,GAAG,IAAI,KAAK,UAAU,KAAK;AAAA,QACjC,OAAO;AACL,cAAI,GAAG,IAAI;AAAA,QACb;AAAA,MACF;AAEA,YAAM,KAAK,GAAG;AACd;AAEA,UAAI,MAAM,UAAU,YAAY;AAC9B,aAAK,QAAQ,OAAO,CAAC,GAAG,SAAS,CAAC;AAClC,gBAAQ,CAAC;AAAA,MACX;AAAA,IACF;AAEA,UAAM,UAAU,CAAC,GAAG,SAAS;AAC7B,QAAI,MAAM,SAAS,GAAG;AACpB,WAAK,QAAQ,OAAO,OAAO;AAAA,IAC7B;AAEA,WAAO,EAAE,SAAS,UAAU;AAAA,EAC9B;AACF;",
+  "names": []
+}

package/dist/data/parsers/jsonl.js

@@ -0,0 +1,84 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var jsonl_exports = {};
+__export(jsonl_exports, {
+  jsonlParser: () => jsonlParser
+});
+module.exports = __toCommonJS(jsonl_exports);
+var fs = __toESM(require("fs"));
+var readline = __toESM(require("readline"));
+const BATCH_SIZE = 1e4;
+const jsonlParser = {
+  name: "jsonl",
+  extensions: [".jsonl", ".ndjson"],
+  isAvailable: () => true,
+  async parse(filePath, opts) {
+    const stream = fs.createReadStream(filePath, { encoding: "utf8" });
+    const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+    const columnSet = /* @__PURE__ */ new Set();
+    let totalRows = 0;
+    let batch = [];
+    for await (const line of rl) {
+      if (line.trim() === "") continue;
+      if (totalRows >= opts.maxRows) break;
+      try {
+        const obj = JSON.parse(line);
+        if (typeof obj !== "object" || obj === null || Array.isArray(obj)) continue;
+        const row = {};
+        for (const [key, value] of Object.entries(obj)) {
+          columnSet.add(key);
+          if (value === null || value === void 0) {
+            row[key] = null;
+          } else if (typeof value === "object") {
+            row[key] = JSON.stringify(value);
+          } else {
+            row[key] = value;
+          }
+        }
+        batch.push(row);
+        totalRows++;
+        if (batch.length >= BATCH_SIZE) {
+          opts.onBatch(batch, [...columnSet]);
+          batch = [];
+        }
+      } catch {
+      }
+    }
+    const columns = [...columnSet];
+    if (batch.length > 0) {
+      opts.onBatch(batch, columns);
+    }
+    return { columns, totalRows };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  jsonlParser
+});
+//# sourceMappingURL=jsonl.js.map