kirograph 0.16.1 → 0.19.0

package/dist/data/linker.js

@@ -0,0 +1,150 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var linker_exports = {};
+__export(linker_exports, {
+  DataCodeLinker: () => DataCodeLinker,
+  detectDataReferences: () => detectDataReferences
+});
+module.exports = __toCommonJS(linker_exports);
+const READ_PATTERNS = [
+  // Node.js / TypeScript
+  /readFileSync\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /readFile\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /createReadStream\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  // Python pandas
+  /read_csv\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /read_excel\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /read_json\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /read_parquet\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /read_table\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  // Python open
+  /open\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet))['"`]/g,
+  // SQL COPY
+  /COPY\s+\w+\s+FROM\s+['"`]([^'"`]+)['"`]/gi,
+  /\\copy\s+\w+\s+from\s+['"`]([^'"`]+)['"`]/gi,
+  // Generic path references in strings (data file extensions)
+  /['"`]((?:\.\.?\/)?(?:[\w\-./]+\/)?[\w\-]+\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet))['"`]/g
+];
+const WRITE_PATTERNS = [
+  /writeFileSync\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|json|xlsx|parquet))['"`]/g,
+  /writeFile\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|json|xlsx|parquet))['"`]/g,
+  /to_csv\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /to_excel\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /to_json\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /to_parquet\s*\(\s*['"`]([^'"`]+)['"`]/g,
+  /createWriteStream\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|json|xlsx|parquet))['"`]/g,
+  /COPY\s+\w+\s+TO\s+['"`]([^'"`]+)['"`]/gi
+];
+function detectDataReferences(sourceContent, knownDataPaths) {
+  const refs = /* @__PURE__ */ new Map();
+  for (const pattern of READ_PATTERNS) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(sourceContent)) !== null) {
+      const filePath = normalizePath(match[1]);
+      if (isDataFilePath(filePath, knownDataPaths)) {
+        const existing = refs.get(filePath);
+        if (!existing || existing.confidence < 0.9) {
+          const confidence = knownDataPaths.has(filePath) ? 1 : 0.7;
+          refs.set(filePath, { refType: "reads", confidence });
+        }
+      }
+    }
+  }
+  for (const pattern of WRITE_PATTERNS) {
+    pattern.lastIndex = 0;
+    let match;
+    while ((match = pattern.exec(sourceContent)) !== null) {
+      const filePath = normalizePath(match[1]);
+      if (isDataFilePath(filePath, knownDataPaths)) {
+        const confidence = knownDataPaths.has(filePath) ? 1 : 0.7;
+        refs.set(filePath, { refType: "writes", confidence });
+      }
+    }
+  }
+  return Array.from(refs.entries()).map(([filePath, info]) => ({
+    filePath,
+    ...info
+  }));
+}
+function generateDatasetId(filePath) {
+  return filePath.replace(/\.[^.]+$/, "").replace(/[/\\]/g, "-").replace(/[^a-zA-Z0-9-_]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").toLowerCase();
+}
+function normalizePath(p) {
+  return p.replace(/^\.\//, "").replace(/\\/g, "/");
+}
+function isDataFilePath(filePath, knownPaths) {
+  if (knownPaths.has(filePath)) return true;
+  const ext = filePath.split(".").pop()?.toLowerCase();
+  return ["csv", "tsv", "jsonl", "ndjson", "json", "xlsx", "xls", "parquet"].includes(ext ?? "");
+}
+class DataCodeLinker {
+  constructor(db, projectRoot) {
+    this.db = db;
+    this.projectRoot = projectRoot;
+  }
+  /**
+   * Populate data_code_refs by scanning source files for data file references.
+   * Returns the number of refs created.
+   */
+  linkAll() {
+    const fs = require("fs");
+    const path = require("path");
+    const datasets = this.db.all("SELECT id, file_path FROM data_datasets");
+    if (datasets.length === 0) return 0;
+    const knownPaths = new Set(datasets.map((d) => d.file_path));
+    const pathToDatasetId = new Map(datasets.map((d) => [d.file_path, d.id]));
+    const sourceFiles = this.db.all(
+      `SELECT file_path FROM files WHERE file_path NOT LIKE '%.csv' AND file_path NOT LIKE '%.tsv' AND file_path NOT LIKE '%.jsonl' AND file_path NOT LIKE '%.ndjson' AND file_path NOT LIKE '%.xlsx' AND file_path NOT LIKE '%.parquet'`
+    );
+    this.db.run("DELETE FROM data_code_refs");
+    let refsCreated = 0;
+    for (const file of sourceFiles) {
+      const absPath = path.join(this.projectRoot, file.file_path);
+      let content;
+      try {
+        content = fs.readFileSync(absPath, "utf-8");
+      } catch {
+        continue;
+      }
+      const refs = detectDataReferences(content, knownPaths);
+      if (refs.length === 0) continue;
+      const qualifiedName = file.file_path;
+      for (const ref of refs) {
+        const datasetId = pathToDatasetId.get(ref.filePath);
+        if (!datasetId) continue;
+        try {
+          this.db.run(
+            `INSERT OR REPLACE INTO data_code_refs (dataset_id, qualified_name, ref_type, confidence) VALUES (?, ?, ?, ?)`,
+            [datasetId, qualifiedName, ref.refType, ref.confidence]
+          );
+          refsCreated++;
+        } catch {
+        }
+      }
+    }
+    return refsCreated;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DataCodeLinker,
+  detectDataReferences
+});
+//# sourceMappingURL=linker.js.map

package/dist/data/linker.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/data/linker.ts"],
+  "sourcesContent": ["/**\n * Data \u2194 Code Linker\n *\n * Detects references to data files in source code and populates data_code_refs.\n * Patterns detected:\n *   - readFileSync('data/users.csv'), readFile('path/to/file.csv')\n *   - pd.read_csv('data/file.csv'), pd.read_excel(...)\n *   - open('data/file.csv'), csv.reader(open(...))\n *   - COPY FROM 'file.csv', \\copy ... from 'file.csv'\n *   - require('./data/file.json'), import ... from './data/file.json'\n *   - fs.createReadStream('data/file.csv')\n *   - String literals matching known data file paths\n */\n\nimport type { DataCodeRef } from './types';\n\n/** Patterns that indicate a code symbol reads a data file */\nconst READ_PATTERNS = [\n  // Node.js / TypeScript\n  /readFileSync\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /readFile\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createReadStream\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python pandas\n  /read_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_table\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python open\n  /open\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet))['\"`]/g,\n  // SQL COPY\n  /COPY\\s+\\w+\\s+FROM\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  /\\\\copy\\s+\\w+\\s+from\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  // Generic path references in strings (data file extensions)\n  /['\"`]((?:\\.\\.?\\/)?(?:[\\w\\-./]+\\/)?[\\w\\-]+\\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet))['\"`]/g,\n];\n\n/** Patterns that indicate a code symbol writes a data file */\nconst WRITE_PATTERNS = [\n  /writeFileSync\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /writeFile\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /to_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createWriteStream\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /COPY\\s+\\w+\\s+TO\\s+['\"`]([^'\"`]+)['\"`]/gi,\n];\n\n/**\n * Scan source code content for references to data files.\n * Returns matched file paths with their reference type.\n */\nexport function detectDataReferences(\n  sourceContent: string,\n  knownDataPaths: Set<string>,\n): Array<{ filePath: string; refType: 'reads' | 'writes'; confidence: number }> {\n  const refs = new Map<string, { refType: 'reads' | 'writes'; confidence: number }>();\n\n  // Check read patterns\n  for (const pattern of READ_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const existing = refs.get(filePath);\n        if (!existing || existing.confidence < 0.9) {\n          const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n          refs.set(filePath, { refType: 'reads', confidence });\n        }\n      }\n    }\n  }\n\n  // Check write patterns\n  for (const pattern of WRITE_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n        refs.set(filePath, { refType: 'writes', confidence });\n      }\n    }\n  }\n\n  return Array.from(refs.entries()).map(([filePath, info]) => ({\n    filePath,\n    ...info,\n  }));\n}\n\n/**\n * Generate dataset ID from a file path (same logic as indexer).\n */\nfunction generateDatasetId(filePath: string): string {\n  return filePath\n    .replace(/\\.[^.]+$/, '')\n    .replace(/[/\\\\]/g, '-')\n    .replace(/[^a-zA-Z0-9-_]/g, '-')\n    .replace(/-+/g, '-')\n    .replace(/^-|-$/g, '')\n    .toLowerCase();\n}\n\n/**\n * Normalize a path reference found in code.\n * Strips leading ./ and resolves simple relative paths.\n */\nfunction normalizePath(p: string): string {\n  return p.replace(/^\\.\\//, '').replace(/\\\\/g, '/');\n}\n\n/**\n * Check if a path looks like a data file (by extension or known path match).\n */\nfunction isDataFilePath(filePath: string, knownPaths: Set<string>): boolean {\n  if (knownPaths.has(filePath)) return true;\n  // Check extension\n  const ext = filePath.split('.').pop()?.toLowerCase();\n  return ['csv', 'tsv', 'jsonl', 'ndjson', 'json', 'xlsx', 'xls', 'parquet'].includes(ext ?? '');\n}\n\n/**\n * Link data files to code symbols.\n * Scans all indexed source files for references to known data file paths.\n */\nexport class DataCodeLinker {\n  private readonly db: any;\n  private readonly projectRoot: string;\n\n  constructor(db: any, projectRoot: string) {\n    this.db = db;\n    this.projectRoot = projectRoot;\n  }\n\n  /**\n   * Populate data_code_refs by scanning source files for data file references.\n   * Returns the number of refs created.\n   */\n  linkAll(): number {\n    const fs = require('fs');\n    const path = require('path');\n\n    // Get all known data file paths\n    const datasets = this.db.all('SELECT id, file_path FROM data_datasets') as Array<{ id: string; file_path: string }>;\n    if (datasets.length === 0) return 0;\n\n    const knownPaths = new Set(datasets.map(d => d.file_path));\n    const pathToDatasetId = new Map(datasets.map(d => [d.file_path, d.id]));\n\n    // Get all indexed source files (from the graph)\n    const sourceFiles = this.db.all(\n      `SELECT file_path FROM files WHERE file_path NOT LIKE '%.csv' AND file_path NOT LIKE '%.tsv' AND file_path NOT LIKE '%.jsonl' AND file_path NOT LIKE '%.ndjson' AND file_path NOT LIKE '%.xlsx' AND file_path NOT LIKE '%.parquet'`\n    ) as Array<{ file_path: string }>;\n\n    // Clear existing refs\n    this.db.run('DELETE FROM data_code_refs');\n\n    let refsCreated = 0;\n\n    for (const file of sourceFiles) {\n      const absPath = path.join(this.projectRoot, file.file_path);\n      let content: string;\n      try {\n        content = fs.readFileSync(absPath, 'utf-8');\n      } catch {\n        continue;\n      }\n\n      const refs = detectDataReferences(content, knownPaths);\n      if (refs.length === 0) continue;\n\n      // Find the qualified_name of the containing symbol (file-level)\n      // Use the file path as the qualified name for file-level refs\n      const qualifiedName = file.file_path;\n\n      for (const ref of refs) {\n        const datasetId = pathToDatasetId.get(ref.filePath);\n        if (!datasetId) continue;\n\n        try {\n          this.db.run(\n            `INSERT OR REPLACE INTO data_code_refs (dataset_id, qualified_name, ref_type, confidence) VALUES (?, ?, ?, ?)`,\n            [datasetId, qualifiedName, ref.refType, ref.confidence],\n          );\n          refsCreated++;\n        } catch {\n          // Ignore duplicates\n        }\n      }\n    }\n\n    return refsCreated;\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBA,MAAM,gBAAgB;AAAA;AAAA,EAEpB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAEA;AACF;AAGA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAMO,SAAS,qBACd,eACA,gBAC8E;AAC9E,QAAM,OAAO,oBAAI,IAAiE;AAGlF,aAAW,WAAW,eAAe;AACnC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,WAAW,KAAK,IAAI,QAAQ;AAClC,YAAI,CAAC,YAAY,SAAS,aAAa,KAAK;AAC1C,gBAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,eAAK,IAAI,UAAU,EAAE,SAAS,SAAS,WAAW,CAAC;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,aAAW,WAAW,gBAAgB;AACpC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,aAAK,IAAI,UAAU,EAAE,SAAS,UAAU,WAAW,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,KAAK,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO;AAAA,IAC3D;AAAA,IACA,GAAG;AAAA,EACL,EAAE;AACJ;AAKA,SAAS,kBAAkB,UAA0B;AACnD,SAAO,SACJ,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE,EACpB,YAAY;AACjB;AAMA,SAAS,cAAc,GAAmB;AACxC,SAAO,EAAE,QAAQ,SAAS,EAAE,EAAE,QAAQ,OAAO,GAAG;AAClD;AAKA,SAAS,eAAe,UAAkB,YAAkC;AAC1E,MAAI,WAAW,IAAI,QAAQ,EAAG,QAAO;AAErC,QAAM,MAAM,SAAS,MAAM,GAAG,EAAE,IAAI,GAAG,YAAY;AACnD,SAAO,CAAC,OAAO,OAAO,SAAS,UAAU,QAAQ,QAAQ,OAAO,SAAS,EAAE,SAAS,OAAO,EAAE;AAC/F;AAMO,MAAM,eAAe;AAAA,EAI1B,YAAY,IAAS,aAAqB;AACxC,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAkB;AAChB,UAAM,KAAK,QAAQ,IAAI;AACvB,UAAM,OAAO,QAAQ,MAAM;AAG3B,UAAM,WAAW,KAAK,GAAG,IAAI,yCAAyC;AACtE,QAAI,SAAS,WAAW,EAAG,QAAO;AAElC,UAAM,aAAa,IAAI,IAAI,SAAS,IAAI,OAAK,EAAE,SAAS,CAAC;AACzD,UAAM,kBAAkB,IAAI,IAAI,SAAS,IAAI,OAAK,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;AAGtE,UAAM,cAAc,KAAK,GAAG;AAAA,MAC1B;AAAA,IACF;AAGA,SAAK,GAAG,IAAI,4BAA4B;AAExC,QAAI,cAAc;AAElB,eAAW,QAAQ,aAAa;AAC9B,YAAM,UAAU,KAAK,KAAK,KAAK,aAAa,KAAK,SAAS;AAC1D,UAAI;AACJ,UAAI;AACF,kBAAU,GAAG,aAAa,SAAS,OAAO;AAAA,MAC5C,QAAQ;AACN;AAAA,MACF;AAEA,YAAM,OAAO,qBAAqB,SAAS,UAAU;AACrD,UAAI,KAAK,WAAW,EAAG;AAIvB,YAAM,gBAAgB,KAAK;AAE3B,iBAAW,OAAO,MAAM;AACtB,cAAM,YAAY,gBAAgB,IAAI,IAAI,QAAQ;AAClD,YAAI,CAAC,UAAW;AAEhB,YAAI;AACF,eAAK,GAAG;AAAA,YACN;AAAA,YACA,CAAC,WAAW,eAAe,IAAI,SAAS,IAAI,UAAU;AAAA,UACxD;AACA;AAAA,QACF,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/data/lint.js

@@ -0,0 +1,109 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var lint_exports = {};
+__export(lint_exports, {
+  DataLinter: () => DataLinter
+});
+module.exports = __toCommonJS(lint_exports);
+var fs = __toESM(require("fs"));
+var path = __toESM(require("path"));
+var crypto = __toESM(require("crypto"));
+class DataLinter {
+  constructor(db, projectRoot) {
+    this.db = db;
+    this.projectRoot = projectRoot;
+  }
+  lint() {
+    const issues = [];
+    const datasets = this.db.all("SELECT * FROM data_datasets");
+    let healthy = 0;
+    for (const ds of datasets) {
+      const dsIssues = [];
+      const absPath = path.join(this.projectRoot, ds.file_path);
+      if (!fs.existsSync(absPath)) {
+        dsIssues.push({ severity: "error", dataset: ds.id, message: `Source file missing: ${ds.file_path}` });
+      } else {
+        try {
+          const content = fs.readFileSync(absPath);
+          const currentHash = crypto.createHash("sha256").update(content).digest("hex");
+          if (currentHash !== ds.content_hash) {
+            dsIssues.push({ severity: "warning", dataset: ds.id, message: `Content changed since last index (hash mismatch). Run kirograph data reindex.` });
+          }
+        } catch {
+          dsIssues.push({ severity: "warning", dataset: ds.id, message: `Cannot read source file: ${ds.file_path}` });
+        }
+      }
+      const tableName = `data_rows_${ds.id.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+      try {
+        const countRow = this.db.get(`SELECT COUNT(*) as cnt FROM "${tableName}"`);
+        const actualRows = countRow?.cnt ?? 0;
+        if (actualRows !== ds.row_count) {
+          dsIssues.push({ severity: "error", dataset: ds.id, message: `Row count mismatch: metadata says ${ds.row_count}, table has ${actualRows}` });
+        }
+      } catch {
+        dsIssues.push({ severity: "error", dataset: ds.id, message: `Row table "${tableName}" does not exist` });
+      }
+      const profileCount = this.db.get("SELECT COUNT(*) as cnt FROM data_columns WHERE dataset_id = ?", [ds.id])?.cnt ?? 0;
+      if (profileCount !== ds.column_count) {
+        dsIssues.push({ severity: "warning", dataset: ds.id, message: `Column count mismatch: metadata says ${ds.column_count}, profiles has ${profileCount}` });
+      }
+      if (dsIssues.length === 0) {
+        healthy++;
+      }
+      issues.push(...dsIssues);
+    }
+    const orphanRefs = this.db.all(`
+      SELECT r.dataset_id, r.qualified_name FROM data_code_refs r
+      LEFT JOIN data_datasets d ON r.dataset_id = d.id
+      WHERE d.id IS NULL
+    `);
+    if (orphanRefs.length > 0) {
+      issues.push({ severity: "warning", message: `${orphanRefs.length} orphan code ref(s) pointing to deleted datasets` });
+    }
+    try {
+      require.resolve("xlsx");
+    } catch {
+      const xlsxFiles = this.db.all(`SELECT file_path FROM data_datasets WHERE format IN ('xlsx', 'xls')`);
+      if (xlsxFiles.length === 0) {
+        issues.push({ severity: "info", message: `Optional dep 'xlsx' not installed. Excel files (.xlsx/.xls) will be skipped during indexing.` });
+      }
+    }
+    try {
+      require.resolve("parquetjs-lite");
+    } catch {
+      issues.push({ severity: "info", message: `Optional dep 'parquetjs-lite' not installed. Parquet files will be skipped during indexing.` });
+    }
+    return { issues, datasetsChecked: datasets.length, healthy };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DataLinter
+});
+//# sourceMappingURL=lint.js.map

package/dist/data/lint.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/data/lint.ts"],
+  "sourcesContent": ["/**\n * Data Lint\n *\n * Integrity checks for the data index:\n * - Row count mismatch (stored vs actual table rows)\n * - Schema consistency (columns in profile vs actual table)\n * - Stale datasets (source file deleted or changed)\n * - Missing optional deps (files skipped due to missing xlsx/parquetjs-lite)\n * - Orphan code refs (refs pointing to deleted datasets)\n */\n\nimport * as fs from 'fs';\nimport * as path from 'path';\nimport * as crypto from 'crypto';\n\nexport interface LintIssue {\n  severity: 'error' | 'warning' | 'info';\n  dataset?: string;\n  message: string;\n}\n\nexport interface LintResult {\n  issues: LintIssue[];\n  datasetsChecked: number;\n  healthy: number;\n}\n\nexport class DataLinter {\n  private readonly db: any;\n  private readonly projectRoot: string;\n\n  constructor(db: any, projectRoot: string) {\n    this.db = db;\n    this.projectRoot = projectRoot;\n  }\n\n  lint(): LintResult {\n    const issues: LintIssue[] = [];\n    const datasets = this.db.all('SELECT * FROM data_datasets') as any[];\n    let healthy = 0;\n\n    for (const ds of datasets) {\n      const dsIssues: LintIssue[] = [];\n\n      // Check source file exists\n      const absPath = path.join(this.projectRoot, ds.file_path);\n      if (!fs.existsSync(absPath)) {\n        dsIssues.push({ severity: 'error', dataset: ds.id, message: `Source file missing: ${ds.file_path}` });\n      } else {\n        // Check content hash (stale detection)\n        try {\n          const content = fs.readFileSync(absPath);\n          const currentHash = crypto.createHash('sha256').update(content).digest('hex');\n          if (currentHash !== ds.content_hash) {\n            dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Content changed since last index (hash mismatch). Run kirograph data reindex.` });\n          }\n        } catch {\n          dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Cannot read source file: ${ds.file_path}` });\n        }\n      }\n\n      // Check row count matches actual table\n      const tableName = `data_rows_${ds.id.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n      try {\n        const countRow = this.db.get(`SELECT COUNT(*) as cnt FROM \"${tableName}\"`);\n        const actualRows = countRow?.cnt ?? 0;\n        if (actualRows !== ds.row_count) {\n          dsIssues.push({ severity: 'error', dataset: ds.id, message: `Row count mismatch: metadata says ${ds.row_count}, table has ${actualRows}` });\n        }\n      } catch {\n        dsIssues.push({ severity: 'error', dataset: ds.id, message: `Row table \"${tableName}\" does not exist` });\n      }\n\n      // Check column count matches profiles\n      const profileCount = this.db.get('SELECT COUNT(*) as cnt FROM data_columns WHERE dataset_id = ?', [ds.id])?.cnt ?? 0;\n      if (profileCount !== ds.column_count) {\n        dsIssues.push({ severity: 'warning', dataset: ds.id, message: `Column count mismatch: metadata says ${ds.column_count}, profiles has ${profileCount}` });\n      }\n\n      if (dsIssues.length === 0) {\n        healthy++;\n      }\n      issues.push(...dsIssues);\n    }\n\n    // Check for orphan code refs\n    const orphanRefs = this.db.all(`\n      SELECT r.dataset_id, r.qualified_name FROM data_code_refs r\n      LEFT JOIN data_datasets d ON r.dataset_id = d.id\n      WHERE d.id IS NULL\n    `) as any[];\n\n    if (orphanRefs.length > 0) {\n      issues.push({ severity: 'warning', message: `${orphanRefs.length} orphan code ref(s) pointing to deleted datasets` });\n    }\n\n    // Check for missing optional deps\n    try {\n      require.resolve('xlsx');\n    } catch {\n      // Check if any xlsx/xls files exist in include patterns\n      const xlsxFiles = this.db.all(`SELECT file_path FROM data_datasets WHERE format IN ('xlsx', 'xls')`) as any[];\n      if (xlsxFiles.length === 0) {\n        // Check if there are xlsx files on disk that were skipped\n        issues.push({ severity: 'info', message: `Optional dep 'xlsx' not installed. Excel files (.xlsx/.xls) will be skipped during indexing.` });\n      }\n    }\n\n    try {\n      require.resolve('parquetjs-lite');\n    } catch {\n      issues.push({ severity: 'info', message: `Optional dep 'parquetjs-lite' not installed. Parquet files will be skipped during indexing.` });\n    }\n\n    return { issues, datasetsChecked: datasets.length, healthy };\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAWA,SAAoB;AACpB,WAAsB;AACtB,aAAwB;AAcjB,MAAM,WAAW;AAAA,EAItB,YAAY,IAAS,aAAqB;AACxC,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,OAAmB;AACjB,UAAM,SAAsB,CAAC;AAC7B,UAAM,WAAW,KAAK,GAAG,IAAI,6BAA6B;AAC1D,QAAI,UAAU;AAEd,eAAW,MAAM,UAAU;AACzB,YAAM,WAAwB,CAAC;AAG/B,YAAM,UAAU,KAAK,KAAK,KAAK,aAAa,GAAG,SAAS;AACxD,UAAI,CAAC,GAAG,WAAW,OAAO,GAAG;AAC3B,iBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,wBAAwB,GAAG,SAAS,GAAG,CAAC;AAAA,MACtG,OAAO;AAEL,YAAI;AACF,gBAAM,UAAU,GAAG,aAAa,OAAO;AACvC,gBAAM,cAAc,OAAO,WAAW,QAAQ,EAAE,OAAO,OAAO,EAAE,OAAO,KAAK;AAC5E,cAAI,gBAAgB,GAAG,cAAc;AACnC,qBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,gFAAgF,CAAC;AAAA,UACjJ;AAAA,QACF,QAAQ;AACN,mBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,4BAA4B,GAAG,SAAS,GAAG,CAAC;AAAA,QAC5G;AAAA,MACF;AAGA,YAAM,YAAY,aAAa,GAAG,GAAG,QAAQ,kBAAkB,GAAG,CAAC;AACnE,UAAI;AACF,cAAM,WAAW,KAAK,GAAG,IAAI,gCAAgC,SAAS,GAAG;AACzE,cAAM,aAAa,UAAU,OAAO;AACpC,YAAI,eAAe,GAAG,WAAW;AAC/B,mBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,qCAAqC,GAAG,SAAS,eAAe,UAAU,GAAG,CAAC;AAAA,QAC5I;AAAA,MACF,QAAQ;AACN,iBAAS,KAAK,EAAE,UAAU,SAAS,SAAS,GAAG,IAAI,SAAS,cAAc,SAAS,mBAAmB,CAAC;AAAA,MACzG;AAGA,YAAM,eAAe,KAAK,GAAG,IAAI,iEAAiE,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO;AACnH,UAAI,iBAAiB,GAAG,cAAc;AACpC,iBAAS,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,IAAI,SAAS,wCAAwC,GAAG,YAAY,kBAAkB,YAAY,GAAG,CAAC;AAAA,MACzJ;AAEA,UAAI,SAAS,WAAW,GAAG;AACzB;AAAA,MACF;AACA,aAAO,KAAK,GAAG,QAAQ;AAAA,IACzB;AAGA,UAAM,aAAa,KAAK,GAAG,IAAI;AAAA;AAAA;AAAA;AAAA,KAI9B;AAED,QAAI,WAAW,SAAS,GAAG;AACzB,aAAO,KAAK,EAAE,UAAU,WAAW,SAAS,GAAG,WAAW,MAAM,mDAAmD,CAAC;AAAA,IACtH;AAGA,QAAI;AACF,sBAAgB,MAAM;AAAA,IACxB,QAAQ;AAEN,YAAM,YAAY,KAAK,GAAG,IAAI,qEAAqE;AACnG,UAAI,UAAU,WAAW,GAAG;AAE1B,eAAO,KAAK,EAAE,UAAU,QAAQ,SAAS,+FAA+F,CAAC;AAAA,MAC3I;AAAA,IACF;AAEA,QAAI;AACF,sBAAgB,gBAAgB;AAAA,IAClC,QAAQ;AACN,aAAO,KAAK,EAAE,UAAU,QAAQ,SAAS,8FAA8F,CAAC;AAAA,IAC1I;AAEA,WAAO,EAAE,QAAQ,iBAAiB,SAAS,QAAQ,QAAQ;AAAA,EAC7D;AACF;",
+  "names": []
+}

package/dist/data/parsers/csv.js

@@ -0,0 +1,132 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var csv_exports = {};
+__export(csv_exports, {
+  csvParser: () => csvParser,
+  tsvParser: () => tsvParser
+});
+module.exports = __toCommonJS(csv_exports);
+var fs = __toESM(require("fs"));
+var readline = __toESM(require("readline"));
+const BATCH_SIZE = 1e4;
+function parseCSVLine(line, delimiter) {
+  const fields = [];
+  let current = "";
+  let inQuotes = false;
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i];
+    if (inQuotes) {
+      if (char === '"') {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        current += char;
+      }
+    } else {
+      if (char === '"') {
+        inQuotes = true;
+      } else if (char === delimiter) {
+        fields.push(current.trim());
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+  }
+  fields.push(current.trim());
+  return fields;
+}
+function inferValue(raw) {
+  if (raw === "" || raw.toLowerCase() === "null" || raw.toLowerCase() === "na" || raw.toLowerCase() === "n/a") return null;
+  if (raw.toLowerCase() === "true") return true;
+  if (raw.toLowerCase() === "false") return false;
+  if (/^-?\d+$/.test(raw)) {
+    const n = parseInt(raw, 10);
+    if (n >= Number.MIN_SAFE_INTEGER && n <= Number.MAX_SAFE_INTEGER) return n;
+  }
+  if (/^-?\d+\.\d+$/.test(raw) || /^-?\d+[eE][+-]?\d+$/.test(raw)) {
+    const f = parseFloat(raw);
+    if (!isNaN(f)) return f;
+  }
+  return raw;
+}
+async function parseFile(filePath, delimiter, opts) {
+  const stream = fs.createReadStream(filePath, { encoding: "utf8" });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+  let columns = [];
+  let totalRows = 0;
+  let batch = [];
+  let isHeader = true;
+  for await (const line of rl) {
+    if (line.trim() === "") continue;
+    if (isHeader) {
+      columns = parseCSVLine(line, delimiter);
+      isHeader = false;
+      continue;
+    }
+    if (totalRows >= opts.maxRows) break;
+    const values = parseCSVLine(line, delimiter);
+    const row = {};
+    for (let i = 0; i < columns.length; i++) {
+      row[columns[i]] = inferValue(values[i] ?? "");
+    }
+    batch.push(row);
+    totalRows++;
+    if (batch.length >= BATCH_SIZE) {
+      opts.onBatch(batch, columns);
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    opts.onBatch(batch, columns);
+  }
+  return { columns, totalRows };
+}
+const csvParser = {
+  name: "csv",
+  extensions: [".csv"],
+  isAvailable: () => true,
+  parse: (filePath, opts) => parseFile(filePath, ",", opts)
+};
+const tsvParser = {
+  name: "tsv",
+  extensions: [".tsv"],
+  isAvailable: () => true,
+  parse: (filePath, opts) => parseFile(filePath, "	", opts)
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  csvParser,
+  tsvParser
+});
+//# sourceMappingURL=csv.js.map

package/dist/data/parsers/csv.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/csv.ts"],
+  "sourcesContent": ["/**\n * CSV/TSV Streaming Parser\n *\n * Handles: .csv, .tsv\n * Parses line-by-line without loading the full file into memory.\n * Handles quoted fields, newlines within quotes, and various delimiters.\n */\n\nimport * as fs from 'fs';\nimport * as readline from 'readline';\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nconst BATCH_SIZE = 10_000;\n\nfunction parseCSVLine(line: string, delimiter: string): string[] {\n  const fields: string[] = [];\n  let current = '';\n  let inQuotes = false;\n\n  for (let i = 0; i < line.length; i++) {\n    const char = line[i];\n\n    if (inQuotes) {\n      if (char === '\"') {\n        if (i + 1 < line.length && line[i + 1] === '\"') {\n          current += '\"';\n          i++; // skip escaped quote\n        } else {\n          inQuotes = false;\n        }\n      } else {\n        current += char;\n      }\n    } else {\n      if (char === '\"') {\n        inQuotes = true;\n      } else if (char === delimiter) {\n        fields.push(current.trim());\n        current = '';\n      } else {\n        current += char;\n      }\n    }\n  }\n  fields.push(current.trim());\n  return fields;\n}\n\nfunction inferValue(raw: string): string | number | boolean | null {\n  if (raw === '' || raw.toLowerCase() === 'null' || raw.toLowerCase() === 'na' || raw.toLowerCase() === 'n/a') return null;\n  if (raw.toLowerCase() === 'true') return true;\n  if (raw.toLowerCase() === 'false') return false;\n\n  // Try integer\n  if (/^-?\\d+$/.test(raw)) {\n    const n = parseInt(raw, 10);\n    if (n >= Number.MIN_SAFE_INTEGER && n <= Number.MAX_SAFE_INTEGER) return n;\n  }\n\n  // Try float\n  if (/^-?\\d+\\.\\d+$/.test(raw) || /^-?\\d+[eE][+-]?\\d+$/.test(raw)) {\n    const f = parseFloat(raw);\n    if (!isNaN(f)) return f;\n  }\n\n  return raw;\n}\n\nasync function parseFile(\n  filePath: string,\n  delimiter: string,\n  opts: { maxRows: number; onBatch: (rows: ParsedRow[], columns: string[]) => void },\n): Promise<{ columns: string[]; totalRows: number }> {\n  const stream = fs.createReadStream(filePath, { encoding: 'utf8' });\n  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });\n\n  let columns: string[] = [];\n  let totalRows = 0;\n  let batch: ParsedRow[] = [];\n  let isHeader = true;\n\n  for await (const line of rl) {\n    if (line.trim() === '') continue;\n\n    if (isHeader) {\n      columns = parseCSVLine(line, delimiter);\n      isHeader = false;\n      continue;\n    }\n\n    if (totalRows >= opts.maxRows) break;\n\n    const values = parseCSVLine(line, delimiter);\n    const row: ParsedRow = {};\n    for (let i = 0; i < columns.length; i++) {\n      row[columns[i]] = inferValue(values[i] ?? '');\n    }\n\n    batch.push(row);\n    totalRows++;\n\n    if (batch.length >= BATCH_SIZE) {\n      opts.onBatch(batch, columns);\n      batch = [];\n    }\n  }\n\n  if (batch.length > 0) {\n    opts.onBatch(batch, columns);\n  }\n\n  return { columns, totalRows };\n}\n\nexport const csvParser: DataFormatParser = {\n  name: 'csv',\n  extensions: ['.csv'],\n  isAvailable: () => true,\n  parse: (filePath, opts) => parseFile(filePath, ',', opts),\n};\n\nexport const tsvParser: DataFormatParser = {\n  name: 'tsv',\n  extensions: ['.tsv'],\n  isAvailable: () => true,\n  parse: (filePath, opts) => parseFile(filePath, '\\t', opts),\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,SAAoB;AACpB,eAA0B;AAG1B,MAAM,aAAa;AAEnB,SAAS,aAAa,MAAc,WAA6B;AAC/D,QAAM,SAAmB,CAAC;AAC1B,MAAI,UAAU;AACd,MAAI,WAAW;AAEf,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,OAAO,KAAK,CAAC;AAEnB,QAAI,UAAU;AACZ,UAAI,SAAS,KAAK;AAChB,YAAI,IAAI,IAAI,KAAK,UAAU,KAAK,IAAI,CAAC,MAAM,KAAK;AAC9C,qBAAW;AACX;AAAA,QACF,OAAO;AACL,qBAAW;AAAA,QACb;AAAA,MACF,OAAO;AACL,mBAAW;AAAA,MACb;AAAA,IACF,OAAO;AACL,UAAI,SAAS,KAAK;AAChB,mBAAW;AAAA,MACb,WAAW,SAAS,WAAW;AAC7B,eAAO,KAAK,QAAQ,KAAK,CAAC;AAC1B,kBAAU;AAAA,MACZ,OAAO;AACL,mBAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AACA,SAAO,KAAK,QAAQ,KAAK,CAAC;AAC1B,SAAO;AACT;AAEA,SAAS,WAAW,KAA+C;AACjE,MAAI,QAAQ,MAAM,IAAI,YAAY,MAAM,UAAU,IAAI,YAAY,MAAM,QAAQ,IAAI,YAAY,MAAM,MAAO,QAAO;AACpH,MAAI,IAAI,YAAY,MAAM,OAAQ,QAAO;AACzC,MAAI,IAAI,YAAY,MAAM,QAAS,QAAO;AAG1C,MAAI,UAAU,KAAK,GAAG,GAAG;AACvB,UAAM,IAAI,SAAS,KAAK,EAAE;AAC1B,QAAI,KAAK,OAAO,oBAAoB,KAAK,OAAO,iBAAkB,QAAO;AAAA,EAC3E;AAGA,MAAI,eAAe,KAAK,GAAG,KAAK,sBAAsB,KAAK,GAAG,GAAG;AAC/D,UAAM,IAAI,WAAW,GAAG;AACxB,QAAI,CAAC,MAAM,CAAC,EAAG,QAAO;AAAA,EACxB;AAEA,SAAO;AACT;AAEA,eAAe,UACb,UACA,WACA,MACmD;AACnD,QAAM,SAAS,GAAG,iBAAiB,UAAU,EAAE,UAAU,OAAO,CAAC;AACjE,QAAM,KAAK,SAAS,gBAAgB,EAAE,OAAO,QAAQ,WAAW,SAAS,CAAC;AAE1E,MAAI,UAAoB,CAAC;AACzB,MAAI,YAAY;AAChB,MAAI,QAAqB,CAAC;AAC1B,MAAI,WAAW;AAEf,mBAAiB,QAAQ,IAAI;AAC3B,QAAI,KAAK,KAAK,MAAM,GAAI;AAExB,QAAI,UAAU;AACZ,gBAAU,aAAa,MAAM,SAAS;AACtC,iBAAW;AACX;AAAA,IACF;AAEA,QAAI,aAAa,KAAK,QAAS;AAE/B,UAAM,SAAS,aAAa,MAAM,SAAS;AAC3C,UAAM,MAAiB,CAAC;AACxB,aAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAI,QAAQ,CAAC,CAAC,IAAI,WAAW,OAAO,CAAC,KAAK,EAAE;AAAA,IAC9C;AAEA,UAAM,KAAK,GAAG;AACd;AAEA,QAAI,MAAM,UAAU,YAAY;AAC9B,WAAK,QAAQ,OAAO,OAAO;AAC3B,cAAQ,CAAC;AAAA,IACX;AAAA,EACF;AAEA,MAAI,MAAM,SAAS,GAAG;AACpB,SAAK,QAAQ,OAAO,OAAO;AAAA,EAC7B;AAEA,SAAO,EAAE,SAAS,UAAU;AAC9B;AAEO,MAAM,YAA8B;AAAA,EACzC,MAAM;AAAA,EACN,YAAY,CAAC,MAAM;AAAA,EACnB,aAAa,MAAM;AAAA,EACnB,OAAO,CAAC,UAAU,SAAS,UAAU,UAAU,KAAK,IAAI;AAC1D;AAEO,MAAM,YAA8B;AAAA,EACzC,MAAM;AAAA,EACN,YAAY,CAAC,MAAM;AAAA,EACnB,aAAa,MAAM;AAAA,EACnB,OAAO,CAAC,UAAU,SAAS,UAAU,UAAU,KAAM,IAAI;AAC3D;",
+  "names": []
+}

package/dist/data/parsers/excel.js

@@ -0,0 +1,88 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var excel_exports = {};
+__export(excel_exports, {
+  excelParser: () => excelParser
+});
+module.exports = __toCommonJS(excel_exports);
+const BATCH_SIZE = 1e4;
+let xlsxModule = null;
+let xlsxChecked = false;
+function tryLoadXlsx() {
+  if (xlsxChecked) return xlsxModule !== null;
+  xlsxChecked = true;
+  try {
+    xlsxModule = require("xlsx");
+    return true;
+  } catch {
+    return false;
+  }
+}
+const excelParser = {
+  name: "xlsx",
+  extensions: [".xlsx", ".xls"],
+  isAvailable() {
+    return tryLoadXlsx();
+  },
+  async parse(filePath, opts) {
+    if (!tryLoadXlsx()) {
+      return { columns: [], totalRows: 0 };
+    }
+    const workbook = xlsxModule.readFile(filePath, { type: "file", cellDates: true });
+    const sheetName = workbook.SheetNames[0];
+    if (!sheetName) return { columns: [], totalRows: 0 };
+    const sheet = workbook.Sheets[sheetName];
+    const jsonData = xlsxModule.utils.sheet_to_json(sheet, { defval: null });
+    if (jsonData.length === 0) return { columns: [], totalRows: 0 };
+    const columns = Object.keys(jsonData[0]);
+    let totalRows = 0;
+    let batch = [];
+    for (const item of jsonData) {
+      if (totalRows >= opts.maxRows) break;
+      const row = {};
+      for (const col of columns) {
+        const val = item[col];
+        if (val === null || val === void 0) {
+          row[col] = null;
+        } else if (val instanceof Date) {
+          row[col] = val.toISOString();
+        } else if (typeof val === "object") {
+          row[col] = JSON.stringify(val);
+        } else {
+          row[col] = val;
+        }
+      }
+      batch.push(row);
+      totalRows++;
+      if (batch.length >= BATCH_SIZE) {
+        opts.onBatch(batch, columns);
+        batch = [];
+      }
+    }
+    if (batch.length > 0) {
+      opts.onBatch(batch, columns);
+    }
+    return { columns, totalRows };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  excelParser
+});
+//# sourceMappingURL=excel.js.map

package/dist/data/parsers/excel.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/excel.ts"],
+  "sourcesContent": ["/**\n * Excel Parser (Optional Dependency)\n *\n * Handles: .xlsx, .xls\n * Requires: xlsx package (optional dependency)\n * Gracefully returns isAvailable()=false if not installed.\n */\n\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nconst BATCH_SIZE = 10_000;\n\nlet xlsxModule: any = null;\nlet xlsxChecked = false;\n\nfunction tryLoadXlsx(): boolean {\n  if (xlsxChecked) return xlsxModule !== null;\n  xlsxChecked = true;\n  try {\n    xlsxModule = require('xlsx');\n    return true;\n  } catch {\n    return false;\n  }\n}\n\nexport const excelParser: DataFormatParser = {\n  name: 'xlsx',\n  extensions: ['.xlsx', '.xls'],\n\n  isAvailable(): boolean {\n    return tryLoadXlsx();\n  },\n\n  async parse(filePath, opts) {\n    if (!tryLoadXlsx()) {\n      return { columns: [], totalRows: 0 };\n    }\n\n    const workbook = xlsxModule.readFile(filePath, { type: 'file', cellDates: true });\n    const sheetName = workbook.SheetNames[0];\n    if (!sheetName) return { columns: [], totalRows: 0 };\n\n    const sheet = workbook.Sheets[sheetName];\n    const jsonData = xlsxModule.utils.sheet_to_json(sheet, { defval: null }) as Record<string, any>[];\n\n    if (jsonData.length === 0) return { columns: [], totalRows: 0 };\n\n    const columns = Object.keys(jsonData[0]);\n    let totalRows = 0;\n    let batch: ParsedRow[] = [];\n\n    for (const item of jsonData) {\n      if (totalRows >= opts.maxRows) break;\n\n      const row: ParsedRow = {};\n      for (const col of columns) {\n        const val = item[col];\n        if (val === null || val === undefined) {\n          row[col] = null;\n        } else if (val instanceof Date) {\n          row[col] = val.toISOString();\n        } else if (typeof val === 'object') {\n          row[col] = JSON.stringify(val);\n        } else {\n          row[col] = val;\n        }\n      }\n\n      batch.push(row);\n      totalRows++;\n\n      if (batch.length >= BATCH_SIZE) {\n        opts.onBatch(batch, columns);\n        batch = [];\n      }\n    }\n\n    if (batch.length > 0) {\n      opts.onBatch(batch, columns);\n    }\n\n    return { columns, totalRows };\n  },\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAUA,MAAM,aAAa;AAEnB,IAAI,aAAkB;AACtB,IAAI,cAAc;AAElB,SAAS,cAAuB;AAC9B,MAAI,YAAa,QAAO,eAAe;AACvC,gBAAc;AACd,MAAI;AACF,iBAAa,QAAQ,MAAM;AAC3B,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAgC;AAAA,EAC3C,MAAM;AAAA,EACN,YAAY,CAAC,SAAS,MAAM;AAAA,EAE5B,cAAuB;AACrB,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAM,MAAM,UAAU,MAAM;AAC1B,QAAI,CAAC,YAAY,GAAG;AAClB,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAEA,UAAM,WAAW,WAAW,SAAS,UAAU,EAAE,MAAM,QAAQ,WAAW,KAAK,CAAC;AAChF,UAAM,YAAY,SAAS,WAAW,CAAC;AACvC,QAAI,CAAC,UAAW,QAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAEnD,UAAM,QAAQ,SAAS,OAAO,SAAS;AACvC,UAAM,WAAW,WAAW,MAAM,cAAc,OAAO,EAAE,QAAQ,KAAK,CAAC;AAEvE,QAAI,SAAS,WAAW,EAAG,QAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAE9D,UAAM,UAAU,OAAO,KAAK,SAAS,CAAC,CAAC;AACvC,QAAI,YAAY;AAChB,QAAI,QAAqB,CAAC;AAE1B,eAAW,QAAQ,UAAU;AAC3B,UAAI,aAAa,KAAK,QAAS;AAE/B,YAAM,MAAiB,CAAC;AACxB,iBAAW,OAAO,SAAS;AACzB,cAAM,MAAM,KAAK,GAAG;AACpB,YAAI,QAAQ,QAAQ,QAAQ,QAAW;AACrC,cAAI,GAAG,IAAI;AAAA,QACb,WAAW,eAAe,MAAM;AAC9B,cAAI,GAAG,IAAI,IAAI,YAAY;AAAA,QAC7B,WAAW,OAAO,QAAQ,UAAU;AAClC,cAAI,GAAG,IAAI,KAAK,UAAU,GAAG;AAAA,QAC/B,OAAO;AACL,cAAI,GAAG,IAAI;AAAA,QACb;AAAA,MACF;AAEA,YAAM,KAAK,GAAG;AACd;AAEA,UAAI,MAAM,UAAU,YAAY;AAC9B,aAAK,QAAQ,OAAO,OAAO;AAC3B,gBAAQ,CAAC;AAAA,MACX;AAAA,IACF;AAEA,QAAI,MAAM,SAAS,GAAG;AACpB,WAAK,QAAQ,OAAO,OAAO;AAAA,IAC7B;AAEA,WAAO,EAAE,SAAS,UAAU;AAAA,EAC9B;AACF;",
+  "names": []
+}

package/dist/data/parsers/index.js

@@ -0,0 +1,79 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parsers_exports = {};
+__export(parsers_exports, {
+  getAvailableExtensions: () => getAvailableExtensions,
+  getDataParser: () => getDataParser,
+  getUnavailableExtensions: () => getUnavailableExtensions,
+  isSupportedDataFormat: () => isSupportedDataFormat
+});
+module.exports = __toCommonJS(parsers_exports);
+var import_csv = require("./csv");
+var import_jsonl = require("./jsonl");
+var import_json_array = require("./json-array");
+var import_excel = require("./excel");
+var import_parquet = require("./parquet");
+const allParsers = [
+  import_csv.csvParser,
+  import_csv.tsvParser,
+  import_jsonl.jsonlParser,
+  import_json_array.jsonArrayParser,
+  import_excel.excelParser,
+  import_parquet.parquetParser
+];
+const extensionMap = /* @__PURE__ */ new Map();
+for (const parser of allParsers) {
+  for (const ext of parser.extensions) {
+    extensionMap.set(ext.toLowerCase(), parser);
+  }
+}
+function getDataParser(filePath) {
+  const ext = filePath.slice(filePath.lastIndexOf(".")).toLowerCase();
+  const parser = extensionMap.get(ext) ?? null;
+  if (!parser) return null;
+  if (!parser.isAvailable()) return null;
+  return parser;
+}
+function isSupportedDataFormat(filePath) {
+  return getDataParser(filePath) !== null;
+}
+function getAvailableExtensions() {
+  const available = [];
+  for (const [ext, parser] of extensionMap) {
+    if (parser.isAvailable()) available.push(ext);
+  }
+  return available;
+}
+function getUnavailableExtensions() {
+  const unavailable = [];
+  for (const [ext, parser] of extensionMap) {
+    if (!parser.isAvailable()) {
+      unavailable.push({ ext, parser: parser.name });
+    }
+  }
+  return unavailable;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getAvailableExtensions,
+  getDataParser,
+  getUnavailableExtensions,
+  isSupportedDataFormat
+});
+//# sourceMappingURL=index.js.map

package/dist/data/parsers/index.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/index.ts"],
+  "sourcesContent": ["/**\n * Data Format Registry\n *\n * Maps file extensions to their appropriate parser.\n * Gracefully skips formats whose optional dependencies are not installed.\n */\n\nimport type { DataFormatParser } from '../types';\nimport { csvParser, tsvParser } from './csv';\nimport { jsonlParser } from './jsonl';\nimport { jsonArrayParser } from './json-array';\nimport { excelParser } from './excel';\nimport { parquetParser } from './parquet';\n\n// \u2500\u2500 Registry \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nconst allParsers: DataFormatParser[] = [\n  csvParser,\n  tsvParser,\n  jsonlParser,\n  jsonArrayParser,\n  excelParser,\n  parquetParser,\n];\n\n/** Extension \u2192 parser lookup */\nconst extensionMap = new Map<string, DataFormatParser>();\nfor (const parser of allParsers) {\n  for (const ext of parser.extensions) {\n    extensionMap.set(ext.toLowerCase(), parser);\n  }\n}\n\n// \u2500\u2500 Public API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n/**\n * Get the parser for a given file extension.\n * Returns null if no parser is registered or the optional dep is missing.\n */\nexport function getDataParser(filePath: string): DataFormatParser | null {\n  const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();\n  const parser = extensionMap.get(ext) ?? null;\n  if (!parser) return null;\n  if (!parser.isAvailable()) return null;\n  return parser;\n}\n\n/**\n * Check if a file extension is supported for data parsing.\n */\nexport function isSupportedDataFormat(filePath: string): boolean {\n  return getDataParser(filePath) !== null;\n}\n\n/**\n * Get all supported extensions (only those with available deps).\n */\nexport function getAvailableExtensions(): string[] {\n  const available: string[] = [];\n  for (const [ext, parser] of extensionMap) {\n    if (parser.isAvailable()) available.push(ext);\n  }\n  return available;\n}\n\n/**\n * Get extensions that are registered but have missing optional deps.\n */\nexport function getUnavailableExtensions(): Array<{ ext: string; parser: string }> {\n  const unavailable: Array<{ ext: string; parser: string }> = [];\n  for (const [ext, parser] of extensionMap) {\n    if (!parser.isAvailable()) {\n      unavailable.push({ ext, parser: parser.name });\n    }\n  }\n  return unavailable;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,iBAAqC;AACrC,mBAA4B;AAC5B,wBAAgC;AAChC,mBAA4B;AAC5B,qBAA8B;AAI9B,MAAM,aAAiC;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,eAAe,oBAAI,IAA8B;AACvD,WAAW,UAAU,YAAY;AAC/B,aAAW,OAAO,OAAO,YAAY;AACnC,iBAAa,IAAI,IAAI,YAAY,GAAG,MAAM;AAAA,EAC5C;AACF;AAQO,SAAS,cAAc,UAA2C;AACvE,QAAM,MAAM,SAAS,MAAM,SAAS,YAAY,GAAG,CAAC,EAAE,YAAY;AAClE,QAAM,SAAS,aAAa,IAAI,GAAG,KAAK;AACxC,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI,CAAC,OAAO,YAAY,EAAG,QAAO;AAClC,SAAO;AACT;AAKO,SAAS,sBAAsB,UAA2B;AAC/D,SAAO,cAAc,QAAQ,MAAM;AACrC;AAKO,SAAS,yBAAmC;AACjD,QAAM,YAAsB,CAAC;AAC7B,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,OAAO,YAAY,EAAG,WAAU,KAAK,GAAG;AAAA,EAC9C;AACA,SAAO;AACT;AAKO,SAAS,2BAAmE;AACjF,QAAM,cAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,CAAC,OAAO,YAAY,GAAG;AACzB,kBAAY,KAAK,EAAE,KAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC/C;AAAA,EACF;AACA,SAAO;AACT;",
+  "names": []
+}