kirograph 0.20.0 → 0.22.0

package/dist/data/indexer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/data/indexer.ts"],
-  "sourcesContent": ["/**\n * Data Indexer\n *\n * Orchestrates: scan data files \u2192 parse \u2192 profile columns \u2192 persist rows to SQLite.\n * Supports incremental re-indexing via content hashes.\n */\n\nimport * as fs from 'fs';\nimport * as path from 'path';\nimport * as crypto from 'crypto';\nimport picomatch from 'picomatch';\nimport type { KiroGraphConfig } from '../config';\nimport type { DataSet, DataColumn, DataIndexResult, ParsedRow } from './types';\nimport { getDataParser } from './parsers/index';\nimport { ColumnProfiler } from './profiler';\n\n/**\n * Generate a dataset ID from a file path.\n * Slugifies the relative path: tests/fixtures/users.csv \u2192 tests-fixtures-users\n */\nfunction generateDatasetId(filePath: string): string {\n  return filePath\n    .replace(/\\.[^.]+$/, '')  // remove extension\n    .replace(/[/\\\\]/g, '-')   // replace path separators\n    .replace(/[^a-zA-Z0-9-_]/g, '-') // replace special chars\n    .replace(/-+/g, '-')      // collapse multiple dashes\n    .replace(/^-|-$/g, '')    // trim leading/trailing dashes\n    .toLowerCase();\n}\n\nexport class DataIndexer {\n  private readonly db: any;\n  private readonly config: KiroGraphConfig;\n  private readonly projectRoot: string;\n\n  constructor(db: any, config: KiroGraphConfig, projectRoot: string) {\n    this.db = db;\n    this.config = config;\n    this.projectRoot = projectRoot;\n  }\n\n  async indexAll(opts?: { force?: boolean; onProgress?: (msg: string) => void }): Promise<DataIndexResult> {\n    const start = Date.now();\n    const result: DataIndexResult = {\n      datasetsIndexed: 0, rowsIndexed: 0, columnsProfiled: 0, errors: [], duration: 0,\n    };\n\n    const files = this.scanDataFiles();\n    opts?.onProgress?.(`data: found ${files.length} data files`);\n\n    for (const relPath of files) {\n      try {\n        const indexed = await this.indexFile(relPath, opts?.force ?? false);\n        if (indexed) {\n          result.datasetsIndexed++;\n          result.rowsIndexed += indexed.rows;\n          result.columnsProfiled += indexed.columns;\n        }\n      } catch (err) {\n        result.errors.push(`${relPath}: ${err instanceof Error ? err.message : String(err)}`);\n      }\n    }\n\n    // Remove datasets for files that no longer exist\n    this.removeDeletedDatasets(files);\n\n    // Link data files to code (if enabled)\n    if ((this.config as any).dataLinkCode !== false && result.datasetsIndexed > 0) {\n      try {\n        const { DataCodeLinker } = await Promise.resolve().then(() => require('./linker.js'));\n        const linker = new DataCodeLinker(this.db, this.projectRoot);\n        const refsCreated = linker.linkAll();\n        opts?.onProgress?.(`data: linked ${refsCreated} code references`);\n      } catch {\n        // Non-critical \u2014 linking failure shouldn't block indexing\n      }\n    }\n\n    result.duration = Date.now() - start;\n    return result;\n  }\n\n  async indexFile(relPath: string, force = false): Promise<{ rows: number; columns: number } | null> {\n    const absPath = path.join(this.projectRoot, relPath);\n    if (!fs.existsSync(absPath)) return null;\n\n    const stat = fs.statSync(absPath);\n    if (stat.size > (this.config as any).dataMaxFileSize) return null;\n\n    // Check content hash for incremental\n    const fileBuffer = fs.readFileSync(absPath);\n    const contentHash = crypto.createHash('sha256').update(fileBuffer).digest('hex');\n\n    const datasetId = generateDatasetId(relPath);\n\n    if (!force) {\n      const existing = this.db.get('SELECT content_hash FROM data_datasets WHERE id = ?', [datasetId]);\n      if (existing?.content_hash === contentHash) return null; // unchanged\n    }\n\n    // Get parser\n    const parser = getDataParser(relPath);\n    if (!parser) return null;\n\n    // Parse and profile\n    const profiler = new ColumnProfiler();\n    const allRows: ParsedRow[] = [];\n    const maxRows = (this.config as any).dataMaxRows ?? 1_000_000;\n\n    const { columns, totalRows } = await parser.parse(absPath, {\n      maxRows,\n      onBatch: (rows, cols) => {\n        profiler.addBatch(rows, cols);\n        allRows.push(...rows);\n      },\n    });\n\n    if (columns.length === 0) return null;\n\n    // Finalize profiles\n    const columnProfiles = profiler.finalize(datasetId);\n\n    // Persist in transaction\n    this.db.run('BEGIN');\n    try {\n      // Delete old dataset data\n      const rowTableName = `data_rows_${datasetId.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n      this.db.run(`DROP TABLE IF EXISTS \"${rowTableName}\"`);\n      this.db.run('DELETE FROM data_columns WHERE dataset_id = ?', [datasetId]);\n      this.db.run('DELETE FROM data_datasets WHERE id = ?', [datasetId]);\n\n      // Insert dataset\n      this.db.run(`\n        INSERT INTO data_datasets (id, file_path, format, row_count, column_count, file_size, content_hash, summary, indexed_at)\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n      `, [datasetId, relPath, parser.name, totalRows, columns.length, stat.size, contentHash, null, Date.now()]);\n\n      // Insert column profiles\n      for (const col of columnProfiles) {\n        this.db.run(`\n          INSERT INTO data_columns (id, dataset_id, name, position, inferred_type, nullable, null_count, null_pct, cardinality, min_value, max_value, mean_value, sample_values, summary, updated_at)\n          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n        `, [\n          col.id, col.datasetId, col.name, col.position, col.inferredType,\n          col.nullable ? 1 : 0, col.nullCount, col.nullPct, col.cardinality,\n          col.minValue, col.maxValue, col.meanValue,\n          JSON.stringify(col.sampleValues), col.summary, col.updatedAt,\n        ]);\n      }\n\n      // Create row table with dynamic schema\n      const colDefs = columns.map(c => `\"${c.replace(/\"/g, '\"\"')}\" TEXT`).join(', ');\n      this.db.run(`CREATE TABLE \"${rowTableName}\" (rowid INTEGER PRIMARY KEY AUTOINCREMENT, ${colDefs})`);\n\n      // Insert rows in batches\n      const colNames = columns.map(c => `\"${c.replace(/\"/g, '\"\"')}\"`).join(', ');\n      const placeholders = columns.map(() => '?').join(', ');\n      const insertStmt = `INSERT INTO \"${rowTableName}\" (${colNames}) VALUES (${placeholders})`;\n\n      for (const row of allRows) {\n        const values = columns.map(c => {\n          const v = row[c];\n          if (v === null || v === undefined) return null;\n          if (typeof v === 'object') return JSON.stringify(v);\n          return v;\n        });\n        this.db.run(insertStmt, values);\n      }\n\n      // Save history snapshot for drift detection\n      const historyColumns = columnProfiles.map(col => ({\n        name: col.name,\n        type: col.inferredType,\n        cardinality: col.cardinality,\n        nullPct: col.nullPct,\n      }));\n      this.db.run(`\n        INSERT INTO data_dataset_history (dataset_id, snapshot_at, row_count, column_count, columns_json, content_hash)\n        VALUES (?, ?, ?, ?, ?, ?)\n      `, [datasetId, Date.now(), totalRows, columns.length, JSON.stringify(historyColumns), contentHash]);\n\n      this.db.run('COMMIT');\n    } catch (err) {\n      this.db.run('ROLLBACK');\n      throw err;\n    }\n\n    return { rows: totalRows, columns: columns.length };\n  }\n\n  // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  private scanDataFiles(): string[] {\n    const includePatterns = (this.config as any).dataInclude ?? [];\n    const excludePatterns = (this.config as any).dataExclude ?? [];\n    const includeMatchers = includePatterns.map((p: string) => picomatch(p));\n    const excludeMatchers = excludePatterns.map((p: string) => picomatch(p));\n    const results: string[] = [];\n\n    const walk = (dir: string) => {\n      let entries: fs.Dirent[];\n      try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }\n\n      for (const entry of entries) {\n        const full = path.join(dir, entry.name);\n        const rel = path.relative(this.projectRoot, full).replace(/\\\\/g, '/');\n\n        if (entry.isDirectory()) {\n          if (excludeMatchers.some((m: any) => m(rel + '/'))) continue;\n          walk(full);\n        } else if (entry.isFile()) {\n          if (excludeMatchers.some((m: any) => m(rel))) continue;\n          if (includeMatchers.some((m: any) => m(rel))) {\n            // Verify a parser exists for this format\n            if (getDataParser(rel)) {\n              results.push(rel);\n            }\n          }\n        }\n      }\n    };\n\n    walk(this.projectRoot);\n    return results;\n  }\n\n  private removeDeletedDatasets(currentFiles: string[]): void {\n    const currentSet = new Set(currentFiles.map(f => generateDatasetId(f)));\n    const indexed = (this.db.all('SELECT id FROM data_datasets') as Array<{ id: string }>).map(r => r.id);\n\n    for (const id of indexed) {\n      if (!currentSet.has(id)) {\n        const rowTableName = `data_rows_${id.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n        this.db.run(`DROP TABLE IF EXISTS \"${rowTableName}\"`);\n        this.db.run('DELETE FROM data_columns WHERE dataset_id = ?', [id]);\n        this.db.run('DELETE FROM data_code_refs WHERE dataset_id = ?', [id]);\n        this.db.run('DELETE FROM data_datasets WHERE id = ?', [id]);\n      }\n    }\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,SAAoB;AACpB,WAAsB;AACtB,aAAwB;AACxB,uBAAsB;AAGtB,qBAA8B;AAC9B,sBAA+B;AAM/B,SAAS,kBAAkB,UAA0B;AACnD,SAAO,SACJ,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE,EACpB,YAAY;AACjB;AAEO,MAAM,YAAY;AAAA,EAKvB,YAAY,IAAS,QAAyB,aAAqB;AACjE,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAM,SAAS,MAA0F;AACvG,UAAM,QAAQ,KAAK,IAAI;AACvB,UAAM,SAA0B;AAAA,MAC9B,iBAAiB;AAAA,MAAG,aAAa;AAAA,MAAG,iBAAiB;AAAA,MAAG,QAAQ,CAAC;AAAA,MAAG,UAAU;AAAA,IAChF;AAEA,UAAM,QAAQ,KAAK,cAAc;AACjC,UAAM,aAAa,eAAe,MAAM,MAAM,aAAa;AAE3D,eAAW,WAAW,OAAO;AAC3B,UAAI;AACF,cAAM,UAAU,MAAM,KAAK,UAAU,SAAS,MAAM,SAAS,KAAK;AAClE,YAAI,SAAS;AACX,iBAAO;AACP,iBAAO,eAAe,QAAQ;AAC9B,iBAAO,mBAAmB,QAAQ;AAAA,QACpC;AAAA,MACF,SAAS,KAAK;AACZ,eAAO,OAAO,KAAK,GAAG,OAAO,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AAGA,SAAK,sBAAsB,KAAK;AAGhC,QAAK,KAAK,OAAe,iBAAiB,SAAS,OAAO,kBAAkB,GAAG;AAC7E,UAAI;AACF,cAAM,EAAE,eAAe,IAAI,MAAM,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ,aAAa,CAAC;AACpF,cAAM,SAAS,IAAI,eAAe,KAAK,IAAI,KAAK,WAAW;AAC3D,cAAM,cAAc,OAAO,QAAQ;AACnC,cAAM,aAAa,gBAAgB,WAAW,kBAAkB;AAAA,MAClE,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,WAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,SAAiB,QAAQ,OAA0D;AACjG,UAAM,UAAU,KAAK,KAAK,KAAK,aAAa,OAAO;AACnD,QAAI,CAAC,GAAG,WAAW,OAAO,EAAG,QAAO;AAEpC,UAAM,OAAO,GAAG,SAAS,OAAO;AAChC,QAAI,KAAK,OAAQ,KAAK,OAAe,gBAAiB,QAAO;AAG7D,UAAM,aAAa,GAAG,aAAa,OAAO;AAC1C,UAAM,cAAc,OAAO,WAAW,QAAQ,EAAE,OAAO,UAAU,EAAE,OAAO,KAAK;AAE/E,UAAM,YAAY,kBAAkB,OAAO;AAE3C,QAAI,CAAC,OAAO;AACV,YAAM,WAAW,KAAK,GAAG,IAAI,uDAAuD,CAAC,SAAS,CAAC;AAC/F,UAAI,UAAU,iBAAiB,YAAa,QAAO;AAAA,IACrD;AAGA,UAAM,aAAS,8BAAc,OAAO;AACpC,QAAI,CAAC,OAAQ,QAAO;AAGpB,UAAM,WAAW,IAAI,+BAAe;AACpC,UAAM,UAAuB,CAAC;AAC9B,UAAM,UAAW,KAAK,OAAe,eAAe;AAEpD,UAAM,EAAE,SAAS,UAAU,IAAI,MAAM,OAAO,MAAM,SAAS;AAAA,MACzD;AAAA,MACA,SAAS,CAAC,MAAM,SAAS;AACvB,iBAAS,SAAS,MAAM,IAAI;AAC5B,gBAAQ,KAAK,GAAG,IAAI;AAAA,MACtB;AAAA,IACF,CAAC;AAED,QAAI,QAAQ,WAAW,EAAG,QAAO;AAGjC,UAAM,iBAAiB,SAAS,SAAS,SAAS;AAGlD,SAAK,GAAG,IAAI,OAAO;AACnB,QAAI;AAEF,YAAM,eAAe,aAAa,UAAU,QAAQ,kBAAkB,GAAG,CAAC;AAC1E,WAAK,GAAG,IAAI,yBAAyB,YAAY,GAAG;AACpD,WAAK,GAAG,IAAI,iDAAiD,CAAC,SAAS,CAAC;AACxE,WAAK,GAAG,IAAI,0CAA0C,CAAC,SAAS,CAAC;AAGjE,WAAK,GAAG,IAAI;AAAA;AAAA;AAAA,SAGT,CAAC,WAAW,SAAS,OAAO,MAAM,WAAW,QAAQ,QAAQ,KAAK,MAAM,aAAa,MAAM,KAAK,IAAI,CAAC,CAAC;AAGzG,iBAAW,OAAO,gBAAgB;AAChC,aAAK,GAAG,IAAI;AAAA;AAAA;AAAA,WAGT;AAAA,UACD,IAAI;AAAA,UAAI,IAAI;AAAA,UAAW,IAAI;AAAA,UAAM,IAAI;AAAA,UAAU,IAAI;AAAA,UACnD,IAAI,WAAW,IAAI;AAAA,UAAG,IAAI;AAAA,UAAW,IAAI;AAAA,UAAS,IAAI;AAAA,UACtD,IAAI;AAAA,UAAU,IAAI;AAAA,UAAU,IAAI;AAAA,UAChC,KAAK,UAAU,IAAI,YAAY;AAAA,UAAG,IAAI;AAAA,UAAS,IAAI;AAAA,QACrD,CAAC;AAAA,MACH;AAGA,YAAM,UAAU,QAAQ,IAAI,OAAK,IAAI,EAAE,QAAQ,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI;AAC7E,WAAK,GAAG,IAAI,iBAAiB,YAAY,+CAA+C,OAAO,GAAG;AAGlG,YAAM,WAAW,QAAQ,IAAI,OAAK,IAAI,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI;AACzE,YAAM,eAAe,QAAQ,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AACrD,YAAM,aAAa,gBAAgB,YAAY,MAAM,QAAQ,aAAa,YAAY;AAEtF,iBAAW,OAAO,SAAS;AACzB,cAAM,SAAS,QAAQ,IAAI,OAAK;AAC9B,gBAAM,IAAI,IAAI,CAAC;AACf,cAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,cAAI,OAAO,MAAM,SAAU,QAAO,KAAK,UAAU,CAAC;AAClD,iBAAO;AAAA,QACT,CAAC;AACD,aAAK,GAAG,IAAI,YAAY,MAAM;AAAA,MAChC;AAGA,YAAM,iBAAiB,eAAe,IAAI,UAAQ;AAAA,QAChD,MAAM,IAAI;AAAA,QACV,MAAM,IAAI;AAAA,QACV,aAAa,IAAI;AAAA,QACjB,SAAS,IAAI;AAAA,MACf,EAAE;AACF,WAAK,GAAG,IAAI;AAAA;AAAA;AAAA,SAGT,CAAC,WAAW,KAAK,IAAI,GAAG,WAAW,QAAQ,QAAQ,KAAK,UAAU,cAAc,GAAG,WAAW,CAAC;AAElG,WAAK,GAAG,IAAI,QAAQ;AAAA,IACtB,SAAS,KAAK;AACZ,WAAK,GAAG,IAAI,UAAU;AACtB,YAAM;AAAA,IACR;AAEA,WAAO,EAAE,MAAM,WAAW,SAAS,QAAQ,OAAO;AAAA,EACpD;AAAA;AAAA,EAIQ,gBAA0B;AAChC,UAAM,kBAAmB,KAAK,OAAe,eAAe,CAAC;AAC7D,UAAM,kBAAmB,KAAK,OAAe,eAAe,CAAC;AAC7D,UAAM,kBAAkB,gBAAgB,IAAI,CAAC,UAAc,iBAAAA,SAAU,CAAC,CAAC;AACvE,UAAM,kBAAkB,gBAAgB,IAAI,CAAC,UAAc,iBAAAA,SAAU,CAAC,CAAC;AACvE,UAAM,UAAoB,CAAC;AAE3B,UAAM,OAAO,CAAC,QAAgB;AAC5B,UAAI;AACJ,UAAI;AAAE,kBAAU,GAAG,YAAY,KAAK,EAAE,eAAe,KAAK,CAAC;AAAA,MAAG,QAAQ;AAAE;AAAA,MAAQ;AAEhF,iBAAW,SAAS,SAAS;AAC3B,cAAM,OAAO,KAAK,KAAK,KAAK,MAAM,IAAI;AACtC,cAAM,MAAM,KAAK,SAAS,KAAK,aAAa,IAAI,EAAE,QAAQ,OAAO,GAAG;AAEpE,YAAI,MAAM,YAAY,GAAG;AACvB,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,MAAM,GAAG,CAAC,EAAG;AACpD,eAAK,IAAI;AAAA,QACX,WAAW,MAAM,OAAO,GAAG;AACzB,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,GAAG,CAAC,EAAG;AAC9C,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,GAAG,CAAC,GAAG;AAE5C,oBAAI,8BAAc,GAAG,GAAG;AACtB,sBAAQ,KAAK,GAAG;AAAA,YAClB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,KAAK,WAAW;AACrB,WAAO;AAAA,EACT;AAAA,EAEQ,sBAAsB,cAA8B;AAC1D,UAAM,aAAa,IAAI,IAAI,aAAa,IAAI,OAAK,kBAAkB,CAAC,CAAC,CAAC;AACtE,UAAM,UAAW,KAAK,GAAG,IAAI,8BAA8B,EAA4B,IAAI,OAAK,EAAE,EAAE;AAEpG,eAAW,MAAM,SAAS;AACxB,UAAI,CAAC,WAAW,IAAI,EAAE,GAAG;AACvB,cAAM,eAAe,aAAa,GAAG,QAAQ,kBAAkB,GAAG,CAAC;AACnE,aAAK,GAAG,IAAI,yBAAyB,YAAY,GAAG;AACpD,aAAK,GAAG,IAAI,iDAAiD,CAAC,EAAE,CAAC;AACjE,aAAK,GAAG,IAAI,mDAAmD,CAAC,EAAE,CAAC;AACnE,aAAK,GAAG,IAAI,0CAA0C,CAAC,EAAE,CAAC;AAAA,MAC5D;AAAA,IACF;AAAA,EACF;AACF;",
+  "sourcesContent": ["/**\n * Data Indexer\n *\n * Orchestrates: scan data files \u2192 parse \u2192 profile columns \u2192 persist rows to SQLite.\n * Supports incremental re-indexing via content hashes.\n */\n\nimport * as fs from 'fs';\nimport * as path from 'path';\nimport * as crypto from 'crypto';\nimport picomatch from 'picomatch';\nimport type { KiroGraphConfig } from '../config';\nimport type { DataSet, DataColumn, DataIndexResult, ParsedRow } from './types';\nimport { getDataParser } from './parsers/index';\nimport { ColumnProfiler } from './profiler';\n\n/**\n * Generate a dataset ID from a file path.\n * Slugifies the relative path: tests/fixtures/users.csv \u2192 tests-fixtures-users\n */\nfunction generateDatasetId(filePath: string): string {\n  return filePath\n    .replace(/\\.[^.]+$/, '')  // remove extension\n    .replace(/[/\\\\]/g, '-')   // replace path separators\n    .replace(/[^a-zA-Z0-9-_]/g, '-') // replace special chars\n    .replace(/-+/g, '-')      // collapse multiple dashes\n    .replace(/^-|-$/g, '')    // trim leading/trailing dashes\n    .toLowerCase();\n}\n\nexport class DataIndexer {\n  private readonly db: any;\n  private readonly config: KiroGraphConfig;\n  private readonly projectRoot: string;\n\n  constructor(db: any, config: KiroGraphConfig, projectRoot: string) {\n    this.db = db;\n    this.config = config;\n    this.projectRoot = projectRoot;\n  }\n\n  async indexAll(opts?: { force?: boolean; onProgress?: (msg: string) => void }): Promise<DataIndexResult> {\n    const start = Date.now();\n    const result: DataIndexResult = {\n      datasetsIndexed: 0, rowsIndexed: 0, columnsProfiled: 0, errors: [], duration: 0,\n    };\n\n    const files = this.scanDataFiles();\n    opts?.onProgress?.(`data: found ${files.length} data files`);\n\n    for (const relPath of files) {\n      try {\n        const indexed = await this.indexFile(relPath, opts?.force ?? false);\n        if (indexed) {\n          result.datasetsIndexed++;\n          result.rowsIndexed += indexed.rows;\n          result.columnsProfiled += indexed.columns;\n        }\n      } catch (err) {\n        result.errors.push(`${relPath}: ${err instanceof Error ? err.message : String(err)}`);\n      }\n    }\n\n    // Remove datasets for files that no longer exist\n    this.removeDeletedDatasets(files);\n\n    // Link data files to code (if enabled)\n    if ((this.config as any).dataLinkCode !== false && result.datasetsIndexed > 0) {\n      try {\n        const { DataCodeLinker } = await Promise.resolve().then(() => require('./linker.js'));\n        const linker = new DataCodeLinker(this.db, this.projectRoot);\n        const refsCreated = linker.linkAll();\n        opts?.onProgress?.(`data: linked ${refsCreated} code references`);\n      } catch {\n        // Non-critical \u2014 linking failure shouldn't block indexing\n      }\n    }\n\n    result.duration = Date.now() - start;\n    return result;\n  }\n\n  async indexFile(relPath: string, force = false): Promise<{ rows: number; columns: number } | null> {\n    const absPath = path.join(this.projectRoot, relPath);\n    if (!fs.existsSync(absPath)) return null;\n\n    const stat = fs.statSync(absPath);\n    if (stat.size > (this.config as any).dataMaxFileSize) return null;\n\n    // Check content hash for incremental\n    const fileBuffer = fs.readFileSync(absPath);\n    const contentHash = crypto.createHash('sha256').update(fileBuffer).digest('hex');\n\n    const datasetId = generateDatasetId(relPath);\n\n    if (!force) {\n      const existing = this.db.get('SELECT content_hash FROM data_datasets WHERE id = ?', [datasetId]);\n      if (existing?.content_hash === contentHash) return null; // unchanged\n    }\n\n    // Get parser\n    const parser = getDataParser(relPath);\n    if (!parser) return null;\n\n    // Parse and profile\n    const profiler = new ColumnProfiler();\n    const allRows: ParsedRow[] = [];\n    const maxRows = (this.config as any).dataMaxRows ?? 1_000_000;\n\n    const { columns, totalRows, metadataJson } = await parser.parse(absPath, {\n      maxRows,\n      onBatch: (rows, cols) => {\n        profiler.addBatch(rows, cols);\n        allRows.push(...rows);\n      },\n    });\n\n    if (columns.length === 0) return null;\n\n    // Finalize profiles\n    const columnProfiles = profiler.finalize(datasetId);\n\n    // Persist in transaction\n    this.db.run('BEGIN');\n    try {\n      // Delete old dataset data\n      const rowTableName = `data_rows_${datasetId.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n      this.db.run(`DROP TABLE IF EXISTS \"${rowTableName}\"`);\n      this.db.run('DELETE FROM data_columns WHERE dataset_id = ?', [datasetId]);\n      this.db.run('DELETE FROM data_datasets WHERE id = ?', [datasetId]);\n\n      // Insert dataset\n      this.db.run(`\n        INSERT INTO data_datasets (id, file_path, format, row_count, column_count, file_size, content_hash, summary, indexed_at, metadata_json)\n        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n      `, [datasetId, relPath, parser.name, totalRows, columns.length, stat.size, contentHash, null, Date.now(), metadataJson ?? null]);\n\n      // Insert column profiles\n      for (const col of columnProfiles) {\n        this.db.run(`\n          INSERT INTO data_columns (id, dataset_id, name, position, inferred_type, nullable, null_count, null_pct, cardinality, min_value, max_value, mean_value, sample_values, summary, updated_at)\n          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n        `, [\n          col.id, col.datasetId, col.name, col.position, col.inferredType,\n          col.nullable ? 1 : 0, col.nullCount, col.nullPct, col.cardinality,\n          col.minValue, col.maxValue, col.meanValue,\n          JSON.stringify(col.sampleValues), col.summary, col.updatedAt,\n        ]);\n      }\n\n      // Create row table with dynamic schema\n      const colDefs = columns.map(c => `\"${c.replace(/\"/g, '\"\"')}\" TEXT`).join(', ');\n      this.db.run(`CREATE TABLE \"${rowTableName}\" (rowid INTEGER PRIMARY KEY AUTOINCREMENT, ${colDefs})`);\n\n      // Insert rows in batches\n      const colNames = columns.map(c => `\"${c.replace(/\"/g, '\"\"')}\"`).join(', ');\n      const placeholders = columns.map(() => '?').join(', ');\n      const insertStmt = `INSERT INTO \"${rowTableName}\" (${colNames}) VALUES (${placeholders})`;\n\n      for (const row of allRows) {\n        const values = columns.map(c => {\n          const v = row[c];\n          if (v === null || v === undefined) return null;\n          if (typeof v === 'object') return JSON.stringify(v);\n          return v;\n        });\n        this.db.run(insertStmt, values);\n      }\n\n      // Save history snapshot for drift detection\n      const historyColumns = columnProfiles.map(col => ({\n        name: col.name,\n        type: col.inferredType,\n        cardinality: col.cardinality,\n        nullPct: col.nullPct,\n      }));\n      this.db.run(`\n        INSERT INTO data_dataset_history (dataset_id, snapshot_at, row_count, column_count, columns_json, content_hash)\n        VALUES (?, ?, ?, ?, ?, ?)\n      `, [datasetId, Date.now(), totalRows, columns.length, JSON.stringify(historyColumns), contentHash]);\n\n      this.db.run('COMMIT');\n    } catch (err) {\n      this.db.run('ROLLBACK');\n      throw err;\n    }\n\n    return { rows: totalRows, columns: columns.length };\n  }\n\n  // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  private scanDataFiles(): string[] {\n    const includePatterns = (this.config as any).dataInclude ?? [];\n    const excludePatterns = (this.config as any).dataExclude ?? [];\n    const includeMatchers = includePatterns.map((p: string) => picomatch(p));\n    const excludeMatchers = excludePatterns.map((p: string) => picomatch(p));\n    const results: string[] = [];\n\n    const walk = (dir: string) => {\n      let entries: fs.Dirent[];\n      try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }\n\n      for (const entry of entries) {\n        const full = path.join(dir, entry.name);\n        const rel = path.relative(this.projectRoot, full).replace(/\\\\/g, '/');\n\n        if (entry.isDirectory()) {\n          if (excludeMatchers.some((m: any) => m(rel + '/'))) continue;\n          walk(full);\n        } else if (entry.isFile()) {\n          if (excludeMatchers.some((m: any) => m(rel))) continue;\n          if (includeMatchers.some((m: any) => m(rel))) {\n            // Verify a parser exists for this format\n            if (getDataParser(rel)) {\n              results.push(rel);\n            }\n          }\n        }\n      }\n    };\n\n    walk(this.projectRoot);\n    return results;\n  }\n\n  private removeDeletedDatasets(currentFiles: string[]): void {\n    const currentSet = new Set(currentFiles.map(f => generateDatasetId(f)));\n    const indexed = (this.db.all('SELECT id FROM data_datasets') as Array<{ id: string }>).map(r => r.id);\n\n    for (const id of indexed) {\n      if (!currentSet.has(id)) {\n        const rowTableName = `data_rows_${id.replace(/[^a-zA-Z0-9_]/g, '_')}`;\n        this.db.run(`DROP TABLE IF EXISTS \"${rowTableName}\"`);\n        this.db.run('DELETE FROM data_columns WHERE dataset_id = ?', [id]);\n        this.db.run('DELETE FROM data_code_refs WHERE dataset_id = ?', [id]);\n        this.db.run('DELETE FROM data_datasets WHERE id = ?', [id]);\n      }\n    }\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOA,SAAoB;AACpB,WAAsB;AACtB,aAAwB;AACxB,uBAAsB;AAGtB,qBAA8B;AAC9B,sBAA+B;AAM/B,SAAS,kBAAkB,UAA0B;AACnD,SAAO,SACJ,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE,EACpB,YAAY;AACjB;AAEO,MAAM,YAAY;AAAA,EAKvB,YAAY,IAAS,QAAyB,aAAqB;AACjE,SAAK,KAAK;AACV,SAAK,SAAS;AACd,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,MAAM,SAAS,MAA0F;AACvG,UAAM,QAAQ,KAAK,IAAI;AACvB,UAAM,SAA0B;AAAA,MAC9B,iBAAiB;AAAA,MAAG,aAAa;AAAA,MAAG,iBAAiB;AAAA,MAAG,QAAQ,CAAC;AAAA,MAAG,UAAU;AAAA,IAChF;AAEA,UAAM,QAAQ,KAAK,cAAc;AACjC,UAAM,aAAa,eAAe,MAAM,MAAM,aAAa;AAE3D,eAAW,WAAW,OAAO;AAC3B,UAAI;AACF,cAAM,UAAU,MAAM,KAAK,UAAU,SAAS,MAAM,SAAS,KAAK;AAClE,YAAI,SAAS;AACX,iBAAO;AACP,iBAAO,eAAe,QAAQ;AAC9B,iBAAO,mBAAmB,QAAQ;AAAA,QACpC;AAAA,MACF,SAAS,KAAK;AACZ,eAAO,OAAO,KAAK,GAAG,OAAO,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC,EAAE;AAAA,MACtF;AAAA,IACF;AAGA,SAAK,sBAAsB,KAAK;AAGhC,QAAK,KAAK,OAAe,iBAAiB,SAAS,OAAO,kBAAkB,GAAG;AAC7E,UAAI;AACF,cAAM,EAAE,eAAe,IAAI,MAAM,QAAQ,QAAQ,EAAE,KAAK,MAAM,QAAQ,aAAa,CAAC;AACpF,cAAM,SAAS,IAAI,eAAe,KAAK,IAAI,KAAK,WAAW;AAC3D,cAAM,cAAc,OAAO,QAAQ;AACnC,cAAM,aAAa,gBAAgB,WAAW,kBAAkB;AAAA,MAClE,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,WAAO,WAAW,KAAK,IAAI,IAAI;AAC/B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,UAAU,SAAiB,QAAQ,OAA0D;AACjG,UAAM,UAAU,KAAK,KAAK,KAAK,aAAa,OAAO;AACnD,QAAI,CAAC,GAAG,WAAW,OAAO,EAAG,QAAO;AAEpC,UAAM,OAAO,GAAG,SAAS,OAAO;AAChC,QAAI,KAAK,OAAQ,KAAK,OAAe,gBAAiB,QAAO;AAG7D,UAAM,aAAa,GAAG,aAAa,OAAO;AAC1C,UAAM,cAAc,OAAO,WAAW,QAAQ,EAAE,OAAO,UAAU,EAAE,OAAO,KAAK;AAE/E,UAAM,YAAY,kBAAkB,OAAO;AAE3C,QAAI,CAAC,OAAO;AACV,YAAM,WAAW,KAAK,GAAG,IAAI,uDAAuD,CAAC,SAAS,CAAC;AAC/F,UAAI,UAAU,iBAAiB,YAAa,QAAO;AAAA,IACrD;AAGA,UAAM,aAAS,8BAAc,OAAO;AACpC,QAAI,CAAC,OAAQ,QAAO;AAGpB,UAAM,WAAW,IAAI,+BAAe;AACpC,UAAM,UAAuB,CAAC;AAC9B,UAAM,UAAW,KAAK,OAAe,eAAe;AAEpD,UAAM,EAAE,SAAS,WAAW,aAAa,IAAI,MAAM,OAAO,MAAM,SAAS;AAAA,MACvE;AAAA,MACA,SAAS,CAAC,MAAM,SAAS;AACvB,iBAAS,SAAS,MAAM,IAAI;AAC5B,gBAAQ,KAAK,GAAG,IAAI;AAAA,MACtB;AAAA,IACF,CAAC;AAED,QAAI,QAAQ,WAAW,EAAG,QAAO;AAGjC,UAAM,iBAAiB,SAAS,SAAS,SAAS;AAGlD,SAAK,GAAG,IAAI,OAAO;AACnB,QAAI;AAEF,YAAM,eAAe,aAAa,UAAU,QAAQ,kBAAkB,GAAG,CAAC;AAC1E,WAAK,GAAG,IAAI,yBAAyB,YAAY,GAAG;AACpD,WAAK,GAAG,IAAI,iDAAiD,CAAC,SAAS,CAAC;AACxE,WAAK,GAAG,IAAI,0CAA0C,CAAC,SAAS,CAAC;AAGjE,WAAK,GAAG,IAAI;AAAA;AAAA;AAAA,SAGT,CAAC,WAAW,SAAS,OAAO,MAAM,WAAW,QAAQ,QAAQ,KAAK,MAAM,aAAa,MAAM,KAAK,IAAI,GAAG,gBAAgB,IAAI,CAAC;AAG/H,iBAAW,OAAO,gBAAgB;AAChC,aAAK,GAAG,IAAI;AAAA;AAAA;AAAA,WAGT;AAAA,UACD,IAAI;AAAA,UAAI,IAAI;AAAA,UAAW,IAAI;AAAA,UAAM,IAAI;AAAA,UAAU,IAAI;AAAA,UACnD,IAAI,WAAW,IAAI;AAAA,UAAG,IAAI;AAAA,UAAW,IAAI;AAAA,UAAS,IAAI;AAAA,UACtD,IAAI;AAAA,UAAU,IAAI;AAAA,UAAU,IAAI;AAAA,UAChC,KAAK,UAAU,IAAI,YAAY;AAAA,UAAG,IAAI;AAAA,UAAS,IAAI;AAAA,QACrD,CAAC;AAAA,MACH;AAGA,YAAM,UAAU,QAAQ,IAAI,OAAK,IAAI,EAAE,QAAQ,MAAM,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI;AAC7E,WAAK,GAAG,IAAI,iBAAiB,YAAY,+CAA+C,OAAO,GAAG;AAGlG,YAAM,WAAW,QAAQ,IAAI,OAAK,IAAI,EAAE,QAAQ,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI;AACzE,YAAM,eAAe,QAAQ,IAAI,MAAM,GAAG,EAAE,KAAK,IAAI;AACrD,YAAM,aAAa,gBAAgB,YAAY,MAAM,QAAQ,aAAa,YAAY;AAEtF,iBAAW,OAAO,SAAS;AACzB,cAAM,SAAS,QAAQ,IAAI,OAAK;AAC9B,gBAAM,IAAI,IAAI,CAAC;AACf,cAAI,MAAM,QAAQ,MAAM,OAAW,QAAO;AAC1C,cAAI,OAAO,MAAM,SAAU,QAAO,KAAK,UAAU,CAAC;AAClD,iBAAO;AAAA,QACT,CAAC;AACD,aAAK,GAAG,IAAI,YAAY,MAAM;AAAA,MAChC;AAGA,YAAM,iBAAiB,eAAe,IAAI,UAAQ;AAAA,QAChD,MAAM,IAAI;AAAA,QACV,MAAM,IAAI;AAAA,QACV,aAAa,IAAI;AAAA,QACjB,SAAS,IAAI;AAAA,MACf,EAAE;AACF,WAAK,GAAG,IAAI;AAAA;AAAA;AAAA,SAGT,CAAC,WAAW,KAAK,IAAI,GAAG,WAAW,QAAQ,QAAQ,KAAK,UAAU,cAAc,GAAG,WAAW,CAAC;AAElG,WAAK,GAAG,IAAI,QAAQ;AAAA,IACtB,SAAS,KAAK;AACZ,WAAK,GAAG,IAAI,UAAU;AACtB,YAAM;AAAA,IACR;AAEA,WAAO,EAAE,MAAM,WAAW,SAAS,QAAQ,OAAO;AAAA,EACpD;AAAA;AAAA,EAIQ,gBAA0B;AAChC,UAAM,kBAAmB,KAAK,OAAe,eAAe,CAAC;AAC7D,UAAM,kBAAmB,KAAK,OAAe,eAAe,CAAC;AAC7D,UAAM,kBAAkB,gBAAgB,IAAI,CAAC,UAAc,iBAAAA,SAAU,CAAC,CAAC;AACvE,UAAM,kBAAkB,gBAAgB,IAAI,CAAC,UAAc,iBAAAA,SAAU,CAAC,CAAC;AACvE,UAAM,UAAoB,CAAC;AAE3B,UAAM,OAAO,CAAC,QAAgB;AAC5B,UAAI;AACJ,UAAI;AAAE,kBAAU,GAAG,YAAY,KAAK,EAAE,eAAe,KAAK,CAAC;AAAA,MAAG,QAAQ;AAAE;AAAA,MAAQ;AAEhF,iBAAW,SAAS,SAAS;AAC3B,cAAM,OAAO,KAAK,KAAK,KAAK,MAAM,IAAI;AACtC,cAAM,MAAM,KAAK,SAAS,KAAK,aAAa,IAAI,EAAE,QAAQ,OAAO,GAAG;AAEpE,YAAI,MAAM,YAAY,GAAG;AACvB,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,MAAM,GAAG,CAAC,EAAG;AACpD,eAAK,IAAI;AAAA,QACX,WAAW,MAAM,OAAO,GAAG;AACzB,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,GAAG,CAAC,EAAG;AAC9C,cAAI,gBAAgB,KAAK,CAAC,MAAW,EAAE,GAAG,CAAC,GAAG;AAE5C,oBAAI,8BAAc,GAAG,GAAG;AACtB,sBAAQ,KAAK,GAAG;AAAA,YAClB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,SAAK,KAAK,WAAW;AACrB,WAAO;AAAA,EACT;AAAA,EAEQ,sBAAsB,cAA8B;AAC1D,UAAM,aAAa,IAAI,IAAI,aAAa,IAAI,OAAK,kBAAkB,CAAC,CAAC,CAAC;AACtE,UAAM,UAAW,KAAK,GAAG,IAAI,8BAA8B,EAA4B,IAAI,OAAK,EAAE,EAAE;AAEpG,eAAW,MAAM,SAAS;AACxB,UAAI,CAAC,WAAW,IAAI,EAAE,GAAG;AACvB,cAAM,eAAe,aAAa,GAAG,QAAQ,kBAAkB,GAAG,CAAC;AACnE,aAAK,GAAG,IAAI,yBAAyB,YAAY,GAAG;AACpD,aAAK,GAAG,IAAI,iDAAiD,CAAC,EAAE,CAAC;AACjE,aAAK,GAAG,IAAI,mDAAmD,CAAC,EAAE,CAAC;AACnE,aAAK,GAAG,IAAI,0CAA0C,CAAC,EAAE,CAAC;AAAA,MAC5D;AAAA,IACF;AAAA,EACF;AACF;",
   "names": ["picomatch"]
 }

package/dist/data/linker.js

@@ -33,13 +33,17 @@ const READ_PATTERNS = [
   /read_json\s*\(\s*['"`]([^'"`]+)['"`]/g,
   /read_parquet\s*\(\s*['"`]([^'"`]+)['"`]/g,
   /read_table\s*\(\s*['"`]([^'"`]+)['"`]/g,
-  // Python open
-  /open\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet))['"`]/g,
+  // Python open (includes PDF)
+  /open\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet|pdf))['"`]/g,
+  // Python PDF libraries
+  /PdfReader\s*\(\s*['"`]([^'"`]+\.pdf)['"`]/g,
+  /pdfplumber\.open\s*\(\s*['"`]([^'"`]+\.pdf)['"`]/g,
+  /fitz\.open\s*\(\s*['"`]([^'"`]+\.pdf)['"`]/g,
   // SQL COPY
   /COPY\s+\w+\s+FROM\s+['"`]([^'"`]+)['"`]/gi,
   /\\copy\s+\w+\s+from\s+['"`]([^'"`]+)['"`]/gi,
   // Generic path references in strings (data file extensions)
-  /['"`]((?:\.\.?\/)?(?:[\w\-./]+\/)?[\w\-]+\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet))['"`]/g
+  /['"`]((?:\.\.?\/)?(?:[\w\-./]+\/)?[\w\-]+\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet|pdf))['"`]/g
 ];
 const WRITE_PATTERNS = [
   /writeFileSync\s*\(\s*['"`]([^'"`]+\.(?:csv|tsv|jsonl|json|xlsx|parquet))['"`]/g,
@@ -92,7 +96,7 @@ function normalizePath(p) {
 function isDataFilePath(filePath, knownPaths) {
   if (knownPaths.has(filePath)) return true;
   const ext = filePath.split(".").pop()?.toLowerCase();
-  return ["csv", "tsv", "jsonl", "ndjson", "json", "xlsx", "xls", "parquet"].includes(ext ?? "");
+  return ["csv", "tsv", "jsonl", "ndjson", "json", "xlsx", "xls", "parquet", "pdf"].includes(ext ?? "");
 }
 class DataCodeLinker {
   constructor(db, projectRoot) {
@@ -111,7 +115,7 @@ class DataCodeLinker {
     const knownPaths = new Set(datasets.map((d) => d.file_path));
     const pathToDatasetId = new Map(datasets.map((d) => [d.file_path, d.id]));
     const sourceFiles = this.db.all(
-      `SELECT file_path FROM files WHERE file_path NOT LIKE '%.csv' AND file_path NOT LIKE '%.tsv' AND file_path NOT LIKE '%.jsonl' AND file_path NOT LIKE '%.ndjson' AND file_path NOT LIKE '%.xlsx' AND file_path NOT LIKE '%.parquet'`
+      `SELECT path AS file_path FROM files WHERE path NOT LIKE '%.csv' AND path NOT LIKE '%.tsv' AND path NOT LIKE '%.jsonl' AND path NOT LIKE '%.ndjson' AND path NOT LIKE '%.xlsx' AND path NOT LIKE '%.parquet' AND path NOT LIKE '%.pdf'`
     );
     this.db.run("DELETE FROM data_code_refs");
     let refsCreated = 0;

package/dist/data/linker.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../src/data/linker.ts"],
-  "sourcesContent": ["/**\n * Data \u2194 Code Linker\n *\n * Detects references to data files in source code and populates data_code_refs.\n * Patterns detected:\n *   - readFileSync('data/users.csv'), readFile('path/to/file.csv')\n *   - pd.read_csv('data/file.csv'), pd.read_excel(...)\n *   - open('data/file.csv'), csv.reader(open(...))\n *   - COPY FROM 'file.csv', \\copy ... from 'file.csv'\n *   - require('./data/file.json'), import ... from './data/file.json'\n *   - fs.createReadStream('data/file.csv')\n *   - String literals matching known data file paths\n */\n\nimport type { DataCodeRef } from './types';\n\n/** Patterns that indicate a code symbol reads a data file */\nconst READ_PATTERNS = [\n  // Node.js / TypeScript\n  /readFileSync\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /readFile\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createReadStream\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python pandas\n  /read_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_table\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python open\n  /open\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet))['\"`]/g,\n  // SQL COPY\n  /COPY\\s+\\w+\\s+FROM\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  /\\\\copy\\s+\\w+\\s+from\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  // Generic path references in strings (data file extensions)\n  /['\"`]((?:\\.\\.?\\/)?(?:[\\w\\-./]+\\/)?[\\w\\-]+\\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet))['\"`]/g,\n];\n\n/** Patterns that indicate a code symbol writes a data file */\nconst WRITE_PATTERNS = [\n  /writeFileSync\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /writeFile\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /to_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createWriteStream\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /COPY\\s+\\w+\\s+TO\\s+['\"`]([^'\"`]+)['\"`]/gi,\n];\n\n/**\n * Scan source code content for references to data files.\n * Returns matched file paths with their reference type.\n */\nexport function detectDataReferences(\n  sourceContent: string,\n  knownDataPaths: Set<string>,\n): Array<{ filePath: string; refType: 'reads' | 'writes'; confidence: number }> {\n  const refs = new Map<string, { refType: 'reads' | 'writes'; confidence: number }>();\n\n  // Check read patterns\n  for (const pattern of READ_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const existing = refs.get(filePath);\n        if (!existing || existing.confidence < 0.9) {\n          const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n          refs.set(filePath, { refType: 'reads', confidence });\n        }\n      }\n    }\n  }\n\n  // Check write patterns\n  for (const pattern of WRITE_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n        refs.set(filePath, { refType: 'writes', confidence });\n      }\n    }\n  }\n\n  return Array.from(refs.entries()).map(([filePath, info]) => ({\n    filePath,\n    ...info,\n  }));\n}\n\n/**\n * Generate dataset ID from a file path (same logic as indexer).\n */\nfunction generateDatasetId(filePath: string): string {\n  return filePath\n    .replace(/\\.[^.]+$/, '')\n    .replace(/[/\\\\]/g, '-')\n    .replace(/[^a-zA-Z0-9-_]/g, '-')\n    .replace(/-+/g, '-')\n    .replace(/^-|-$/g, '')\n    .toLowerCase();\n}\n\n/**\n * Normalize a path reference found in code.\n * Strips leading ./ and resolves simple relative paths.\n */\nfunction normalizePath(p: string): string {\n  return p.replace(/^\\.\\//, '').replace(/\\\\/g, '/');\n}\n\n/**\n * Check if a path looks like a data file (by extension or known path match).\n */\nfunction isDataFilePath(filePath: string, knownPaths: Set<string>): boolean {\n  if (knownPaths.has(filePath)) return true;\n  // Check extension\n  const ext = filePath.split('.').pop()?.toLowerCase();\n  return ['csv', 'tsv', 'jsonl', 'ndjson', 'json', 'xlsx', 'xls', 'parquet'].includes(ext ?? '');\n}\n\n/**\n * Link data files to code symbols.\n * Scans all indexed source files for references to known data file paths.\n */\nexport class DataCodeLinker {\n  private readonly db: any;\n  private readonly projectRoot: string;\n\n  constructor(db: any, projectRoot: string) {\n    this.db = db;\n    this.projectRoot = projectRoot;\n  }\n\n  /**\n   * Populate data_code_refs by scanning source files for data file references.\n   * Returns the number of refs created.\n   */\n  linkAll(): number {\n    const fs = require('fs');\n    const path = require('path');\n\n    // Get all known data file paths\n    const datasets = this.db.all('SELECT id, file_path FROM data_datasets') as Array<{ id: string; file_path: string }>;\n    if (datasets.length === 0) return 0;\n\n    const knownPaths = new Set(datasets.map(d => d.file_path));\n    const pathToDatasetId = new Map(datasets.map(d => [d.file_path, d.id]));\n\n    // Get all indexed source files (from the graph)\n    const sourceFiles = this.db.all(\n      `SELECT file_path FROM files WHERE file_path NOT LIKE '%.csv' AND file_path NOT LIKE '%.tsv' AND file_path NOT LIKE '%.jsonl' AND file_path NOT LIKE '%.ndjson' AND file_path NOT LIKE '%.xlsx' AND file_path NOT LIKE '%.parquet'`\n    ) as Array<{ file_path: string }>;\n\n    // Clear existing refs\n    this.db.run('DELETE FROM data_code_refs');\n\n    let refsCreated = 0;\n\n    for (const file of sourceFiles) {\n      const absPath = path.join(this.projectRoot, file.file_path);\n      let content: string;\n      try {\n        content = fs.readFileSync(absPath, 'utf-8');\n      } catch {\n        continue;\n      }\n\n      const refs = detectDataReferences(content, knownPaths);\n      if (refs.length === 0) continue;\n\n      // Find the qualified_name of the containing symbol (file-level)\n      // Use the file path as the qualified name for file-level refs\n      const qualifiedName = file.file_path;\n\n      for (const ref of refs) {\n        const datasetId = pathToDatasetId.get(ref.filePath);\n        if (!datasetId) continue;\n\n        try {\n          this.db.run(\n            `INSERT OR REPLACE INTO data_code_refs (dataset_id, qualified_name, ref_type, confidence) VALUES (?, ?, ?, ?)`,\n            [datasetId, qualifiedName, ref.refType, ref.confidence],\n          );\n          refsCreated++;\n        } catch {\n          // Ignore duplicates\n        }\n      }\n    }\n\n    return refsCreated;\n  }\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBA,MAAM,gBAAgB;AAAA;AAAA,EAEpB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAEA;AACF;AAGA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAMO,SAAS,qBACd,eACA,gBAC8E;AAC9E,QAAM,OAAO,oBAAI,IAAiE;AAGlF,aAAW,WAAW,eAAe;AACnC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,WAAW,KAAK,IAAI,QAAQ;AAClC,YAAI,CAAC,YAAY,SAAS,aAAa,KAAK;AAC1C,gBAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,eAAK,IAAI,UAAU,EAAE,SAAS,SAAS,WAAW,CAAC;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,aAAW,WAAW,gBAAgB;AACpC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,aAAK,IAAI,UAAU,EAAE,SAAS,UAAU,WAAW,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,KAAK,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO;AAAA,IAC3D;AAAA,IACA,GAAG;AAAA,EACL,EAAE;AACJ;AAKA,SAAS,kBAAkB,UAA0B;AACnD,SAAO,SACJ,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE,EACpB,YAAY;AACjB;AAMA,SAAS,cAAc,GAAmB;AACxC,SAAO,EAAE,QAAQ,SAAS,EAAE,EAAE,QAAQ,OAAO,GAAG;AAClD;AAKA,SAAS,eAAe,UAAkB,YAAkC;AAC1E,MAAI,WAAW,IAAI,QAAQ,EAAG,QAAO;AAErC,QAAM,MAAM,SAAS,MAAM,GAAG,EAAE,IAAI,GAAG,YAAY;AACnD,SAAO,CAAC,OAAO,OAAO,SAAS,UAAU,QAAQ,QAAQ,OAAO,SAAS,EAAE,SAAS,OAAO,EAAE;AAC/F;AAMO,MAAM,eAAe;AAAA,EAI1B,YAAY,IAAS,aAAqB;AACxC,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAkB;AAChB,UAAM,KAAK,QAAQ,IAAI;AACvB,UAAM,OAAO,QAAQ,MAAM;AAG3B,UAAM,WAAW,KAAK,GAAG,IAAI,yCAAyC;AACtE,QAAI,SAAS,WAAW,EAAG,QAAO;AAElC,UAAM,aAAa,IAAI,IAAI,SAAS,IAAI,OAAK,EAAE,SAAS,CAAC;AACzD,UAAM,kBAAkB,IAAI,IAAI,SAAS,IAAI,OAAK,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;AAGtE,UAAM,cAAc,KAAK,GAAG;AAAA,MAC1B;AAAA,IACF;AAGA,SAAK,GAAG,IAAI,4BAA4B;AAExC,QAAI,cAAc;AAElB,eAAW,QAAQ,aAAa;AAC9B,YAAM,UAAU,KAAK,KAAK,KAAK,aAAa,KAAK,SAAS;AAC1D,UAAI;AACJ,UAAI;AACF,kBAAU,GAAG,aAAa,SAAS,OAAO;AAAA,MAC5C,QAAQ;AACN;AAAA,MACF;AAEA,YAAM,OAAO,qBAAqB,SAAS,UAAU;AACrD,UAAI,KAAK,WAAW,EAAG;AAIvB,YAAM,gBAAgB,KAAK;AAE3B,iBAAW,OAAO,MAAM;AACtB,cAAM,YAAY,gBAAgB,IAAI,IAAI,QAAQ;AAClD,YAAI,CAAC,UAAW;AAEhB,YAAI;AACF,eAAK,GAAG;AAAA,YACN;AAAA,YACA,CAAC,WAAW,eAAe,IAAI,SAAS,IAAI,UAAU;AAAA,UACxD;AACA;AAAA,QACF,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;",
+  "sourcesContent": ["/**\n * Data \u2194 Code Linker\n *\n * Detects references to data files in source code and populates data_code_refs.\n * Patterns detected:\n *   - readFileSync('data/users.csv'), readFile('path/to/file.csv')\n *   - pd.read_csv('data/file.csv'), pd.read_excel(...)\n *   - open('data/file.csv'), csv.reader(open(...))\n *   - COPY FROM 'file.csv', \\copy ... from 'file.csv'\n *   - require('./data/file.json'), import ... from './data/file.json'\n *   - fs.createReadStream('data/file.csv')\n *   - String literals matching known data file paths\n */\n\nimport type { DataCodeRef } from './types';\n\n/** Patterns that indicate a code symbol reads a data file */\nconst READ_PATTERNS = [\n  // Node.js / TypeScript\n  /readFileSync\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /readFile\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createReadStream\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python pandas\n  /read_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /read_table\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  // Python open (includes PDF)\n  /open\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|ndjson|json|xlsx|xls|parquet|pdf))['\"`]/g,\n  // Python PDF libraries\n  /PdfReader\\s*\\(\\s*['\"`]([^'\"`]+\\.pdf)['\"`]/g,\n  /pdfplumber\\.open\\s*\\(\\s*['\"`]([^'\"`]+\\.pdf)['\"`]/g,\n  /fitz\\.open\\s*\\(\\s*['\"`]([^'\"`]+\\.pdf)['\"`]/g,\n  // SQL COPY\n  /COPY\\s+\\w+\\s+FROM\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  /\\\\copy\\s+\\w+\\s+from\\s+['\"`]([^'\"`]+)['\"`]/gi,\n  // Generic path references in strings (data file extensions)\n  /['\"`]((?:\\.\\.?\\/)?(?:[\\w\\-./]+\\/)?[\\w\\-]+\\.(?:csv|tsv|jsonl|ndjson|xlsx|xls|parquet|pdf))['\"`]/g,\n];\n\n/** Patterns that indicate a code symbol writes a data file */\nconst WRITE_PATTERNS = [\n  /writeFileSync\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /writeFile\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /to_csv\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_excel\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_json\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /to_parquet\\s*\\(\\s*['\"`]([^'\"`]+)['\"`]/g,\n  /createWriteStream\\s*\\(\\s*['\"`]([^'\"`]+\\.(?:csv|tsv|jsonl|json|xlsx|parquet))['\"`]/g,\n  /COPY\\s+\\w+\\s+TO\\s+['\"`]([^'\"`]+)['\"`]/gi,\n];\n\n/**\n * Scan source code content for references to data files.\n * Returns matched file paths with their reference type.\n */\nexport function detectDataReferences(\n  sourceContent: string,\n  knownDataPaths: Set<string>,\n): Array<{ filePath: string; refType: 'reads' | 'writes'; confidence: number }> {\n  const refs = new Map<string, { refType: 'reads' | 'writes'; confidence: number }>();\n\n  // Check read patterns\n  for (const pattern of READ_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const existing = refs.get(filePath);\n        if (!existing || existing.confidence < 0.9) {\n          const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n          refs.set(filePath, { refType: 'reads', confidence });\n        }\n      }\n    }\n  }\n\n  // Check write patterns\n  for (const pattern of WRITE_PATTERNS) {\n    pattern.lastIndex = 0;\n    let match: RegExpExecArray | null;\n    while ((match = pattern.exec(sourceContent)) !== null) {\n      const filePath = normalizePath(match[1]!);\n      if (isDataFilePath(filePath, knownDataPaths)) {\n        const confidence = knownDataPaths.has(filePath) ? 1.0 : 0.7;\n        refs.set(filePath, { refType: 'writes', confidence });\n      }\n    }\n  }\n\n  return Array.from(refs.entries()).map(([filePath, info]) => ({\n    filePath,\n    ...info,\n  }));\n}\n\n/**\n * Generate dataset ID from a file path (same logic as indexer).\n */\nfunction generateDatasetId(filePath: string): string {\n  return filePath\n    .replace(/\\.[^.]+$/, '')\n    .replace(/[/\\\\]/g, '-')\n    .replace(/[^a-zA-Z0-9-_]/g, '-')\n    .replace(/-+/g, '-')\n    .replace(/^-|-$/g, '')\n    .toLowerCase();\n}\n\n/**\n * Normalize a path reference found in code.\n * Strips leading ./ and resolves simple relative paths.\n */\nfunction normalizePath(p: string): string {\n  return p.replace(/^\\.\\//, '').replace(/\\\\/g, '/');\n}\n\n/**\n * Check if a path looks like a data file (by extension or known path match).\n */\nfunction isDataFilePath(filePath: string, knownPaths: Set<string>): boolean {\n  if (knownPaths.has(filePath)) return true;\n  // Check extension\n  const ext = filePath.split('.').pop()?.toLowerCase();\n  return ['csv', 'tsv', 'jsonl', 'ndjson', 'json', 'xlsx', 'xls', 'parquet', 'pdf'].includes(ext ?? '');\n}\n\n/**\n * Link data files to code symbols.\n * Scans all indexed source files for references to known data file paths.\n */\nexport class DataCodeLinker {\n  private readonly db: any;\n  private readonly projectRoot: string;\n\n  constructor(db: any, projectRoot: string) {\n    this.db = db;\n    this.projectRoot = projectRoot;\n  }\n\n  /**\n   * Populate data_code_refs by scanning source files for data file references.\n   * Returns the number of refs created.\n   */\n  linkAll(): number {\n    const fs = require('fs');\n    const path = require('path');\n\n    // Get all known data file paths\n    const datasets = this.db.all('SELECT id, file_path FROM data_datasets') as Array<{ id: string; file_path: string }>;\n    if (datasets.length === 0) return 0;\n\n    const knownPaths = new Set(datasets.map(d => d.file_path));\n    const pathToDatasetId = new Map(datasets.map(d => [d.file_path, d.id]));\n\n    // Get all indexed source files (from the graph)\n    // Note: files table uses 'path' as the column name, aliased to file_path for readability.\n    const sourceFiles = this.db.all(\n      `SELECT path AS file_path FROM files WHERE path NOT LIKE '%.csv' AND path NOT LIKE '%.tsv' AND path NOT LIKE '%.jsonl' AND path NOT LIKE '%.ndjson' AND path NOT LIKE '%.xlsx' AND path NOT LIKE '%.parquet' AND path NOT LIKE '%.pdf'`\n    ) as Array<{ file_path: string }>;\n\n    // Clear existing refs\n    this.db.run('DELETE FROM data_code_refs');\n\n    let refsCreated = 0;\n\n    for (const file of sourceFiles) {\n      const absPath = path.join(this.projectRoot, file.file_path);\n      let content: string;\n      try {\n        content = fs.readFileSync(absPath, 'utf-8');\n      } catch {\n        continue;\n      }\n\n      const refs = detectDataReferences(content, knownPaths);\n      if (refs.length === 0) continue;\n\n      // Find the qualified_name of the containing symbol (file-level)\n      // Use the file path as the qualified name for file-level refs\n      const qualifiedName = file.file_path;\n\n      for (const ref of refs) {\n        const datasetId = pathToDatasetId.get(ref.filePath);\n        if (!datasetId) continue;\n\n        try {\n          this.db.run(\n            `INSERT OR REPLACE INTO data_code_refs (dataset_id, qualified_name, ref_type, confidence) VALUES (?, ?, ?, ?)`,\n            [datasetId, qualifiedName, ref.refType, ref.confidence],\n          );\n          refsCreated++;\n        } catch {\n          // Ignore duplicates\n        }\n      }\n    }\n\n    return refsCreated;\n  }\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiBA,MAAM,gBAAgB;AAAA;AAAA,EAEpB;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAEA;AACF;AAGA,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAMO,SAAS,qBACd,eACA,gBAC8E;AAC9E,QAAM,OAAO,oBAAI,IAAiE;AAGlF,aAAW,WAAW,eAAe;AACnC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,WAAW,KAAK,IAAI,QAAQ;AAClC,YAAI,CAAC,YAAY,SAAS,aAAa,KAAK;AAC1C,gBAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,eAAK,IAAI,UAAU,EAAE,SAAS,SAAS,WAAW,CAAC;AAAA,QACrD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,aAAW,WAAW,gBAAgB;AACpC,YAAQ,YAAY;AACpB,QAAI;AACJ,YAAQ,QAAQ,QAAQ,KAAK,aAAa,OAAO,MAAM;AACrD,YAAM,WAAW,cAAc,MAAM,CAAC,CAAE;AACxC,UAAI,eAAe,UAAU,cAAc,GAAG;AAC5C,cAAM,aAAa,eAAe,IAAI,QAAQ,IAAI,IAAM;AACxD,aAAK,IAAI,UAAU,EAAE,SAAS,UAAU,WAAW,CAAC;AAAA,MACtD;AAAA,IACF;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,KAAK,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO;AAAA,IAC3D;AAAA,IACA,GAAG;AAAA,EACL,EAAE;AACJ;AAKA,SAAS,kBAAkB,UAA0B;AACnD,SAAO,SACJ,QAAQ,YAAY,EAAE,EACtB,QAAQ,UAAU,GAAG,EACrB,QAAQ,mBAAmB,GAAG,EAC9B,QAAQ,OAAO,GAAG,EAClB,QAAQ,UAAU,EAAE,EACpB,YAAY;AACjB;AAMA,SAAS,cAAc,GAAmB;AACxC,SAAO,EAAE,QAAQ,SAAS,EAAE,EAAE,QAAQ,OAAO,GAAG;AAClD;AAKA,SAAS,eAAe,UAAkB,YAAkC;AAC1E,MAAI,WAAW,IAAI,QAAQ,EAAG,QAAO;AAErC,QAAM,MAAM,SAAS,MAAM,GAAG,EAAE,IAAI,GAAG,YAAY;AACnD,SAAO,CAAC,OAAO,OAAO,SAAS,UAAU,QAAQ,QAAQ,OAAO,WAAW,KAAK,EAAE,SAAS,OAAO,EAAE;AACtG;AAMO,MAAM,eAAe;AAAA,EAI1B,YAAY,IAAS,aAAqB;AACxC,SAAK,KAAK;AACV,SAAK,cAAc;AAAA,EACrB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,UAAkB;AAChB,UAAM,KAAK,QAAQ,IAAI;AACvB,UAAM,OAAO,QAAQ,MAAM;AAG3B,UAAM,WAAW,KAAK,GAAG,IAAI,yCAAyC;AACtE,QAAI,SAAS,WAAW,EAAG,QAAO;AAElC,UAAM,aAAa,IAAI,IAAI,SAAS,IAAI,OAAK,EAAE,SAAS,CAAC;AACzD,UAAM,kBAAkB,IAAI,IAAI,SAAS,IAAI,OAAK,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;AAItE,UAAM,cAAc,KAAK,GAAG;AAAA,MAC1B;AAAA,IACF;AAGA,SAAK,GAAG,IAAI,4BAA4B;AAExC,QAAI,cAAc;AAElB,eAAW,QAAQ,aAAa;AAC9B,YAAM,UAAU,KAAK,KAAK,KAAK,aAAa,KAAK,SAAS;AAC1D,UAAI;AACJ,UAAI;AACF,kBAAU,GAAG,aAAa,SAAS,OAAO;AAAA,MAC5C,QAAQ;AACN;AAAA,MACF;AAEA,YAAM,OAAO,qBAAqB,SAAS,UAAU;AACrD,UAAI,KAAK,WAAW,EAAG;AAIvB,YAAM,gBAAgB,KAAK;AAE3B,iBAAW,OAAO,MAAM;AACtB,cAAM,YAAY,gBAAgB,IAAI,IAAI,QAAQ;AAClD,YAAI,CAAC,UAAW;AAEhB,YAAI;AACF,eAAK,GAAG;AAAA,YACN;AAAA,YACA,CAAC,WAAW,eAAe,IAAI,SAAS,IAAI,UAAU;AAAA,UACxD;AACA;AAAA,QACF,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/data/linker.test.js

@@ -0,0 +1,60 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_linker = require("./linker");
+const KNOWN_PATHS = /* @__PURE__ */ new Set([
+  "data/report.pdf",
+  "docs/manual.pdf",
+  "data/users.csv"
+]);
+(0, import_vitest.describe)("detectDataReferences \u2014 PDF patterns", () => {
+  (0, import_vitest.it)("detects readFileSync with .pdf path", () => {
+    const src = `const buf = readFileSync('data/report.pdf')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/report.pdf" && r.refType === "reads")).toBe(true);
+  });
+  (0, import_vitest.it)("detects createReadStream with .pdf path", () => {
+    const src = `fs.createReadStream('data/report.pdf')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/report.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("detects Python open() with .pdf extension", () => {
+    const src = `f = open('docs/manual.pdf', 'rb')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "docs/manual.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("detects PdfReader() call", () => {
+    const src = `reader = PdfReader('data/report.pdf')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/report.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("detects pdfplumber.open() call", () => {
+    const src = `with pdfplumber.open('data/report.pdf') as pdf:`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/report.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("detects fitz.open() call", () => {
+    const src = `doc = fitz.open('docs/manual.pdf')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "docs/manual.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("detects generic string literal with .pdf extension", () => {
+    const src = `const path = 'data/report.pdf';`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/report.pdf")).toBe(true);
+  });
+  (0, import_vitest.it)("gives confidence 1.0 for known paths, lower for unknown", () => {
+    const src = `readFileSync('data/report.pdf'); readFileSync('other/unknown.pdf')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    const known = refs.find((r) => r.filePath === "data/report.pdf");
+    const unknown = refs.find((r) => r.filePath === "other/unknown.pdf");
+    (0, import_vitest.expect)(known?.confidence).toBe(1);
+    (0, import_vitest.expect)(unknown?.confidence).toBeLessThan(1);
+  });
+  (0, import_vitest.it)("does not confuse .pdf with .csv patterns", () => {
+    const src = `readFileSync('data/users.csv')`;
+    const refs = (0, import_linker.detectDataReferences)(src, KNOWN_PATHS);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath === "data/users.csv")).toBe(true);
+    (0, import_vitest.expect)(refs.some((r) => r.filePath?.endsWith(".pdf"))).toBe(false);
+  });
+});
+//# sourceMappingURL=linker.test.js.map

package/dist/data/linker.test.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/data/linker.test.ts"],
+  "sourcesContent": ["import { describe, it, expect } from 'vitest';\nimport { detectDataReferences } from './linker';\n\nconst KNOWN_PATHS = new Set([\n  'data/report.pdf',\n  'docs/manual.pdf',\n  'data/users.csv',\n]);\n\ndescribe('detectDataReferences \u2014 PDF patterns', () => {\n  it('detects readFileSync with .pdf path', () => {\n    const src = `const buf = readFileSync('data/report.pdf')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/report.pdf' && r.refType === 'reads')).toBe(true);\n  });\n\n  it('detects createReadStream with .pdf path', () => {\n    const src = `fs.createReadStream('data/report.pdf')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/report.pdf')).toBe(true);\n  });\n\n  it('detects Python open() with .pdf extension', () => {\n    const src = `f = open('docs/manual.pdf', 'rb')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'docs/manual.pdf')).toBe(true);\n  });\n\n  it('detects PdfReader() call', () => {\n    const src = `reader = PdfReader('data/report.pdf')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/report.pdf')).toBe(true);\n  });\n\n  it('detects pdfplumber.open() call', () => {\n    const src = `with pdfplumber.open('data/report.pdf') as pdf:`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/report.pdf')).toBe(true);\n  });\n\n  it('detects fitz.open() call', () => {\n    const src = `doc = fitz.open('docs/manual.pdf')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'docs/manual.pdf')).toBe(true);\n  });\n\n  it('detects generic string literal with .pdf extension', () => {\n    const src = `const path = 'data/report.pdf';`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/report.pdf')).toBe(true);\n  });\n\n  it('gives confidence 1.0 for known paths, lower for unknown', () => {\n    const src = `readFileSync('data/report.pdf'); readFileSync('other/unknown.pdf')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    const known = refs.find(r => r.filePath === 'data/report.pdf');\n    const unknown = refs.find(r => r.filePath === 'other/unknown.pdf');\n    expect(known?.confidence).toBe(1.0);\n    expect(unknown?.confidence).toBeLessThan(1.0);\n  });\n\n  it('does not confuse .pdf with .csv patterns', () => {\n    const src = `readFileSync('data/users.csv')`;\n    const refs = detectDataReferences(src, KNOWN_PATHS);\n    expect(refs.some(r => r.filePath === 'data/users.csv')).toBe(true);\n    expect(refs.some(r => r.filePath?.endsWith('.pdf'))).toBe(false);\n  });\n});\n"],
+  "mappings": ";AAAA,oBAAqC;AACrC,oBAAqC;AAErC,MAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAAA,IAED,wBAAS,4CAAuC,MAAM;AACpD,wBAAG,uCAAuC,MAAM;AAC9C,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,qBAAqB,EAAE,YAAY,OAAO,CAAC,EAAE,KAAK,IAAI;AAAA,EAC7F,CAAC;AAED,wBAAG,2CAA2C,MAAM;AAClD,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,6CAA6C,MAAM;AACpD,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,4BAA4B,MAAM;AACnC,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,kCAAkC,MAAM;AACzC,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,4BAA4B,MAAM;AACnC,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,sDAAsD,MAAM;AAC7D,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB,CAAC,EAAE,KAAK,IAAI;AAAA,EACpE,CAAC;AAED,wBAAG,2DAA2D,MAAM;AAClE,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,UAAM,QAAQ,KAAK,KAAK,OAAK,EAAE,aAAa,iBAAiB;AAC7D,UAAM,UAAU,KAAK,KAAK,OAAK,EAAE,aAAa,mBAAmB;AACjE,8BAAO,OAAO,UAAU,EAAE,KAAK,CAAG;AAClC,8BAAO,SAAS,UAAU,EAAE,aAAa,CAAG;AAAA,EAC9C,CAAC;AAED,wBAAG,4CAA4C,MAAM;AACnD,UAAM,MAAM;AACZ,UAAM,WAAO,oCAAqB,KAAK,WAAW;AAClD,8BAAO,KAAK,KAAK,OAAK,EAAE,aAAa,gBAAgB,CAAC,EAAE,KAAK,IAAI;AACjE,8BAAO,KAAK,KAAK,OAAK,EAAE,UAAU,SAAS,MAAM,CAAC,CAAC,EAAE,KAAK,KAAK;AAAA,EACjE,CAAC;AACH,CAAC;",
+  "names": []
+}

package/dist/data/parsers/__fixtures__/pdf-inspector-stub.js

@@ -0,0 +1,41 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pdf_inspector_stub_exports = {};
+__export(pdf_inspector_stub_exports, {
+  classifyPdf: () => classifyPdf,
+  extractPagesMarkdown: () => extractPagesMarkdown,
+  processPdf: () => processPdf
+});
+module.exports = __toCommonJS(pdf_inspector_stub_exports);
+function extractPagesMarkdown(_buf) {
+  throw new Error("@firecrawl/pdf-inspector is not installed");
+}
+function processPdf(_buf) {
+  throw new Error("@firecrawl/pdf-inspector is not installed");
+}
+function classifyPdf(_buf) {
+  throw new Error("@firecrawl/pdf-inspector is not installed");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  classifyPdf,
+  extractPagesMarkdown,
+  processPdf
+});
+//# sourceMappingURL=pdf-inspector-stub.js.map

package/dist/data/parsers/__fixtures__/pdf-inspector-stub.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../../src/data/parsers/__fixtures__/pdf-inspector-stub.ts"],
+  "sourcesContent": ["// Stub for @firecrawl/pdf-inspector used in tests.\n// Real package not installed \u2014 this stub allows require() to resolve.\n// vi.mock('@firecrawl/pdf-inspector', factory) overrides this in individual tests.\nexport function extractPagesMarkdown(_buf: Buffer): never {\n  throw new Error('@firecrawl/pdf-inspector is not installed');\n}\nexport function processPdf(_buf: Buffer): never {\n  throw new Error('@firecrawl/pdf-inspector is not installed');\n}\nexport function classifyPdf(_buf: Buffer): never {\n  throw new Error('@firecrawl/pdf-inspector is not installed');\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGO,SAAS,qBAAqB,MAAqB;AACxD,QAAM,IAAI,MAAM,2CAA2C;AAC7D;AACO,SAAS,WAAW,MAAqB;AAC9C,QAAM,IAAI,MAAM,2CAA2C;AAC7D;AACO,SAAS,YAAY,MAAqB;AAC/C,QAAM,IAAI,MAAM,2CAA2C;AAC7D;",
+  "names": []
+}

package/dist/data/parsers/index.js

@@ -29,13 +29,15 @@ var import_jsonl = require("./jsonl");
 var import_json_array = require("./json-array");
 var import_excel = require("./excel");
 var import_parquet = require("./parquet");
+var import_pdf = require("./pdf");
 const allParsers = [
   import_csv.csvParser,
   import_csv.tsvParser,
   import_jsonl.jsonlParser,
   import_json_array.jsonArrayParser,
   import_excel.excelParser,
-  import_parquet.parquetParser
+  import_parquet.parquetParser,
+  import_pdf.pdfParser
 ];
 const extensionMap = /* @__PURE__ */ new Map();
 for (const parser of allParsers) {

package/dist/data/parsers/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../../../src/data/parsers/index.ts"],
-  "sourcesContent": ["/**\n * Data Format Registry\n *\n * Maps file extensions to their appropriate parser.\n * Gracefully skips formats whose optional dependencies are not installed.\n */\n\nimport type { DataFormatParser } from '../types';\nimport { csvParser, tsvParser } from './csv';\nimport { jsonlParser } from './jsonl';\nimport { jsonArrayParser } from './json-array';\nimport { excelParser } from './excel';\nimport { parquetParser } from './parquet';\n\n// \u2500\u2500 Registry \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nconst allParsers: DataFormatParser[] = [\n  csvParser,\n  tsvParser,\n  jsonlParser,\n  jsonArrayParser,\n  excelParser,\n  parquetParser,\n];\n\n/** Extension \u2192 parser lookup */\nconst extensionMap = new Map<string, DataFormatParser>();\nfor (const parser of allParsers) {\n  for (const ext of parser.extensions) {\n    extensionMap.set(ext.toLowerCase(), parser);\n  }\n}\n\n// \u2500\u2500 Public API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n/**\n * Get the parser for a given file extension.\n * Returns null if no parser is registered or the optional dep is missing.\n */\nexport function getDataParser(filePath: string): DataFormatParser | null {\n  const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();\n  const parser = extensionMap.get(ext) ?? null;\n  if (!parser) return null;\n  if (!parser.isAvailable()) return null;\n  return parser;\n}\n\n/**\n * Check if a file extension is supported for data parsing.\n */\nexport function isSupportedDataFormat(filePath: string): boolean {\n  return getDataParser(filePath) !== null;\n}\n\n/**\n * Get all supported extensions (only those with available deps).\n */\nexport function getAvailableExtensions(): string[] {\n  const available: string[] = [];\n  for (const [ext, parser] of extensionMap) {\n    if (parser.isAvailable()) available.push(ext);\n  }\n  return available;\n}\n\n/**\n * Get extensions that are registered but have missing optional deps.\n */\nexport function getUnavailableExtensions(): Array<{ ext: string; parser: string }> {\n  const unavailable: Array<{ ext: string; parser: string }> = [];\n  for (const [ext, parser] of extensionMap) {\n    if (!parser.isAvailable()) {\n      unavailable.push({ ext, parser: parser.name });\n    }\n  }\n  return unavailable;\n}\n"],
-  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,iBAAqC;AACrC,mBAA4B;AAC5B,wBAAgC;AAChC,mBAA4B;AAC5B,qBAA8B;AAI9B,MAAM,aAAiC;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,eAAe,oBAAI,IAA8B;AACvD,WAAW,UAAU,YAAY;AAC/B,aAAW,OAAO,OAAO,YAAY;AACnC,iBAAa,IAAI,IAAI,YAAY,GAAG,MAAM;AAAA,EAC5C;AACF;AAQO,SAAS,cAAc,UAA2C;AACvE,QAAM,MAAM,SAAS,MAAM,SAAS,YAAY,GAAG,CAAC,EAAE,YAAY;AAClE,QAAM,SAAS,aAAa,IAAI,GAAG,KAAK;AACxC,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI,CAAC,OAAO,YAAY,EAAG,QAAO;AAClC,SAAO;AACT;AAKO,SAAS,sBAAsB,UAA2B;AAC/D,SAAO,cAAc,QAAQ,MAAM;AACrC;AAKO,SAAS,yBAAmC;AACjD,QAAM,YAAsB,CAAC;AAC7B,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,OAAO,YAAY,EAAG,WAAU,KAAK,GAAG;AAAA,EAC9C;AACA,SAAO;AACT;AAKO,SAAS,2BAAmE;AACjF,QAAM,cAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,CAAC,OAAO,YAAY,GAAG;AACzB,kBAAY,KAAK,EAAE,KAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC/C;AAAA,EACF;AACA,SAAO;AACT;",
+  "sourcesContent": ["/**\n * Data Format Registry\n *\n * Maps file extensions to their appropriate parser.\n * Gracefully skips formats whose optional dependencies are not installed.\n */\n\nimport type { DataFormatParser } from '../types';\nimport { csvParser, tsvParser } from './csv';\nimport { jsonlParser } from './jsonl';\nimport { jsonArrayParser } from './json-array';\nimport { excelParser } from './excel';\nimport { parquetParser } from './parquet';\nimport { pdfParser } from './pdf';\n\n// \u2500\u2500 Registry \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nconst allParsers: DataFormatParser[] = [\n  csvParser,\n  tsvParser,\n  jsonlParser,\n  jsonArrayParser,\n  excelParser,\n  parquetParser,\n  pdfParser,\n];\n\n/** Extension \u2192 parser lookup */\nconst extensionMap = new Map<string, DataFormatParser>();\nfor (const parser of allParsers) {\n  for (const ext of parser.extensions) {\n    extensionMap.set(ext.toLowerCase(), parser);\n  }\n}\n\n// \u2500\u2500 Public API \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n/**\n * Get the parser for a given file extension.\n * Returns null if no parser is registered or the optional dep is missing.\n */\nexport function getDataParser(filePath: string): DataFormatParser | null {\n  const ext = filePath.slice(filePath.lastIndexOf('.')).toLowerCase();\n  const parser = extensionMap.get(ext) ?? null;\n  if (!parser) return null;\n  if (!parser.isAvailable()) return null;\n  return parser;\n}\n\n/**\n * Check if a file extension is supported for data parsing.\n */\nexport function isSupportedDataFormat(filePath: string): boolean {\n  return getDataParser(filePath) !== null;\n}\n\n/**\n * Get all supported extensions (only those with available deps).\n */\nexport function getAvailableExtensions(): string[] {\n  const available: string[] = [];\n  for (const [ext, parser] of extensionMap) {\n    if (parser.isAvailable()) available.push(ext);\n  }\n  return available;\n}\n\n/**\n * Get extensions that are registered but have missing optional deps.\n */\nexport function getUnavailableExtensions(): Array<{ ext: string; parser: string }> {\n  const unavailable: Array<{ ext: string; parser: string }> = [];\n  for (const [ext, parser] of extensionMap) {\n    if (!parser.isAvailable()) {\n      unavailable.push({ ext, parser: parser.name });\n    }\n  }\n  return unavailable;\n}\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAQA,iBAAqC;AACrC,mBAA4B;AAC5B,wBAAgC;AAChC,mBAA4B;AAC5B,qBAA8B;AAC9B,iBAA0B;AAI1B,MAAM,aAAiC;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGA,MAAM,eAAe,oBAAI,IAA8B;AACvD,WAAW,UAAU,YAAY;AAC/B,aAAW,OAAO,OAAO,YAAY;AACnC,iBAAa,IAAI,IAAI,YAAY,GAAG,MAAM;AAAA,EAC5C;AACF;AAQO,SAAS,cAAc,UAA2C;AACvE,QAAM,MAAM,SAAS,MAAM,SAAS,YAAY,GAAG,CAAC,EAAE,YAAY;AAClE,QAAM,SAAS,aAAa,IAAI,GAAG,KAAK;AACxC,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI,CAAC,OAAO,YAAY,EAAG,QAAO;AAClC,SAAO;AACT;AAKO,SAAS,sBAAsB,UAA2B;AAC/D,SAAO,cAAc,QAAQ,MAAM;AACrC;AAKO,SAAS,yBAAmC;AACjD,QAAM,YAAsB,CAAC;AAC7B,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,OAAO,YAAY,EAAG,WAAU,KAAK,GAAG;AAAA,EAC9C;AACA,SAAO;AACT;AAKO,SAAS,2BAAmE;AACjF,QAAM,cAAsD,CAAC;AAC7D,aAAW,CAAC,KAAK,MAAM,KAAK,cAAc;AACxC,QAAI,CAAC,OAAO,YAAY,GAAG;AACzB,kBAAY,KAAK,EAAE,KAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC/C;AAAA,EACF;AACA,SAAO;AACT;",
   "names": []
 }

package/dist/data/parsers/pdf.js

@@ -0,0 +1,95 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pdf_exports = {};
+__export(pdf_exports, {
+  _resetPdfInspectorForTesting: () => _resetPdfInspectorForTesting,
+  _setPdfInspectorForTesting: () => _setPdfInspectorForTesting,
+  pdfParser: () => pdfParser
+});
+module.exports = __toCommonJS(pdf_exports);
+var import_fs = require("fs");
+let pdfModule = null;
+let pdfChecked = false;
+function tryLoadPdfInspector() {
+  if (pdfChecked) return pdfModule !== null;
+  pdfChecked = true;
+  try {
+    pdfModule = require("@firecrawl/pdf-inspector");
+    return true;
+  } catch {
+    return false;
+  }
+}
+function _setPdfInspectorForTesting(mod) {
+  pdfModule = mod;
+  pdfChecked = true;
+}
+function _resetPdfInspectorForTesting() {
+  pdfModule = null;
+  pdfChecked = false;
+}
+const pdfParser = {
+  name: "pdf",
+  extensions: [".pdf"],
+  isAvailable() {
+    return tryLoadPdfInspector();
+  },
+  async parse(filePath, opts) {
+    if (!tryLoadPdfInspector()) {
+      return { columns: [], totalRows: 0 };
+    }
+    const { extractPagesMarkdown, processPdf } = pdfModule;
+    const buffer = (0, import_fs.readFileSync)(filePath);
+    const result = extractPagesMarkdown(buffer);
+    let metadataJson = null;
+    try {
+      const meta = processPdf(buffer);
+      metadataJson = JSON.stringify({
+        pdfType: meta.pdfType ?? null,
+        confidence: meta.confidence ?? null,
+        title: meta.title ?? null,
+        hasEncodingIssues: meta.hasEncodingIssues ?? false,
+        isComplexLayout: meta.isComplexLayout ?? false
+      });
+    } catch {
+    }
+    const columns = ["page", "content", "needs_ocr", "has_tables", "has_columns"];
+    const batch = [];
+    for (const p of result.pages) {
+      batch.push({
+        page: String(p.page),
+        content: p.markdown ?? "",
+        needs_ocr: String(p.needsOcr ?? false),
+        has_tables: String(result.pagesWithTables?.includes(p.page) ?? false),
+        has_columns: String(result.pagesWithColumns?.includes(p.page) ?? false)
+      });
+    }
+    if (batch.length > 0) {
+      opts.onBatch(batch, columns);
+    }
+    return { columns, totalRows: batch.length, metadataJson };
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  _resetPdfInspectorForTesting,
+  _setPdfInspectorForTesting,
+  pdfParser
+});
+//# sourceMappingURL=pdf.js.map

package/dist/data/parsers/pdf.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/data/parsers/pdf.ts"],
+  "sourcesContent": ["/**\n * PDF Parser (Optional Dependency)\n *\n * Handles: .pdf\n * Requires: @firecrawl/pdf-inspector (optional dependency, Rust/NAPI)\n * Gracefully returns isAvailable()=false if not installed or unsupported platform.\n *\n * Each page becomes one row: page, content, needs_ocr, has_tables, has_columns.\n * Text-based PDFs process in under 200ms locally.\n * Scanned/mixed pages get needs_ocr='true' and are included (not skipped).\n */\n\nimport { readFileSync } from 'fs';\nimport type { DataFormatParser, ParsedRow } from '../types';\n\nlet pdfModule: any = null;\nlet pdfChecked = false;\n\nfunction tryLoadPdfInspector(): boolean {\n  if (pdfChecked) return pdfModule !== null;\n  pdfChecked = true;\n  try {\n    // eslint-disable-next-line @typescript-eslint/no-require-imports\n    pdfModule = require('@firecrawl/pdf-inspector');\n    return true;\n  } catch {\n    return false;\n  }\n}\n\n/** Testing hook \u2014 allows injecting a mock module without module system tricks. */\nexport function _setPdfInspectorForTesting(mod: any): void {\n  pdfModule = mod;\n  pdfChecked = true;\n}\n\nexport function _resetPdfInspectorForTesting(): void {\n  pdfModule = null;\n  pdfChecked = false;\n}\n\nexport const pdfParser: DataFormatParser = {\n  name: 'pdf',\n  extensions: ['.pdf'],\n\n  isAvailable(): boolean {\n    return tryLoadPdfInspector();\n  },\n\n  async parse(filePath, opts) {\n    if (!tryLoadPdfInspector()) {\n      return { columns: [], totalRows: 0 };\n    }\n\n    const { extractPagesMarkdown, processPdf } = pdfModule;\n    const buffer = readFileSync(filePath);\n\n    // Per-page content + OCR flags\n    const result = extractPagesMarkdown(buffer);\n\n    // Document-level metadata (non-critical)\n    let metadataJson: string | null = null;\n    try {\n      const meta = processPdf(buffer);\n      metadataJson = JSON.stringify({\n        pdfType: meta.pdfType ?? null,\n        confidence: meta.confidence ?? null,\n        title: meta.title ?? null,\n        hasEncodingIssues: meta.hasEncodingIssues ?? false,\n        isComplexLayout: meta.isComplexLayout ?? false,\n      });\n    } catch {\n      // non-critical \u2014 index proceeds without metadata\n    }\n\n    const columns = ['page', 'content', 'needs_ocr', 'has_tables', 'has_columns'];\n    const batch: ParsedRow[] = [];\n\n    for (const p of result.pages) {\n      batch.push({\n        page: String(p.page),\n        content: p.markdown ?? '',\n        needs_ocr: String(p.needsOcr ?? false),\n        has_tables: String(result.pagesWithTables?.includes(p.page) ?? false),\n        has_columns: String(result.pagesWithColumns?.includes(p.page) ?? false),\n      });\n    }\n\n    if (batch.length > 0) {\n      opts.onBatch(batch, columns);\n    }\n\n    return { columns, totalRows: batch.length, metadataJson };\n  },\n};\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYA,gBAA6B;AAG7B,IAAI,YAAiB;AACrB,IAAI,aAAa;AAEjB,SAAS,sBAA+B;AACtC,MAAI,WAAY,QAAO,cAAc;AACrC,eAAa;AACb,MAAI;AAEF,gBAAY,QAAQ,0BAA0B;AAC9C,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGO,SAAS,2BAA2B,KAAgB;AACzD,cAAY;AACZ,eAAa;AACf;AAEO,SAAS,+BAAqC;AACnD,cAAY;AACZ,eAAa;AACf;AAEO,MAAM,YAA8B;AAAA,EACzC,MAAM;AAAA,EACN,YAAY,CAAC,MAAM;AAAA,EAEnB,cAAuB;AACrB,WAAO,oBAAoB;AAAA,EAC7B;AAAA,EAEA,MAAM,MAAM,UAAU,MAAM;AAC1B,QAAI,CAAC,oBAAoB,GAAG;AAC1B,aAAO,EAAE,SAAS,CAAC,GAAG,WAAW,EAAE;AAAA,IACrC;AAEA,UAAM,EAAE,sBAAsB,WAAW,IAAI;AAC7C,UAAM,aAAS,wBAAa,QAAQ;AAGpC,UAAM,SAAS,qBAAqB,MAAM;AAG1C,QAAI,eAA8B;AAClC,QAAI;AACF,YAAM,OAAO,WAAW,MAAM;AAC9B,qBAAe,KAAK,UAAU;AAAA,QAC5B,SAAS,KAAK,WAAW;AAAA,QACzB,YAAY,KAAK,cAAc;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,mBAAmB,KAAK,qBAAqB;AAAA,QAC7C,iBAAiB,KAAK,mBAAmB;AAAA,MAC3C,CAAC;AAAA,IACH,QAAQ;AAAA,IAER;AAEA,UAAM,UAAU,CAAC,QAAQ,WAAW,aAAa,cAAc,aAAa;AAC5E,UAAM,QAAqB,CAAC;AAE5B,eAAW,KAAK,OAAO,OAAO;AAC5B,YAAM,KAAK;AAAA,QACT,MAAM,OAAO,EAAE,IAAI;AAAA,QACnB,SAAS,EAAE,YAAY;AAAA,QACvB,WAAW,OAAO,EAAE,YAAY,KAAK;AAAA,QACrC,YAAY,OAAO,OAAO,iBAAiB,SAAS,EAAE,IAAI,KAAK,KAAK;AAAA,QACpE,aAAa,OAAO,OAAO,kBAAkB,SAAS,EAAE,IAAI,KAAK,KAAK;AAAA,MACxE,CAAC;AAAA,IACH;AAEA,QAAI,MAAM,SAAS,GAAG;AACpB,WAAK,QAAQ,OAAO,OAAO;AAAA,IAC7B;AAEA,WAAO,EAAE,SAAS,WAAW,MAAM,QAAQ,aAAa;AAAA,EAC1D;AACF;",
+  "names": []
+}

package/dist/data/parsers/pdf.test.js

@@ -0,0 +1,125 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_pdf = require("./pdf");
+import_vitest.vi.mock("fs", async (importOriginal) => {
+  const actual = await importOriginal();
+  return { ...actual, readFileSync: import_vitest.vi.fn().mockReturnValue(Buffer.from("fake-pdf")) };
+});
+const mockExtractPages = import_vitest.vi.fn();
+const mockProcessPdf = import_vitest.vi.fn();
+function makeMockResult(pages, opts) {
+  return {
+    pages,
+    pagesWithTables: opts?.pagesWithTables ?? [],
+    pagesWithColumns: opts?.pagesWithColumns ?? []
+  };
+}
+(0, import_vitest.describe)("pdfParser", () => {
+  (0, import_vitest.beforeEach)(() => {
+    import_vitest.vi.clearAllMocks();
+    (0, import_pdf._setPdfInspectorForTesting)({ extractPagesMarkdown: mockExtractPages, processPdf: mockProcessPdf });
+  });
+  (0, import_vitest.afterEach)(() => {
+    (0, import_pdf._resetPdfInspectorForTesting)();
+  });
+  (0, import_vitest.it)("isAvailable returns true when module is injected", () => {
+    (0, import_vitest.expect)(import_pdf.pdfParser.isAvailable()).toBe(true);
+  });
+  (0, import_vitest.it)("isAvailable returns false when module is not available", () => {
+    (0, import_pdf._resetPdfInspectorForTesting)();
+    (0, import_vitest.expect)(import_pdf.pdfParser.isAvailable()).toBe(false);
+  });
+  (0, import_vitest.it)("parse returns empty result when library is unavailable", async () => {
+    (0, import_pdf._resetPdfInspectorForTesting)();
+    const batches = [];
+    const result = await import_pdf.pdfParser.parse("/fake/doc.pdf", {
+      maxRows: 1e3,
+      onBatch: (rows, cols) => batches.push({ rows, cols })
+    });
+    (0, import_vitest.expect)(result.columns).toEqual([]);
+    (0, import_vitest.expect)(result.totalRows).toBe(0);
+    (0, import_vitest.expect)(batches).toHaveLength(0);
+  });
+  (0, import_vitest.it)("happy path: text-based PDF produces correct rows and metadata", async () => {
+    mockExtractPages.mockReturnValue(makeMockResult([
+      { page: 0, markdown: "# Title\n\nSome content here.", needsOcr: false },
+      { page: 1, markdown: "Page two content.", needsOcr: false }
+    ], { pagesWithTables: [1], pagesWithColumns: [] }));
+    mockProcessPdf.mockReturnValue({
+      pdfType: "TextBased",
+      confidence: 0.97,
+      title: "Test Doc",
+      hasEncodingIssues: false,
+      isComplexLayout: false
+    });
+    const batches = [];
+    const result = await import_pdf.pdfParser.parse("/fake/report.pdf", {
+      maxRows: 1e3,
+      onBatch: (rows, cols) => batches.push({ rows, cols })
+    });
+    (0, import_vitest.expect)(result.columns).toEqual(["page", "content", "needs_ocr", "has_tables", "has_columns"]);
+    (0, import_vitest.expect)(result.totalRows).toBe(2);
+    (0, import_vitest.expect)(batches).toHaveLength(1);
+    (0, import_vitest.expect)(batches[0].rows[0]).toMatchObject({ page: "0", needs_ocr: "false", has_tables: "false" });
+    (0, import_vitest.expect)(batches[0].rows[1]).toMatchObject({ page: "1", needs_ocr: "false", has_tables: "true" });
+    (0, import_vitest.expect)(result.metadataJson).toBeTruthy();
+    const meta = JSON.parse(result.metadataJson);
+    (0, import_vitest.expect)(meta.pdfType).toBe("TextBased");
+    (0, import_vitest.expect)(meta.confidence).toBe(0.97);
+    (0, import_vitest.expect)(meta.hasEncodingIssues).toBe(false);
+  });
+  (0, import_vitest.it)("mixed PDF: text pages indexed normally, scanned pages flagged", async () => {
+    mockExtractPages.mockReturnValue(makeMockResult([
+      { page: 0, markdown: "Text content", needsOcr: false },
+      { page: 1, markdown: "", needsOcr: true },
+      { page: 2, markdown: "More text", needsOcr: false },
+      { page: 3, markdown: "", needsOcr: true }
+    ]));
+    mockProcessPdf.mockReturnValue({ pdfType: "Mixed", confidence: 0.85, hasEncodingIssues: false, isComplexLayout: false });
+    const batches = [];
+    await import_pdf.pdfParser.parse("/fake/mixed.pdf", {
+      maxRows: 1e3,
+      onBatch: (rows2) => batches.push({ rows: rows2 })
+    });
+    (0, import_vitest.expect)(batches).toHaveLength(1);
+    const rows = batches[0].rows;
+    (0, import_vitest.expect)(rows).toHaveLength(4);
+    (0, import_vitest.expect)(rows.filter((r) => r.needs_ocr === "true")).toHaveLength(2);
+    (0, import_vitest.expect)(rows.filter((r) => r.needs_ocr === "false")).toHaveLength(2);
+  });
+  (0, import_vitest.it)("hasEncodingIssues is captured in metadataJson", async () => {
+    mockExtractPages.mockReturnValue(makeMockResult([{ page: 0, markdown: "garbled", needsOcr: false }]));
+    mockProcessPdf.mockReturnValue({ pdfType: "TextBased", confidence: 0.4, hasEncodingIssues: true, isComplexLayout: false });
+    const result = await import_pdf.pdfParser.parse("/fake/garbled.pdf", { maxRows: 1e3, onBatch: () => {
+    } });
+    (0, import_vitest.expect)(result.metadataJson).toBeTruthy();
+    const meta = JSON.parse(result.metadataJson);
+    (0, import_vitest.expect)(meta.hasEncodingIssues).toBe(true);
+  });
+  (0, import_vitest.it)("empty PDF returns no rows and no batch", async () => {
+    mockExtractPages.mockReturnValue({ pages: [], pagesWithTables: [], pagesWithColumns: [] });
+    mockProcessPdf.mockReturnValue({ pdfType: "TextBased", confidence: 1, hasEncodingIssues: false, isComplexLayout: false });
+    const batches = [];
+    const result = await import_pdf.pdfParser.parse("/fake/empty.pdf", {
+      maxRows: 1e3,
+      onBatch: (rows, cols) => batches.push({ rows, cols })
+    });
+    (0, import_vitest.expect)(result.totalRows).toBe(0);
+    (0, import_vitest.expect)(batches).toHaveLength(0);
+  });
+  (0, import_vitest.it)("gracefully handles processPdf failure \u2014 still returns rows", async () => {
+    mockExtractPages.mockReturnValue(makeMockResult([{ page: 0, markdown: "content", needsOcr: false }]));
+    mockProcessPdf.mockImplementation(() => {
+      throw new Error("processPdf failed");
+    });
+    const batches = [];
+    const result = await import_pdf.pdfParser.parse("/fake/broken-meta.pdf", {
+      maxRows: 1e3,
+      onBatch: (rows, cols) => batches.push({ rows, cols })
+    });
+    (0, import_vitest.expect)(result.totalRows).toBe(1);
+    (0, import_vitest.expect)(batches).toHaveLength(1);
+    (0, import_vitest.expect)(result.metadataJson).toBeNull();
+  });
+});
+//# sourceMappingURL=pdf.test.js.map