kirograph 0.16.1 → 0.17.0

package/dist/mcp/tools.js

@@ -480,11 +480,119 @@ const tools = [
         projectPath: { type: "string", description: "Project root path (optional)" }
       }
     }
+  },
+  // ── Data tools (require enableData=true) ────────────────────────────────────
+  {
+    name: "kirograph_data_list",
+    description: "List all indexed datasets with row counts, column counts, and file sizes.",
+    inputSchema: { type: "object", properties: { projectPath: { type: "string", description: "Project root path (optional)" } } }
+  },
+  {
+    name: "kirograph_data_describe",
+    description: "Full schema profile of a dataset: column names, types, cardinality, null%, sample values. Use to orient on a dataset without reading any rows.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID (from kirograph_data_list)" },
+        column: { type: "string", description: "Optional: deep-dive on a single column" },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset"]
+    }
+  },
+  {
+    name: "kirograph_data_query",
+    description: "Filtered row retrieval with structured operators. Returns only matching rows (max 500). Use instead of reading raw data files.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID" },
+        filters: { type: "array", description: "Array of {column, op, value} filters. Ops: eq, neq, gt, gte, lt, lte, contains, in, is_null, between" },
+        columns: { type: "array", description: "Column projection (only return these columns)" },
+        limit: { type: "number", description: "Max rows (default: 100, hard cap: 500)", default: 100 },
+        offset: { type: "number", description: "Pagination offset", default: 0 },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset"]
+    }
+  },
+  {
+    name: "kirograph_data_aggregate",
+    description: "Server-side GROUP BY aggregation. Computation runs in SQLite \u2014 only the result set enters context. Use for count, sum, avg, min, max questions.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID" },
+        groupBy: { type: "array", description: "Columns to group by" },
+        metrics: { type: "array", description: "Array of {column, op} metrics. Ops: count, sum, avg, min, max, count_distinct" },
+        filters: { type: "array", description: "Optional pre-filters (same format as kirograph_data_query)" },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset", "groupBy", "metrics"]
+    }
+  },
+  {
+    name: "kirograph_data_search",
+    description: "Search column names and sample values by keyword. Tells you which column holds the answer without loading data.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID" },
+        query: { type: "string", description: "Search keyword" },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset", "query"]
+    }
+  },
+  {
+    name: "kirograph_data_join",
+    description: "SQL JOIN across two indexed datasets. Combines data without loading either file into context.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        left: { type: "string", description: "Left dataset ID" },
+        right: { type: "string", description: "Right dataset ID" },
+        leftColumn: { type: "string", description: "Join column from left dataset" },
+        rightColumn: { type: "string", description: "Join column from right dataset" },
+        type: { type: "string", description: "Join type: inner (default), left, right", enum: ["inner", "left", "right"], default: "inner" },
+        columns: { type: "array", description: "Column projection (prefix with dataset ID)" },
+        limit: { type: "number", description: "Max rows (default: 100, hard cap: 500)", default: 100 },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["left", "right", "leftColumn", "rightColumn"]
+    }
+  },
+  {
+    name: "kirograph_data_correlations",
+    description: "Pairwise Pearson correlations between numeric columns. Discovers hidden relationships without loading data.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID" },
+        threshold: { type: "number", description: "Min absolute correlation to include (default: 0.3)", default: 0.3 },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset"]
+    }
+  },
+  {
+    name: "kirograph_data_quality",
+    description: "Data quality triage: rank columns by risk (null rate, cardinality anomalies, type issues). Identifies problematic columns without loading data.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        dataset: { type: "string", description: "Dataset ID" },
+        projectPath: { type: "string", description: "Project root path (optional)" }
+      },
+      required: ["dataset"]
+    }
   }
 ];
 class ToolHandler {
   constructor(cg) {
     this.connections = /* @__PURE__ */ new Map();
+    /** Anti-loop: track recent data_query calls per dataset for pagination detection. */
+    this.queryTracker = /* @__PURE__ */ new Map();
     this.defaultCg = cg;
   }
   setDefaultKiroGraph(cg) {
@@ -500,6 +608,34 @@ class ToolHandler {
     }
     this.connections.clear();
   }
+  /** Anti-loop: detect pagination patterns and warn the agent. */
+  checkPaginationLoop(dataset, offset, response) {
+    const now = Date.now();
+    const key = dataset;
+    const currentOffset = offset ?? 0;
+    for (const [k, v] of this.queryTracker) {
+      if (now - v.lastCall > 6e4) this.queryTracker.delete(k);
+    }
+    const entry = this.queryTracker.get(key) ?? { offsets: [], lastCall: 0 };
+    entry.offsets.push(currentOffset);
+    entry.lastCall = now;
+    if (entry.offsets.length > 10) entry.offsets = entry.offsets.slice(-10);
+    this.queryTracker.set(key, entry);
+    if (entry.offsets.length > 5) {
+      const recent = entry.offsets.slice(-6);
+      let isIncrementing = true;
+      for (let i = 1; i < recent.length; i++) {
+        if (recent[i] <= recent[i - 1]) {
+          isIncrementing = false;
+          break;
+        }
+      }
+      if (isIncrementing) {
+        return response + "\n\n\u26A0 Pagination detected. Consider using kirograph_data_aggregate for summary statistics instead of paginating through all rows.";
+      }
+    }
+    return response;
+  }
   async getConnection(projectPath) {
     if (!projectPath) return this.defaultCg;
     const resolved = path.resolve(projectPath);
@@ -529,6 +665,8 @@ class ToolHandler {
               tracker.recordMemorySaving(toolName, outputTokens, naiveCost);
             } else if (toolName.startsWith("kirograph_docs_")) {
               tracker.recordDocsSaving(toolName, outputTokens, naiveCost);
+            } else if (toolName.startsWith("kirograph_data_")) {
+              tracker.recordDataSaving(toolName, outputTokens, naiveCost);
             } else {
               tracker.recordGraphSaving(toolName, outputTokens, naiveCost);
             }
@@ -606,6 +744,9 @@ class ToolHandler {
         if (stats.bySource.docs.count > 0) {
           lines.push(`  Docs tools: ${stats.bySource.docs.count} calls, ~${stats.bySource.docs.saved.toLocaleString()} tokens saved (vs reading full doc files)`);
         }
+        if (stats.bySource.data.count > 0) {
+          lines.push(`  Data tools: ${stats.bySource.data.count} calls, ~${stats.bySource.data.saved.toLocaleString()} tokens saved (vs loading raw data files)`);
+        }
         if (stats.bySource.exec.count > 0) {
           lines.push(`  Compression: ${stats.bySource.exec.count} calls, ~${stats.bySource.exec.saved.toLocaleString()} tokens saved (vs raw output)`);
         }
@@ -732,6 +873,39 @@ class ToolHandler {
           }
         } catch {
         }
+        try {
+          const projectRoot3 = cg.getProjectRoot();
+          const config3 = await (await Promise.resolve().then(() => require("../config.js"))).loadConfig(projectRoot3);
+          if (config3.enableData && config3.dataContextLimit > 0) {
+            const db3 = cg.getDatabase();
+            db3.applyDataSchema();
+            const entryFiles = ctx.entryPoints.map((n) => n.filePath).filter(Boolean);
+            if (entryFiles.length > 0) {
+              const placeholders = entryFiles.map(() => "?").join(", ");
+              const dataRefs = db3.getRawDb().all(
+                `SELECT DISTINCT d.id, d.file_path, d.row_count, d.column_count
+                 FROM data_code_refs r JOIN data_datasets d ON r.dataset_id = d.id
+                 WHERE r.qualified_name IN (${placeholders})`,
+                entryFiles
+              );
+              if (dataRefs.length > 0) {
+                const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+                const dq = new DataQueries(db3.getRawDb());
+                const limit = config3.dataContextLimit;
+                lines.push("", "## Related Data");
+                for (const ref of dataRefs.slice(0, limit)) {
+                  const info = dq.describeDataset(ref.id);
+                  if (info) {
+                    const colSummary = info.columns.map((c) => `${c.name}:${c.inferredType}`).join(", ");
+                    lines.push(`- **${ref.id}** (${ref.file_path}) \u2014 ${ref.row_count} rows, ${ref.column_count} cols`);
+                    lines.push(`  Schema: ${colSummary}`);
+                  }
+                }
+              }
+            }
+          }
+        } catch {
+        }
         return lines.join("\n");
       }
       case "kirograph_callers": {
@@ -841,6 +1015,26 @@ class ToolHandler {
           }
         } catch {
         }
+        let dataLine = "  Data: disabled";
+        try {
+          const { loadConfig: loadCfg2 } = await Promise.resolve().then(() => require("../config.js"));
+          const cfg2 = await loadCfg2(cg.getProjectRoot());
+          if (cfg2.enableData) {
+            const rawDb2 = cg.getDatabase().getRawDb();
+            cg.getDatabase().applyDataSchema();
+            const datasetCount = rawDb2.get("SELECT COUNT(*) as cnt FROM data_datasets")?.cnt ?? 0;
+            if (datasetCount > 0) {
+              const totalRows = rawDb2.get("SELECT SUM(row_count) as total FROM data_datasets")?.total ?? 0;
+              const totalCols = rawDb2.get("SELECT SUM(column_count) as total FROM data_datasets")?.total ?? 0;
+              const totalSize = rawDb2.get("SELECT SUM(file_size) as total FROM data_datasets")?.total ?? 0;
+              const sizeMb = (totalSize / 1024 / 1024).toFixed(2);
+              dataLine = `  Data: enabled \u2014 ${datasetCount} datasets, ${totalRows.toLocaleString()} rows, ${totalCols} columns (${sizeMb} MB source)`;
+            } else {
+              dataLine = `  Data: enabled (no datasets indexed yet \u2014 run kirograph index)`;
+            }
+          }
+        } catch {
+        }
         const threshold = stats.syncWarningThreshold ?? 10;
         const pendingFiles = stats.pendingFiles ?? 0;
         const syncRunning = stats.syncRunning ?? false;
@@ -870,6 +1064,7 @@ class ToolHandler {
           frameworkLine,
           archLine,
           docsLine,
+          dataLine,
           `  DB size: ${dbMb} MB`,
           ...semanticLines,
           ...syncLines,
@@ -1407,6 +1602,234 @@ class ToolHandler {
           return `[${r.refType}] ${direction} (confidence: ${r.confidence.toFixed(2)})`;
         }).join("\n");
       }
+      // ── Data tools ────────────────────────────────────────────────────────────
+      case "kirograph_data_list": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const datasets = dq.listDatasets();
+        if (datasets.length === 0) return "No datasets indexed. Run kirograph index or kirograph data reindex.";
+        return datasets.map((ds) => {
+          const sizeMb = (ds.fileSize / 1024 / 1024).toFixed(2);
+          return `${ds.id} (${ds.format})
+  File: ${ds.filePath}
+  Rows: ${ds.rowCount.toLocaleString()} | Columns: ${ds.columnCount} | Size: ${sizeMb} MB`;
+        }).join("\n\n");
+      }
+      case "kirograph_data_describe": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        if (args.column) {
+          const col = dq.describeColumn(args.dataset, args.column);
+          if (!col) return `Column "${args.column}" not found in dataset "${args.dataset}".`;
+          return [
+            `Column: ${col.name}`,
+            `Type: ${col.inferredType}`,
+            `Nullable: ${col.nullable} (${(col.nullPct * 100).toFixed(1)}% null)`,
+            `Cardinality: ${col.cardinality}`,
+            col.minValue ? `Min: ${col.minValue}` : "",
+            col.maxValue ? `Max: ${col.maxValue}` : "",
+            col.meanValue !== null ? `Mean: ${col.meanValue.toFixed(2)}` : "",
+            `Samples: ${col.sampleValues.join(", ")}`
+          ].filter(Boolean).join("\n");
+        }
+        const result = dq.describeDataset(args.dataset);
+        if (!result) return `Dataset "${args.dataset}" not found. Use kirograph_data_list to see available datasets.`;
+        const lines = [
+          `Dataset: ${result.dataset.id} (${result.dataset.format})`,
+          `File: ${result.dataset.filePath}`,
+          `Rows: ${result.dataset.rowCount.toLocaleString()} | Columns: ${result.dataset.columnCount}`,
+          "",
+          "Columns:"
+        ];
+        for (const col of result.columns) {
+          const nullInfo = col.nullable ? ` (${(col.nullPct * 100).toFixed(0)}% null)` : "";
+          const samples = col.sampleValues.length > 0 ? ` \u2014 samples: ${col.sampleValues.slice(0, 3).join(", ")}` : "";
+          const summary = col.summary ? ` [${col.summary}]` : "";
+          lines.push(`  ${col.name}: ${col.inferredType}${nullInfo} [${col.cardinality} distinct]${samples}${summary}`);
+        }
+        const rules = dq.validationRules(args.dataset);
+        if (rules && rules.length > 0) {
+          lines.push("", "Validation rules:");
+          for (const r of rules.slice(0, 10)) {
+            lines.push(`  ${r.column}: ${r.rules.join("; ")}`);
+          }
+        }
+        const hints = dq.sampleHints(args.dataset);
+        if (hints && hints.length > 0) {
+          lines.push("", "Sample data hints:");
+          for (const h of hints.slice(0, 10)) {
+            lines.push(`  ${h.column}: ${h.hint}`);
+          }
+        }
+        return lines.join("\n");
+      }
+      case "kirograph_data_query": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const result = dq.queryRows(args.dataset, {
+          filters: args.filters,
+          columns: args.columns,
+          limit: args.limit ?? 100,
+          offset: args.offset ?? 0
+        });
+        if (!result) return `Dataset "${args.dataset}" not found.`;
+        if (result.rows.length === 0) return `No rows match the given filters (${result.totalMatching} total in dataset).`;
+        const header = `${result.rows.length} rows returned (${result.totalMatching} total matching):
+`;
+        const rowStrs = result.rows.slice(0, 50).map((row, i) => {
+          const vals = Object.entries(row).map(([k, v]) => `${k}=${v ?? "null"}`).join(", ");
+          return `  ${i + 1}. ${vals}`;
+        });
+        if (result.rows.length > 50) rowStrs.push(`  \u2026and ${result.rows.length - 50} more rows`);
+        let response = header + rowStrs.join("\n");
+        const maxChars = config.dataMaxResponseTokens * 4;
+        if (response.length > maxChars) {
+          response = response.slice(0, maxChars) + "\n\n[truncated: response exceeded token budget]";
+        }
+        response = this.checkPaginationLoop(args.dataset, args.offset, response);
+        return response;
+      }
+      case "kirograph_data_aggregate": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const result = dq.aggregate(args.dataset, {
+          groupBy: args.groupBy ?? [],
+          metrics: args.metrics ?? [],
+          filters: args.filters
+        });
+        if (!result) return `Dataset "${args.dataset}" not found.`;
+        if (result.rows.length === 0) return "No results (empty dataset or all rows filtered out).";
+        const keys = Object.keys(result.rows[0]);
+        const header = keys.join(" | ");
+        const separator = keys.map(() => "---").join(" | ");
+        const rows = result.rows.slice(0, 100).map((row) => keys.map((k) => row[k] ?? "null").join(" | "));
+        let response = `${header}
+${separator}
+${rows.join("\n")}${result.rows.length > 100 ? `
+\u2026and ${result.rows.length - 100} more groups` : ""}`;
+        const maxChars = config.dataMaxResponseTokens * 4;
+        if (response.length > maxChars) {
+          response = response.slice(0, maxChars) + "\n\n[truncated: response exceeded token budget]";
+        }
+        return response;
+      }
+      case "kirograph_data_search": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const cols = dq.searchColumns(args.dataset, args.query);
+        if (cols.length === 0) return `No columns matching "${args.query}" in dataset "${args.dataset}".`;
+        return cols.map((c) => {
+          const samples = c.sampleValues.length > 0 ? ` \u2014 samples: ${c.sampleValues.slice(0, 3).join(", ")}` : "";
+          return `${c.name}: ${c.inferredType} [${c.cardinality} distinct]${samples}`;
+        }).join("\n");
+      }
+      case "kirograph_data_join": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        try {
+          const result = dq.join({
+            left: args.left,
+            right: args.right,
+            leftColumn: args.leftColumn,
+            rightColumn: args.rightColumn,
+            type: args.type ?? "inner",
+            columns: args.columns,
+            limit: args.limit
+          });
+          if (!result) return `Dataset not found. Verify both dataset IDs with kirograph_data_list.`;
+          const joinTypeStr = String(args.type ?? "inner").toUpperCase();
+          const header = `Join: ${args.left}.${args.leftColumn} ${joinTypeStr} JOIN ${args.right}.${args.rightColumn}
+Matching rows: ${result.totalMatching} (showing ${result.rows.length})`;
+          if (result.rows.length === 0) return `${header}
+
+No matching rows.`;
+          const lines = result.rows.map((r) => JSON.stringify(r));
+          let response = `${header}
+
+${lines.join("\n")}`;
+          const maxChars = config.dataMaxResponseTokens * 4;
+          if (response.length > maxChars) {
+            response = response.slice(0, maxChars) + "\n\n[truncated: response exceeded token budget]";
+          }
+          return response;
+        } catch (err) {
+          return `Error: ${err instanceof Error ? err.message : String(err)}`;
+        }
+      }
+      case "kirograph_data_correlations": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const pairs = dq.correlations(args.dataset, args.threshold);
+        if (pairs === null) return `Dataset "${args.dataset}" not found.`;
+        if (pairs.length === 0) return `No correlations above threshold ${args.threshold ?? 0.3} found. The dataset may have fewer than 2 numeric columns or no significant correlations.`;
+        const lines = pairs.map(
+          (p) => `${p.column1} \u2194 ${p.column2}: ${p.correlation > 0 ? "+" : ""}${p.correlation.toFixed(4)} (${p.strength})`
+        );
+        return `Correlations for "${args.dataset}" (threshold: ${args.threshold ?? 0.3}):
+
+${lines.join("\n")}`;
+      }
+      case "kirograph_data_quality": {
+        const { loadConfig } = await Promise.resolve().then(() => require("../config.js"));
+        const projectRoot = cg.getProjectRoot();
+        const config = await loadConfig(projectRoot);
+        if (!config.enableData) return "Data indexing is not enabled. Set enableData: true in .kirograph/config.json and run kirograph index.";
+        const { DataQueries } = await Promise.resolve().then(() => require("../data/queries.js"));
+        const db = cg.getDatabase();
+        db.applyDataSchema();
+        const dq = new DataQueries(db.getRawDb());
+        const quality = dq.quality(args.dataset);
+        if (quality === null) return `Dataset "${args.dataset}" not found.`;
+        if (quality.length === 0) return `No quality issues detected in "${args.dataset}". All columns look healthy.`;
+        const lines = quality.map(
+          (q) => `${q.column} (risk: ${(q.riskScore * 100).toFixed(0)}%): ${q.issues.join("; ")}`
+        );
+        return `Quality report for "${args.dataset}" (${quality.length} columns with issues):
+
+${lines.join("\n")}`;
+      }
       default:
         return `Unknown tool: ${toolName}`;
     }