ai-first-cli 1.1.1 → 1.1.3

package/src/core/incrementalAnalyzer.ts

@@ -0,0 +1,382 @@
+/**
+ * Incremental Analyzer
+ * 
+ * Performs incremental updates to repository intelligence when files change,
+ * without requiring a full re-analysis.
+ */
+
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import { writeFile, readJsonFile } from "../utils/fileUtils.js";
+import { computeFileHash, loadIndexState, saveIndexState, FileState } from "./indexState.js";
+import { extractSymbols } from "../analyzers/symbols.js";
+import { FileInfo } from "./repoScanner.js";
+import { buildKnowledgeGraph } from "./knowledgeGraphBuilder.js";
+
+export interface ChangedFile {
+  path: string;
+  status: "added" | "modified" | "deleted";
+  hash?: string;
+}
+
+export interface IncrementalUpdateResult {
+  changedFiles: ChangedFile[];
+  updatedSymbols: number;
+  updatedDependencies: number;
+  updatedFeatures: string[];
+  updatedFlows: string[];
+  graphUpdated: boolean;
+  errors: string[];
+}
+
+// ============================================================
+// File Change Detection
+// ============================================================
+
+export function detectChangedFiles(rootDir: string, useGit: boolean = true): ChangedFile[] {
+  if (useGit && isGitRepository(rootDir)) {
+    return detectChangesWithGit(rootDir);
+  }
+  return detectChangesWithTimestamps(rootDir);
+}
+
+function isGitRepository(rootDir: string): boolean {
+  return fs.existsSync(path.join(rootDir, ".git"));
+}
+
+function detectChangesWithGit(rootDir: string): ChangedFile[] {
+  const changes: ChangedFile[] = [];
+  
+  try {
+    const output = execSync("git diff --name-status HEAD", {
+      cwd: rootDir,
+      encoding: "utf-8"
+    });
+    
+    for (const line of output.split("\n")) {
+      if (!line.trim()) continue;
+      const [status, filePath] = line.split("\t").map(s => s.trim());
+      if (!filePath) continue;
+      
+      let changeStatus: "added" | "modified" | "deleted" = "modified";
+      if (status.startsWith("A") || status === "??") changeStatus = "added";
+      else if (status.startsWith("D")) changeStatus = "deleted";
+      
+      changes.push({ path: filePath, status: changeStatus });
+    }
+    
+    const stagedOutput = execSync("git diff --name-status --cached", {
+      cwd: rootDir,
+      encoding: "utf-8"
+    });
+    
+    for (const line of stagedOutput.split("\n")) {
+      if (!line.trim()) continue;
+      const [status, filePath] = line.split("\t").map(s => s.trim());
+      if (!filePath || changes.find(c => c.path === filePath)) continue;
+      changes.push({ path: filePath, status: "modified" });
+    }
+  } catch {
+    return detectChangesWithTimestamps(rootDir);
+  }
+  
+  return changes;
+}
+
+function detectChangesWithTimestamps(rootDir: string): ChangedFile[] {
+  const changes: ChangedFile[] = [];
+  const aiDir = path.join(rootDir, "ai");
+  const state = loadIndexState(aiDir);
+  
+  if (!state) return [];
+  
+  for (const [filePath, fileState] of Object.entries(state.files)) {
+    const fullPath = path.join(rootDir, filePath);
+    
+    if (!fs.existsSync(fullPath)) {
+      changes.push({ path: filePath, status: "deleted" });
+      continue;
+    }
+    
+    const currentHash = computeFileHash(fullPath);
+    if (currentHash && currentHash.hash !== fileState.hash) {
+      changes.push({ path: filePath, status: "modified", hash: currentHash.hash });
+    }
+  }
+  
+  return changes;
+}
+
+// ============================================================
+// Symbol Update
+// ============================================================
+
+export function updateSymbols(rootDir: string, changedFiles: ChangedFile[], aiDir: string): number {
+  const symbolsPath = path.join(aiDir, "symbols.json");
+  let existingSymbols: Array<{name: string; file: string; type?: string}> = [];
+  
+  if (fs.existsSync(symbolsPath)) {
+    try {
+      const raw = readJsonFile(symbolsPath);
+      if (Array.isArray(raw)) existingSymbols = raw as typeof existingSymbols;
+    } catch { existingSymbols = []; }
+  }
+  
+  const changedPaths = new Set(changedFiles.map(f => f.path));
+  existingSymbols = existingSymbols.filter(s => !changedPaths.has(s.file));
+  
+  for (const changed of changedFiles) {
+    if (changed.status === "deleted") continue;
+    const fullPath = path.join(rootDir, changed.path);
+    if (!fs.existsSync(fullPath)) continue;
+    
+    try {
+      const fileInfo: FileInfo = {
+        path: changed.path,
+        relativePath: changed.path,
+        extension: path.extname(changed.path),
+        name: path.basename(changed.path, path.extname(changed.path))
+      };
+      const symbols = extractSymbols([fileInfo]);
+      for (const symbol of symbols.symbols || []) {
+        existingSymbols.push({ name: symbol.name, file: changed.path, type: symbol.type });
+      }
+    } catch { /* skip */ }
+  }
+  
+  writeFile(symbolsPath, JSON.stringify(existingSymbols, null, 2));
+  return existingSymbols.length;
+}
+
+// ============================================================
+// Dependency Update
+// ============================================================
+
+export function updateDependencies(rootDir: string, changedFiles: ChangedFile[], aiDir: string): number {
+  const depsPath = path.join(aiDir, "dependencies.json");
+  let existingDeps: Record<string, unknown> = {};
+  
+  if (fs.existsSync(depsPath)) {
+    try { existingDeps = readJsonFile(depsPath) as typeof existingDeps; } catch { /* ignore */ }
+  }
+  
+  const packageFiles = changedFiles.filter(f => 
+    f.path.endsWith("package.json") || f.path.endsWith("requirements.txt")
+  );
+  
+  for (const pkgFile of packageFiles) {
+    const fullPath = path.join(rootDir, pkgFile.path);
+    if (!fs.existsSync(fullPath)) continue;
+    
+    try {
+      if (pkgFile.path.endsWith("package.json")) {
+        const pkg = JSON.parse(fs.readFileSync(fullPath, "utf-8"));
+        existingDeps.dependencies = pkg.dependencies || {};
+        existingDeps.devDependencies = pkg.devDependencies || {};
+      }
+    } catch { /* skip */ }
+  }
+  
+  writeFile(depsPath, JSON.stringify(existingDeps, null, 2));
+  return Object.keys(existingDeps.dependencies as object || {}).length;
+}
+
+// ============================================================
+// Feature Update
+// ============================================================
+
+export function updateFeatures(rootDir: string, changedFiles: ChangedFile[], aiDir: string): string[] {
+  const changedPaths = new Set(changedFiles.map(f => f.path));
+  const featuresDir = path.join(aiDir, "context", "features");
+  const updatedFeatures: string[] = [];
+  
+  if (!fs.existsSync(featuresDir)) return updatedFeatures;
+  
+  try {
+    for (const featureFile of fs.readdirSync(featuresDir)) {
+      if (!featureFile.endsWith(".json")) continue;
+      
+      const featurePath = path.join(featuresDir, featureFile);
+      const featureData = readJsonFile(featurePath) as {name: string; files?: string[]};
+      if (!featureData?.files) continue;
+      
+      const featureFileSet = new Set(featureData.files);
+      const affected = [...changedPaths].some(f => featureFileSet.has(f));
+      
+      if (affected || changedFiles.some(c => c.status === "deleted")) {
+        const featureFilesList = featureData.files.filter(f => 
+          !changedFiles.some(c => c.path === f && c.status === "deleted")
+        );
+        
+        if (featureFilesList.length > 0) {
+          featureData.files = featureFilesList;
+          writeFile(featurePath, JSON.stringify(featureData, null, 2));
+        } else {
+          fs.unlinkSync(featurePath);
+        }
+        updatedFeatures.push(featureData.name);
+      }
+    }
+  } catch { /* ignore */ }
+  
+  return updatedFeatures;
+}
+
+// ============================================================
+// Flow Update
+// ============================================================
+
+export function updateFlows(rootDir: string, changedFiles: ChangedFile[], aiDir: string): string[] {
+  const changedPaths = new Set(changedFiles.map(f => f.path));
+  const flowsDir = path.join(aiDir, "context", "flows");
+  const updatedFlows: string[] = [];
+  
+  if (!fs.existsSync(flowsDir)) return updatedFlows;
+  
+  try {
+    for (const flowFile of fs.readdirSync(flowsDir)) {
+      if (!flowFile.endsWith(".json")) continue;
+      
+      const flowPath = path.join(flowsDir, flowFile);
+      const flowData = readJsonFile(flowPath) as {name: string; files?: string[]};
+      if (!flowData?.files) continue;
+      
+      const flowFileSet = new Set(flowData.files);
+      const affected = [...changedPaths].some(f => flowFileSet.has(f));
+      
+      if (affected || changedFiles.some(c => c.status === "deleted")) {
+        const flowFilesList = flowData.files.filter(f => 
+          !changedFiles.some(c => c.path === f && c.status === "deleted")
+        );
+        
+        if (flowFilesList.length > 0) {
+          flowData.files = flowFilesList;
+          writeFile(flowPath, JSON.stringify(flowData, null, 2));
+        } else {
+          fs.unlinkSync(flowPath);
+        }
+        updatedFlows.push(flowData.name);
+      }
+    }
+  } catch { /* ignore */ }
+  
+  return updatedFlows;
+}
+
+// ============================================================
+// Knowledge Graph Update
+// ============================================================
+
+export function updateKnowledgeGraph(rootDir: string, aiDir: string): boolean {
+  try {
+    buildKnowledgeGraph(rootDir, aiDir);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ============================================================
+// Main Incremental Update
+// ============================================================
+
+export function runIncrementalUpdate(rootDir: string, aiDir?: string): IncrementalUpdateResult {
+  const targetAiDir = aiDir || path.join(rootDir, "ai");
+  const errors: string[] = [];
+  
+  if (!fs.existsSync(targetAiDir)) {
+    return {
+      changedFiles: [],
+      updatedSymbols: 0,
+      updatedDependencies: 0,
+      updatedFeatures: [],
+      updatedFlows: [],
+      graphUpdated: false,
+      errors: ["AI context directory does not exist. Run 'ai-first init' first."]
+    };
+  }
+  
+  const changedFiles = detectChangedFiles(rootDir);
+  
+  if (changedFiles.length === 0) {
+    return {
+      changedFiles: [],
+      updatedSymbols: 0,
+      updatedDependencies: 0,
+      updatedFeatures: [],
+      updatedFlows: [],
+      graphUpdated: false,
+      errors: []
+    };
+  }
+  
+  let updatedSymbols = 0;
+  try { updatedSymbols = updateSymbols(rootDir, changedFiles, targetAiDir); }
+  catch (e) { errors.push(`Failed to update symbols: ${e}`); }
+  
+  let updatedDependencies = 0;
+  try { updatedDependencies = updateDependencies(rootDir, changedFiles, targetAiDir); }
+  catch (e) { errors.push(`Failed to update dependencies: ${e}`); }
+  
+  let updatedFeatures: string[] = [];
+  try { updatedFeatures = updateFeatures(rootDir, changedFiles, targetAiDir); }
+  catch (e) { errors.push(`Failed to update features: ${e}`); }
+  
+  let updatedFlows: string[] = [];
+  try { updatedFlows = updateFlows(rootDir, changedFiles, targetAiDir); }
+  catch (e) { errors.push(`Failed to update flows: ${e}`); }
+  
+  let graphUpdated = false;
+  try { graphUpdated = updateKnowledgeGraph(rootDir, targetAiDir); }
+  catch (e) { errors.push(`Failed to update knowledge graph: ${e}`); }
+  
+  try { updateIndexState(rootDir, changedFiles); }
+  catch { /* non-critical */ }
+  
+  return {
+    changedFiles,
+    updatedSymbols,
+    updatedDependencies,
+    updatedFeatures,
+    updatedFlows,
+    graphUpdated,
+    errors
+  };
+}
+
+function updateIndexState(rootDir: string, changedFiles: ChangedFile[]): void {
+  const aiDir = path.join(rootDir, "ai");
+  let state = loadIndexState(aiDir);
+  
+  if (!state) {
+    state = { version: "1.0.0", lastIndexed: new Date().toISOString(), totalFiles: 0, files: {} };
+  }
+  
+  const filesMap = new Map<string, FileState>(Object.entries(state.files));
+  
+  for (const changed of changedFiles) {
+    const fullPath = path.join(rootDir, changed.path);
+    
+    if (changed.status === "deleted") {
+      filesMap.delete(changed.path);
+    } else if (fs.existsSync(fullPath)) {
+      const hashData = computeFileHash(fullPath);
+      if (hashData) {
+        filesMap.set(changed.path, {
+          path: changed.path,
+          hash: hashData.hash,
+          mtime: hashData.mtime,
+          size: hashData.size,
+          indexedAt: new Date().toISOString()
+        });
+      }
+    }
+  }
+  
+  state.files = Object.fromEntries(filesMap);
+  state.totalFiles = filesMap.size;
+  state.lastIndexed = new Date().toISOString();
+  
+  saveIndexState(aiDir, filesMap);
+}

package/src/core/knowledgeGraphBuilder.ts

@@ -0,0 +1,181 @@
+import fs from "fs";
+import path from "path";
+import { ensureDir, writeFile, readJsonFile } from "../utils/fileUtils.js";
+
+export type NodeType = "feature" | "flow" | "file" | "symbol" | "commit";
+export type EdgeType = "contains" | "implements" | "declares" | "references" | "modifies";
+
+export interface KnowledgeNode {
+  id: string;
+  type: NodeType;
+  label?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface KnowledgeEdge {
+  from: string;
+  to: string;
+  type: EdgeType;
+}
+
+export interface KnowledgeGraph {
+  nodes: KnowledgeNode[];
+  edges: KnowledgeEdge[];
+  metadata: {
+    generated: string;
+    sources: string[];
+    nodeCount: number;
+    edgeCount: number;
+  };
+}
+
+function loadFeatures(aiDir: string) {
+  const featuresPath = path.join(aiDir, "context", "features");
+  if (!fs.existsSync(featuresPath)) return [];
+  const features: Array<{name: string; path: string; files: string[]}> = [];
+  try {
+    for (const file of fs.readdirSync(featuresPath)) {
+      if (!file.endsWith(".json")) continue;
+      const data = readJsonFile(path.join(featuresPath, file)) as {name?: unknown; path?: unknown; files?: unknown};
+      if (data?.name && Array.isArray(data.files)) {
+        features.push({ name: String(data.name), path: String(data.path || ""), files: data.files.map(String) });
+      }
+    }
+  } catch { return []; }
+  return features;
+}
+
+function loadFlows(aiDir: string) {
+  const flowsPath = path.join(aiDir, "context", "flows");
+  if (!fs.existsSync(flowsPath)) return [];
+  const flows: Array<{name: string; entrypoint: string; files: string[]}> = [];
+  try {
+    for (const file of fs.readdirSync(flowsPath)) {
+      if (!file.endsWith(".json")) continue;
+      const data = readJsonFile(path.join(flowsPath, file)) as {name?: unknown; entrypoint?: unknown; files?: unknown};
+      if (data?.name && Array.isArray(data.files)) {
+        flows.push({ name: String(data.name), entrypoint: String(data.entrypoint || ""), files: data.files.map(String) });
+      }
+    }
+  } catch { return []; }
+  return flows;
+}
+
+function loadGitActivity(aiDir: string) {
+  const gitPath = path.join(aiDir, "git", "commit-activity.json");
+  if (!fs.existsSync(gitPath)) return null;
+  try {
+    const data = readJsonFile(gitPath) as {totalCommits?: unknown; files?: unknown};
+    if (typeof data?.totalCommits === "number" && typeof data?.files === "object") {
+      return { totalCommits: data.totalCommits, files: data.files as Record<string, number> };
+    }
+  } catch { return null; }
+  return null;
+}
+
+function loadSymbols(aiDir: string) {
+  const symbolsPath = path.join(aiDir, "symbols.json");
+  if (!fs.existsSync(symbolsPath)) return [];
+  try {
+    const data = readJsonFile(symbolsPath);
+    if (Array.isArray(data)) {
+      return data.slice(0, 1000).map((s: unknown) => {
+        const sym = s as {name?: unknown; file?: unknown; type?: unknown; references?: unknown};
+        return { name: String(sym.name || ""), file: String(sym.file || ""), type: sym.type ? String(sym.type) : undefined, references: Array.isArray(sym.references) ? sym.references.map(String) : undefined };
+      });
+    }
+  } catch { return []; }
+  return [];
+}
+
+function loadFiles(aiDir: string) {
+  const filesPath = path.join(aiDir, "files.json");
+  if (!fs.existsSync(filesPath)) return [];
+  try {
+    const data = readJsonFile(filesPath);
+    if (Array.isArray(data)) {
+      return data.slice(0, 500).map((f: unknown) => typeof f === "string" ? f : (f as {path?: unknown})?.path ? String((f as {path: unknown}).path) : String(f));
+    }
+  } catch { return []; }
+  return [];
+}
+
+export function createNodes(aiDir: string): KnowledgeNode[] {
+  const nodes: KnowledgeNode[] = [];
+  const nodeIds = new Set<string>();
+  const addNode = (id: string, type: NodeType, label?: string, metadata?: Record<string, unknown>) => {
+    if (nodeIds.has(id)) return;
+    nodeIds.add(id);
+    nodes.push({ id, type, label: label || id, metadata });
+  };
+  for (const f of loadFeatures(aiDir)) addNode(f.name, "feature", f.name, { path: f.path, fileCount: f.files.length });
+  for (const f of loadFlows(aiDir)) addNode(f.name, "flow", f.name, { entrypoint: f.entrypoint, fileCount: f.files.length });
+  for (const f of loadFiles(aiDir)) addNode(f, "file", path.basename(f));
+  for (const s of loadSymbols(aiDir)) if (s.file && s.name) addNode(`${s.file}#${s.name}`, "symbol", s.name, { file: s.file, symbolType: s.type });
+  const git = loadGitActivity(aiDir);
+  if (git) {
+    for (const file of Object.entries(git.files).sort((a, b) => b[1] - a[1]).slice(0, 50).map(x => x[0])) {
+      addNode(`commit:${file}`, "commit", path.basename(file), { file, commitCount: git.files[file] });
+    }
+  }
+  return nodes;
+}
+
+export function createEdges(aiDir: string): KnowledgeEdge[] {
+  const edges: KnowledgeEdge[] = [];
+  const edgeKeys = new Set<string>();
+  const addEdge = (from: string, to: string, type: EdgeType) => {
+    const key = `${from}|${to}|${type}`;
+    if (edgeKeys.has(key)) return;
+    edgeKeys.add(key);
+    edges.push({ from, to, type });
+  };
+  const features = loadFeatures(aiDir);
+  const flows = loadFlows(aiDir);
+  for (const flow of flows) {
+    for (const feature of features) {
+      const intersection = flow.files.filter(f => feature.files.includes(f));
+      if (intersection.length > 0) addEdge(feature.name, flow.name, "contains");
+    }
+  }
+  for (const flow of flows) for (const file of flow.files) addEdge(flow.name, file, "implements");
+  for (const s of loadSymbols(aiDir)) if (s.file && s.name) addEdge(s.file, `${s.file}#${s.name}`, "declares");
+  for (const s of loadSymbols(aiDir)) if (s.file && s.references) for (const ref of s.references) addEdge(`${s.file}#${s.name}`, ref, "references");
+  const git = loadGitActivity(aiDir);
+  if (git) for (const file of Object.keys(git.files)) addEdge(`commit:${file}`, file, "modifies");
+  return edges;
+}
+
+export function buildKnowledgeGraph(rootDir: string, aiDir?: string): KnowledgeGraph {
+  const targetAiDir = aiDir || path.join(rootDir, "ai");
+  const graphDir = path.join(targetAiDir, "graph");
+  ensureDir(graphDir);
+  const nodes = createNodes(targetAiDir);
+  const edges = createEdges(targetAiDir);
+  const sources: string[] = [];
+  if (fs.existsSync(path.join(targetAiDir, "context", "features"))) sources.push("features");
+  if (fs.existsSync(path.join(targetAiDir, "context", "flows"))) sources.push("flows");
+  if (fs.existsSync(path.join(targetAiDir, "git"))) sources.push("git");
+  if (fs.existsSync(path.join(targetAiDir, "symbols.json"))) sources.push("symbols");
+  if (fs.existsSync(path.join(targetAiDir, "dependencies.json"))) sources.push("dependencies");
+  const graph: KnowledgeGraph = { nodes, edges, metadata: { generated: new Date().toISOString(), sources, nodeCount: nodes.length, edgeCount: edges.length } };
+  writeFile(path.join(graphDir, "knowledge-graph.json"), JSON.stringify(graph, null, 2));
+  return graph;
+}
+
+export function loadKnowledgeGraph(aiDir: string): KnowledgeGraph | null {
+  const graphPath = path.join(aiDir, "graph", "knowledge-graph.json");
+  if (!fs.existsSync(graphPath)) return null;
+  try { return readJsonFile(graphPath) as unknown as KnowledgeGraph; } catch { return null; }
+}
+
+export function getNodesByType(graph: KnowledgeGraph, type: NodeType): KnowledgeNode[] { return graph.nodes.filter(n => n.type === type); }
+export function getEdgesByType(graph: KnowledgeGraph, type: EdgeType): KnowledgeEdge[] { return graph.edges.filter(e => e.type === type); }
+export function getNeighbors(graph: KnowledgeGraph, nodeId: string): KnowledgeNode[] {
+  const neighborIds = new Set<string>();
+  for (const edge of graph.edges) {
+    if (edge.from === nodeId) neighborIds.add(edge.to);
+    if (edge.to === nodeId) neighborIds.add(edge.from);
+  }
+  return graph.nodes.filter(n => neighborIds.has(n.id));
+}