ai-first-cli 1.2.2 → 1.3.0

package/src/core/parsers/typescriptParser.ts

@@ -0,0 +1,445 @@
+import * as ts from "typescript";
+
+export interface ParsedSymbol {
+  name: string;
+  type: "class" | "interface" | "function" | "method" | "property" | "enum" | "type" | "const" | "variable";
+  line: number;
+  character: number;
+  isExported: boolean;
+  modifiers?: string[];
+  heritage?: {
+    extends?: string[];
+    implements?: string[];
+  };
+  members?: ParsedSymbol[];
+  parameters?: {
+    name: string;
+    type?: string;
+    optional?: boolean;
+  }[];
+  returnType?: string;
+  jsDoc?: string;
+}
+
+export interface ParsedFile {
+  filePath: string;
+  symbols: ParsedSymbol[];
+  imports: {
+    name: string;
+    module: string;
+    isDefault?: boolean;
+    isNamespace?: boolean;
+  }[];
+  exports: string[];
+}
+
+export class TypeScriptParser {
+  private program: ts.Program | null = null;
+
+  parseFile(filePath: string, sourceText: string): ParsedFile {
+    const sourceFile = ts.createSourceFile(
+      filePath,
+      sourceText,
+      ts.ScriptTarget.Latest,
+      true,
+      ts.ScriptKind.TS
+    );
+
+    const symbols: ParsedSymbol[] = [];
+    const imports: ParsedFile["imports"] = [];
+    const exports: string[] = [];
+
+    this.visitNode(sourceFile, sourceFile, symbols, imports, exports);
+
+    return {
+      filePath,
+      symbols,
+      imports,
+      exports,
+    };
+  }
+
+  private visitNode(
+    node: ts.Node,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[],
+    imports: ParsedFile["imports"],
+    exports: string[]
+  ): void {
+    const { line, character } = sourceFile.getLineAndCharacterOfPosition(node.getStart());
+
+    switch (node.kind) {
+      case ts.SyntaxKind.ClassDeclaration:
+        this.handleClassDeclaration(node as ts.ClassDeclaration, line, character, sourceFile, symbols);
+        break;
+
+      case ts.SyntaxKind.InterfaceDeclaration:
+        this.handleInterfaceDeclaration(node as ts.InterfaceDeclaration, line, character, sourceFile, symbols);
+        break;
+
+      case ts.SyntaxKind.FunctionDeclaration:
+        this.handleFunctionDeclaration(node as ts.FunctionDeclaration, line, character, sourceFile, symbols, exports);
+        break;
+
+      case ts.SyntaxKind.EnumDeclaration:
+        this.handleEnumDeclaration(node as ts.EnumDeclaration, line, character, sourceFile, symbols);
+        break;
+
+      case ts.SyntaxKind.TypeAliasDeclaration:
+        this.handleTypeAliasDeclaration(node as ts.TypeAliasDeclaration, line, character, sourceFile, symbols);
+        break;
+
+      case ts.SyntaxKind.VariableStatement:
+        this.handleVariableStatement(node as ts.VariableStatement, line, character, sourceFile, symbols, exports);
+        break;
+
+      case ts.SyntaxKind.ImportDeclaration:
+        this.handleImportDeclaration(node as ts.ImportDeclaration, sourceFile, imports);
+        break;
+
+      case ts.SyntaxKind.ExportAssignment:
+      case ts.SyntaxKind.ExportDeclaration:
+        this.handleExport(node, sourceFile, exports);
+        break;
+    }
+
+    ts.forEachChild(node, (child) =>
+      this.visitNode(child, sourceFile, symbols, imports, exports)
+    );
+  }
+
+  private handleClassDeclaration(
+    node: ts.ClassDeclaration,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[]
+  ): void {
+    if (!node.name) return;
+
+    const symbol: ParsedSymbol = {
+      name: node.name.text,
+      type: "class",
+      line: line + 1,
+      character: character + 1,
+      isExported: this.hasExportModifier(node),
+      modifiers: this.getModifiers(node),
+      heritage: this.getHeritage(node),
+      members: [],
+      jsDoc: this.getJsDoc(node, sourceFile),
+    };
+
+    node.members.forEach((member) => {
+      const memberSymbol = this.extractClassMember(member, sourceFile);
+      if (memberSymbol) {
+        symbol.members!.push(memberSymbol);
+      }
+    });
+
+    symbols.push(symbol);
+  }
+
+  private handleInterfaceDeclaration(
+    node: ts.InterfaceDeclaration,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[]
+  ): void {
+    const symbol: ParsedSymbol = {
+      name: node.name.text,
+      type: "interface",
+      line: line + 1,
+      character: character + 1,
+      isExported: this.hasExportModifier(node),
+      modifiers: this.getModifiers(node),
+      heritage: { extends: node.heritageClauses?.flatMap((h) =>
+        h.types.map((t) => t.getText(sourceFile))
+      ) },
+      members: [],
+      jsDoc: this.getJsDoc(node, sourceFile),
+    };
+
+    node.members.forEach((member) => {
+      const memberSymbol = this.extractInterfaceMember(member, sourceFile);
+      if (memberSymbol) {
+        symbol.members!.push(memberSymbol);
+      }
+    });
+
+    symbols.push(symbol);
+  }
+
+  private handleFunctionDeclaration(
+    node: ts.FunctionDeclaration,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[],
+    exports: string[]
+  ): void {
+    if (!node.name) return;
+
+    const symbol: ParsedSymbol = {
+      name: node.name.text,
+      type: "function",
+      line: line + 1,
+      character: character + 1,
+      isExported: this.hasExportModifier(node),
+      modifiers: this.getModifiers(node),
+      parameters: this.getParameters(node.parameters, sourceFile),
+      returnType: node.type?.getText(sourceFile),
+      jsDoc: this.getJsDoc(node, sourceFile),
+    };
+
+    symbols.push(symbol);
+
+    if (symbol.isExported) {
+      exports.push(symbol.name);
+    }
+  }
+
+  private handleEnumDeclaration(
+    node: ts.EnumDeclaration,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[]
+  ): void {
+    const symbol: ParsedSymbol = {
+      name: node.name.text,
+      type: "enum",
+      line: line + 1,
+      character: character + 1,
+      isExported: this.hasExportModifier(node),
+      modifiers: this.getModifiers(node),
+      jsDoc: this.getJsDoc(node, sourceFile),
+    };
+
+    symbols.push(symbol);
+  }
+
+  private handleTypeAliasDeclaration(
+    node: ts.TypeAliasDeclaration,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[]
+  ): void {
+    const symbol: ParsedSymbol = {
+      name: node.name.text,
+      type: "type",
+      line: line + 1,
+      character: character + 1,
+      isExported: this.hasExportModifier(node),
+      modifiers: this.getModifiers(node),
+      jsDoc: this.getJsDoc(node, sourceFile),
+    };
+
+    symbols.push(symbol);
+  }
+
+  private handleVariableStatement(
+    node: ts.VariableStatement,
+    line: number,
+    character: number,
+    sourceFile: ts.SourceFile,
+    symbols: ParsedSymbol[],
+    exports: string[]
+  ): void {
+    const isExported = this.hasExportModifier(node);
+
+    node.declarationList.declarations.forEach((decl) => {
+      if (ts.isIdentifier(decl.name)) {
+        const symbol: ParsedSymbol = {
+          name: decl.name.text,
+          type: this.isConst(node) ? "const" : "variable",
+          line: line + 1,
+          character: character + 1,
+          isExported,
+          modifiers: this.getModifiers(node),
+          jsDoc: this.getJsDoc(node, sourceFile),
+        };
+
+        symbols.push(symbol);
+
+        if (isExported) {
+          exports.push(symbol.name);
+        }
+      }
+    });
+  }
+
+  private handleImportDeclaration(
+    node: ts.ImportDeclaration,
+    sourceFile: ts.SourceFile,
+    imports: ParsedFile["imports"]
+  ): void {
+    const moduleSpecifier = (node.moduleSpecifier as ts.StringLiteral).text;
+
+    if (node.importClause) {
+      if (node.importClause.name) {
+        imports.push({
+          name: node.importClause.name.text,
+          module: moduleSpecifier,
+          isDefault: true,
+        });
+      }
+
+      if (node.importClause.namedBindings) {
+        if (ts.isNamedImports(node.importClause.namedBindings)) {
+          node.importClause.namedBindings.elements.forEach((element) => {
+            imports.push({
+              name: element.name.text,
+              module: moduleSpecifier,
+              isDefault: false,
+            });
+          });
+        } else if (ts.isNamespaceImport(node.importClause.namedBindings)) {
+          imports.push({
+            name: node.importClause.namedBindings.name.text,
+            module: moduleSpecifier,
+            isNamespace: true,
+          });
+        }
+      }
+    }
+  }
+
+  private handleExport(
+    node: ts.Node,
+    sourceFile: ts.SourceFile,
+    exports: string[]
+  ): void {
+    if (ts.isExportAssignment(node)) {
+      if (ts.isIdentifier(node.expression)) {
+        exports.push(node.expression.text);
+      }
+    }
+  }
+
+  private extractClassMember(
+    member: ts.ClassElement,
+    sourceFile: ts.SourceFile
+  ): ParsedSymbol | null {
+    const { line, character } = sourceFile.getLineAndCharacterOfPosition(member.getStart());
+
+    if (ts.isMethodDeclaration(member) && member.name) {
+      return {
+        name: member.name.getText(sourceFile),
+        type: "method",
+        line: line + 1,
+        character: character + 1,
+        isExported: false,
+        modifiers: this.getModifiers(member),
+        parameters: this.getParameters(member.parameters, sourceFile),
+        returnType: member.type?.getText(sourceFile),
+      };
+    }
+
+    if (ts.isPropertyDeclaration(member) && member.name) {
+      return {
+        name: member.name.getText(sourceFile),
+        type: "property",
+        line: line + 1,
+        character: character + 1,
+        isExported: false,
+        modifiers: this.getModifiers(member),
+      };
+    }
+
+    return null;
+  }
+
+  private extractInterfaceMember(
+    member: ts.TypeElement,
+    sourceFile: ts.SourceFile
+  ): ParsedSymbol | null {
+    const { line, character } = sourceFile.getLineAndCharacterOfPosition(member.getStart());
+
+    if (ts.isMethodSignature(member) && member.name) {
+      return {
+        name: member.name.getText(sourceFile),
+        type: "method",
+        line: line + 1,
+        character: character + 1,
+        isExported: false,
+        parameters: this.getParameters(member.parameters, sourceFile),
+        returnType: member.type?.getText(sourceFile),
+      };
+    }
+
+    if (ts.isPropertySignature(member) && member.name) {
+      return {
+        name: member.name.getText(sourceFile),
+        type: "property",
+        line: line + 1,
+        character: character + 1,
+        isExported: false,
+      };
+    }
+
+    return null;
+  }
+
+  private getModifiers(node: ts.Node): string[] {
+    const modifiers: string[] = [];
+    if (ts.canHaveModifiers(node)) {
+      const nodeModifiers = ts.getModifiers(node);
+      nodeModifiers?.forEach((mod) => {
+        modifiers.push(ts.SyntaxKind[mod.kind].toLowerCase().replace("keyword", ""));
+      });
+    }
+    return modifiers;
+  }
+
+  private hasExportModifier(node: ts.Node): boolean {
+    if (ts.canHaveModifiers(node)) {
+      const modifiers = ts.getModifiers(node);
+      return modifiers?.some((mod) => mod.kind === ts.SyntaxKind.ExportKeyword) ?? false;
+    }
+    return false;
+  }
+
+  private isConst(node: ts.VariableStatement): boolean {
+    return (node.declarationList.flags & ts.NodeFlags.Const) !== 0;
+  }
+
+  private getHeritage(node: ts.ClassDeclaration): { extends?: string[]; implements?: string[] } {
+    const heritage: { extends?: string[]; implements?: string[] } = {};
+
+    if (node.heritageClauses) {
+      node.heritageClauses.forEach((clause) => {
+        const types = clause.types.map((t) => t.getText());
+        if (clause.token === ts.SyntaxKind.ExtendsKeyword) {
+          heritage.extends = types;
+        } else if (clause.token === ts.SyntaxKind.ImplementsKeyword) {
+          heritage.implements = types;
+        }
+      });
+    }
+
+    return heritage;
+  }
+
+  private getParameters(
+    params: ts.NodeArray<ts.ParameterDeclaration>,
+    sourceFile: ts.SourceFile
+  ): { name: string; type?: string; optional?: boolean }[] {
+    return params.map((param) => ({
+      name: param.name.getText(sourceFile),
+      type: param.type?.getText(sourceFile),
+      optional: !!param.questionToken,
+    }));
+  }
+
+  private getJsDoc(node: ts.Node, sourceFile: ts.SourceFile): string | undefined {
+    const jsDoc = ts.getJSDocCommentsAndTags(node);
+    if (jsDoc && jsDoc.length > 0) {
+      return jsDoc[0].getText(sourceFile);
+    }
+    return undefined;
+  }
+}
+
+export const typescriptParser = new TypeScriptParser();

package/src/core/pipeline.ts

@@ -0,0 +1,241 @@
+import { FileInfo } from "./repoScanner.js";
+import { extractSymbols, type SymbolsAnalysis, type Symbol as CodeSymbol } from "../analyzers/symbols.js";
+import { parserRegistry, createSymbolFromParsed } from "./parsers/index.js";
+import { architectureDetector, type ArchitectureAnalysis } from "./analysis/architectureDetector.js";
+import { dependencyAnalyzer, type DependencyGraph } from "./analysis/dependencyAnalyzer.js";
+import { aiContextGenerator } from "./generation/aiContextGenerator.js";
+import { flowGenerator } from "./generation/flowGenerator.js";
+import { architectureGenerator } from "./generation/architectureGenerator.js";
+import { readFile } from "../utils/fileUtils.js";
+
+export interface AnalysisResult {
+  symbols: SymbolsAnalysis;
+  architecture: ArchitectureAnalysis;
+  dependencyGraph: DependencyGraph;
+  aiContext: string;
+  architectureDoc: string;
+  flows: Array<{
+    name: string;
+    type: string;
+    files: string[];
+    layers: string[];
+  }>;
+  metrics: {
+    duration: number;
+    filesProcessed: number;
+    symbolsExtracted: number;
+    cacheHitRate: number;
+  };
+}
+
+interface CacheEntry {
+  symbols: CodeSymbol[];
+  mtime: number;
+}
+
+export class AnalysisPipeline {
+  private cache: Map<string, CacheEntry> = new Map();
+  private stats = { cacheHits: 0, cacheMisses: 0 };
+
+  async run(files: FileInfo[], projectName: string): Promise<AnalysisResult> {
+    const startTime = Date.now();
+
+    try {
+      const symbols = this.extractSymbolsWithAST(files);
+      
+      const cacheHitRate = this.stats.cacheHits + this.stats.cacheMisses > 0 
+        ? Math.round((this.stats.cacheHits / (this.stats.cacheHits + this.stats.cacheMisses)) * 100)
+        : 0;
+      
+      const dependencyGraph = this.buildDependencyGraph(symbols);
+      
+      const architecture = architectureDetector.detect(
+        symbols.symbols,
+        { edges: dependencyGraph.edges }
+      );
+
+      const flows = flowGenerator.generateFlows(
+        architecture.entryPoints,
+        symbols.symbols,
+        { edges: dependencyGraph.edges },
+        { edges: [] },
+        { edges: [] }
+      );
+
+      const aiContext = aiContextGenerator.generate({
+        architecture,
+        symbols: symbols.symbols,
+        dependencies: {
+          totalDependencies: dependencyGraph.edges.length,
+          dependencies: dependencyGraph.edges.map((e) => ({
+            source: e.from,
+            target: e.to,
+            type: e.type,
+          })),
+        },
+        projectName,
+      });
+
+      const architectureDoc = architectureGenerator.generate(architecture, dependencyGraph);
+
+      const duration = Date.now() - startTime;
+
+      return {
+        symbols,
+        architecture,
+        dependencyGraph,
+        aiContext,
+        architectureDoc,
+        flows: flows.map((f) => ({
+          name: f.name,
+          type: f.type,
+          files: f.files.map((fi) => fi.path),
+          layers: f.layers,
+        })),
+        metrics: {
+          duration,
+          filesProcessed: files.length,
+          symbolsExtracted: symbols.symbols.length,
+          cacheHitRate,
+        },
+      };
+    } catch (error) {
+      console.error("Analysis pipeline failed:", error);
+      throw error;
+    }
+  }
+
+  private extractSymbolsWithAST(files: FileInfo[]): SymbolsAnalysis {
+    const allSymbols: SymbolsAnalysis["symbols"] = [];
+    const byFile: Record<string, SymbolsAnalysis["symbols"]> = {};
+
+    const BATCH_SIZE = 50;
+    for (let i = 0; i < files.length; i += BATCH_SIZE) {
+      const batch = files.slice(i, i + BATCH_SIZE);
+      
+      const batchResults = batch.map(file => {
+        const cached = this.getCachedSymbols(file);
+        if (cached) {
+          return { filePath: file.relativePath, symbols: cached };
+        }
+        return this.processFile(file);
+      });
+
+      for (const result of batchResults) {
+        if (result && result.symbols.length > 0) {
+          allSymbols.push(...result.symbols);
+          byFile[result.filePath] = result.symbols;
+        }
+      }
+    }
+
+    const byId: Record<string, (typeof allSymbols)[0]> = {};
+    for (const sym of allSymbols) {
+      byId[sym.id] = sym;
+    }
+
+    const byType: Record<string, typeof allSymbols> = {};
+    for (const sym of allSymbols) {
+      if (!byType[sym.type]) byType[sym.type] = [];
+      byType[sym.type].push(sym);
+    }
+
+    return {
+      symbols: allSymbols,
+      byId,
+      byFile,
+      byType,
+    };
+  }
+
+  private getCachedSymbols(file: FileInfo): CodeSymbol[] | null {
+    const cached = this.cache.get(file.path);
+    if (cached) {
+      this.stats.cacheHits++;
+      return cached.symbols;
+    }
+    return null;
+  }
+
+  private processFile(file: FileInfo): { filePath: string; symbols: CodeSymbol[] } | null {
+    this.stats.cacheMisses++;
+    const ext = "." + file.extension;
+    let symbols: CodeSymbol[] = [];
+
+    try {
+      if (parserRegistry.hasParser(ext)) {
+        const content = readFile(file.path);
+        const parsed = parserRegistry.parse(file.relativePath, content, ext);
+        
+        if (parsed) {
+          symbols = createSymbolFromParsed(parsed, file.relativePath);
+        }
+      } else {
+        const existing = extractSymbols([file]);
+        symbols = existing.symbols;
+      }
+
+      this.cache.set(file.path, { symbols, mtime: Date.now() });
+      
+      if (this.cache.size > 1000) {
+        const firstKey = this.cache.keys().next().value;
+        if (firstKey !== undefined) {
+          this.cache.delete(firstKey);
+        }
+      }
+
+      return { filePath: file.relativePath, symbols };
+    } catch (error) {
+      console.warn(`Failed to process ${file.path}:`, error);
+      const fallbackSymbols = this.fallbackToRegex(file);
+      return { filePath: file.relativePath, symbols: fallbackSymbols };
+    }
+  }
+
+  private fallbackToRegex(file: FileInfo): CodeSymbol[] {
+    const existing = extractSymbols([file]);
+    return existing.symbols;
+  }
+
+  private buildDependencyGraph(symbols: SymbolsAnalysis): DependencyGraph {
+    const files: Array<{ filePath: string; imports: Array<{ name: string; module: string; isDefault?: boolean }>; exports: string[] }> = [];
+
+    for (const [filePath, fileSymbols] of Object.entries(symbols.byFile)) {
+      const imports: Array<{ name: string; module: string; isDefault?: boolean }> = [];
+      const exports: string[] = [];
+
+      for (const sym of fileSymbols) {
+        if (sym.export) {
+          exports.push(sym.name);
+        }
+      }
+
+      files.push({ filePath, imports, exports });
+    }
+
+    return dependencyAnalyzer.buildDependencyGraph(files);
+  }
+
+  clearCache(): void {
+    this.cache.clear();
+    this.stats = { cacheHits: 0, cacheMisses: 0 };
+  }
+
+  getMetrics(): { 
+    cacheSize: number; 
+    cacheHits: number; 
+    cacheMisses: number;
+    cacheHitRate: number;
+  } {
+    const total = this.stats.cacheHits + this.stats.cacheMisses;
+    const cacheHitRate = total > 0 ? Math.round((this.stats.cacheHits / total) * 100) : 0;
+    return { 
+      cacheSize: this.cache.size, 
+      cacheHits: this.stats.cacheHits, 
+      cacheMisses: this.stats.cacheMisses,
+      cacheHitRate,
+    };
+  }
+}
+
+export const analysisPipeline = new AnalysisPipeline();