@thoughtflow/codebase 0.2.0

package/src/tools/workspace/analyze-project.tool.ts

@@ -0,0 +1,429 @@
+import { z } from "zod";
+import * as fs from "fs";
+import * as path from "path";
+import { ToolBase, ToolContext, ToolExecutionResult, KnowledgeGraph, resolveSafePath } from "@thoughtflow/core";
+
+/** Depth levels for project analysis. */
+export type AnalysisDepth = "shallow" | "standard" | "deep";
+
+/** Directories always skipped during scanning. */
+const SKIP_DIRS = new Set([
+  "node_modules",
+  "vendor",
+  "dist",
+  "build",
+  ".git",
+  "coverage",
+  "__pycache__",
+  ".next",
+  "target",
+  ".turbo",
+  ".cache",
+  "out",
+  ".svelte-kit",
+]);
+
+/** Config files used to identify project characteristics. */
+const CONFIG_FILES: Array<{ file: string; key: string }> = [
+  { file: "package.json",       key: "packageJson" },
+  { file: "tsconfig.json",      key: "tsconfig" },
+  { file: "nest-cli.json",      key: "nestCli" },
+  { file: "docker-compose.yml", key: "dockerCompose" },
+  { file: "docker-compose.yaml",key: "dockerCompose" },
+  { file: "Cargo.toml",         key: "cargoToml" },
+  { file: "pyproject.toml",     key: "pyprojectToml" },
+  { file: "go.mod",             key: "goMod" },
+  { file: ".eslintrc.json",     key: "eslint" },
+  { file: "jest.config.ts",     key: "jestConfig" },
+  { file: "jest.config.js",     key: "jestConfig" },
+  { file: "vitest.config.ts",   key: "vitestConfig" },
+];
+
+export interface ProjectAnalysisResult {
+  language: string | null;
+  framework: string | null;
+  testFramework: string | null;
+  entryPoints: string[];
+  dependencies: string[];
+  devDependencies: string[];
+  dockerized: boolean;
+  ports: number[];
+  cicd: string[];
+  structure?: string;   // shallow directory tree (standard+deep)
+  httpEndpoints?: string[]; // deep only
+  exports?: string[];       // deep only
+  configFiles: string[];
+}
+
+export class AnalyzeProjectTool extends ToolBase {
+  readonly name = "analyzeProject";
+  override readonly readonly = true;
+  readonly description =
+    "Analyse an existing project to determine its language, framework, test framework, entry points, " +
+    "dependencies, Docker setup, and (in deep mode) HTTP endpoints and API exports. " +
+    "Depth: 'shallow' = config files only; 'standard' = config + directory tree up to depth 3 (default); " +
+    "'deep' = standard + HTTP endpoint and export extraction. " +
+    "Always skips node_modules, vendor, dist, build, .git, coverage, __pycache__, .next, target.";
+
+  constructor(private readonly rootDir: string, private readonly knowledgeGraph?: KnowledgeGraph) {
+    super(
+      z.object({
+        path:  z.string().optional().describe("Subdirectory to analyse (relative to rootDir). Defaults to rootDir."),
+        depth: z.enum(["shallow", "standard", "deep"]).optional().describe(
+          "Analysis depth: 'shallow' (config files only), 'standard' (default; + directory tree ≤3 deep), 'deep' (+ HTTP endpoints and exports)"
+        ),
+      })
+    );
+  }
+
+  async run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult> {
+    const depth: AnalysisDepth = ((input["depth"] as string) ?? "standard") as AnalysisDepth;
+    const relPath = (input["path"] as string | undefined) ?? ".";
+    const analysisRoot = resolveSafePath(this.rootDir, relPath);
+    if (!analysisRoot) {
+      return { success: false, error: "Path traversal not allowed." };
+    }
+    if (!fs.existsSync(analysisRoot)) {
+      return { success: false, error: `Directory not found: ${relPath}` };
+    }
+
+    const result: ProjectAnalysisResult = {
+      language: null,
+      framework: null,
+      testFramework: null,
+      entryPoints: [],
+      dependencies: [],
+      devDependencies: [],
+      dockerized: false,
+      ports: [],
+      cicd: [],
+      configFiles: [],
+    };
+
+    // ── Read config files ──────────────────────────────────────────────────
+    const configs: Record<string, unknown> = {};
+    for (const { file, key } of CONFIG_FILES) {
+      const fullPath = path.join(analysisRoot, file);
+      if (fs.existsSync(fullPath)) {
+        result.configFiles.push(file);
+        try {
+          const raw = fs.readFileSync(fullPath, "utf-8");
+          if (file.endsWith(".json") || file.endsWith(".toml")) {
+            configs[key] = file.endsWith(".json") ? JSON.parse(raw) : raw;
+          } else {
+            configs[key] = raw;
+          }
+        } catch {
+          // parse error — keep the filename in configFiles but skip data
+        }
+      }
+    }
+
+    // ── Language + Framework detection ────────────────────────────────────
+    const pkg = configs["packageJson"] as Record<string, unknown> | undefined;
+    const deps: Record<string, string> = {
+      ...((pkg?.["dependencies"] ?? {}) as Record<string, string>),
+    };
+    const devDeps: Record<string, string> = {
+      ...((pkg?.["devDependencies"] ?? {}) as Record<string, string>),
+    };
+
+    if (pkg) {
+      result.language = "javascript";
+      if (configs["tsconfig"] || devDeps["typescript"] || devDeps["ts-node"]) {
+        result.language = "typescript";
+      }
+      result.dependencies = Object.keys(deps);
+      result.devDependencies = Object.keys(devDeps);
+
+      // Framework
+      if (deps["@nestjs/core"] || deps["@nestjs/common"]) {
+        result.framework = "nestjs";
+      } else if (deps["express"]) {
+        result.framework = "express";
+      } else if (deps["fastify"]) {
+        result.framework = "fastify";
+      } else if (deps["next"]) {
+        result.framework = "nextjs";
+      } else if (deps["react"]) {
+        result.framework = "react";
+      } else if (deps["vue"]) {
+        result.framework = "vue";
+      } else if (deps["svelte"] || deps["@sveltejs/kit"]) {
+        result.framework = "svelte";
+      } else if (deps["hono"]) {
+        result.framework = "hono";
+      }
+
+      // Test framework
+      if (devDeps["jest"] || deps["jest"]) {
+        result.testFramework = "jest";
+      } else if (devDeps["vitest"] || deps["vitest"]) {
+        result.testFramework = "vitest";
+      } else if (devDeps["mocha"] || deps["mocha"]) {
+        result.testFramework = "mocha";
+      } else if (devDeps["@playwright/test"]) {
+        result.testFramework = "playwright";
+      }
+
+      // Entry points from package.json scripts
+      const scripts = (pkg["scripts"] ?? {}) as Record<string, string>;
+      const mainField = pkg["main"] as string | undefined;
+      if (mainField) result.entryPoints.push(mainField);
+      const startScript = scripts["start"] ?? scripts["dev"] ?? scripts["serve"];
+      if (startScript && result.entryPoints.length === 0) {
+        result.entryPoints.push(`script:start → ${startScript}`);
+      }
+    }
+
+    if (configs["cargoToml"]) {
+      result.language = "rust";
+    }
+    if (configs["pyprojectToml"]) {
+      result.language = "python";
+    }
+    if (configs["goMod"]) {
+      result.language = "go";
+    }
+
+    // ── Docker / CI ────────────────────────────────────────────────────────
+    if (configs["dockerCompose"]) {
+      result.dockerized = true;
+      // extract exposed ports
+      const composeRaw = configs["dockerCompose"] as string;
+      const portMatches = composeRaw.matchAll(/["']?(\d{2,5}):(\d{2,5})["']?/g);
+      const portSet = new Set<number>();
+      for (const m of portMatches) {
+        const p = Number(m[2]);
+        if (p > 0 && p < 65536) portSet.add(p);
+      }
+      result.ports = Array.from(portSet);
+    }
+    if (fs.existsSync(path.join(analysisRoot, "Dockerfile"))) {
+      result.dockerized = true;
+    }
+
+    const ciFiles = [
+      ".github/workflows",
+      ".gitlab-ci.yml",
+      ".circleci/config.yml",
+      "Jenkinsfile",
+      ".travis.yml",
+      "azure-pipelines.yml",
+      ".drone.yml",
+    ];
+    for (const f of ciFiles) {
+      const full = path.join(analysisRoot, f);
+      if (fs.existsSync(full)) {
+        result.cicd.push(f);
+      }
+    }
+
+    // ── Directory structure (standard+deep) ───────────────────────────────
+    if (depth !== "shallow") {
+      result.structure = this.buildTree(analysisRoot, analysisRoot, 0, 3);
+    }
+
+    // ── Deep analysis: HTTP endpoints + exports ───────────────────────────
+    if (depth === "deep") {
+      result.httpEndpoints = this.extractHttpEndpoints(analysisRoot);
+      result.exports = this.extractExports(analysisRoot);
+    }
+
+    // ── Build knowledge graph (if available) ─────────────────────────────
+    if (this.knowledgeGraph) {
+      await this.buildKnowledgeGraph(result, analysisRoot);
+    }
+
+    const summary =
+      `language: ${result.language ?? "unknown"}\n` +
+      `framework: ${result.framework ?? "none"}\n` +
+      `testFramework: ${result.testFramework ?? "none"}\n` +
+      `entryPoints: ${result.entryPoints.join(", ") || "none"}\n` +
+      `dependencies (${result.dependencies.length}): ${result.dependencies.slice(0, 20).join(", ")}${result.dependencies.length > 20 ? "…" : ""}\n` +
+      `devDependencies (${result.devDependencies.length}): ${result.devDependencies.slice(0, 20).join(", ")}${result.devDependencies.length > 20 ? "…" : ""}\n` +
+      `dockerized: ${result.dockerized}\n` +
+      `ports: ${result.ports.join(", ") || "none"}\n` +
+      `cicd: ${result.cicd.join(", ") || "none"}\n` +
+      `configFiles: ${result.configFiles.join(", ") || "none"}\n` +
+      (result.structure ? `\nStructure:\n${result.structure}` : "") +
+      (result.httpEndpoints ? `\nHTTP endpoints (${result.httpEndpoints.length}): ${result.httpEndpoints.slice(0, 30).join(", ")}` : "") +
+      (result.exports ? `\nExports (${result.exports.length}): ${result.exports.slice(0, 30).join(", ")}` : "");
+
+    return { success: true, output: summary, data: result };
+  }
+
+  /** Build an ASCII directory tree skipping SKIP_DIRS, up to maxDepth. */
+  private buildTree(root: string, dir: string, currentDepth: number, maxDepth: number): string {
+    if (currentDepth > maxDepth) return "";
+    let entries: fs.Dirent[];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return "";
+    }
+    const lines: string[] = [];
+    const indent = "  ".repeat(currentDepth);
+    for (const entry of entries) {
+      if (SKIP_DIRS.has(entry.name)) continue;
+      if (entry.isDirectory()) {
+        lines.push(`${indent}${entry.name}/`);
+        if (currentDepth < maxDepth) {
+          lines.push(this.buildTree(root, path.join(dir, entry.name), currentDepth + 1, maxDepth));
+        }
+      } else {
+        lines.push(`${indent}${entry.name}`);
+      }
+    }
+    return lines.filter(Boolean).join("\n");
+  }
+
+  /** Scan TS/JS files for route definitions like router.get('/...'), @Get('/...'). */
+  private extractHttpEndpoints(dir: string): string[] {
+    const endpoints: string[] = [];
+    const routeRegex = /\.(get|post|put|patch|delete)\s*\(\s*['"`]([^'"`]+)['"`]/gi;
+    const decoratorRegex = /@(Get|Post|Put|Patch|Delete|All)\s*\(\s*['"`]([^'"`]*)['"`]/g;
+
+    const walk = (d: string) => {
+      if (endpoints.length > 100) return;
+      let entries: fs.Dirent[];
+      try { entries = fs.readdirSync(d, { withFileTypes: true }); }
+      catch { return; }
+      for (const entry of entries) {
+        if (SKIP_DIRS.has(entry.name)) continue;
+        const full = path.join(d, entry.name);
+        if (entry.isDirectory()) {
+          walk(full);
+        } else if (entry.isFile() && /\.(ts|js|tsx|jsx)$/.test(entry.name)) {
+          let src: string;
+          try { src = fs.readFileSync(full, "utf-8"); }
+          catch { continue; }
+          for (const m of src.matchAll(routeRegex)) {
+            endpoints.push(`${m[1].toUpperCase()} ${m[2]}`);
+          }
+          for (const m of src.matchAll(decoratorRegex)) {
+            endpoints.push(`${m[1].toUpperCase()} ${m[2] || "/"}`);
+          }
+        }
+      }
+    };
+    walk(dir);
+    return [...new Set(endpoints)];
+  }
+
+  /** Scan TS/JS index files for named exports. */
+  private extractExports(dir: string): string[] {
+    const exports: string[] = [];
+    const exportRegex = /export\s+(?:(?:default\s+)?(?:class|function|const|let|var|type|interface|enum)\s+)?(\w+)/g;
+
+    const walk = (d: string, depth: number) => {
+      if (exports.length > 200 || depth > 5) return;
+      let entries: fs.Dirent[];
+      try { entries = fs.readdirSync(d, { withFileTypes: true }); }
+      catch { return; }
+      for (const entry of entries) {
+        if (SKIP_DIRS.has(entry.name)) continue;
+        const full = path.join(d, entry.name);
+        if (entry.isDirectory()) {
+          walk(full, depth + 1);
+        } else if (entry.isFile() && /\.(ts|tsx)$/.test(entry.name) && !entry.name.endsWith(".d.ts")) {
+          let src: string;
+          try { src = fs.readFileSync(full, "utf-8"); }
+          catch { continue; }
+          for (const m of src.matchAll(exportRegex)) {
+            if (m[1] && m[1] !== "from" && m[1] !== "default") {
+              exports.push(m[1]);
+            }
+          }
+        }
+      }
+    };
+    walk(dir, 0);
+    return [...new Set(exports)];
+  }
+
+  /** Build knowledge graph nodes and edges from project analysis. */
+  private async buildKnowledgeGraph(result: ProjectAnalysisResult, analysisRoot: string): Promise<void> {
+    if (!this.knowledgeGraph) return;
+
+    // Create project root node
+    const projectNode = await this.knowledgeGraph.addNode({
+      type: "module",
+      name: path.basename(analysisRoot),
+      description: `Project root: ${result.language} ${result.framework} application`,
+      source: "code",
+      sourceFile: ".",
+      confidence: 1.0,
+      tags: ["project", "root"],
+      metadata: {
+        language: result.language,
+        framework: result.framework,
+        testFramework: result.testFramework,
+        dockerized: result.dockerized,
+      },
+    });
+
+    // Create dependency nodes and edges
+    for (const dep of result.dependencies) {
+      const depNode = await this.knowledgeGraph.addNode({
+        type: "dependency",
+        name: dep,
+        description: `NPM dependency: ${dep}`,
+        source: "code",
+        sourceFile: "package.json",
+        confidence: 1.0,
+        tags: ["dependency", "npm"],
+      });
+
+      await this.knowledgeGraph.addEdge({
+        fromId: projectNode.id,
+        toId: depNode.id,
+        type: "depends_on",
+        confidence: 1.0,
+        evidence: "Listed in package.json dependencies",
+      });
+    }
+
+    // Create entry point nodes
+    for (const entry of result.entryPoints) {
+      const entryNode = await this.knowledgeGraph.addNode({
+        type: "service",
+        name: entry,
+        description: `Application entry point: ${entry}`,
+        source: "code",
+        sourceFile: entry,
+        confidence: 1.0,
+        tags: ["entry-point"],
+      });
+
+      await this.knowledgeGraph.addEdge({
+        fromId: projectNode.id,
+        toId: entryNode.id,
+        type: "contains",
+        confidence: 1.0,
+        evidence: "Defined as entry point in package.json",
+      });
+    }
+
+    // Create config file nodes
+    for (const configFile of result.configFiles) {
+      const configNode = await this.knowledgeGraph.addNode({
+        type: "module",
+        name: configFile,
+        description: `Configuration file: ${configFile}`,
+        source: "code",
+        sourceFile: configFile,
+        confidence: 1.0,
+        tags: ["config"],
+      });
+
+      await this.knowledgeGraph.addEdge({
+        fromId: projectNode.id,
+        toId: configNode.id,
+        type: "contains",
+        confidence: 1.0,
+        evidence: "Configuration file in project root",
+      });
+    }
+  }
+}

package/src/tools/workspace/codebase-research.tool.ts

@@ -0,0 +1,100 @@
+import { z } from "zod";
+import { ToolBase, ToolContext, ToolExecutionResult, DependencyAnalyzer } from "@thoughtflow/core";
+import { CodebaseIndex } from "../../libs/codebase-index.class";
+
+const ACTIONS = ["search", "dependencies", "duplicates", "stats"] as const;
+
+export class CodebaseResearchTool extends ToolBase {
+  readonly name = "codebaseResearch";
+  override readonly readonly = true;
+  readonly description =
+    "Task-flow helper over CodebaseIndex and dependency analysis. " +
+    "search: code/path/symbol search. dependencies: build import graph for provided files. " +
+    "duplicates: check whether proposed symbols already exist. stats: index metadata.";
+
+  constructor(private readonly rootDir: string) {
+    super(
+      z.object({
+        action: z.enum(ACTIONS),
+        query: z.string().optional(),
+        mode: z.enum(["hybrid", "content", "path", "symbol"]).optional(),
+        files: z.array(z.string()).optional(),
+        symbols: z.array(z.string()).optional(),
+        limit: z.number().int().min(1).max(200).optional(),
+        reindex: z.boolean().optional(),
+      })
+    );
+  }
+
+  async run(input: Record<string, unknown>, _ctx: ToolContext): Promise<ToolExecutionResult> {
+    const action = input["action"] as (typeof ACTIONS)[number];
+
+    const index = await CodebaseIndex.forRoot(this.rootDir);
+    if (!index.isIndexed() || input["reindex"] === true) {
+      index.rebuild();
+    }
+
+    if (action === "stats") {
+      return {
+        success: true,
+        output: JSON.stringify(index.getStats()),
+      };
+    }
+
+    if (action === "search") {
+      const query = String(input["query"] ?? "").trim();
+      const mode = (input["mode"] as "hybrid" | "content" | "path" | "symbol" | undefined) ?? "hybrid";
+      const limit = Number(input["limit"] ?? 20);
+
+      const hits = index.search({
+        query: query || undefined,
+        mode,
+        limit,
+        offset: 0,
+      });
+
+      return {
+        success: true,
+        output: JSON.stringify({ count: hits.length, hits }),
+      };
+    }
+
+    if (action === "dependencies") {
+      const files = (input["files"] as string[] | undefined) ?? [];
+      if (files.length === 0) {
+        return { success: false, error: "dependencies action requires non-empty 'files'" };
+      }
+
+      const analyzer = new DependencyAnalyzer();
+      const report = analyzer.analyzeFromFiles(this.rootDir, files);
+      return {
+        success: true,
+        output: JSON.stringify(report),
+      };
+    }
+
+    const symbols = (input["symbols"] as string[] | undefined) ?? [];
+    if (symbols.length === 0) {
+      return { success: false, error: "duplicates action requires non-empty 'symbols'" };
+    }
+
+    const duplicates: Record<string, unknown> = {};
+    for (const symbol of symbols) {
+      const hits = index.search({
+        query: symbol,
+        mode: "symbol",
+        limit: 50,
+        offset: 0,
+      });
+
+      if (hits.length > 0) {
+        duplicates[symbol] = hits;
+      }
+    }
+
+    return {
+      success: true,
+      output: JSON.stringify({ duplicateCount: Object.keys(duplicates).length, duplicates }),
+    };
+  }
+}