@anirudw/repolens 0.1.0

package/src/parser/strategies/javascript.ts

@@ -0,0 +1,199 @@
+import type { ParserStrategy, ExtractedDependency, ParsedFile } from "../types.js";
+
+type TreeSitterParser = {
+  parse(content: string): { rootNode: TreeNode };
+  setLanguage(lang: unknown): void;
+};
+
+let TreeSitterParser: new () => TreeSitterParser;
+let JavaScript: unknown;
+
+async function ensureLoaded(): Promise<void> {
+  if (!TreeSitterParser) {
+    const [ts, js] = await Promise.all([
+      import("tree-sitter"),
+      import("tree-sitter-javascript"),
+    ]);
+    TreeSitterParser = ts.default as new () => TreeSitterParser;
+    JavaScript = js.default;
+  }
+}
+
+type TreeNode = {
+  type: string;
+  text: string;
+  startPosition: { row: number; column: number };
+  endPosition: { row: number; column: number };
+  children: TreeNode[];
+  childForFieldName(name: string): TreeNode | null;
+};
+
+export class JavaScriptParser implements ParserStrategy {
+  private parser: TreeSitterParser | null = null;
+
+  private async ensureParser(): Promise<TreeSitterParser> {
+    if (!this.parser) {
+      await ensureLoaded();
+      this.parser = new TreeSitterParser();
+      this.parser.setLanguage(JavaScript);
+    }
+    return this.parser;
+  }
+
+  async parse(filePath: string, content: string): Promise<ParsedFile> {
+    const parser = await this.ensureParser();
+    const tree = parser.parse(content);
+    const rootNode = tree.rootNode as TreeNode;
+    const dependencies: ExtractedDependency[] = [];
+    const imports: Set<string> = new Set();
+    const sourceModules: string[] = [];
+
+    this.extractImports(rootNode, dependencies, imports, sourceModules);
+
+    const heuristics: Record<string, boolean> = {};
+    const allImports = [...imports, ...sourceModules].map((s) => s.toLowerCase());
+    heuristics.isReact = allImports.some((i) => i === "react" || i === "@types/react");
+    heuristics.isReactNative = allImports.some((i) => i === "react-native");
+    heuristics.isNodejs = allImports.some((i) => i.startsWith("node:"));
+    heuristics.hasDefaultExport = this.hasDefaultExport(rootNode);
+    heuristics.hasNamedExports = this.hasNamedExports(rootNode);
+
+    return {
+      id: filePath,
+      relativePath: filePath.split("/").pop() ?? filePath,
+      language: "javascript",
+      dependencies,
+      metadata: {
+        sizeBytes: Buffer.byteLength(content, "utf-8"),
+        heuristics,
+      },
+    };
+  }
+
+  private extractImports(
+    node: TreeNode,
+    dependencies: ExtractedDependency[],
+    imports: Set<string>,
+    sourceModules: string[]
+  ): void {
+    if (node.type === "import_statement") {
+      this.extractImportStatement(node, dependencies, imports, sourceModules);
+    } else if (node.type === "call_expression" && this.isRequireCall(node)) {
+      this.extractRequireCall(node, dependencies, sourceModules);
+    }
+
+    for (const child of node.children) {
+      this.extractImports(child, dependencies, imports, sourceModules);
+    }
+  }
+
+  private extractImportStatement(
+    node: TreeNode,
+    dependencies: ExtractedDependency[],
+    imports: Set<string>,
+    sourceModules: string[]
+  ): void {
+    const sourceNode = node.childForFieldName("source");
+    if (sourceNode && sourceNode.type === "string") {
+      const rawSpecifier = sourceNode.text.slice(1, -1);
+      sourceModules.push(rawSpecifier);
+      dependencies.push({
+        rawSpecifier,
+        type: "import",
+        location: {
+          line: node.startPosition.row + 1,
+          column: node.startPosition.column + 1,
+        },
+        resolvedPath: rawSpecifier,
+      });
+    }
+
+    for (const child of node.children) {
+      if (child.type === "import_specifier") {
+        const nameNode = child.childForFieldName("name");
+        if (nameNode) {
+          imports.add(nameNode.text);
+        }
+      } else if (child.type === "import_default_specifier") {
+        const nameNode = child.childForFieldName("name");
+        if (nameNode) {
+          imports.add(nameNode.text);
+        }
+      } else if (child.type === "import_namespace_specifier") {
+        const nameNode = child.childForFieldName("name");
+        if (nameNode) {
+          imports.add(nameNode.text);
+        }
+      }
+    }
+  }
+
+  private extractRequireCall(
+    node: TreeNode,
+    dependencies: ExtractedDependency[],
+    sourceModules: string[]
+  ): void {
+    const args = node.childForFieldName("arguments");
+    if (args) {
+      for (const arg of args.children) {
+        if (arg.type === "string") {
+          const rawSpecifier = arg.text.slice(1, -1);
+          sourceModules.push(rawSpecifier);
+          dependencies.push({
+            rawSpecifier,
+            type: "require",
+            location: {
+              line: node.startPosition.row + 1,
+              column: node.startPosition.column + 1,
+            },
+            resolvedPath: rawSpecifier,
+          });
+        }
+      }
+    }
+  }
+
+  private isRequireCall(node: TreeNode): boolean {
+    const functionExpr = node.childForFieldName("function");
+    if (!functionExpr) return false;
+
+    if (functionExpr.type === "identifier" && functionExpr.text === "require") {
+      return true;
+    }
+
+    if (functionExpr.type === "member_expression") {
+      const object = functionExpr.childForFieldName("object");
+      const property = functionExpr.childForFieldName("property");
+      if (object?.text === "require" && property?.text === "resolve") {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  private hasDefaultExport(node: TreeNode): boolean {
+    if (node.type === "export_statement") {
+      return node.children.some((c) => c.type === "default");
+    }
+    for (const child of node.children) {
+      if (this.hasDefaultExport(child)) return true;
+    }
+    return false;
+  }
+
+  private hasNamedExports(node: TreeNode): boolean {
+    if (node.type === "export_clause") return true;
+    if (node.type === "export_statement") {
+      const hasDefault = node.children.some((c) => c.type === "default");
+      if (hasDefault) return false;
+      return node.children.some(
+        (c) => c.type === "variable_declaration" || c.type === "function_declaration" || c.type === "class_declaration"
+      );
+    }
+    for (const child of node.children) {
+      if (this.hasNamedExports(child)) return true;
+    }
+    return false;
+  }
+}

package/src/parser/strategies/markdown.ts

@@ -0,0 +1,83 @@
+import { remark } from "remark";
+import { visit } from "unist-util-visit";
+import type { Node } from "unist";
+import type { ParserStrategy, ExtractedDependency, ParsedFile } from "../types.js";
+
+interface LinkNode extends Node {
+  type: string;
+  url?: string;
+  children?: Node[];
+  position?: {
+    start: { line: number; column: number };
+    end: { line: number; column: number };
+  };
+}
+
+export class MarkdownParser implements ParserStrategy {
+  private processor = remark();
+
+  async parse(filePath: string, content: string): Promise<ParsedFile> {
+    const tree = this.processor.parse(content);
+    const dependencies: ExtractedDependency[] = [];
+
+    visit(tree, "link", (node: LinkNode) => {
+      if (node.url) {
+        dependencies.push({
+          rawSpecifier: node.url,
+          type: "markdown-link",
+          location: {
+            line: node.position?.start.line ?? 1,
+            column: node.position?.start.column ?? 1,
+          },
+          resolvedPath: this.resolvePath(node.url),
+        });
+      }
+    });
+
+    const wikilinkRegex = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
+    let match;
+    while ((match = wikilinkRegex.exec(content)) !== null) {
+      const lineInfo = this.getLineNumber(content, match.index);
+      dependencies.push({
+        rawSpecifier: match[1],
+        type: "wikilink",
+        location: {
+          line: lineInfo.line,
+          column: lineInfo.column,
+        },
+        resolvedPath: this.resolvePath(match[1]),
+      });
+    }
+
+    return {
+      id: filePath,
+      relativePath: filePath.split("/").pop() ?? filePath,
+      language: "markdown",
+      dependencies,
+      metadata: {
+        sizeBytes: Buffer.byteLength(content, "utf-8"),
+        heuristics: {
+          hasWikilinks: dependencies.some((d) => d.type === "wikilink"),
+          hasExternalLinks: dependencies.some((d) =>
+            d.rawSpecifier.startsWith("http")
+          ),
+        },
+      },
+    };
+  }
+
+  private getLineNumber(content: string, index: number): { line: number; column: number } {
+    const lines = content.substring(0, index).split("\n");
+    return {
+      line: lines.length,
+      column: (lines[lines.length - 1]?.length ?? 0) + 1,
+    };
+  }
+
+  private resolvePath(specifier: string): string | null {
+    if (specifier.startsWith("http") || specifier.startsWith("//")) {
+      return null;
+    }
+    return specifier;
+  }
+}

package/src/parser/strategies/python.ts

@@ -0,0 +1,184 @@
+import type { ParserStrategy, ExtractedDependency, ParsedFile } from "../types.js";
+
+type TreeSitterParser = {
+  parse(content: string): { rootNode: TreeNode };
+  setLanguage(lang: unknown): void;
+};
+
+let TreeSitterParser: new () => TreeSitterParser;
+let Python: unknown;
+
+async function ensureLoaded(): Promise<void> {
+  if (!TreeSitterParser) {
+    const [ts, py] = await Promise.all([
+      import("tree-sitter"),
+      import("tree-sitter-python"),
+    ]);
+    TreeSitterParser = ts.default as new () => TreeSitterParser;
+    Python = py.default;
+  }
+}
+
+type TreeNode = {
+  type: string;
+  text: string;
+  startPosition: { row: number; column: number };
+  endPosition: { row: number; column: number };
+  children: TreeNode[];
+  childForFieldName(name: string): TreeNode | null;
+};
+
+export class PythonParser implements ParserStrategy {
+  private parser: TreeSitterParser | null = null;
+
+  private async ensureParser(): Promise<TreeSitterParser> {
+    if (!this.parser) {
+      await ensureLoaded();
+      this.parser = new TreeSitterParser();
+      this.parser.setLanguage(Python);
+    }
+    return this.parser;
+  }
+
+  async parse(filePath: string, content: string): Promise<ParsedFile> {
+    const parser = await this.ensureParser();
+    const tree = parser.parse(content);
+    const rootNode = tree.rootNode as TreeNode;
+    const dependencies: ExtractedDependency[] = [];
+    const heuristics: Record<string, boolean> = {};
+
+    this.extractImports(rootNode, dependencies);
+    heuristics.hasEntrypoint = this.detectMainEntrypoint(rootNode);
+
+    return {
+      id: filePath,
+      relativePath: filePath.split("/").pop() ?? filePath,
+      language: "python",
+      dependencies,
+      metadata: {
+        sizeBytes: Buffer.byteLength(content, "utf-8"),
+        heuristics,
+      },
+    };
+  }
+
+  private extractImports(node: TreeNode, dependencies: ExtractedDependency[]): void {
+    if (node.type === "import_statement") {
+      for (const child of node.children) {
+        if (child.type === "dotted_name") {
+          dependencies.push({
+            rawSpecifier: child.text,
+            type: "import",
+            location: {
+              line: node.startPosition.row + 1,
+              column: node.startPosition.column + 1,
+            },
+            resolvedPath: child.text,
+          });
+        } else if (child.type === "aliased_name") {
+          const dottedName = child.childForFieldName("name");
+          if (dottedName) {
+            dependencies.push({
+              rawSpecifier: dottedName.text,
+              type: "import",
+              location: {
+                line: node.startPosition.row + 1,
+                column: node.startPosition.column + 1,
+              },
+              resolvedPath: dottedName.text,
+            });
+          }
+        }
+      }
+    } else if (node.type === "import_from_statement") {
+      let moduleName = "";
+      const importedNames: string[] = [];
+      let foundImportKeyword = false;
+
+      for (const child of node.children) {
+        if (child.type === "dotted_name") {
+          if (!foundImportKeyword) {
+            moduleName = child.text;
+          } else {
+            importedNames.push(child.text);
+          }
+        } else if (child.type === "identifier") {
+          if (foundImportKeyword) {
+            importedNames.push(child.text);
+          }
+        } else if (child.type === "import") {
+          foundImportKeyword = true;
+        } else if (child.type === "wildcard_import") {
+          importedNames.push("*");
+        }
+      }
+
+      if (importedNames.length === 0 && moduleName) {
+        dependencies.push({
+          rawSpecifier: moduleName,
+          type: "import",
+          location: {
+            line: node.startPosition.row + 1,
+            column: node.startPosition.column + 1,
+          },
+          resolvedPath: moduleName,
+        });
+      } else {
+        for (const name of importedNames) {
+          dependencies.push({
+            rawSpecifier: `${moduleName}.${name}`,
+            type: "import",
+            location: {
+              line: node.startPosition.row + 1,
+              column: node.startPosition.column + 1,
+            },
+            resolvedPath: `${moduleName}.${name}`,
+          });
+        }
+      }
+    }
+
+    for (const child of node.children) {
+      this.extractImports(child, dependencies);
+    }
+  }
+
+  private detectMainEntrypoint(node: TreeNode): boolean {
+    if (node.type === "if_statement") {
+      const condition = node.childForFieldName("condition");
+      if (condition && this.isNameEqualsMainCheck(condition)) {
+        return true;
+      }
+    }
+
+    for (const child of node.children) {
+      if (this.detectMainEntrypoint(child)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  private isNameEqualsMainCheck(node: TreeNode): boolean {
+    if (node.type === "comparison_operator") {
+      let hasName = false;
+      let hasMainString = false;
+
+      for (const child of node.children) {
+        if (child.type === "identifier" && child.text === "__name__") {
+          hasName = true;
+        } else if (child.type === "string") {
+          const innerText = child.text.replace(/['"]/g, "");
+          if (innerText === "__main__") {
+            hasMainString = true;
+          }
+        }
+      }
+
+      return hasName && hasMainString;
+    }
+
+    return false;
+  }
+}

package/src/parser/types.ts

@@ -0,0 +1,18 @@
+export interface ExtractedDependency {
+  rawSpecifier: string;
+  type: 'import' | 'require' | 'export' | 'markdown-link' | 'wikilink';
+  location: { line: number; column: number };
+  resolvedPath: string | null;
+}
+
+export interface ParsedFile {
+  id: string;
+  relativePath: string;
+  language: string;
+  dependencies: ExtractedDependency[];
+  metadata: { sizeBytes: number; heuristics: Record<string, boolean> };
+}
+
+export interface ParserStrategy {
+  parse(filePath: string, content: string): Promise<ParsedFile>;
+}

package/src/renderers/json/exporter.ts

@@ -0,0 +1,56 @@
+import { writeFile } from "node:fs/promises";
+import type { Graph } from "../../graph/network.js";
+import type { ParsedFile } from "../../parser/types.js";
+
+export interface JsonNode {
+  id: string;
+  relativePath: string;
+  language: string;
+  sizeBytes: number;
+  heuristics: Record<string, boolean>;
+}
+
+export interface JsonEdge {
+  source: string;
+  target: string;
+  type: string;
+}
+
+export interface GraphJson {
+  nodes: JsonNode[];
+  edges: JsonEdge[];
+}
+
+export async function exportGraphToJson(
+  graph: Graph,
+  parsedFiles: ParsedFile[],
+  outputPath: string
+): Promise<void> {
+  const fileMap = new Map<string, ParsedFile>();
+  for (const file of parsedFiles) {
+    fileMap.set(file.id, file);
+  }
+
+  const nodes: JsonNode[] = [];
+  for (const [nodeId] of graph.getNodes()) {
+    const parsedFile = fileMap.get(nodeId);
+    if (parsedFile) {
+      nodes.push({
+        id: parsedFile.id,
+        relativePath: parsedFile.relativePath,
+        language: parsedFile.language,
+        sizeBytes: parsedFile.metadata.sizeBytes,
+        heuristics: parsedFile.metadata.heuristics,
+      });
+    }
+  }
+
+  const edges: JsonEdge[] = graph.getEdges().map((edge) => ({
+    source: edge.source,
+    target: edge.target,
+    type: edge.type,
+  }));
+
+  const output: GraphJson = { nodes, edges };
+  await writeFile(outputPath, JSON.stringify(output, null, 2));
+}

package/src/scanner/file-types.ts

@@ -0,0 +1,42 @@
+export enum SupportedLanguage {
+  JavaScript = "javascript",
+  TypeScript = "typescript",
+  Python = "python",
+  Java = "java",
+  Markdown = "markdown",
+  Unknown = "unknown",
+}
+
+export const LANGUAGE_EXTENSIONS: Record<SupportedLanguage, string[]> = {
+  [SupportedLanguage.JavaScript]: [".js", ".jsx", ".mjs", ".cjs"],
+  [SupportedLanguage.TypeScript]: [".ts", ".tsx", ".mts", ".cts"],
+  [SupportedLanguage.Python]: [".py"],
+  [SupportedLanguage.Java]: [".java"],
+  [SupportedLanguage.Markdown]: [".md", ".markdown"],
+  [SupportedLanguage.Unknown]: [],
+};
+
+export const EXTENSION_TO_LANGUAGE: Map<string, SupportedLanguage> = new Map();
+
+for (const [lang, extensions] of Object.entries(LANGUAGE_EXTENSIONS)) {
+  if (lang !== SupportedLanguage.Unknown) {
+    for (const ext of extensions) {
+      EXTENSION_TO_LANGUAGE.set(ext, lang as SupportedLanguage);
+    }
+  }
+}
+
+export function getLanguageFromPath(filePath: string): SupportedLanguage {
+  const ext = getExtension(filePath);
+  return EXTENSION_TO_LANGUAGE.get(ext) ?? SupportedLanguage.Unknown;
+}
+
+export function getExtension(filePath: string): string {
+  const lastDot = filePath.lastIndexOf(".");
+  if (lastDot === -1) return "";
+  return filePath.slice(lastDot);
+}
+
+export function isTargetLanguage(filePath: string): boolean {
+  return getLanguageFromPath(filePath) !== SupportedLanguage.Unknown;
+}

package/src/scanner/ignore.ts

@@ -0,0 +1,58 @@
+import { readFileSync } from "node:fs";
+import ignore, { Ignore } from "ignore";
+
+export const DEFAULT_IGNORES = [
+  "node_modules",
+  "bower_components",
+  "dist",
+  "build",
+  "out",
+  "target",
+  ".git",
+  ".svn",
+  ".hg",
+  ".DS_Store",
+  "Thumbs.db",
+  "*.pyc",
+  "__pycache__",
+  ".pytest_cache",
+  ".mypy_cache",
+  ".next",
+  ".nuxt",
+  ".cache",
+  ".parcel-cache",
+  "coverage",
+  ".nyc_output",
+  ".turbo",
+];
+
+export function createIgnoreRules(rootDir: string): Ignore {
+  const ig = ignore();
+
+  for (const pattern of DEFAULT_IGNORES) {
+    ig.add(pattern);
+  }
+
+  const gitignorePath = `${rootDir}/.gitignore`;
+  try {
+    const content = readFileSync(gitignorePath, "utf-8");
+    const rules = content
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line && !line.startsWith("#"));
+    ig.add(rules);
+  } catch {
+    // .gitignore doesn't exist, continue with defaults
+  }
+
+  return ig;
+}
+
+export function shouldIgnore(
+  ig: Ignore,
+  relativePath: string,
+  isDirectory: boolean
+): boolean {
+  const pathWithTrailingSlash = isDirectory ? `${relativePath}/` : relativePath;
+  return ig.test(pathWithTrailingSlash).ignored;
+}

package/src/scanner/index.ts

@@ -0,0 +1,9 @@
+export { scanDirectory, type ScanResult, type ScannedFile } from "./walker.js";
+export {
+  SupportedLanguage,
+  LANGUAGE_EXTENSIONS,
+  getLanguageFromPath,
+  getExtension,
+  isTargetLanguage,
+} from "./file-types.js";
+export { createIgnoreRules, shouldIgnore, DEFAULT_IGNORES } from "./ignore.js";