@monoes/graph 1.0.0

package/src/extract/index.ts

@@ -0,0 +1,190 @@
+import { readFileSync, mkdirSync } from 'fs';
+import { extname } from 'path';
+import type { ClassifiedFile, ExtractionResult, BuildOptions, GraphNode, GraphEdge } from '../types.js';
+import type { LanguageExtractor } from './types.js';
+import { FileCache } from '../cache.js';
+import { typescriptExtractor } from './languages/typescript.js';
+import { pythonExtractor } from './languages/python.js';
+import { goExtractor } from './languages/go.js';
+import { rustExtractor } from './languages/rust.js';
+
+// ---- registry ----
+
+const EXTRACTORS: LanguageExtractor[] = [
+  typescriptExtractor,
+  pythonExtractor,
+  goExtractor,
+  rustExtractor,
+];
+
+/** Build a lookup from file extension → extractor */
+const EXT_MAP = new Map<string, LanguageExtractor>(
+  EXTRACTORS.flatMap((e) => e.extensions.map((ext) => [ext, e])),
+);
+
+/** Build a lookup from language name → extractor */
+const LANG_MAP = new Map<string, LanguageExtractor>(
+  EXTRACTORS.map((e) => [e.language, e]),
+);
+
+function resolveExtractor(file: ClassifiedFile): LanguageExtractor | null {
+  if (file.language) {
+    const byLang = LANG_MAP.get(file.language.toLowerCase());
+    if (byLang) return byLang;
+  }
+  const ext = extname(file.path).toLowerCase();
+  return EXT_MAP.get(ext) ?? null;
+}
+
+// ---- result merging ----
+
+function mergeResults(results: ExtractionResult[]): ExtractionResult {
+  const nodeMap = new Map<string, GraphNode>();
+  const edgeSet = new Set<string>();
+  const edges: GraphEdge[] = [];
+  const errors: string[] = [];
+  let filesProcessed = 0;
+  let fromCache = 0;
+
+  for (const r of results) {
+    filesProcessed += r.filesProcessed;
+    fromCache += r.fromCache;
+    errors.push(...r.errors);
+
+    for (const node of r.nodes) {
+      // Deduplicate by id — keep the first occurrence
+      if (!nodeMap.has(node.id)) {
+        nodeMap.set(node.id, node);
+      }
+    }
+
+    for (const edge of r.edges) {
+      // Deduplicate edges by source+target+relation key
+      const key = `${edge.source}||${edge.target}||${edge.relation}`;
+      if (!edgeSet.has(key)) {
+        edgeSet.add(key);
+        edges.push(edge);
+      }
+    }
+  }
+
+  return {
+    nodes: Array.from(nodeMap.values()),
+    edges,
+    filesProcessed,
+    fromCache,
+    errors,
+  };
+}
+
+// ---- public API ----
+
+/**
+ * Extract graph nodes and edges from all classified files.
+ *
+ * Reads each file, checks the on-disk cache keyed by SHA-256 content hash,
+ * dispatches to the appropriate language extractor, and merges all results
+ * into a single ExtractionResult (nodes deduplicated by id, edges by source+target+relation).
+ *
+ * @param files      - Classified files to process (from the classify layer)
+ * @param outputDir  - Directory used for cache storage (.monobrain/graph by default)
+ * @param options    - Build options (languages filter, maxFileSizeBytes, etc.)
+ */
+export async function extractAll(
+  files: ClassifiedFile[],
+  outputDir: string,
+  options?: BuildOptions,
+): Promise<ExtractionResult> {
+  const maxSize = options?.maxFileSizeBytes ?? 500 * 1024; // 500 KB
+  const langFilter = options?.languages?.map((l) => l.toLowerCase());
+
+  mkdirSync(outputDir, { recursive: true });
+  const cache = new FileCache(outputDir);
+
+  const results: ExtractionResult[] = [];
+
+  for (const file of files) {
+    // Skip files that are too large
+    if (file.sizeBytes > maxSize) {
+      results.push({
+        nodes: [],
+        edges: [],
+        filesProcessed: 1,
+        fromCache: 0,
+        errors: [`Skipped ${file.path}: file size ${file.sizeBytes} exceeds limit ${maxSize}`],
+      });
+      continue;
+    }
+
+    // Apply language filter
+    if (langFilter && langFilter.length > 0) {
+      const extractor = resolveExtractor(file);
+      const fileLang = (file.language ?? '').toLowerCase();
+      const extractorLang = extractor?.language ?? '';
+      if (!langFilter.includes(fileLang) && !langFilter.includes(extractorLang)) {
+        continue;
+      }
+    }
+
+    const extractor = resolveExtractor(file);
+    if (!extractor) {
+      // No extractor for this file type — skip silently
+      continue;
+    }
+
+    // Read content
+    let content: string;
+    try {
+      content = readFileSync(file.path, 'utf8');
+    } catch (err) {
+      results.push({
+        nodes: [],
+        edges: [],
+        filesProcessed: 1,
+        fromCache: 0,
+        errors: [`Failed to read ${file.path}: ${err instanceof Error ? err.message : String(err)}`],
+      });
+      continue;
+    }
+
+    // Check cache using FileCache (keyed on filePath + content hash)
+    const cacheKey = cache.key(file.path, content);
+    const cached = cache.get(cacheKey);
+    if (cached) {
+      results.push({
+        ...cached,
+        filesProcessed: 1,
+        fromCache: 1,
+      });
+      continue;
+    }
+
+    // Extract
+    let result: ExtractionResult;
+    try {
+      result = extractor.extract(file.path, content);
+    } catch (err) {
+      result = {
+        nodes: [],
+        edges: [],
+        filesProcessed: 1,
+        fromCache: 0,
+        errors: [`Extractor error for ${file.path}: ${err instanceof Error ? err.message : String(err)}`],
+      };
+    }
+
+    // Store in cache
+    cache.set(cacheKey, result);
+    results.push(result);
+  }
+
+  return mergeResults(results);
+}
+
+// Re-export types and utilities for consumers
+export type { LanguageExtractor } from './types.js';
+export { typescriptExtractor } from './languages/typescript.js';
+export { pythonExtractor } from './languages/python.js';
+export { goExtractor } from './languages/go.js';
+export { rustExtractor } from './languages/rust.js';
+export { isTreeSitterAvailable, tryLoadParser, walk, parseFile, parseFileFromDisk } from './tree-sitter-runner.js';

package/src/extract/languages/go.ts

@@ -0,0 +1,206 @@
+import { basename } from 'path';
+import type { GraphNode, GraphEdge, ExtractionResult } from '../../types.js';
+import type { LanguageExtractor } from '../types.js';
+import {
+  tryLoadParser,
+  walk,
+  type SyntaxNodeLike,
+} from '../tree-sitter-runner.js';
+
+// ---- helpers ----
+
+function nodeName(node: SyntaxNodeLike): string {
+  const nameNode = node.childForFieldName('name');
+  return nameNode?.text ?? '';
+}
+
+function loc(node: SyntaxNodeLike): string {
+  return `L${node.startPosition.row + 1}`;
+}
+
+// ---- tree-sitter extraction ----
+
+function extractWithTreeSitter(filePath: string, content: string): ExtractionResult {
+  const nodes: GraphNode[] = [];
+  const edges: GraphEdge[] = [];
+  const errors: string[] = [];
+
+  const parser = tryLoadParser('go');
+  if (!parser) {
+    return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+  }
+
+  let tree: { rootNode: SyntaxNodeLike };
+  try {
+    tree = parser.parse(content);
+  } catch (err) {
+    errors.push(`tree-sitter parse error in ${filePath}: ${String(err)}`);
+    return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+  }
+
+  walk(tree.rootNode, (n) => {
+    // ---- function declarations ----
+    if (n.type === 'function_declaration' || n.type === 'method_declaration') {
+      const name = nodeName(n);
+      if (name) {
+        // For methods, qualify with receiver type
+        let qualifiedName = name;
+        if (n.type === 'method_declaration') {
+          const receiver = n.childForFieldName('receiver');
+          if (receiver) {
+            // receiver text looks like "(r *MyType)" — extract type name
+            const receiverType = receiver.text.replace(/^\(|\)$/g, '').trim().replace(/^\w+\s+\*?/, '');
+            if (receiverType) qualifiedName = `${receiverType}.${name}`;
+          }
+        }
+        nodes.push({
+          id: qualifiedName,
+          label: qualifiedName,
+          fileType: 'code',
+          sourceFile: filePath,
+          sourceLocation: loc(n),
+        });
+      }
+      return;
+    }
+
+    // ---- type declarations (struct, interface) ----
+    if (n.type === 'type_declaration') {
+      for (const child of n.children) {
+        if (child.type === 'type_spec') {
+          const nameNode = child.childForFieldName('name');
+          const typeNode = child.childForFieldName('type');
+          const typeName = nameNode?.text ?? '';
+
+          if (!typeName) continue;
+
+          const typeKind =
+            typeNode?.type === 'struct_type' ? 'struct'
+            : typeNode?.type === 'interface_type' ? 'interface'
+            : 'type';
+
+          nodes.push({
+            id: typeName,
+            label: typeName,
+            fileType: 'code',
+            sourceFile: filePath,
+            sourceLocation: loc(child),
+            nodeKind: typeKind,
+          });
+        }
+      }
+      return;
+    }
+
+    // ---- import declarations ----
+    if (n.type === 'import_declaration') {
+      for (const child of n.children) {
+        if (child.type === 'import_spec_list') {
+          for (const spec of child.children) {
+            if (spec.type === 'import_spec') {
+              const pathNode = spec.childForFieldName('path');
+              if (pathNode) {
+                const importPath = pathNode.text.replace(/^"|"$/g, '');
+                edges.push({
+                  source: basename(filePath),
+                  target: importPath,
+                  relation: 'imports',
+                  confidence: 'EXTRACTED',
+                  sourceFile: filePath,
+                  sourceLocation: loc(spec),
+                });
+              }
+            }
+          }
+        }
+        if (child.type === 'import_spec') {
+          const pathNode = child.childForFieldName('path');
+          if (pathNode) {
+            const importPath = pathNode.text.replace(/^"|"$/g, '');
+            edges.push({
+              source: basename(filePath),
+              target: importPath,
+              relation: 'imports',
+              confidence: 'EXTRACTED',
+              sourceFile: filePath,
+              sourceLocation: loc(child),
+            });
+          }
+        }
+      }
+    }
+  });
+
+  return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+}
+
+// ---- regex fallback ----
+
+function extractWithRegex(filePath: string, content: string): ExtractionResult {
+  const nodes: GraphNode[] = [];
+  const edges: GraphEdge[] = [];
+
+  const lines = content.split('\n');
+  let inImportBlock = false;
+
+  lines.forEach((line, idx) => {
+    const location = `L${idx + 1}`;
+    const trimmed = line.trim();
+
+    // Detect import blocks
+    if (trimmed === 'import (') { inImportBlock = true; return; }
+    if (inImportBlock && trimmed === ')') { inImportBlock = false; return; }
+
+    if (inImportBlock) {
+      const importPathMatch = trimmed.match(/"([^"]+)"/);
+      if (importPathMatch) {
+        edges.push({ source: basename(filePath), target: importPathMatch[1], relation: 'imports', confidence: 'EXTRACTED', sourceFile: filePath, sourceLocation: location });
+      }
+      return;
+    }
+
+    // Single-line import: import "pkg"
+    const singleImportMatch = trimmed.match(/^import\s+"([^"]+)"/);
+    if (singleImportMatch) {
+      edges.push({ source: basename(filePath), target: singleImportMatch[1], relation: 'imports', confidence: 'EXTRACTED', sourceFile: filePath, sourceLocation: location });
+    }
+
+    // Function: func (r *Recv) MethodName(...) or func FuncName(...)
+    const funcMatch = trimmed.match(/^func\s+(?:\(\w+\s+\*?(\w+)\)\s+)?(\w+)/);
+    if (funcMatch) {
+      const receiver = funcMatch[1];
+      const name = funcMatch[2];
+      const qualifiedName = receiver ? `${receiver}.${name}` : name;
+      nodes.push({ id: qualifiedName, label: qualifiedName, fileType: 'code', sourceFile: filePath, sourceLocation: location });
+    }
+
+    // Struct: type Name struct
+    const structMatch = trimmed.match(/^type\s+(\w+)\s+struct/);
+    if (structMatch) {
+      nodes.push({ id: structMatch[1], label: structMatch[1], fileType: 'code', sourceFile: filePath, sourceLocation: location, nodeKind: 'struct' });
+    }
+
+    // Interface: type Name interface
+    const ifaceMatch = trimmed.match(/^type\s+(\w+)\s+interface/);
+    if (ifaceMatch) {
+      nodes.push({ id: ifaceMatch[1], label: ifaceMatch[1], fileType: 'code', sourceFile: filePath, sourceLocation: location, nodeKind: 'interface' });
+    }
+  });
+
+  return { nodes, edges, filesProcessed: 1, fromCache: 0, errors: [] };
+}
+
+// ---- extractor implementation ----
+
+export const goExtractor: LanguageExtractor = {
+  language: 'go',
+  extensions: ['.go'],
+
+  extract(filePath: string, content: string): ExtractionResult {
+    const tsResult = extractWithTreeSitter(filePath, content);
+    if (tsResult.nodes.length > 0 || tsResult.edges.length > 0 || tsResult.errors.length > 0) {
+      return tsResult;
+    }
+    return extractWithRegex(filePath, content);
+  },
+};

package/src/extract/languages/python.ts

@@ -0,0 +1,270 @@
+import { basename } from 'path';
+import type { GraphNode, GraphEdge, ExtractionResult } from '../../types.js';
+import type { LanguageExtractor } from '../types.js';
+import {
+  tryLoadParser,
+  walk,
+  type SyntaxNodeLike,
+} from '../tree-sitter-runner.js';
+
+// ---- helpers ----
+
+function nodeName(node: SyntaxNodeLike): string {
+  const nameNode = node.childForFieldName('name');
+  return nameNode?.text ?? '';
+}
+
+function loc(node: SyntaxNodeLike): string {
+  return `L${node.startPosition.row + 1}`;
+}
+
+// ---- tree-sitter extraction ----
+
+function extractWithTreeSitter(filePath: string, content: string): ExtractionResult {
+  const nodes: GraphNode[] = [];
+  const edges: GraphEdge[] = [];
+  const errors: string[] = [];
+
+  const parser = tryLoadParser('python');
+  if (!parser) {
+    return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+  }
+
+  let tree: { rootNode: SyntaxNodeLike };
+  try {
+    tree = parser.parse(content);
+  } catch (err) {
+    errors.push(`tree-sitter parse error in ${filePath}: ${String(err)}`);
+    return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+  }
+
+  // Track method context for call attribution
+  const functionStack: string[] = [];
+  const classStack: string[] = [];
+
+  walk(tree.rootNode, (n) => {
+    // ---- class definitions ----
+    if (n.type === 'class_definition') {
+      const name = nodeName(n);
+      if (name) {
+        nodes.push({
+          id: name,
+          label: name,
+          fileType: 'code',
+          sourceFile: filePath,
+          sourceLocation: loc(n),
+        });
+        classStack.push(name);
+
+        // Inheritance: class A(B, C):
+        const argList = n.childForFieldName('superclasses');
+        if (argList) {
+          for (const child of argList.children) {
+            if (child.type === 'identifier' || child.type === 'attribute') {
+              edges.push({
+                source: name,
+                target: child.text,
+                relation: 'extends',
+                confidence: 'EXTRACTED',
+                sourceFile: filePath,
+                sourceLocation: loc(argList),
+              });
+            }
+          }
+        }
+      }
+      return;
+    }
+
+    // ---- function / async function definitions ----
+    if (n.type === 'function_definition' || n.type === 'decorated_definition') {
+      // For decorated_definition, descend to find the inner function_definition
+      const funcNode =
+        n.type === 'decorated_definition'
+          ? (n.children.find((c) => c.type === 'function_definition') ?? n)
+          : n;
+
+      const name = nodeName(funcNode);
+      if (name) {
+        const qualifiedName =
+          classStack.length > 0
+            ? `${classStack[classStack.length - 1]}.${name}`
+            : name;
+
+        nodes.push({
+          id: qualifiedName,
+          label: qualifiedName,
+          fileType: 'code',
+          sourceFile: filePath,
+          sourceLocation: loc(funcNode),
+        });
+        functionStack.push(qualifiedName);
+      }
+      return;
+    }
+
+    // ---- import statements ----
+    if (n.type === 'import_statement') {
+      // import X, import X as Y
+      for (const child of n.children) {
+        if (child.type === 'dotted_name' || child.type === 'aliased_import') {
+          const importedName =
+            child.type === 'aliased_import'
+              ? child.childForFieldName('name')?.text ?? child.text
+              : child.text;
+          if (importedName) {
+            edges.push({
+              source: basename(filePath),
+              target: importedName,
+              relation: 'imports',
+              confidence: 'EXTRACTED',
+              sourceFile: filePath,
+              sourceLocation: loc(n),
+            });
+          }
+        }
+      }
+      return;
+    }
+
+    // ---- from X import Y ----
+    if (n.type === 'import_from_statement') {
+      const moduleNode = n.childForFieldName('module_name');
+      const moduleName = moduleNode?.text ?? '';
+
+      if (moduleName) {
+        edges.push({
+          source: basename(filePath),
+          target: moduleName,
+          relation: 'imports',
+          confidence: 'EXTRACTED',
+          sourceFile: filePath,
+          sourceLocation: loc(n),
+        });
+      }
+
+      // Also emit edges for individual imported names
+      for (const child of n.children) {
+        if (child.type === 'dotted_name' || child.type === 'identifier') {
+          // Skip the module_name we already handled
+          if (child === moduleNode) continue;
+          edges.push({
+            source: filePath,
+            target: child.text,
+            relation: 'imports',
+            confidence: 'EXTRACTED',
+            sourceFile: filePath,
+            sourceLocation: loc(n),
+          });
+        }
+        if (child.type === 'aliased_import') {
+          const importedName = child.childForFieldName('name')?.text ?? child.text;
+          if (importedName) {
+            edges.push({
+              source: filePath,
+              target: importedName,
+              relation: 'imports',
+              confidence: 'EXTRACTED',
+              sourceFile: filePath,
+              sourceLocation: loc(n),
+            });
+          }
+        }
+      }
+      return;
+    }
+
+    // ---- call expressions ----
+    if (n.type === 'call') {
+      const fnNode = n.childForFieldName('function');
+      if (!fnNode) return;
+
+      const calleeName = fnNode.text;
+      const caller =
+        functionStack[functionStack.length - 1] ??
+        classStack[classStack.length - 1] ??
+        basename(filePath);
+
+      if (calleeName && caller && calleeName !== caller) {
+        edges.push({
+          source: caller,
+          target: calleeName,
+          relation: 'calls',
+          confidence: 'INFERRED',
+          confidenceScore: 0.7,
+          sourceFile: filePath,
+          sourceLocation: loc(n),
+        });
+      }
+    }
+  });
+
+  return { nodes, edges, filesProcessed: 1, fromCache: 0, errors };
+}
+
+// ---- regex fallback ----
+
+function extractWithRegex(filePath: string, content: string): ExtractionResult {
+  const nodes: GraphNode[] = [];
+  const edges: GraphEdge[] = [];
+
+  const lines = content.split('\n');
+
+  lines.forEach((line, idx) => {
+    const location = `L${idx + 1}`;
+
+    // class
+    const classMatch = line.match(/^class\s+(\w+)/);
+    if (classMatch) {
+      const name = classMatch[1];
+      nodes.push({ id: name, label: name, fileType: 'code', sourceFile: filePath, sourceLocation: location });
+
+      // inheritance: class A(B, C):
+      const inheritMatch = line.match(/^class\s+\w+\(([^)]+)\)/);
+      if (inheritMatch) {
+        for (const base of inheritMatch[1].split(',')) {
+          const baseName = base.trim();
+          if (baseName && baseName !== 'object') {
+            edges.push({ source: name, target: baseName, relation: 'extends', confidence: 'EXTRACTED', sourceFile: filePath, sourceLocation: location });
+          }
+        }
+      }
+    }
+
+    // function
+    const funcMatch = line.match(/^(?:async\s+)?def\s+(\w+)/);
+    if (funcMatch) {
+      const name = funcMatch[1];
+      nodes.push({ id: name, label: name, fileType: 'code', sourceFile: filePath, sourceLocation: location });
+    }
+
+    // import
+    const importMatch = line.match(/^import\s+(\S+)/);
+    if (importMatch) {
+      edges.push({ source: basename(filePath), target: importMatch[1], relation: 'imports', confidence: 'EXTRACTED', sourceFile: filePath, sourceLocation: location });
+    }
+
+    // from X import Y
+    const fromImportMatch = line.match(/^from\s+(\S+)\s+import/);
+    if (fromImportMatch) {
+      edges.push({ source: basename(filePath), target: fromImportMatch[1], relation: 'imports', confidence: 'EXTRACTED', sourceFile: filePath, sourceLocation: location });
+    }
+  });
+
+  return { nodes, edges, filesProcessed: 1, fromCache: 0, errors: [] };
+}
+
+// ---- extractor implementation ----
+
+export const pythonExtractor: LanguageExtractor = {
+  language: 'python',
+  extensions: ['.py', '.pyw'],
+
+  extract(filePath: string, content: string): ExtractionResult {
+    const tsResult = extractWithTreeSitter(filePath, content);
+    if (tsResult.nodes.length > 0 || tsResult.edges.length > 0 || tsResult.errors.length > 0) {
+      return tsResult;
+    }
+    return extractWithRegex(filePath, content);
+  },
+};