@monoes/graph 1.0.0

package/src/extract/tree-sitter-runner.ts

@@ -0,0 +1,165 @@
+import { readFileSync } from 'fs';
+import type { ExtractionResult } from '../types.js';
+import type { LanguageExtractor } from './types.js';
+
+// Dynamic type references — the real shapes come from node-tree-sitter at runtime.
+// We use `unknown` here so that the module compiles without the optional dep installed.
+type ParserInstance = {
+  setLanguage(lang: unknown): void;
+  parse(src: string): { rootNode: SyntaxNodeLike };
+};
+
+export type SyntaxNodeLike = {
+  type: string;
+  text: string;
+  startPosition: { row: number; column: number };
+  endPosition: { row: number; column: number };
+  children: SyntaxNodeLike[];
+  childForFieldName(name: string): SyntaxNodeLike | null;
+  descendantsOfType(type: string | string[]): SyntaxNodeLike[];
+};
+
+// ---- availability probe ----
+
+let _treeSitterAvailable: boolean | null = null;
+let _ParserCtor: (new () => ParserInstance) | null = null;
+
+function probeTreeSitter(): void {
+  if (_treeSitterAvailable !== null) return;
+
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const mod = require('node-tree-sitter') as { default?: unknown } | unknown;
+    const ctor =
+      (mod as { default?: unknown }).default !== undefined
+        ? (mod as { default: unknown }).default
+        : mod;
+    if (typeof ctor === 'function') {
+      _ParserCtor = ctor as new () => ParserInstance;
+      _treeSitterAvailable = true;
+    } else {
+      _treeSitterAvailable = false;
+    }
+  } catch {
+    _treeSitterAvailable = false;
+  }
+}
+
+/**
+ * Returns true when node-tree-sitter is installed and loadable.
+ */
+export function isTreeSitterAvailable(): boolean {
+  probeTreeSitter();
+  return _treeSitterAvailable === true;
+}
+
+// ---- language grammar loader ----
+
+const LANGUAGE_MODULE_MAP: Record<string, string> = {
+  typescript: 'tree-sitter-typescript',
+  tsx: 'tree-sitter-typescript',
+  javascript: 'tree-sitter-javascript',
+  jsx: 'tree-sitter-javascript',
+  python: 'tree-sitter-python',
+  go: 'tree-sitter-go',
+  rust: 'tree-sitter-rust',
+};
+
+/**
+ * Attempts to create a configured Parser for the given language identifier.
+ * Returns null when tree-sitter or the grammar is not installed.
+ */
+export function tryLoadParser(language: string): ParserInstance | null {
+  if (!isTreeSitterAvailable() || _ParserCtor === null) return null;
+
+  const moduleName = LANGUAGE_MODULE_MAP[language];
+  if (!moduleName) return null;
+
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const grammarMod = require(moduleName) as Record<string, unknown>;
+
+    // tree-sitter-typescript exposes { typescript, tsx } sub-grammars
+    let grammar: unknown;
+    if (language === 'typescript' || language === 'tsx') {
+      grammar =
+        (grammarMod as { typescript?: unknown; tsx?: unknown })[language] ??
+        grammarMod['default'] ??
+        grammarMod;
+    } else {
+      grammar = grammarMod['default'] ?? grammarMod;
+    }
+
+    const parser = new _ParserCtor!();
+    parser.setLanguage(grammar);
+    return parser;
+  } catch {
+    return null;
+  }
+}
+
+// ---- depth-first AST walker ----
+
+export function walk(
+  node: SyntaxNodeLike,
+  visitor: (n: SyntaxNodeLike) => void,
+): void {
+  visitor(node);
+  for (const child of node.children) {
+    walk(child, visitor);
+  }
+}
+
+// ---- main entry point ----
+
+/**
+ * Parses a source file and delegates to the given LanguageExtractor.
+ * The extractor receives the file path and raw content; it owns the AST
+ * traversal internally (using tryLoadParser / walk from this module).
+ *
+ * Falls back gracefully: if tree-sitter cannot be loaded the extractor is still
+ * called with the raw content and is expected to use its regex fallback.
+ */
+export function parseFile(
+  filePath: string,
+  content: string,
+  extractor: LanguageExtractor,
+): ExtractionResult {
+  try {
+    return extractor.extract(filePath, content);
+  } catch (err) {
+    return {
+      nodes: [],
+      edges: [],
+      filesProcessed: 1,
+      fromCache: 0,
+      errors: [
+        `parseFile error for ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
+      ],
+    };
+  }
+}
+
+/**
+ * Convenience helper: read a file from disk and parse it.
+ */
+export function parseFileFromDisk(
+  filePath: string,
+  extractor: LanguageExtractor,
+): ExtractionResult {
+  let content: string;
+  try {
+    content = readFileSync(filePath, 'utf8');
+  } catch (err) {
+    return {
+      nodes: [],
+      edges: [],
+      filesProcessed: 1,
+      fromCache: 0,
+      errors: [
+        `Failed to read ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
+      ],
+    };
+  }
+  return parseFile(filePath, content, extractor);
+}

package/src/extract/types.ts

@@ -0,0 +1,7 @@
+import type { ExtractionResult } from '../types.js';
+
+export interface LanguageExtractor {
+  language: string;
+  extensions: string[];
+  extract(filePath: string, content: string): ExtractionResult;
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export { buildGraph } from './pipeline.js';
+export { FileCache } from './cache.js';
+export { collectFiles } from './detect.js';
+export { buildGraph as buildGraphologyGraph } from './build.js';
+export { detectCommunities } from './cluster.js';
+export { buildAnalysis, godNodes, surprisingConnections, graphStats } from './analyze.js';
+export { saveGraph, loadGraph, graphExists, getGraphPath } from './export.js';
+export { isTreeSitterAvailable, tryLoadParser, parseFile, parseFileFromDisk } from './extract/tree-sitter-runner.js';
+export type { LanguageExtractor } from './extract/types.js';
+export type * from './types.js';

package/src/pipeline.ts

@@ -0,0 +1,166 @@
+import { join } from 'path';
+import { mkdirSync, readFileSync } from 'fs';
+import type {
+  BuildOptions,
+  ExtractionResult,
+  GraphAnalysis,
+  SerializedGraph,
+} from './types.js';
+import { collectFiles } from './detect.js';
+import { FileCache } from './cache.js';
+import { buildGraph as buildGraphologyGraph } from './build.js';
+import { detectCommunities } from './cluster.js';
+import { buildAnalysis } from './analyze.js';
+import { saveGraph } from './export.js';
+import { typescriptExtractor } from './extract/languages/typescript.js';
+import { parseFile } from './extract/tree-sitter-runner.js';
+import type { LanguageExtractor } from './extract/types.js';
+
+const DEFAULT_OUTPUT_SUBDIR = '.monobrain/graph';
+
+// Map language identifiers to the extractors we have available.
+// python and go extractors are loaded lazily when their modules exist.
+const EXTRACTOR_MAP: Record<string, LanguageExtractor> = {
+  typescript: typescriptExtractor,
+  javascript: typescriptExtractor, // TS extractor handles JS via regex + tree-sitter-javascript
+};
+
+/** Attempt to load python/go extractors that may be present in the extract/languages dir. */
+async function tryLoadExtractor(language: string): Promise<LanguageExtractor | undefined> {
+  if (EXTRACTOR_MAP[language]) return EXTRACTOR_MAP[language];
+  try {
+    const mod = await import(`./extract/languages/${language}.js`) as Record<string, LanguageExtractor | undefined>;
+    const extractor = (mod[`${language}Extractor`] ?? mod['default']) as LanguageExtractor | undefined;
+    if (extractor) EXTRACTOR_MAP[language] = extractor;
+    return extractor;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Main entry point for building a knowledge graph from a codebase.
+ *
+ * Orchestrates file collection, per-file extraction (with caching),
+ * graph construction via graphology, community detection, and serialisation.
+ *
+ * @param projectPath - Absolute path to the root of the codebase to analyse.
+ * @param options     - Optional build configuration.
+ * @returns           - Serialized graph + analysis summary.
+ */
+export async function buildGraph(
+  projectPath: string,
+  options: BuildOptions = {},
+): Promise<{ graph: SerializedGraph; analysis: GraphAnalysis }> {
+  // Resolve output directory
+  const outputDir = options.outputDir ?? join(projectPath, DEFAULT_OUTPUT_SUBDIR);
+  mkdirSync(outputDir, { recursive: true });
+
+  const cache = new FileCache(outputDir);
+
+  // 1. Collect files
+  const files = collectFiles(projectPath, options);
+
+  // 2. Extract nodes/edges from each file (cache-aware)
+  const merged: ExtractionResult = {
+    nodes: [],
+    edges: [],
+    hyperedges: [],
+    filesProcessed: 0,
+    fromCache: 0,
+    errors: [],
+  };
+
+  for (const file of files) {
+    let content: string;
+    try {
+      content = readFileSync(file.path, 'utf-8');
+    } catch (err) {
+      merged.errors.push(`Cannot read ${file.path}: ${String(err)}`);
+      continue;
+    }
+
+    const cacheKey = cache.key(file.path, content);
+
+    let result = cache.get(cacheKey);
+    if (result) {
+      merged.fromCache += 1;
+    } else {
+      const extractor = file.language
+        ? await tryLoadExtractor(file.language)
+        : undefined;
+
+      if (extractor) {
+        result = parseFile(file.path, content, extractor);
+      } else {
+        result = extractGeneric(file.path, content);
+      }
+      cache.set(cacheKey, result);
+    }
+
+    merged.nodes.push(...result.nodes);
+    merged.edges.push(...result.edges);
+    if (result.hyperedges) merged.hyperedges!.push(...result.hyperedges);
+    merged.filesProcessed += 1;
+    merged.errors.push(...result.errors);
+  }
+
+  // 3. Build graphology graph (dedup + stub endpoints)
+  const graph = buildGraphologyGraph(merged);
+
+  // 4. Community detection (Louvain with directory-based fallback)
+  await detectCommunities(graph);
+
+  // 5. Degree annotation
+  graph.forEachNode((id) => {
+    graph.setNodeAttribute(id, 'degree', graph.degree(id));
+  });
+
+  // 6. Build analysis (god nodes, surprise edges, communities, stats)
+  const analysis = buildAnalysis(graph, outputDir);
+
+  // 7. Persist to disk
+  saveGraph(graph, outputDir, projectPath);
+
+  // 8. Serialize to the public return type
+  const serialized: SerializedGraph = {
+    version: '1.0.0',
+    builtAt: new Date().toISOString(),
+    projectPath,
+    directed: true,
+    multigraph: false,
+    nodes: graph.nodes().map((id) => ({
+      id,
+      ...graph.getNodeAttributes(id),
+    })),
+    links: graph.edges().map((edgeId) => ({
+      source: graph.source(edgeId),
+      target: graph.target(edgeId),
+      ...graph.getEdgeAttributes(edgeId),
+    })),
+  };
+
+  return { graph: serialized, analysis };
+}
+
+// ---------------------------------------------------------------------------
+// Internal: minimal fallback for languages without a dedicated extractor
+// ---------------------------------------------------------------------------
+
+function extractGeneric(filePath: string, content: string): ExtractionResult {
+  return {
+    nodes: [
+      {
+        id: filePath,
+        label: filePath.split('/').pop() ?? filePath,
+        fileType: 'code',
+        sourceFile: filePath,
+        linesOfCode: content.split('\n').length,
+      },
+    ],
+    edges: [],
+    filesProcessed: 1,
+    fromCache: 0,
+    errors: [],
+  };
+}

package/src/types.ts

@@ -0,0 +1,116 @@
+// Confidence levels for graph edges — matches graphify's audit trail
+export type Confidence = 'EXTRACTED' | 'INFERRED' | 'AMBIGUOUS';
+
+// Node file types
+export type FileType = 'code' | 'document' | 'paper' | 'image' | 'unknown';
+
+// Node in the knowledge graph
+export interface GraphNode {
+  id: string;
+  label: string;
+  fileType: FileType;
+  sourceFile: string;
+  sourceLocation?: string;  // e.g. "L42"
+  community?: number;
+  degree?: number;
+  // Extra attributes for specific node types
+  [key: string]: unknown;
+}
+
+// Edge in the knowledge graph
+export interface GraphEdge {
+  source: string;
+  target: string;
+  relation: string;         // 'calls', 'imports', 'uses', 'contains', 'implements', etc.
+  confidence: Confidence;
+  confidenceScore?: number; // 0.0-1.0 for INFERRED edges
+  sourceFile?: string;
+  sourceLocation?: string;
+  weight?: number;
+}
+
+// Raw extraction result from a single file or set of files
+export interface ExtractionResult {
+  nodes: GraphNode[];
+  edges: GraphEdge[];
+  hyperedges?: HyperEdge[];
+  filesProcessed: number;
+  fromCache: number;
+  errors: string[];
+}
+
+// Hyperedge (group relationship)
+export interface HyperEdge {
+  label: string;
+  nodes: string[];
+  confidence: Confidence;
+  confidenceScore?: number;
+}
+
+// Graph analysis result
+export interface GraphAnalysis {
+  godNodes: GodNode[];
+  surprises: SurpriseEdge[];
+  communities: Record<number, string[]>;
+  stats: GraphStats;
+}
+
+export interface GodNode {
+  id: string;
+  label: string;
+  degree: number;
+  community?: number;
+  sourceFile: string;
+  neighbors: string[];
+}
+
+export interface SurpriseEdge {
+  from: string;
+  fromCommunity: number;
+  fromFile: string;
+  to: string;
+  toCommunity: number;
+  toFile: string;
+  relation: string;
+  confidence: Confidence;
+  score: number;
+}
+
+export interface GraphStats {
+  nodes: number;
+  edges: number;
+  communities: number;
+  confidence: Record<Confidence, number>;
+  fileTypes: Record<string, number>;
+  topRelations: Record<string, number>;
+  isDirected: boolean;
+  graphPath?: string;
+}
+
+// Serialized graph format (JSON-compatible)
+export interface SerializedGraph {
+  version: string;
+  builtAt: string;
+  projectPath: string;
+  nodes: Array<{ id: string } & Record<string, unknown>>;
+  links: Array<{ source: string; target: string } & Record<string, unknown>>;
+  directed: boolean;
+  multigraph: boolean;
+}
+
+// Build options
+export interface BuildOptions {
+  codeOnly?: boolean;
+  outputDir?: string;        // defaults to <projectPath>/.monobrain/graph/
+  maxFileSizeBytes?: number; // defaults to 500KB
+  excludePatterns?: string[];
+  languages?: string[];
+}
+
+// File classification
+export interface ClassifiedFile {
+  path: string;
+  fileType: FileType;
+  language?: string;
+  sizeBytes: number;
+}