@kentwynn/kgraph 0.1.26 → 0.2.0

package/dist/scanner/jvm-symbol-extractor.js

@@ -1,16 +1,17 @@
+import { parseSource } from './tree-sitter-parser.js';
 // Handles Java (.java) and Kotlin (.kt, .kts)
-export function extractJvmSymbols(sourceText, filePath) {
-    const ext = filePath.endsWith('.kt') || filePath.endsWith('.kts') ? 'kotlin' : 'java';
-    const lines = sourceText.split('\n');
+export async function extractJvmSymbols(sourceText, filePath) {
     const symbols = [];
     const dependencies = [];
     const relationships = [];
     const warnings = [];
-    // Stack tracks class/object/interface scope
-    const typeStack = [];
-    let braceDepth = 0;
-    const addSymbol = (name, kind, lineNum, exported, parentName) => {
-        const id = [filePath, kind, parentName, name, lineNum]
+    if (!sourceText.trim()) {
+        return { symbols, dependencies, relationships, warnings };
+    }
+    const lang = filePath.endsWith('.kt') || filePath.endsWith('.kts') ? 'kotlin' : 'java';
+    const tree = await parseSource(sourceText, lang);
+    const addSymbol = (name, kind, startLine, endLine, exported, parentName) => {
+        const id = [filePath, kind, parentName, name, startLine]
             .filter(Boolean)
             .join('#');
         symbols.push({
@@ -18,8 +19,8 @@ export function extractJvmSymbols(sourceText, filePath) {
             name,
             kind,
             filePath,
-            startLine: lineNum,
-            endLine: lineNum,
+            startLine,
+            endLine,
             exported,
             parentName,
         });
@@ -32,74 +33,141 @@ export function extractJvmSymbols(sourceText, filePath) {
             confidence: 'high',
         });
     };
-    for (let i = 0; i < lines.length; i++) {
-        const line = lines[i];
-        const lineNum = i + 1;
-        const trimmed = line.trim();
-        if (!trimmed || trimmed.startsWith('//') || trimmed.startsWith('*'))
-            continue;
-        braceDepth +=
-            (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
-        while (typeStack.length > 0 &&
-            braceDepth < typeStack[typeStack.length - 1].braceDepth) {
-            typeStack.pop();
+    function hasModifier(node, modifier) {
+        const mods = node.childForFieldName('modifiers') ??
+            node.namedChildren.find((c) => c.type === 'modifiers');
+        if (!mods)
+            return false;
+        return mods.text.includes(modifier);
+    }
+    function walkJava(node, parentClassName) {
+        switch (node.type) {
+            case 'import_declaration': {
+                // Java import: scoped_identifier child
+                const scopedId = node.namedChildren.find((c) => c.type === 'scoped_identifier' || c.type === 'identifier');
+                if (scopedId) {
+                    const specifier = scopedId.text.replace(/\.\*$/, '');
+                    dependencies.push({ fromFile: filePath, specifier, kind: 'package' });
+                    relationships.push({
+                        sourceType: 'file',
+                        sourceId: filePath,
+                        targetType: 'package',
+                        targetId: specifier,
+                        relationshipType: 'import',
+                        confidence: 'high',
+                    });
+                }
+                return;
+            }
+            case 'class_declaration':
+            case 'interface_declaration':
+            case 'enum_declaration':
+            case 'annotation_type_declaration': {
+                const nameNode = node.childForFieldName('name');
+                if (nameNode) {
+                    const exported = hasModifier(node, 'public');
+                    addSymbol(nameNode.text, 'class', node.startPosition.row + 1, node.endPosition.row + 1, exported, parentClassName);
+                    // Walk class body for methods and nested types
+                    const body = node.childForFieldName('body') ??
+                        node.namedChildren.find((c) => c.type === 'class_body' ||
+                            c.type === 'interface_body' ||
+                            c.type === 'enum_body');
+                    if (body) {
+                        for (const child of body.namedChildren) {
+                            walkJava(child, nameNode.text);
+                        }
+                    }
+                }
+                return;
+            }
+            case 'method_declaration':
+            case 'constructor_declaration': {
+                const nameNode = node.childForFieldName('name');
+                if (nameNode) {
+                    const exported = hasModifier(node, 'public');
+                    const kind = parentClassName
+                        ? 'method'
+                        : 'function';
+                    addSymbol(nameNode.text, kind, node.startPosition.row + 1, node.endPosition.row + 1, exported, parentClassName);
+                }
+                return;
+            }
         }
-        // import statement
-        const importMatch = trimmed.match(/^import\s+([\w.]+(?:\.\*)?)/);
-        if (importMatch) {
-            const specifier = importMatch[1].replace(/\.\*$/, '');
-            dependencies.push({ fromFile: filePath, specifier, kind: 'package' });
-            relationships.push({
-                sourceType: 'file',
-                sourceId: filePath,
-                targetType: 'package',
-                targetId: specifier,
-                relationshipType: 'import',
-                confidence: 'high',
-            });
-            continue;
+        for (const child of node.namedChildren) {
+            walkJava(child, parentClassName);
         }
-        if (ext === 'java') {
-            // class / interface / enum / @interface
-            const typeMatch = trimmed.match(/\b(?:public\s+|private\s+|protected\s+|abstract\s+|final\s+)*(?:class|interface|enum|@interface)\s+(\w+)/);
-            if (typeMatch) {
-                const parent = typeStack[typeStack.length - 1];
-                const exported = trimmed.includes('public');
-                addSymbol(typeMatch[1], 'class', lineNum, exported, parent?.name);
-                typeStack.push({ name: typeMatch[1], braceDepth });
-                continue;
+    }
+    function walkKotlin(node, parentClassName) {
+        switch (node.type) {
+            case 'import': {
+                // Kotlin import: qualified_identifier child
+                const qualId = node.namedChildren.find((c) => c.type === 'qualified_identifier' || c.type === 'identifier');
+                if (qualId) {
+                    const specifier = qualId.text;
+                    dependencies.push({ fromFile: filePath, specifier, kind: 'package' });
+                    relationships.push({
+                        sourceType: 'file',
+                        sourceId: filePath,
+                        targetType: 'package',
+                        targetId: specifier,
+                        relationshipType: 'import',
+                        confidence: 'high',
+                    });
+                }
+                return;
             }
-            // method: visibility returnType methodName(
-            // Avoid matching field declarations (no parenthesis)
-            const methodMatch = trimmed.match(/\b(?:public|private|protected|static|final|synchronized|abstract|native|default|void|@Override\s+(?:public|protected))\b.*\s+(\w+)\s*\(/);
-            if (methodMatch && !trimmed.startsWith('//')) {
-                const parent = typeStack[typeStack.length - 1];
-                const exported = trimmed.includes('public');
-                const kind = parent ? 'method' : 'function';
-                addSymbol(methodMatch[1], kind, lineNum, exported, parent?.name);
-                continue;
+            case 'class_declaration': {
+                const nameNode = node.childForFieldName('name') ??
+                    node.namedChildren.find((c) => c.type === 'identifier');
+                if (nameNode) {
+                    const exported = !hasModifier(node, 'private') && !hasModifier(node, 'internal');
+                    addSymbol(nameNode.text, 'class', node.startPosition.row + 1, node.endPosition.row + 1, exported, parentClassName);
+                    // Walk class body
+                    const body = node.namedChildren.find((c) => c.type === 'class_body');
+                    if (body) {
+                        for (const child of body.namedChildren) {
+                            walkKotlin(child, nameNode.text);
+                        }
+                    }
+                }
+                return;
             }
-        }
-        if (ext === 'kotlin') {
-            // class / interface / object / data class / sealed class
-            const typeMatch = trimmed.match(/\b(?:data\s+|sealed\s+|abstract\s+|open\s+|inner\s+)?(?:class|interface|object|enum\s+class)\s+(\w+)/);
-            if (typeMatch) {
-                const parent = typeStack[typeStack.length - 1];
-                const exported = !trimmed.startsWith('private') && !trimmed.startsWith('internal');
-                addSymbol(typeMatch[1], 'class', lineNum, exported, parent?.name);
-                typeStack.push({ name: typeMatch[1], braceDepth });
-                continue;
+            case 'object_declaration': {
+                const nameNode = node.namedChildren.find((c) => c.type === 'identifier');
+                if (nameNode) {
+                    const exported = !hasModifier(node, 'private') && !hasModifier(node, 'internal');
+                    addSymbol(nameNode.text, 'class', node.startPosition.row + 1, node.endPosition.row + 1, exported, parentClassName);
+                    const body = node.namedChildren.find((c) => c.type === 'class_body');
+                    if (body) {
+                        for (const child of body.namedChildren) {
+                            walkKotlin(child, nameNode.text);
+                        }
+                    }
+                }
+                return;
             }
-            // fun
-            const funcMatch = trimmed.match(/\bfun\s+(\w+)\s*[(<]/);
-            if (funcMatch) {
-                const parent = typeStack[typeStack.length - 1];
-                const exported = !trimmed.startsWith('private') && !trimmed.startsWith('internal');
-                const kind = parent ? 'method' : 'function';
-                addSymbol(funcMatch[1], kind, lineNum, exported, parent?.name);
-                continue;
+            case 'function_declaration': {
+                const nameNode = node.namedChildren.find((c) => c.type === 'identifier');
+                if (nameNode) {
+                    const exported = !hasModifier(node, 'private') && !hasModifier(node, 'internal');
+                    const kind = parentClassName
+                        ? 'method'
+                        : 'function';
+                    addSymbol(nameNode.text, kind, node.startPosition.row + 1, node.endPosition.row + 1, exported, parentClassName);
+                }
+                return;
             }
         }
+        for (const child of node.namedChildren) {
+            walkKotlin(child, parentClassName);
+        }
+    }
+    if (lang === 'java') {
+        walkJava(tree.rootNode);
+    }
+    else {
+        walkKotlin(tree.rootNode);
     }
+    tree.delete();
     return { symbols, dependencies, relationships, warnings };
 }

package/dist/scanner/python-symbol-extractor.d.ts

@@ -1,2 +1,2 @@
 import type { SymbolExtractionResult } from './ts-symbol-extractor.js';
-export declare function extractPythonSymbols(sourceText: string, filePath: string): SymbolExtractionResult;
+export declare function extractPythonSymbols(sourceText: string, filePath: string): Promise<SymbolExtractionResult>;

package/dist/scanner/python-symbol-extractor.js

@@ -1,13 +1,15 @@
-export function extractPythonSymbols(sourceText, filePath) {
-    const lines = sourceText.split('\n');
+import { parseSource } from './tree-sitter-parser.js';
+export async function extractPythonSymbols(sourceText, filePath) {
     const symbols = [];
     const dependencies = [];
     const relationships = [];
     const warnings = [];
-    // Stack tracks nested class scope: [{name, indent}]
-    const classStack = [];
-    const addSymbol = (name, kind, lineNum, parentName) => {
-        const id = [filePath, kind, parentName, name, lineNum]
+    if (!sourceText.trim()) {
+        return { symbols, dependencies, relationships, warnings };
+    }
+    const tree = await parseSource(sourceText, 'python');
+    const addSymbol = (name, kind, startLine, endLine, exported, parentName) => {
+        const id = [filePath, kind, parentName, name, startLine]
             .filter(Boolean)
             .join('#');
         symbols.push({
@@ -15,9 +17,9 @@ export function extractPythonSymbols(sourceText, filePath) {
             name,
             kind,
             filePath,
-            startLine: lineNum,
-            endLine: lineNum,
-            exported: false,
+            startLine,
+            endLine,
+            exported,
             parentName,
         });
         relationships.push({
@@ -29,70 +31,89 @@ export function extractPythonSymbols(sourceText, filePath) {
             confidence: 'high',
         });
     };
-    for (let i = 0; i < lines.length; i++) {
-        const line = lines[i];
-        const lineNum = i + 1;
-        const trimmed = line.trimStart();
-        if (!trimmed || trimmed.startsWith('#'))
-            continue;
-        const indent = line.length - trimmed.length;
-        // Pop classes we've exited: any class whose indent >= current line's indent
-        // (unless this line IS the class that opened at that indent — handled by re-pushing below)
-        while (classStack.length > 0 &&
-            classStack[classStack.length - 1].indent >= indent) {
-            classStack.pop();
-        }
-        // class definition
-        const classMatch = trimmed.match(/^class\s+([A-Za-z_]\w*)/);
-        if (classMatch) {
-            const name = classMatch[1];
-            const parent = classStack[classStack.length - 1];
-            addSymbol(name, 'class', lineNum, parent?.name);
-            classStack.push({ name, indent });
-            continue;
-        }
-        // function / method definition (sync or async)
-        const funcMatch = trimmed.match(/^(?:async\s+)?def\s+([A-Za-z_]\w*)/);
-        if (funcMatch) {
-            const name = funcMatch[1];
-            const parent = classStack[classStack.length - 1];
-            const kind = parent !== undefined ? 'method' : 'function';
-            addSymbol(name, kind, lineNum, parent?.name);
-            continue;
-        }
-        // import module
-        const importMatch = trimmed.match(/^import\s+([\w.]+)/);
-        if (importMatch) {
-            const specifier = importMatch[1];
-            dependencies.push({ fromFile: filePath, specifier, kind: 'package' });
-            addSymbol(specifier, 'import', lineNum);
-            relationships.push({
-                sourceType: 'file',
-                sourceId: filePath,
-                targetType: 'package',
-                targetId: specifier,
-                relationshipType: 'import',
-                confidence: 'high',
-            });
-            continue;
+    function walk(node, parentClassName) {
+        switch (node.type) {
+            case 'class_definition': {
+                const nameNode = node.childForFieldName('name');
+                if (nameNode) {
+                    const startLine = node.startPosition.row + 1;
+                    const endLine = node.endPosition.row + 1;
+                    addSymbol(nameNode.text, 'class', startLine, endLine, false, parentClassName);
+                    // Walk children for nested classes/methods
+                    const body = node.childForFieldName('body');
+                    if (body) {
+                        for (const child of body.namedChildren) {
+                            walk(child, nameNode.text);
+                        }
+                    }
+                }
+                return; // Don't recurse further from here
+            }
+            case 'function_definition': {
+                const nameNode = node.childForFieldName('name');
+                if (nameNode) {
+                    const startLine = node.startPosition.row + 1;
+                    const endLine = node.endPosition.row + 1;
+                    const kind = parentClassName
+                        ? 'method'
+                        : 'function';
+                    addSymbol(nameNode.text, kind, startLine, endLine, false, parentClassName);
+                }
+                return; // Don't recurse into function bodies
+            }
+            case 'import_statement': {
+                // import os / import os.path
+                const startLine = node.startPosition.row + 1;
+                for (const child of node.namedChildren) {
+                    if (child.type === 'dotted_name' || child.type === 'aliased_import') {
+                        const specifier = child.type === 'aliased_import'
+                            ? (child.childForFieldName('name')?.text ?? child.text)
+                            : child.text;
+                        dependencies.push({
+                            fromFile: filePath,
+                            specifier,
+                            kind: 'package',
+                        });
+                        addSymbol(specifier, 'import', startLine, startLine, false);
+                        relationships.push({
+                            sourceType: 'file',
+                            sourceId: filePath,
+                            targetType: 'package',
+                            targetId: specifier,
+                            relationshipType: 'import',
+                            confidence: 'high',
+                        });
+                    }
+                }
+                return;
+            }
+            case 'import_from_statement': {
+                // from X import Y
+                const moduleNode = node.childForFieldName('module_name');
+                if (moduleNode) {
+                    const specifier = moduleNode.text;
+                    const kind = specifier.startsWith('.')
+                        ? 'local'
+                        : 'package';
+                    dependencies.push({ fromFile: filePath, specifier, kind });
+                    relationships.push({
+                        sourceType: 'file',
+                        sourceId: filePath,
+                        targetType: kind === 'local' ? 'file' : 'package',
+                        targetId: specifier,
+                        relationshipType: 'import',
+                        confidence: kind === 'local' ? 'medium' : 'high',
+                    });
+                }
+                return;
+            }
         }
-        // from X import Y
-        const fromMatch = trimmed.match(/^from\s+(\S+)\s+import/);
-        if (fromMatch) {
-            const specifier = fromMatch[1];
-            const kind = specifier.startsWith('.')
-                ? 'local'
-                : 'package';
-            dependencies.push({ fromFile: filePath, specifier, kind });
-            relationships.push({
-                sourceType: 'file',
-                sourceId: filePath,
-                targetType: kind === 'local' ? 'file' : 'package',
-                targetId: specifier,
-                relationshipType: 'import',
-                confidence: kind === 'local' ? 'medium' : 'high',
-            });
+        // Recurse into children for top-level statements
+        for (const child of node.namedChildren) {
+            walk(child, parentClassName);
         }
     }
+    walk(tree.rootNode);
+    tree.delete();
     return { symbols, dependencies, relationships, warnings };
 }

package/dist/scanner/repo-scanner.js

@@ -2,6 +2,7 @@ import fg from 'fast-glob';
 import crypto from 'node:crypto';
 import { readFile, stat } from 'node:fs/promises';
 import path from 'node:path';
+import { estimateTokens } from '../session/token-estimator.js';
 import { extractCSymbols } from './c-symbol-extractor.js';
 import { extractCSharpSymbols } from './csharp-symbol-extractor.js';
 import { buildFastGlobIgnore, detectLanguage, isPreciseLanguage, readGitignorePatterns, shouldExclude, } from './file-classifier.js';
@@ -10,10 +11,9 @@ import { extractJvmSymbols } from './jvm-symbol-extractor.js';
 import { extractPythonSymbols } from './python-symbol-extractor.js';
 import { extractRustSymbols } from './rust-symbol-extractor.js';
 import { extractTsSymbols } from './ts-symbol-extractor.js';
-import { estimateTokens } from '../session/token-estimator.js';
 const C_EXTS = new Set(['.c', '.h', '.cpp', '.cc', '.cxx', '.hpp', '.hxx']);
 const JVM_EXTS = new Set(['.java', '.kt', '.kts']);
-function extractSymbols(text, repoPath) {
+async function extractSymbols(text, repoPath) {
     const ext = path.extname(repoPath);
     if (ext === '.py' || ext === '.pyw' || ext === '.pyi') {
         return extractPythonSymbols(text, repoPath);
@@ -46,21 +46,71 @@ export async function scanRepository(rootPath, config, previous) {
         unique: true,
         ignore: buildFastGlobIgnore(allExcludes),
     });
+    // Build lookup maps from previous scan for incremental skip
+    const prevFileByPath = new Map((previous?.files ?? []).map((f) => [f.path, f]));
+    const prevSymbolsByFile = new Map();
+    const prevDepsByFile = new Map();
+    const prevRelsBySource = new Map();
+    if (previous) {
+        for (const sym of previous.symbols) {
+            const arr = prevSymbolsByFile.get(sym.filePath) ?? [];
+            arr.push(sym);
+            prevSymbolsByFile.set(sym.filePath, arr);
+        }
+        for (const dep of previous.dependencies) {
+            const arr = prevDepsByFile.get(dep.fromFile) ?? [];
+            arr.push(dep);
+            prevDepsByFile.set(dep.fromFile, arr);
+        }
+        for (const rel of previous.relationships) {
+            if (rel.relationshipType !== 'import' &&
+                rel.relationshipType !== 'moved-from') {
+                const arr = prevRelsBySource.get(rel.sourceId) ?? [];
+                arr.push(rel);
+                prevRelsBySource.set(rel.sourceId, arr);
+            }
+        }
+    }
     const files = [];
     const symbols = [];
     const dependencies = [];
     const relationships = [];
     const warnings = [];
+    let skippedFiles = 0;
     for (const repoPath of entries.sort()) {
         if (shouldExclude(repoPath, mergedConfig)) {
             continue;
         }
         const absolutePath = path.join(rootPath, repoPath);
         try {
-            const [info, content] = await Promise.all([
-                stat(absolutePath),
-                readFile(absolutePath),
-            ]);
+            const info = await stat(absolutePath);
+            // Incremental skip: if mtime and size match previous, carry forward
+            const prevFile = prevFileByPath.get(repoPath);
+            if (prevFile &&
+                prevFile.sizeBytes === info.size &&
+                prevFile.modifiedAt === info.mtime.toISOString()) {
+                files.push({ ...prevFile, modifiedAt: info.mtime.toISOString() });
+                const prevSyms = prevSymbolsByFile.get(repoPath);
+                if (prevSyms)
+                    symbols.push(...prevSyms);
+                const prevDeps = prevDepsByFile.get(repoPath);
+                if (prevDeps)
+                    dependencies.push(...prevDeps);
+                const prevRels = prevRelsBySource.get(repoPath);
+                if (prevRels)
+                    relationships.push(...prevRels);
+                // Also carry forward symbol-sourced relationships
+                if (prevSyms) {
+                    for (const sym of prevSyms) {
+                        const symRels = prevRelsBySource.get(sym.id);
+                        if (symRels)
+                            relationships.push(...symRels);
+                    }
+                }
+                skippedFiles++;
+                continue;
+            }
+            const content = await readFile(absolutePath);
             const text = content.toString('utf8');
             const contentHash = crypto
                 .createHash('sha256')
@@ -79,7 +129,7 @@ export async function scanRepository(rootPath, config, previous) {
                 warnings: [],
             };
             if (isPreciseLanguage(repoPath, config)) {
-                const extracted = extractSymbols(text, repoPath);
+                const extracted = await extractSymbols(text, repoPath);
                 symbols.push(...extracted.symbols);
                 dependencies.push(...extracted.dependencies);
                 relationships.push(...extracted.relationships.filter((relationship) => relationship.relationshipType !== 'import'));
@@ -105,7 +155,14 @@ export async function scanRepository(rootPath, config, previous) {
     resolveLocalDependencies(dependencies, files);
     relationships.push(...buildImportRelationships(dependencies));
     relationships.push(...detectMovedFiles(previous?.files ?? [], files));
-    return { files, symbols, dependencies, relationships, warnings };
+    return {
+        files,
+        symbols,
+        dependencies,
+        relationships,
+        warnings,
+        skippedFiles,
+    };
 }
 const SOURCE_EXTENSIONS = [
     '.ts',

package/dist/scanner/rust-symbol-extractor.d.ts

@@ -1,2 +1,2 @@
 import type { SymbolExtractionResult } from './ts-symbol-extractor.js';
-export declare function extractRustSymbols(sourceText: string, filePath: string): SymbolExtractionResult;
+export declare function extractRustSymbols(sourceText: string, filePath: string): Promise<SymbolExtractionResult>;