@kentwynn/kgraph 0.2.22 → 0.2.24

package/dist/scanner/repo-scanner.js

@@ -3,22 +3,44 @@ import crypto from 'node:crypto';
 import { readFile, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { estimateTokens } from '../session/token-estimator.js';
+import { extractBroadSymbols, supportsBroadExtraction, } from './broad-symbol-extractor.js';
 import { extractCSymbols } from './c-symbol-extractor.js';
 import { extractCSharpSymbols } from './csharp-symbol-extractor.js';
 import { buildFastGlobIgnore, detectLanguage, isPreciseLanguage, readGitignorePatterns, shouldExclude, } from './file-classifier.js';
-import { getChangedFilesSince, isGitRepo } from './git-utils.js';
+import { getChangedFilesSince, getGitIgnoredFiles, isGitRepo, } from './git-utils.js';
 import { extractGoSymbols } from './go-symbol-extractor.js';
 import { extractJvmSymbols } from './jvm-symbol-extractor.js';
+import { extractPhpSymbols } from './php-symbol-extractor.js';
 import { extractPythonSymbols } from './python-symbol-extractor.js';
+import { extractRubySymbols } from './ruby-symbol-extractor.js';
 import { extractRustSymbols } from './rust-symbol-extractor.js';
+import { extractShellSymbols } from './shell-symbol-extractor.js';
+import { extractSqlSymbols } from './sql-symbol-extractor.js';
 import { extractTsSymbols } from './ts-symbol-extractor.js';
 const C_EXTS = new Set(['.c', '.h', '.cpp', '.cc', '.cxx', '.hpp', '.hxx']);
 const JVM_EXTS = new Set(['.java', '.kt', '.kts']);
+const TS_EXTS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts']);
 async function extractSymbols(text, repoPath) {
     const ext = path.extname(repoPath);
+    const language = detectLanguage(repoPath);
+    if (supportsBroadExtraction(language)) {
+        return extractBroadSymbols(text, repoPath, language);
+    }
     if (ext === '.py' || ext === '.pyw' || ext === '.pyi') {
         return extractPythonSymbols(text, repoPath);
     }
+    if (ext === '.php') {
+        return extractPhpSymbols(text, repoPath);
+    }
+    if (ext === '.rb' || ext === '.rake' || language === 'ruby') {
+        return extractRubySymbols(text, repoPath);
+    }
+    if (['.sh', '.bash', '.zsh', '.fish'].includes(ext) || language === 'shell') {
+        return extractShellSymbols(text, repoPath);
+    }
+    if (ext === '.sql') {
+        return extractSqlSymbols(text, repoPath);
+    }
     if (ext === '.go') {
         return extractGoSymbols(text, repoPath);
     }
@@ -34,19 +56,26 @@ async function extractSymbols(text, repoPath) {
     if (ext === '.cs') {
         return extractCSharpSymbols(text, repoPath);
     }
-    return extractTsSymbols(text, repoPath);
+    if (TS_EXTS.has(ext)) {
+        return extractTsSymbols(text, repoPath);
+    }
+    return { symbols: [], dependencies: [], relationships: [], warnings: [] };
 }
 export async function scanRepository(rootPath, config, previous) {
     const gitignorePatterns = await readGitignorePatterns(rootPath);
     const allExcludes = [...config.exclude, ...gitignorePatterns];
     const mergedConfig = { ...config, exclude: allExcludes };
-    const entries = await fg(config.include, {
+    const rawEntries = await fg(config.include, {
         cwd: rootPath,
         dot: true,
         onlyFiles: true,
         unique: true,
         ignore: buildFastGlobIgnore(allExcludes),
     });
+    const gitIgnored = (await isGitRepo(rootPath))
+        ? await getGitIgnoredFiles(rootPath, rawEntries)
+        : new Set();
+    const entries = rawEntries.filter((entry) => !gitIgnored.has(entry));
     // Build lookup maps from previous scan for incremental skip
     const prevFileByPath = new Map((previous?.files ?? []).map((f) => [f.path, f]));
     const prevSymbolsByFile = new Map();
@@ -190,7 +219,7 @@ export async function scanRepository(rootPath, config, previous) {
     }
     resolveLocalDependencies(dependencies, files);
     relationships.push(...buildImportRelationships(dependencies));
-    relationships.push(...detectMovedFiles(previous?.files ?? [], files));
+    relationships.push(...detectMovedFiles((previous?.files ?? []).filter((file) => !shouldExclude(file.path, mergedConfig)), files));
     return {
         files,
         symbols,
@@ -223,6 +252,15 @@ const SOURCE_EXTENSIONS = [
     '.hpp',
     '.hxx',
     '.cs',
+    '.php',
+    '.swift',
+    '.rb',
+    '.rake',
+    '.lua',
+    '.dart',
+    '.ex',
+    '.exs',
+    '.scala',
 ];
 function resolveLocalDependencies(dependencies, files) {
     const filePaths = new Set(files.map((file) => file.path));
@@ -237,6 +275,9 @@ function resolveLocalDependencyPath(fromFile, specifier, filePaths) {
     if (!specifier.startsWith('.')) {
         return undefined;
     }
+    if (path.posix.extname(fromFile) === '.py') {
+        return resolvePythonRelativeImportPath(fromFile, specifier, filePaths);
+    }
     const base = path.posix.normalize(path.posix.join(path.posix.dirname(fromFile), specifier));
     const candidates = path.posix.extname(base)
         ? [base]
@@ -246,6 +287,26 @@ function resolveLocalDependencyPath(fromFile, specifier, filePaths) {
         ];
     return candidates.find((candidate) => filePaths.has(candidate));
 }
+function resolvePythonRelativeImportPath(fromFile, specifier, filePaths) {
+    const match = specifier.match(/^(\.+)(.*)$/);
+    if (!match) {
+        return undefined;
+    }
+    const [, dots, moduleName] = match;
+    const packageParts = path.posix.dirname(fromFile).split('/');
+    const levelsUp = Math.max(0, dots.length - 1);
+    const baseParts = levelsUp > 0 ? packageParts.slice(0, -levelsUp) : packageParts;
+    const modulePath = moduleName
+        .split('.')
+        .filter(Boolean)
+        .join('/');
+    const base = path.posix.normalize(path.posix.join(...baseParts, modulePath));
+    const candidates = [
+        `${base}.py`,
+        path.posix.join(base, '__init__.py'),
+    ];
+    return candidates.find((candidate) => filePaths.has(candidate));
+}
 function buildImportRelationships(dependencies) {
     return dependencies.map((dependency) => ({
         sourceType: 'file',

package/dist/scanner/ruby-symbol-extractor.d.ts

@@ -0,0 +1,2 @@
+import type { SymbolExtractionResult } from './ts-symbol-extractor.js';
+export declare function extractRubySymbols(sourceText: string, filePath: string): Promise<SymbolExtractionResult>;

package/dist/scanner/ruby-symbol-extractor.js

@@ -0,0 +1,75 @@
+import { emptyExtractionResult, ExtractionContext, } from './extraction-context.js';
+import { parseSource } from './tree-sitter-parser.js';
+export async function extractRubySymbols(sourceText, filePath) {
+    if (!sourceText.trim()) {
+        return emptyExtractionResult();
+    }
+    const tree = await parseSource(sourceText, 'ruby');
+    const context = new ExtractionContext(filePath);
+    function addNamedSymbol(node, kind, parentName) {
+        const nameNode = findNameNode(node);
+        if (!nameNode)
+            return undefined;
+        return context.addSymbol({
+            name: nameNode.text,
+            kind,
+            startLine: node.startPosition.row + 1,
+            endLine: node.endPosition.row + 1,
+            exported: true,
+            parentName,
+        });
+    }
+    function walk(node, parentName) {
+        switch (node.type) {
+            case 'call': {
+                const methodName = findCallMethodName(node);
+                if (methodName === 'require' || methodName === 'require_relative') {
+                    const argument = findFirstStringContent(node);
+                    if (argument) {
+                        context.addDependency(argument, methodName === 'require_relative' ? 'local' : 'package');
+                    }
+                    return;
+                }
+                break;
+            }
+            case 'class':
+            case 'module': {
+                const symbol = addNamedSymbol(node, node.type === 'class' ? 'class' : 'type', parentName);
+                const nextParent = symbol?.name ?? parentName;
+                for (const child of node.namedChildren) {
+                    walk(child, nextParent);
+                }
+                return;
+            }
+            case 'method':
+            case 'singleton_method': {
+                const symbol = addNamedSymbol(node, parentName ? 'method' : 'function', parentName);
+                if (symbol && parentName) {
+                    const parent = context.symbols.find((candidate) => candidate.name === parentName &&
+                        (candidate.kind === 'class' || candidate.kind === 'type'));
+                    if (parent)
+                        context.addSymbolContains(parent, symbol);
+                }
+                return;
+            }
+        }
+        for (const child of node.namedChildren) {
+            walk(child, parentName);
+        }
+    }
+    walk(tree.rootNode);
+    tree.delete();
+    return context.toResult();
+}
+function findNameNode(node) {
+    return (node.childForFieldName('name') ??
+        node.namedChildren.find((child) => child.type === 'constant') ??
+        node.namedChildren.find((child) => child.type === 'identifier'));
+}
+function findCallMethodName(node) {
+    return (node.childForFieldName('method')?.text ??
+        node.namedChildren.find((child) => child.type === 'identifier')?.text);
+}
+function findFirstStringContent(node) {
+    return node.descendantsOfType('string_content')[0]?.text;
+}

package/dist/scanner/shell-symbol-extractor.d.ts

@@ -0,0 +1,2 @@
+import type { SymbolExtractionResult } from './ts-symbol-extractor.js';
+export declare function extractShellSymbols(sourceText: string, filePath: string): Promise<SymbolExtractionResult>;

package/dist/scanner/shell-symbol-extractor.js

@@ -0,0 +1,78 @@
+import { emptyExtractionResult, ExtractionContext, } from './extraction-context.js';
+import { parseSource } from './tree-sitter-parser.js';
+export async function extractShellSymbols(sourceText, filePath) {
+    if (!sourceText.trim()) {
+        return emptyExtractionResult();
+    }
+    const tree = await parseSource(sourceText, 'bash');
+    const context = new ExtractionContext(filePath);
+    function walk(node, currentFunctionId) {
+        switch (node.type) {
+            case 'function_definition': {
+                const name = findFunctionName(node);
+                if (name) {
+                    const symbol = context.addSymbol({
+                        name,
+                        kind: 'function',
+                        startLine: node.startPosition.row + 1,
+                        endLine: node.endPosition.row + 1,
+                        exported: true,
+                    });
+                    for (const child of node.namedChildren) {
+                        walk(child, symbol.id);
+                    }
+                    return;
+                }
+                break;
+            }
+            case 'command': {
+                const commandName = findCommandName(node);
+                const firstArgument = findFirstCommandArgument(node);
+                if ((commandName === 'source' || commandName === '.') &&
+                    firstArgument) {
+                    context.addDependency(firstArgument, 'local');
+                    return;
+                }
+                if (currentFunctionId && commandName && isLocalScriptReference(commandName)) {
+                    context.relationships.push({
+                        sourceType: 'symbol',
+                        sourceId: currentFunctionId,
+                        targetType: 'file',
+                        targetId: commandName,
+                        relationshipType: 'calls',
+                        confidence: 'low',
+                    });
+                }
+                break;
+            }
+        }
+        for (const child of node.namedChildren) {
+            walk(child, currentFunctionId);
+        }
+    }
+    walk(tree.rootNode);
+    tree.delete();
+    return context.toResult();
+}
+function findFunctionName(node) {
+    return (node.childForFieldName('name')?.text ??
+        node.namedChildren.find((child) => child.type === 'word')?.text);
+}
+function findCommandName(node) {
+    return (node.childForFieldName('name')?.text ??
+        node.namedChildren
+            .find((child) => child.type === 'command_name')
+            ?.namedChildren.find((child) => child.type === 'word')?.text ??
+        node.namedChildren.find((child) => child.type === 'command_name')?.text ??
+        node.namedChildren.find((child) => child.type === 'word')?.text);
+}
+function findFirstCommandArgument(node) {
+    return node.namedChildren.find((child) => child.type === 'word')?.text;
+}
+function isLocalScriptReference(value) {
+    return (value.startsWith('./') ||
+        value.startsWith('../') ||
+        value.endsWith('.sh') ||
+        value.endsWith('.bash') ||
+        value.endsWith('.zsh'));
+}

package/dist/scanner/sql-symbol-extractor.d.ts

@@ -0,0 +1,2 @@
+import type { SymbolExtractionResult } from './ts-symbol-extractor.js';
+export declare function extractSqlSymbols(sourceText: string, filePath: string): Promise<SymbolExtractionResult>;

package/dist/scanner/sql-symbol-extractor.js

@@ -0,0 +1,166 @@
+import { emptyExtractionResult, ExtractionContext, } from './extraction-context.js';
+export async function extractSqlSymbols(sourceText, filePath) {
+    if (!sourceText.trim()) {
+        return emptyExtractionResult();
+    }
+    const context = new ExtractionContext(filePath);
+    const withoutComments = stripLineComments(sourceText);
+    const statements = splitStatements(withoutComments);
+    for (const statement of statements) {
+        collectCreateStatement(statement, context);
+        collectAlterStatement(statement, context);
+        collectQueryReferences(statement, context);
+    }
+    return context.toResult();
+}
+function collectCreateStatement(statement, context) {
+    const normalized = compact(statement.text);
+    const createMatch = normalized.match(/^CREATE\s+(?:OR\s+REPLACE\s+)?(?:(?:UNIQUE|MATERIALIZED|TEMP|TEMPORARY)\s+)*(TABLE|VIEW|INDEX|FUNCTION|PROCEDURE|TRIGGER|TYPE)\s+(?:IF\s+NOT\s+EXISTS\s+)?("?[\w.]+"?)/i);
+    if (!createMatch?.[1] || !createMatch[2]) {
+        return;
+    }
+    const objectKind = createMatch[1].toLowerCase();
+    const name = normalizeSqlIdentifier(createMatch[2]);
+    const symbol = context.addSymbol({
+        name,
+        kind: objectKind === 'function' || objectKind === 'procedure'
+            ? 'function'
+            : 'type',
+        startLine: statement.startLine,
+        endLine: statement.endLine,
+        exported: true,
+        parentName: objectKind,
+    });
+    const tableForTrigger = normalized.match(/\bON\s+("?[\w.]+"?)/i)?.[1];
+    if (objectKind === 'trigger' && tableForTrigger) {
+        context.relationships.push({
+            sourceType: 'symbol',
+            sourceId: symbol.id,
+            targetType: 'symbol',
+            targetId: normalizeSqlIdentifier(tableForTrigger),
+            relationshipType: 'mentions',
+            confidence: 'high',
+        });
+    }
+    for (const referencedTable of referencedTables(normalized)) {
+        context.relationships.push({
+            sourceType: 'symbol',
+            sourceId: symbol.id,
+            targetType: 'symbol',
+            targetId: referencedTable,
+            relationshipType: 'mentions',
+            confidence: 'medium',
+        });
+    }
+}
+function collectAlterStatement(statement, context) {
+    const normalized = compact(statement.text);
+    const alterMatch = normalized.match(/^ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?("?[\w.]+"?)/i);
+    if (!alterMatch?.[1]) {
+        return;
+    }
+    const tableName = normalizeSqlIdentifier(alterMatch[1]);
+    const symbol = context.addSymbol({
+        name: `alter ${tableName}`,
+        kind: 'type',
+        startLine: statement.startLine,
+        endLine: statement.endLine,
+        exported: true,
+        parentName: 'table',
+    });
+    context.relationships.push({
+        sourceType: 'symbol',
+        sourceId: symbol.id,
+        targetType: 'symbol',
+        targetId: tableName,
+        relationshipType: 'mentions',
+        confidence: 'high',
+    });
+    for (const referencedTable of referencedTables(normalized)) {
+        context.relationships.push({
+            sourceType: 'symbol',
+            sourceId: symbol.id,
+            targetType: 'symbol',
+            targetId: referencedTable,
+            relationshipType: 'mentions',
+            confidence: 'medium',
+        });
+    }
+}
+function collectQueryReferences(statement, context) {
+    const normalized = compact(statement.text);
+    const queryMatch = normalized.match(/^(SELECT|INSERT|UPDATE|DELETE)\b/i);
+    if (!queryMatch?.[1]) {
+        return;
+    }
+    const symbol = context.addSymbol({
+        name: `${queryMatch[1].toLowerCase()} statement ${statement.startLine}`,
+        kind: 'type',
+        startLine: statement.startLine,
+        endLine: statement.endLine,
+    });
+    for (const table of referencedTables(normalized)) {
+        context.relationships.push({
+            sourceType: 'symbol',
+            sourceId: symbol.id,
+            targetType: 'symbol',
+            targetId: table,
+            relationshipType: 'mentions',
+            confidence: 'medium',
+        });
+    }
+}
+function stripLineComments(sourceText) {
+    return sourceText
+        .split(/\r?\n/)
+        .map((line) => line.replace(/--.*$/, ''))
+        .join('\n');
+}
+function splitStatements(sourceText) {
+    const statements = [];
+    let current = '';
+    let startLine = 1;
+    let lineNumber = 1;
+    for (const line of sourceText.split(/\r?\n/)) {
+        if (!current.trim()) {
+            startLine = lineNumber;
+        }
+        current += `${line}\n`;
+        if (line.includes(';')) {
+            statements.push({
+                text: current,
+                startLine,
+                endLine: lineNumber,
+            });
+            current = '';
+        }
+        lineNumber += 1;
+    }
+    if (current.trim()) {
+        statements.push({ text: current, startLine, endLine: lineNumber - 1 });
+    }
+    return statements;
+}
+function compact(value) {
+    return value.replace(/\s+/g, ' ').trim();
+}
+function normalizeSqlIdentifier(value) {
+    return value.replace(/^"+|"+$/g, '').replace(/[;,)]$/, '');
+}
+function referencedTables(statement) {
+    const names = new Set();
+    const patterns = [
+        /\bREFERENCES\s+("?[\w.]+"?)/gi,
+        /\bFROM\s+("?[\w.]+"?)/gi,
+        /\bJOIN\s+("?[\w.]+"?)/gi,
+        /\bUPDATE\s+("?[\w.]+"?)/gi,
+        /\bINTO\s+("?[\w.]+"?)/gi,
+    ];
+    for (const pattern of patterns) {
+        for (const match of statement.matchAll(pattern)) {
+            if (match[1])
+                names.add(normalizeSqlIdentifier(match[1]));
+        }
+    }
+    return [...names];
+}

package/dist/scanner/tree-sitter-parser.d.ts

@@ -1,5 +1,4 @@
 import { Language, Tree } from 'web-tree-sitter';
-type GrammarKey = 'python' | 'java' | 'kotlin' | 'go' | 'rust' | 'c' | 'cpp' | 'c_sharp';
+export type GrammarKey = 'python' | 'java' | 'kotlin' | 'go' | 'rust' | 'c' | 'cpp' | 'c_sharp' | 'php' | 'ruby' | 'bash' | 'yaml' | 'json' | 'html' | 'css' | 'lua' | 'dart' | 'elixir' | 'scala';
 export declare function loadLanguage(grammarKey: GrammarKey): Promise<Language>;
 export declare function parseSource(sourceText: string, grammarKey: GrammarKey): Promise<Tree>;
-export {};

package/dist/scanner/tree-sitter-parser.js

@@ -19,6 +19,20 @@ const GRAMMAR_PACKAGES = {
         pkg: 'tree-sitter-c-sharp',
         wasm: 'tree-sitter-c_sharp.wasm',
     },
+    php: { pkg: 'tree-sitter-php', wasm: 'tree-sitter-php.wasm' },
+    ruby: { pkg: 'tree-sitter-ruby', wasm: 'tree-sitter-ruby.wasm' },
+    bash: { pkg: 'tree-sitter-bash', wasm: 'tree-sitter-bash.wasm' },
+    yaml: {
+        pkg: '@tree-sitter-grammars/tree-sitter-yaml',
+        wasm: 'tree-sitter-yaml.wasm',
+    },
+    json: { pkg: 'tree-sitter-json', wasm: 'tree-sitter-json.wasm' },
+    html: { pkg: 'tree-sitter-html', wasm: 'tree-sitter-html.wasm' },
+    css: { pkg: 'tree-sitter-css', wasm: 'tree-sitter-css.wasm' },
+    lua: { pkg: 'tree-sitter-lua', wasm: 'tree-sitter-lua.wasm' },
+    dart: { pkg: 'tree-sitter-dart', wasm: 'tree-sitter-dart.wasm' },
+    elixir: { pkg: 'tree-sitter-elixir', wasm: 'tree-sitter-elixir.wasm' },
+    scala: { pkg: 'tree-sitter-scala', wasm: 'tree-sitter-scala.wasm' },
 };
 async function ensureInit() {
     if (!initPromise) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kentwynn/kgraph",
-  "version": "0.2.22",
+  "version": "0.2.24",
   "description": "Persistent repo intelligence for AI coding assistants.",
   "type": "module",
   "bin": {
@@ -45,16 +45,27 @@
   "dependencies": {
     "@clack/prompts": "^1.3.0",
     "@tree-sitter-grammars/tree-sitter-kotlin": "^1.1.0",
+    "@tree-sitter-grammars/tree-sitter-yaml": "^0.7.1",
     "chalk": "^5.6.2",
     "commander": "^12.1.0",
     "fast-glob": "^3.3.2",
+    "tree-sitter-bash": "^0.25.1",
     "tree-sitter-c": "^0.24.1",
     "tree-sitter-c-sharp": "^0.23.5",
     "tree-sitter-cpp": "^0.23.4",
+    "tree-sitter-css": "^0.25.0",
+    "tree-sitter-dart": "^1.0.0",
+    "tree-sitter-elixir": "^0.3.5",
     "tree-sitter-go": "^0.25.0",
+    "tree-sitter-html": "^0.23.2",
     "tree-sitter-java": "^0.23.5",
+    "tree-sitter-json": "^0.24.8",
+    "tree-sitter-lua": "^2.1.3",
+    "tree-sitter-php": "^0.24.2",
     "tree-sitter-python": "^0.25.0",
+    "tree-sitter-ruby": "^0.23.1",
     "tree-sitter-rust": "^0.24.0",
+    "tree-sitter-scala": "^0.24.0",
     "typescript": "^5.9.3",
     "web-tree-sitter": "^0.26.8",
     "yaml": "^2.5.1"