@getmikk/core 1.8.0 → 1.8.2

package/src/parser/tree-sitter/parser.ts

@@ -2,18 +2,143 @@ import * as path from 'node:path'
 import { createRequire } from 'node:module'
 import { hashContent } from '../../hash/file-hasher.js'
 import { BaseParser } from '../base-parser.js'
-import type { ParsedFile, ParsedFunction, ParsedClass, ParsedParam, ParsedImport, ParsedExport, ParsedRoute } from '../types.js'
+import type { ParsedFile, ParsedFunction, ParsedClass, ParsedParam, ParsedImport } from '../types.js'
 import * as Queries from './queries.js'
 
 // Safely require web-tree-sitter via CJS
 const getRequire = () => {
-    if (typeof require !== 'undefined') return require;
-    return createRequire(import.meta.url);
-};
-const _require = getRequire();
+    if (typeof require !== 'undefined') return require
+    return createRequire(import.meta.url)
+}
+const _require = getRequire()
 const ParserModule = _require('web-tree-sitter')
 const Parser = ParserModule.Parser || ParserModule
 
+// ---------------------------------------------------------------------------
+// Language-specific export visibility rules
+// ---------------------------------------------------------------------------
+
+/**
+ * Determine whether a function node is exported based on language conventions.
+ * Python: public if name does not start with underscore.
+ * Java/C#/Rust: requires an explicit visibility keyword in the node text.
+ * Go: exported if name starts with an uppercase letter.
+ * All others (C, C++, PHP, Ruby): default to false (no reliable static rule).
+ */
+function isExportedByLanguage(ext: string, name: string, nodeText: string): boolean {
+    switch (ext) {
+        case '.py':
+            return !name.startsWith('_')
+        case '.java':
+        case '.cs':
+            return /\bpublic\b/.test(nodeText)
+        case '.go':
+            return name.length > 0 && name[0] === name[0].toUpperCase() && name[0] !== name[0].toLowerCase()
+        case '.rs':
+            return /\bpub\b/.test(nodeText)
+        default:
+            return false
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Parameter extraction from tree-sitter nodes
+// ---------------------------------------------------------------------------
+
+/**
+ * Best-effort parameter extraction from a function definition node.
+ * Walks child nodes looking for parameter/formal_parameter identifiers.
+ * Returns an empty array on failure — never throws.
+ */
+function extractParamsFromNode(defNode: any): ParsedParam[] {
+    const params: ParsedParam[] = []
+    if (!defNode || !defNode.children) return params
+
+    // Walk all descendants looking for parameter-like nodes
+    const walk = (node: any) => {
+        if (!node) return
+        const t = node.type ?? ''
+        // Common parameter node type names across tree-sitter grammars
+        if (
+            t === 'parameter' || t === 'formal_parameter' || t === 'simple_parameter' ||
+            t === 'variadic_parameter' || t === 'typed_parameter' || t === 'typed_default_parameter' ||
+            t === 'keyword_argument' || t === 'field_declaration'
+        ) {
+            // Try to find the identifier within this param node
+            const identNode = findFirstChild(node, n => n.type === 'identifier' || n.type === 'name')
+            const typeNode = findFirstChild(node, n =>
+                n.type === 'type' || n.type === 'type_annotation' ||
+                n.type === 'type_identifier' || n.type === 'predefined_type'
+            )
+            const name = identNode?.text ?? node.text ?? ''
+            const type = typeNode?.text ?? 'any'
+            if (name && name !== '' && !params.some(p => p.name === name)) {
+                params.push({ name, type, optional: false })
+            }
+            return // Don't recurse into parameter children
+        }
+        if (node.children) {
+            for (const child of node.children) walk(child)
+        }
+    }
+
+    walk(defNode)
+    return params
+}
+
+function findFirstChild(node: any, predicate: (n: any) => boolean): any {
+    if (!node?.children) return null
+    for (const child of node.children) {
+        if (predicate(child)) return child
+    }
+    return null
+}
+
+// ---------------------------------------------------------------------------
+// Scope-aware call resolver
+// ---------------------------------------------------------------------------
+
+/**
+ * Given the ordered list of functions (with startLine/endLine already set)
+ * and a map of callName → line, assign each call to the innermost function
+ * whose line range contains that call's line.
+ * 
+ * Returns an array of call names that were NOT assigned to any function scope 
+ * (these are module-scope calls).
+ */
+function assignCallsToFunctions(
+    functions: ParsedFunction[],
+    callEntries: Array<{ name: string; line: number }>
+): string[] {
+    const unassigned: string[] = []
+    for (const { name, line } of callEntries) {
+        // Find the innermost (smallest range) function that contains this line
+        let best: ParsedFunction | null = null
+        let bestRange = Infinity
+        for (const fn of functions) {
+            if (line >= fn.startLine && line <= fn.endLine) {
+                const range = fn.endLine - fn.startLine
+                if (range < bestRange) {
+                    best = fn
+                    bestRange = range
+                }
+            }
+        }
+        if (best) {
+            if (!best.calls.includes(name)) {
+                best.calls.push(name)
+            }
+        } else {
+            unassigned.push(name)
+        }
+    }
+    return unassigned
+}
+
+// ---------------------------------------------------------------------------
+// Main parser class
+// ---------------------------------------------------------------------------
+
 export class TreeSitterParser extends BaseParser {
     private parser: any = null
     private languages = new Map<string, any>()
@@ -35,7 +160,6 @@ export class TreeSitterParser extends BaseParser {
         const config = await this.getLanguageConfig(ext)
 
         if (!config || !config.lang) {
-            // Fallback to empty if language not supported or grammar failed to load
             return this.buildEmptyFile(filePath, content, ext)
         }
 
@@ -47,7 +171,10 @@ export class TreeSitterParser extends BaseParser {
         const functions: ParsedFunction[] = []
         const classesMap = new Map<string, ParsedClass>()
         const imports: ParsedImport[] = []
-        const calls = new Set<string>()
+        // callEntries stores name + line so we can scope them to the right function
+        const callEntries: Array<{ name: string; line: number }> = []
+        // Track processed function IDs to avoid collisions from overloads
+        const seenFnIds = new Set<string>()
 
         for (const match of matches) {
             const captures: Record<string, any> = {}
@@ -55,13 +182,17 @@ export class TreeSitterParser extends BaseParser {
                 captures[c.name] = c.node
             }
 
-            // Calls
+            // --- Calls: record name and line position ---
             if (captures['call.name']) {
-                calls.add(captures['call.name'].text)
+                const callNode = captures['call.name']
+                callEntries.push({
+                    name: callNode.text,
+                    line: (callNode.startPosition?.row ?? 0) + 1,
+                })
                 continue
             }
 
-            // Imports
+            // --- Imports ---
             if (captures['import.source']) {
                 const src = captures['import.source'].text.replace(/['"]/g, '')
                 imports.push({
@@ -69,31 +200,51 @@ export class TreeSitterParser extends BaseParser {
                     resolvedPath: '',
                     names: [],
                     isDefault: false,
-                    isDynamic: false
+                    isDynamic: false,
                 })
                 continue
             }
 
-            // Functions / Methods
+            // --- Functions / Methods ---
             if (captures['definition.function'] || captures['definition.method']) {
                 const nameNode = captures['name']
                 const defNode = captures['definition.function'] || captures['definition.method']
-                
+
                 if (nameNode && defNode) {
                     const fnName = nameNode.text
+                    const startLine = defNode.startPosition.row + 1
+                    const endLine = defNode.endPosition.row + 1
+                    const nodeText = defNode.text ?? ''
+
+                    // Unique ID: include start line to handle overloads and same-name scoped functions
+                    let fnId = `fn:${filePath}:${fnName}:${startLine}`
+                    if (seenFnIds.has(fnId)) {
+                        // Extremely rare duplicate — skip rather than corrupt
+                        continue
+                    }
+                    seenFnIds.add(fnId)
+
+                    const exported = isExportedByLanguage(ext, fnName, nodeText)
+                    const isAsync = /\basync\b/.test(nodeText)
+
+                    // Detect return type — language-specific heuristics
+                    const returnType = extractReturnType(ext, defNode)
+
+                    const params = extractParamsFromNode(defNode)
+
                     functions.push({
-                        id: `fn:${filePath}:${fnName}`,
+                        id: fnId,
                         name: fnName,
                         file: filePath,
-                        startLine: defNode.startPosition.row + 1,
-                        endLine: defNode.endPosition.row + 1,
-                        params: [],
-                        returnType: 'any',
-                        isExported: true, // simplified for universal parser
-                        isAsync: false,
-                        calls: [], // We aggregate at file level currently
-                        hash: hashContent(defNode.text),
-                        purpose: '',
+                        startLine,
+                        endLine,
+                        params,
+                        returnType,
+                        isExported: exported,
+                        isAsync,
+                        calls: [], // populated after all functions are collected
+                        hash: hashContent(nodeText),
+                        purpose: extractDocComment(content, startLine),
                         edgeCasesHandled: [],
                         errorHandling: [],
                         detailedLines: [],
@@ -101,46 +252,71 @@ export class TreeSitterParser extends BaseParser {
                 }
             }
 
-            // Classes / Structs / Interfaces
-            if (captures['definition.class'] || captures['definition.struct'] || captures['definition.interface']) {
+            // --- Classes / Structs / Interfaces ---
+            if (
+                captures['definition.class'] ||
+                captures['definition.struct'] ||
+                captures['definition.interface']
+            ) {
                 const nameNode = captures['name']
-                const defNode = captures['definition.class'] || captures['definition.struct'] || captures['definition.interface']
-                
+                const defNode =
+                    captures['definition.class'] ||
+                    captures['definition.struct'] ||
+                    captures['definition.interface']
+
                 if (nameNode && defNode) {
                     const clsName = nameNode.text
-                    if (!classesMap.has(clsName)) {
-                        classesMap.set(clsName, {
-                            id: `cls:${filePath}:${clsName}`,
+                    const startLine = defNode.startPosition.row + 1
+                    const endLine = defNode.endPosition.row + 1
+                    const nodeText = defNode.text ?? ''
+                    const clsId = `cls:${filePath}:${clsName}:${startLine}`
+
+                    if (!classesMap.has(clsId)) {
+                        classesMap.set(clsId, {
+                            id: clsId,
                             name: clsName,
                             file: filePath,
-                            startLine: defNode.startPosition.row + 1,
-                            endLine: defNode.endPosition.row + 1,
+                            startLine,
+                            endLine,
                             methods: [],
-                            isExported: true,
+                            isExported: isExportedByLanguage(ext, clsName, nodeText),
                         })
                     }
                 }
             }
         }
 
-        // Attach global calls to the first function as a heuristic, or store in a dummy
-        if (functions.length > 0) {
-            functions[0].calls = Array.from(calls)
-        }
+        // Assign calls to their enclosing function scopes.
+        const unassignedCalls = assignCallsToFunctions(functions, callEntries)
 
-        let finalLang: ParsedFile['language'] = 'go'
-        switch (ext) {
-            case '.py': finalLang = 'python'; break
-            case '.java': finalLang = 'java'; break
-            case '.c': case '.h': finalLang = 'c'; break
-            case '.cpp': case '.cc': case '.hpp': finalLang = 'cpp'; break
-            case '.cs': finalLang = 'csharp'; break
-            case '.go': finalLang = 'go'; break
-            case '.rs': finalLang = 'rust'; break
-            case '.php': finalLang = 'php'; break
-            case '.rb': finalLang = 'ruby'; break
+        // Only add a synthetic module-level function if there are actually calls made outside any function.
+        if (unassignedCalls.length > 0) {
+            const lineCount = content.split('\n').length
+            functions.push({
+                id: `fn:${filePath}:<module>:1`,
+                name: '<module>',
+                file: filePath,
+                startLine: 1,
+                endLine: lineCount || 1,
+                params: [],
+                returnType: 'void',
+                isExported: true,
+                isAsync: false,
+                calls: Array.from(new Set(unassignedCalls)),
+                hash: '',
+                purpose: 'Module-level initialization code',
+                edgeCasesHandled: [],
+                errorHandling: [],
+                detailedLines: [],
+            })
         }
 
+        const finalLang = extensionToLanguage(ext)
+
+        // Link methods: functions whose names contain '.' belong to a class
+        // (Go receiver methods, Java/C# member methods detected via method capture)
+        linkMethodsToClasses(functions, classesMap)
+
         return {
             path: filePath,
             language: finalLang,
@@ -148,45 +324,42 @@ export class TreeSitterParser extends BaseParser {
             classes: Array.from(classesMap.values()),
             generics: [],
             imports,
-            exports: functions.map(f => ({ name: f.name, type: 'function', file: filePath })),
+            exports: functions.filter(f => f.isExported).map(f => ({
+                name: f.name,
+                type: 'function' as const,
+                file: filePath,
+            })),
             routes: [],
             hash: hashContent(content),
-            parsedAt: Date.now()
+            parsedAt: Date.now(),
         }
     }
 
-    resolveImports(files: ParsedFile[], projectRoot: string): ParsedFile[] {
-        // Universal resolver: just link absolute paths if they exist locally
-        // Basic heuristic for all 11 languages
+    resolveImports(files: ParsedFile[], _projectRoot: string): ParsedFile[] {
+        // Tree-sitter resolver: no cross-file resolution implemented.
+        // Imports are left with resolvedPath = '' which signals unresolved to the graph builder.
+        // A future pass can resolve Go/Python/Java imports using language-specific rules.
         return files
     }
 
     private buildEmptyFile(filePath: string, content: string, ext: string): ParsedFile {
-        let finalLang: ParsedFile['language'] = 'unknown'
-        switch (ext) {
-            case '.py': finalLang = 'python'; break
-            case '.java': finalLang = 'java'; break
-            case '.c': case '.h': finalLang = 'c'; break
-            case '.cpp': case '.cc': case '.hpp': finalLang = 'cpp'; break
-            case '.cs': finalLang = 'csharp'; break
-            case '.go': finalLang = 'go'; break
-            case '.rs': finalLang = 'rust'; break
-            case '.php': finalLang = 'php'; break
-            case '.rb': finalLang = 'ruby'; break
-        }
         return {
             path: filePath,
-            language: finalLang,
-            functions: [], classes: [], generics: [], imports: [], exports: [], routes: [],
+            language: extensionToLanguage(ext),
+            functions: [],
+            classes: [],
+            generics: [],
+            imports: [],
+            exports: [],
+            routes: [],
             hash: hashContent(content),
-            parsedAt: Date.now()
+            parsedAt: Date.now(),
         }
     }
 
     private async loadLang(name: string): Promise<any> {
         if (this.languages.has(name)) return this.languages.get(name)
         try {
-            // Get module root path to locate wasms
             const tcPath = _require.resolve('tree-sitter-wasms/package.json')
             const wasmPath = path.join(path.dirname(tcPath), 'out', `tree-sitter-${name}.wasm`)
             const lang = await Parser.Language.load(wasmPath)
@@ -226,3 +399,114 @@ export class TreeSitterParser extends BaseParser {
         }
     }
 }
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function extensionToLanguage(ext: string): ParsedFile['language'] {
+    switch (ext) {
+        case '.py': return 'python'
+        case '.java': return 'java'
+        case '.c': case '.h': return 'c'
+        case '.cpp': case '.cc': case '.hpp': return 'cpp'
+        case '.cs': return 'csharp'
+        case '.go': return 'go'
+        case '.rs': return 'rust'
+        case '.php': return 'php'
+        case '.rb': return 'ruby'
+        default: return 'unknown'
+    }
+}
+
+/**
+ * Extract a simple return type hint from the function node text.
+ * Falls back to 'unknown' rather than 'any' to distinguish "not parsed"
+ * from "genuinely untyped".
+ */
+function extractReturnType(ext: string, defNode: any): string {
+    const text: string = defNode?.text ?? ''
+    // TypeScript/Go/Rust: look for "-> Type" or ": Type" after parameters
+    const arrowMatch = text.match(/\)\s*->\s*([^\s{]+)/)
+    if (arrowMatch) return arrowMatch[1].trim()
+    // Java/C# style: "public int foo(" — type precedes the name
+    // This is too fragile to do reliably here; return 'unknown'
+    if (ext === '.go') {
+        // Go: "func foo() (int, error)" or "func foo() error"
+        const goReturnTuple = text.match(/\)\s+(\([^)]+\))/)
+        if (goReturnTuple) return goReturnTuple[1].trim()
+        const goReturn = text.match(/\)\s+([^\s{(]+)/)
+        if (goReturn) return goReturn[1].trim()
+    }
+    return 'unknown'
+}
+
+/**
+ * Extract a single-line doc comment immediately preceding the given line.
+ * Scans backwards from startLine looking for `#`, `//`, `/**`, or `"""` comments.
+ */
+function extractDocComment(content: string, startLine: number): string {
+    const lines = content.split('\n')
+    const targetIdx = startLine - 2 // 0-indexed line before the function
+    if (targetIdx < 0) return ''
+
+    const prev = lines[targetIdx]?.trim() ?? ''
+    // Single-line comment styles
+    for (const prefix of ['# ', '// ', '/// ']) {
+        if (prev.startsWith(prefix)) return prev.slice(prefix.length).trim()
+    }
+    // JSDoc / block comment end
+    if (prev === '*/') {
+        // Walk back to find the first meaningful JSDoc line
+        for (let i = targetIdx - 1; i >= 0; i--) {
+            const line = lines[i].trim()
+            if (line.startsWith('/*') || line.startsWith('/**')) break
+            const cleaned = line.replace(/^\*+\s?/, '')
+            if (cleaned && !/^[\-_=*]{3,}$/.test(cleaned)) return cleaned
+        }
+    }
+    return ''
+}
+
+/**
+ * Move functions that are class methods (identified by having a receiver or
+ * by being within the line range of a class) into the class's methods array.
+ * This is a best-effort heuristic; direct tree-sitter capture of method
+ * declarations already places them correctly in most languages.
+ */
+function linkMethodsToClasses(
+    functions: ParsedFunction[],
+    classesMap: Map<string, ParsedClass>
+): void {
+    const classes = Array.from(classesMap.values())
+    if (classes.length === 0) return
+
+    for (const fn of functions) {
+        // Already categorised if name contains "." (e.g. "MyClass.method")
+        // and never link the synthetic <module> function to a class.
+        if (fn.name === '<module>' || fn.name.includes('.')) continue
+
+        // Skip functions nested inside other functions (local helpers)
+        const isNestedInFunction = functions.some(f => 
+            f.id !== fn.id && 
+            fn.startLine >= f.startLine && fn.endLine <= f.endLine
+        )
+        if (isNestedInFunction) continue
+
+        // Find the innermost (smallest range) class that contains this function
+        let bestCls: ParsedClass | null = null
+        let bestRange = Infinity
+        for (const cls of classes) {
+            if (fn.startLine > cls.startLine && fn.endLine <= cls.endLine) {
+                const range = cls.endLine - cls.startLine
+                if (range < bestRange) {
+                    bestCls = cls
+                    bestRange = range
+                }
+            }
+        }
+        if (bestCls && !bestCls.methods.some(m => m.id === fn.id)) {
+            bestCls.methods.push(fn)
+        }
+    }
+}

package/src/parser/types.ts

@@ -15,6 +15,7 @@ export interface ParsedFunction {
     id: string              // "fn:auth/verify.ts:verifyToken"
     name: string            // "verifyToken"
     file: string            // "src/auth/verify.ts"
+    moduleId?: string
     startLine: number       // 14
     endLine: number         // 28
     params: ParsedParam[]   // [{name: "token", type: "string"}]
@@ -52,6 +53,7 @@ export interface ParsedClass {
     id: string
     name: string
     file: string
+    moduleId?: string
     startLine: number
     endLine: number
     methods: ParsedFunction[]

package/src/parser/typescript/ts-extractor.ts

@@ -324,7 +324,9 @@ export class TypeScriptExtractor {
         const bodyText = node.getText(this.sourceFile)
 
         return {
-            id: `fn:${this.filePath}:${name}`,
+            // Include startLine in the ID to prevent collision between overload signatures
+            // and same-named functions in different scopes (e.g. two `init` declarations).
+            id: `fn:${this.filePath}:${name}:${startLine}`,
             name,
             file: this.filePath,
             startLine,
@@ -361,7 +363,9 @@ export class TypeScriptExtractor {
         const bodyText = stmt.getText(this.sourceFile)
 
         return {
-            id: `fn:${this.filePath}:${name}`,
+            // Include startLine to prevent collision between same-named const arrow functions
+            // at different scopes (e.g. two `handler` declarations in different blocks).
+            id: `fn:${this.filePath}:${name}:${startLine}`,
             name,
             file: this.filePath,
             startLine,
@@ -399,7 +403,7 @@ export class TypeScriptExtractor {
                 const bodyText = member.getText(this.sourceFile)
 
                 methods.push({
-                    id: `fn:${this.filePath}:${name}.constructor`,
+                    id: `fn:${this.filePath}:${name}.constructor:${mStartLine}`,
                     name: `${name}.constructor`,
                     file: this.filePath,
                     startLine: mStartLine,
@@ -428,7 +432,7 @@ export class TypeScriptExtractor {
                 const bodyText = member.getText(this.sourceFile)
 
                 methods.push({
-                    id: `fn:${this.filePath}:${name}.${methodName}`,
+                    id: `fn:${this.filePath}:${name}.${methodName}:${mStartLine}`,
                     name: `${name}.${methodName}`,
                     file: this.filePath,
                     startLine: mStartLine,
@@ -541,9 +545,16 @@ export class TypeScriptExtractor {
                 const comment = fullText.slice(range.pos, range.end)
                 let clean = ''
                 if (comment.startsWith('/**') || comment.startsWith('/*')) {
-                    clean = comment.replace(/[\/\*]/g, '').trim()
+                    // Strip only the comment delimiters (/* ** */) NOT arbitrary slashes.
+                    // Using /[\/\*]/g was wrong — it removes slashes inside URL paths and
+                    // regex literals embedded in doc comments (e.g. "/api/users" → "apiusers").
+                    clean = comment
+                        .replace(/^\/\*+/, '')   // remove leading /*  or /**
+                        .replace(/\*+\/$/, '')   // remove trailing */
+                        .replace(/^\s*\*+\s?/gm, '') // remove leading * on each line
+                        .trim()
                 } else if (comment.startsWith('//')) {
-                    clean = comment.replace(/\/\//g, '').trim()
+                    clean = comment.replace(/^\/\/+\s?/, '').trim()
                 }
 
                 // Skip divider lines (lines with 3+ repeated special characters)