@getmikk/core 2.0.13 → 2.0.15

package/src/parser/index.ts

@@ -1,17 +1,16 @@
+ 
 import * as nodePath from 'node:path'
-import { BaseParser } from './base-parser.js'
-import { OxcParser } from './oxc-parser.js'
-import { GoParser } from './go/go-parser.js'
-import { UnsupportedLanguageError } from '../utils/errors.js'
-import type { ParsedFile } from './types.js'
+import { LanguageRegistry } from './language-registry.js'
+export { LanguageRegistry } from './language-registry.js'
+import './oxc-parser.js'
+import './tree-sitter/parser.js'
+import './go/go-extractor.js'
+import { BaseExtractor } from './base-extractor.js'
 import { hashContent } from '../hash/file-hasher.js'
-import {
-    parserKindForExtension,
-    languageForExtension,
-    getParserExtensions,
-    isTreeSitterExtension,
-    type ParserKind,
-} from '../utils/language-registry.js'
+import { IncrementalCache } from '../cache/incremental-cache.js'
+import { languageForExtension, toParsedFileLanguage } from '../utils/language-registry.js'
+import { ErrorRecoveryEngine } from './error-recovery.js'
+import type { ParsedFile } from './types.js'
 
 export type {
     ParsedFile,
@@ -25,23 +24,15 @@ export type {
     ParsedGeneric,
     ParsedRoute
 } from './types.js'
-export { BaseParser } from './base-parser.js'
-export { TypeScriptParser } from './typescript/ts-parser.js'
-export { TypeScriptExtractor } from './typescript/ts-extractor.js'
-export { TypeScriptResolver } from './typescript/ts-resolver.js'
-export { GoParser } from './go/go-parser.js'
-export { GoExtractor } from './go/go-extractor.js'
-export { GoResolver } from './go/go-resolver.js'
-export { JavaScriptParser } from './javascript/js-parser.js'
-export { JavaScriptExtractor } from './javascript/js-extractor.js'
-export { JavaScriptResolver } from './javascript/js-resolver.js'
+
+export { BaseExtractor } from './base-extractor.js'
 export { BoundaryChecker } from './boundary-checker.js'
-export { TreeSitterParser } from './tree-sitter/parser.js'
 
 export type ParseDiagnosticStage = 'read' | 'parse' | 'resolve-imports'
 export type ParseDiagnosticReason =
     | 'read-error'
     | 'parse-error'
+    | 'parse-error-recovered'
     | 'resolve-error'
     | 'unsupported-extension'
     | 'parser-unavailable'
@@ -49,7 +40,6 @@ export type ParseDiagnosticReason =
 export interface ParseDiagnostic {
     filePath: string
     extension: string
-    parser: ParserKind
     stage: ParseDiagnosticStage
     reason: ParseDiagnosticReason
     message: string
@@ -70,18 +60,9 @@ export interface ParseFilesResult {
     summary: ParseFilesSummary
 }
 
-const isLikelyParserUnavailable = (parser: ParserKind, message: string): boolean => {
-    if (parser !== 'tree-sitter') return false
-    const normalized = message.toLowerCase()
-    return normalized.includes('web-tree-sitter') ||
-        normalized.includes('tree-sitter') ||
-        normalized.includes('cannot find module')
-}
-
-
 const buildFallbackParsedFile = (filePath: string, content: string, ext: string): ParsedFile => ({
     path: filePath,
-    language: languageForExtension(ext) as ParsedFile['language'],
+    language: toParsedFileLanguage(languageForExtension(ext)),
     functions: [],
     classes: [],
     generics: [],
@@ -100,209 +81,105 @@ const normalizeErrorMessage = (err: unknown): string => {
     return String(err)
 }
 
-/** Get the appropriate parser for a file based on its extension */
-export function getParser(filePath: string): BaseParser {
-    const ext = nodePath.extname(filePath).toLowerCase()
-    const parserKind = parserKindForExtension(ext)
-
-    switch (parserKind) {
-        case 'oxc':
-            return new OxcParser()
-        case 'go':
-            return new GoParser()
-        case 'tree-sitter':
-            return createTreeSitterParser()
-        default:
-            throw new UnsupportedLanguageError(ext || '<no extension>')
-    }
-}
-
-let _treeSitterParserInstance: BaseParser | null = null
-
-const createTreeSitterParser = (): BaseParser => {
-    if (!_treeSitterParserInstance) {
-        // Return a lazy-loading wrapper that handles missing tree-sitter gracefully.
-        _treeSitterParserInstance = new LazyTreeSitterParser()
-    }
-    return _treeSitterParserInstance
-}
-
-class LazyTreeSitterParser extends BaseParser {
-    private parser: any = null
-
-    async init(): Promise<void> {
-        if (this.parser) return
-        try {
-            const { TreeSitterParser } = await import('./tree-sitter/parser.js')
-            this.parser = new TreeSitterParser()
-        } catch {
-            // web-tree-sitter not available
-        }
-    }
-
-    async parse(filePath: string, content: string): Promise<ParsedFile> {
-        await this.init()
-        if (!this.parser) {
-            return this.buildEmptyFile(filePath, content)
-        }
-        return this.parser.parse(filePath, content)
-    }
-
-    async resolveImports(files: ParsedFile[], projectRoot: string): Promise<ParsedFile[]> {
-        await this.init()
-        if (!this.parser) return files
-        return this.parser.resolveImports(files, projectRoot)
-    }
-
-    getSupportedExtensions(): string[] {
-        return [...getParserExtensions('tree-sitter')]
-    }
-
-    private buildEmptyFile(filePath: string, content: string): ParsedFile {
-        const ext = nodePath.extname(filePath).toLowerCase()
-        const lang = languageForExtension(ext)
-        return {
-            path: filePath,
-            language: lang as ParsedFile['language'],
-            functions: [],
-            classes: [],
-            generics: [],
-            imports: [],
-            exports: [],
-            routes: [],
-            variables: [],
-            calls: [],
-            hash: hashContent(content),
-            parsedAt: Date.now(),
-        }
-    }
-}
-
 export interface ParseFilesOptions {
     strictParserPreflight?: boolean
-    treeSitterRuntimeAvailable?: boolean
+    concurrency?: number
 }
 
-async function isTreeSitterRuntimeAvailable(): Promise<boolean> {
-    try {
-        const { TreeSitterParser } = await import('./tree-sitter/parser.js')
-        const parser = new TreeSitterParser()
-        if (typeof (parser as any).isRuntimeAvailable !== 'function') {
-            return true
+const DEFAULT_CONCURRENCY = 32
+
+async function parallelBatch<T, R>(
+    items: T[],
+    processor: (item: T) => Promise<R>,
+    concurrency: number
+): Promise<R[]> {
+    const results: R[] = new Array(items.length)
+    const errors: (Error | null)[] = new Array(items.length)
+    
+    for (let i = 0; i < items.length; i += concurrency) {
+        const batch = items.slice(i, i + concurrency)
+        const batchResults = await Promise.allSettled(batch.map((item, _j) => processor(item)))
+        
+        for (let j = 0; j < batchResults.length; j++) {
+            const result = batchResults[j]
+            if (result.status === 'fulfilled') {
+                results[i + j] = result.value
+            } else {
+                errors[i + j] = result.reason
+            }
         }
-        return await (parser as any).isRuntimeAvailable()
-    } catch {
-        return false
     }
+    
+    return results
 }
 
+/**
+ * Main entry point for scanning and parsing multiple files.
+ * Uses LanguageRegistry to dispatch to the correct extractor.
+ * PARALLELIZED: Files are parsed concurrently for better performance.
+ */
 export async function parseFilesWithDiagnostics(
     filePaths: string[],
     projectRoot: string,
     readFile: (fp: string) => Promise<string>,
     options: ParseFilesOptions = {},
 ): Promise<ParseFilesResult> {
-    // Shared parser instances — avoid re-initialisation overhead per file.
-    const oxcParser = new OxcParser()
-    const goParser = new GoParser()
-
-    // Lazily loaded to avoid mandatory dependency on tree-sitter for TS/JS-only projects.
-    let treeSitterParser: BaseParser | null = null
-    const getTreeSitter = async (): Promise<BaseParser> => {
-        if (!treeSitterParser) {
-            const { TreeSitterParser } = await import('./tree-sitter/parser.js')
-            treeSitterParser = new TreeSitterParser()
-        }
-        return treeSitterParser
-    }
-
     const diagnostics: ParseDiagnostic[] = []
-    const addDiagnostic = (diagnostic: ParseDiagnostic) => diagnostics.push(diagnostic)
-
-    const treeSitterNeeded = filePaths.some(fp => {
-        const ext = nodePath.extname(fp).toLowerCase()
-        return isTreeSitterExtension(ext)
-    })
-
-    let treeSitterAvailable = true
-    if (treeSitterNeeded) {
-        treeSitterAvailable =
-            typeof options.treeSitterRuntimeAvailable === 'boolean'
-                ? options.treeSitterRuntimeAvailable
-                : await isTreeSitterRuntimeAvailable()
-        if (!treeSitterAvailable) {
-            addDiagnostic({
-                filePath: '*',
-                extension: '*',
-                parser: 'tree-sitter',
-                stage: 'parse',
-                reason: 'parser-unavailable',
-                message: 'Tree-sitter runtime unavailable. Install web-tree-sitter and language grammars.',
-            })
-            if (options.strictParserPreflight) {
-                return {
-                    files: [],
-                    diagnostics,
-                    summary: {
-                        requestedFiles: filePaths.length,
-                        parsedFiles: 0,
-                        fallbackFiles: 0,
-                        unreadableFiles: 0,
-                        unsupportedFiles: 0,
-                        diagnostics: diagnostics.length,
-                    },
-                }
-            }
-        }
-    }
-
-    // Normalized project root for absolute path construction.
     const normalizedRoot = nodePath.resolve(projectRoot).replace(/\\/g, '/')
-
-    // Group by parser to enable batch resolveImports.
-    const oxcFiles: ParsedFile[] = []
-    const goFiles: ParsedFile[] = []
-    const treeFiles: ParsedFile[] = []
+    const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY
+    
+    const cache = new IncrementalCache(projectRoot)
+    const registry = LanguageRegistry.getInstance()
+    
+    const filesByExtractor = new Map<BaseExtractor, ParsedFile[]>()
     const fallbackFiles: ParsedFile[] = []
-
+    const _pendingDiagnostics: Array<{ filePath: string; ext: string; stage: ParseDiagnosticStage; reason: ParseDiagnosticReason; message: string }> = []
+    
     let parsedFilesCount = 0
     let fallbackFilesCount = 0
     let unreadableFiles = 0
     let unsupportedFiles = 0
 
-    // Parse sequentially to avoid races in parser implementations that keep
-    // mutable per-instance state (e.g. language switching/counters).
-    for (const fp of filePaths) {
-        const ext = nodePath.extname(fp).toLowerCase()
-        const parserKind = parserKindForExtension(ext)
-
-        // Build absolute posix path — this is the single source of truth for all IDs.
+    const fileResults = await parallelBatch(filePaths, async (fp) => {
         const absoluteFp = nodePath.resolve(normalizedRoot, fp).replace(/\\/g, '/')
-
-        let content: string
+        const ext = nodePath.extname(absoluteFp).toLowerCase()
+        const langDef = registry.getForFile(absoluteFp)
+        
         try {
-            content = await readFile(absoluteFp)
+            const content = await readFile(absoluteFp)
+            return { absoluteFp, ext, langDef, content }
         } catch (err: unknown) {
+            return { 
+                absoluteFp, ext, langDef, content: null, 
+                error: normalizeErrorMessage(err) 
+            }
+        }
+    }, concurrency)
+
+    for (const result of fileResults) {
+        if (result.error) {
             unreadableFiles += 1
-            addDiagnostic({
-                filePath: absoluteFp,
-                extension: ext,
-                parser: parserKind,
+            diagnostics.push({
+                filePath: result.absoluteFp,
+                extension: result.ext,
                 stage: 'read',
                 reason: 'read-error',
-                message: normalizeErrorMessage(err),
+                message: result.error,
             })
             continue
         }
 
-        if (parserKind === 'unknown') {
+        if (result.content === null) continue
+
+        const { absoluteFp, ext, langDef, content } = result
+
+        if (!langDef) {
             unsupportedFiles += 1
             fallbackFilesCount += 1
-            fallbackFiles.push(buildFallbackParsedFile(absoluteFp, content, ext))
-            addDiagnostic({
+            fallbackFiles.push(buildFallbackParsedFile(absoluteFp, content!, ext))
+            diagnostics.push({
                 filePath: absoluteFp,
                 extension: ext,
-                parser: parserKind,
                 stage: 'parse',
                 reason: 'unsupported-extension',
                 message: `Unsupported extension: ${ext || '<none>'}`,
@@ -311,111 +188,86 @@ export async function parseFilesWithDiagnostics(
         }
 
         try {
-            if (parserKind === 'oxc') {
-                const parsed = await oxcParser.parse(absoluteFp, content)
-                oxcFiles.push(parsed)
-                parsedFilesCount += 1
-            } else if (parserKind === 'go') {
-                const parsed = await goParser.parse(absoluteFp, content)
-                goFiles.push(parsed)
-                parsedFilesCount += 1
-            } else {
-                if (!treeSitterAvailable) {
-                    fallbackFilesCount += 1
-                    fallbackFiles.push(buildFallbackParsedFile(absoluteFp, content, ext))
-                    addDiagnostic({
-                        filePath: absoluteFp,
-                        extension: ext,
-                        parser: 'tree-sitter',
-                        stage: 'parse',
-                        reason: 'parser-unavailable',
-                        message: 'Tree-sitter runtime unavailable. Falling back to empty parsed file.',
-                    })
-                    continue
-                }
-                const ts = await getTreeSitter()
-                const parsed = await ts.parse(absoluteFp, content)
-                treeFiles.push(parsed)
+            const contentHash = hashContent(content!)
+            const cached = await cache.get(absoluteFp, contentHash)
+            
+            if (cached) {
+                const group = filesByExtractor.get(langDef.extractor) || []
+                group.push(cached)
+                filesByExtractor.set(langDef.extractor, group)
                 parsedFilesCount += 1
+                continue
             }
-        } catch (err: unknown) {
-            fallbackFilesCount += 1
-            const message = normalizeErrorMessage(err)
-            const reason: ParseDiagnosticReason = isLikelyParserUnavailable(parserKind, message)
-                ? 'parser-unavailable'
-                : 'parse-error'
-
-            fallbackFiles.push(buildFallbackParsedFile(absoluteFp, content, ext))
-            addDiagnostic({
-                filePath: absoluteFp,
-                extension: ext,
-                parser: parserKind,
-                stage: 'parse',
-                reason,
-                message,
-            })
-        }
-    }
-
-    // Resolve imports batch-wise per parser (each has its own resolver).
-    let resolvedOxcFiles = oxcFiles
-    if (oxcFiles.length > 0) {
-        try {
-            resolvedOxcFiles = await oxcParser.resolveImports(oxcFiles, normalizedRoot)
-        } catch (err: unknown) {
-            addDiagnostic({
-                filePath: '*',
-                extension: '*',
-                parser: 'oxc',
-                stage: 'resolve-imports',
-                reason: 'resolve-error',
-                message: normalizeErrorMessage(err),
-            })
-        }
-    }
 
-    let resolvedGoFiles = goFiles
-    if (goFiles.length > 0) {
-        try {
-            resolvedGoFiles = await goParser.resolveImports(goFiles, normalizedRoot)
+            const parsed = await langDef.extractor.extract(absoluteFp, content!)
+            await cache.set(absoluteFp, contentHash, parsed)
+            
+            const group = filesByExtractor.get(langDef.extractor) || []
+            group.push(parsed)
+            filesByExtractor.set(langDef.extractor, group)
+            parsedFilesCount += 1
         } catch (err: unknown) {
-            addDiagnostic({
-                filePath: '*',
-                extension: '*',
-                parser: 'go',
-                stage: 'resolve-imports',
-                reason: 'resolve-error',
-                message: normalizeErrorMessage(err),
-            })
+            if (process.env.MIKK_DEBUG) {
+                console.error(`[parser] Error extracting ${absoluteFp}:`, err instanceof Error ? err.message : String(err))
+            }
+            fallbackFilesCount += 1
+            
+            const errorMessage = normalizeErrorMessage(err)
+            const language = langDef?.name ?? languageForExtension(ext) ?? 'unknown'
+            
+            const recoveryEngine = new ErrorRecoveryEngine()
+            const recoveryResult = await recoveryEngine.recover(absoluteFp, content!, language)
+            
+            if (recoveryResult.success && recoveryResult.confidence > 0.2) {
+                fallbackFiles.push(recoveryResult.parsed)
+                diagnostics.push({
+                    filePath: absoluteFp,
+                    extension: ext,
+                    stage: 'parse',
+                    reason: 'parse-error-recovered',
+                    message: `${errorMessage} | Recovered ${recoveryResult.strategy} (confidence: ${(recoveryResult.confidence * 100).toFixed(0)}%) | ${recoveryResult.parsed.functions.length} fns, ${recoveryResult.parsed.classes.length} classes`,
+                })
+            } else {
+                fallbackFiles.push(buildFallbackParsedFile(absoluteFp, content!, ext))
+                diagnostics.push({
+                    filePath: absoluteFp,
+                    extension: ext,
+                    stage: 'parse',
+                    reason: 'parse-error',
+                    message: errorMessage,
+                })
+            }
         }
     }
 
-    let resolvedTreeFiles = treeFiles
-    if (treeFiles.length > 0) {
+    for (const [extractor, files] of filesByExtractor.entries()) {
         try {
-            const treeParser = treeSitterParser ?? await getTreeSitter()
-            resolvedTreeFiles = await treeParser.resolveImports(treeFiles, normalizedRoot)
+            const resolved = await extractor.resolveImports(files, normalizedRoot)
+            
+            for (let i = 0; i < files.length; i++) {
+                const originalFile = files[i]
+                const resolvedFile = resolved[i]
+                if (resolvedFile && resolvedFile !== originalFile) {
+                    files[i] = resolvedFile
+                }
+            }
+            fallbackFiles.push(...files)
         } catch (err: unknown) {
-            addDiagnostic({
+            diagnostics.push({
                 filePath: '*',
                 extension: '*',
-                parser: 'tree-sitter',
                 stage: 'resolve-imports',
                 reason: 'resolve-error',
                 message: normalizeErrorMessage(err),
             })
+            fallbackFiles.push(...files)
         }
     }
 
-    const resolved: ParsedFile[] = [
-        ...resolvedOxcFiles,
-        ...resolvedGoFiles,
-        ...resolvedTreeFiles,
-        ...fallbackFiles,
-    ]
+    cache.flush()
 
     return {
-        files: resolved,
+        files: fallbackFiles,
         diagnostics,
         summary: {
             requestedFiles: filePaths.length,
@@ -428,15 +280,6 @@ export async function parseFilesWithDiagnostics(
     }
 }
 
-/**
- * Parse multiple files, resolve their imports, and return ParsedFile[].
- *
- * Path contract (critical for graph correctness):
- *   - filePaths come from discoverFiles() as project-root-relative strings
- *   - We resolve them to ABSOLUTE posix paths before passing to parse()
- *   - ParsedFile.path is therefore always absolute + forward-slash
- *   - OxcResolver also returns absolute paths → import edges always consistent
- */
 export async function parseFiles(
     filePaths: string[],
     projectRoot: string,

package/src/parser/language-registry.ts

@@ -0,0 +1,57 @@
+import type { SyntaxNode } from 'web-tree-sitter';
+import { BaseExtractor } from './base-extractor.js';
+
+export interface LanguageDefinition {
+  name: string;
+  extensions: string[];
+  treeSitterGrammar: string;
+  extractor: BaseExtractor;
+  semanticFeatures: {
+    hasTypeSystem: boolean;
+    hasGenerics: boolean;
+    hasMacros: boolean;
+    hasAnnotations: boolean;
+    hasPatternMatching: boolean;
+  };
+  specialHandling?: {
+    importResolution?: (source: string, filePath: string) => Promise<string[]>;
+    exportDetection?: (node: SyntaxNode) => boolean;
+  };
+}
+
+export class LanguageRegistry {
+  private static instance: LanguageRegistry;
+  private languages: Map<string, LanguageDefinition> = new Map();
+  private extMap: Map<string, string> = new Map();
+
+  private constructor() {}
+
+  public static getInstance(): LanguageRegistry {
+    if (!LanguageRegistry.instance) {
+      LanguageRegistry.instance = new LanguageRegistry();
+    }
+    return LanguageRegistry.instance;
+  }
+
+  public register(lang: LanguageDefinition): void {
+    this.languages.set(lang.name, lang);
+    for (const ext of lang.extensions) {
+      this.extMap.set(ext, lang.name);
+    }
+  }
+
+  public getForFile(filePath: string): LanguageDefinition | null {
+    const ext = filePath.slice(filePath.lastIndexOf('.'));
+    const langName = this.extMap.get(ext);
+    if (!langName) return null;
+    return this.languages.get(langName) || null;
+  }
+
+  public getAllSupportedExtensions(): string[] {
+    return Array.from(this.extMap.keys());
+  }
+
+  public listLanguages(): string[] {
+    return Array.from(this.languages.keys());
+  }
+}