@plebeiantech/lat.md 0.12.3-fork.13 → 0.12.3-fork.14

package/dist/src/source-parser.js

@@ -1,7 +1,10 @@
-import { readFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { createRequire } from 'node:module';
+import { performance } from 'node:perf_hooks';
+import { PARSER_CACHE_VERSION, hashParserContent, parsedCachePath, parserCacheIdentity, readParsedCache, writeParsedCache, } from './parser-cache.js';
 import { Parser, Language, } from 'web-tree-sitter';
+import { sourceFileExtension, } from './source-formats.js';
 // Lazy singleton for the parser
 let parserReady = null;
 let parserInstance = null;
@@ -21,25 +24,22 @@ async function ensureParser() {
     }
     return parserInstance;
 }
-/** Extension → tree-sitter WASM grammar mapping. This is the single source of
- *  truth for which source file extensions lat supports. */
+/** Every supported source extension must declare a tree-sitter grammar. */
 const grammarMap = {
-    '.ts': 'tree-sitter-typescript.wasm',
-    '.tsx': 'tree-sitter-tsx.wasm',
+    '.c': 'tree-sitter-c.wasm',
+    '.dart': 'tree-sitter-dart.wasm',
+    '.go': 'tree-sitter-go.wasm',
+    '.h': 'tree-sitter-c.wasm',
+    '.java': 'tree-sitter-java.wasm',
     '.js': 'tree-sitter-javascript.wasm',
     '.jsx': 'tree-sitter-javascript.wasm',
     '.py': 'tree-sitter-python.wasm',
     '.rs': 'tree-sitter-rust.wasm',
-    '.go': 'tree-sitter-go.wasm',
-    '.c': 'tree-sitter-c.wasm',
-    '.h': 'tree-sitter-c.wasm',
+    '.ts': 'tree-sitter-typescript.wasm',
+    '.tsx': 'tree-sitter-tsx.wasm',
 };
-/** All source file extensions that lat can parse (derived from grammarMap). */
-export const SOURCE_EXTENSIONS = new Set(Object.keys(grammarMap));
 async function getLanguage(ext) {
     const wasmFile = grammarMap[ext];
-    if (!wasmFile)
-        return null;
     // Ensure WASM runtime is initialized before loading languages
     await ensureParser();
     if (!languages.has(wasmFile)) {
@@ -246,6 +246,314 @@ function extractPySymbols(tree) {
     }
     return symbols;
 }
+function dartDirectChild(node, type) {
+    return node.namedChildren.find((child) => child.type === type) ?? null;
+}
+function dartDefinitionName(node) {
+    const fieldName = extractName(node);
+    if (fieldName)
+        return fieldName;
+    if (node.type === 'class_definition') {
+        const alias = dartDirectChild(node, 'mixin_application_class');
+        return alias ? (dartDirectChild(alias, 'identifier')?.text ?? null) : null;
+    }
+    if (node.type === 'type_alias') {
+        return dartDirectChild(node, 'type_identifier')?.text ?? null;
+    }
+    return dartDirectChild(node, 'identifier')?.text ?? null;
+}
+function dartCallableNode(node) {
+    if (node.type !== 'method_signature' && node.type !== 'declaration') {
+        return node;
+    }
+    return (node.namedChildren.find((child) => [
+        'constructor_signature',
+        'factory_constructor_signature',
+        'function_signature',
+        'getter_signature',
+        'operator_signature',
+        'redirecting_factory_constructor_signature',
+        'setter_signature',
+    ].includes(child.type)) ?? null);
+}
+function dartCallableName(node, parent) {
+    const callable = dartCallableNode(node);
+    if (!callable)
+        return null;
+    if (callable.type === 'operator_signature') {
+        const operator = callable.namedChildren.find((child) => child.type.endsWith('_operator'));
+        return operator ? `operator ${operator.text}` : null;
+    }
+    if (callable.type === 'constructor_signature' ||
+        callable.type === 'factory_constructor_signature' ||
+        callable.type === 'redirecting_factory_constructor_signature') {
+        const names = callable.namedChildren
+            .filter((child) => child.type === 'identifier')
+            .map((child) => child.text);
+        if (names.length === 0)
+            return null;
+        if (names.length === 1)
+            return names[0];
+        return names[0] === parent ? names[1] : names.at(-1);
+    }
+    return extractName(callable);
+}
+function dartAnnotatedStart(siblings, index) {
+    let start = siblings[index];
+    for (let i = index - 1; i >= 0 && siblings[i].type === 'annotation'; i--) {
+        start = siblings[i];
+    }
+    return start;
+}
+function dartDefinitionEnd(siblings, index) {
+    return siblings[index + 1]?.type === 'function_body'
+        ? siblings[index + 1]
+        : siblings[index];
+}
+function dartSignature(sourceLines, node) {
+    return sourceLines[node.startPosition.row]?.trim() ?? '';
+}
+function pushDartSymbol(sourceLines, symbols, name, kind, node, options = {}) {
+    symbols.push({
+        name,
+        kind,
+        ...(options.parent ? { parent: options.parent } : {}),
+        startLine: (options.start ?? node).startPosition.row + 1,
+        endLine: (options.end ?? node).endPosition.row + 1,
+        signature: dartSignature(sourceLines, node),
+    });
+}
+function dartVariableKind(siblings, index, container) {
+    if (container?.namedChildren.some((child) => child.type === 'const_builtin')) {
+        return 'const';
+    }
+    const row = siblings[index].startPosition.row;
+    for (let i = index - 1; i >= 0 && siblings[i].endPosition.row === row; i--) {
+        if (siblings[i].type === 'const_builtin')
+            return 'const';
+    }
+    return 'variable';
+}
+function extractDartVariables(sourceLines, symbols, node, kind, parent) {
+    const lists = node.type === 'initialized_identifier_list' ||
+        node.type === 'static_final_declaration_list'
+        ? [node]
+        : node.namedChildren.filter((child) => child.type === 'initialized_identifier_list' ||
+            child.type === 'static_final_declaration_list');
+    for (const list of lists) {
+        for (const declaration of list.namedChildren) {
+            if (declaration.type !== 'initialized_identifier' &&
+                declaration.type !== 'static_final_declaration') {
+                continue;
+            }
+            const name = dartDirectChild(declaration, 'identifier')?.text;
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, kind, node, { parent });
+            }
+        }
+    }
+}
+function extractDartMembers(sourceLines, body, parent, symbols) {
+    const children = body.namedChildren;
+    for (let i = 0; i < children.length; i++) {
+        const node = children[i];
+        if (node.type === 'method_signature') {
+            const name = dartCallableName(node, parent);
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, 'method', node, {
+                    parent,
+                    start: dartAnnotatedStart(children, i),
+                    end: dartDefinitionEnd(children, i),
+                });
+            }
+        }
+        else if (node.type === 'declaration') {
+            const name = dartCallableName(node, parent);
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, 'method', node, { parent });
+            }
+            extractDartVariables(sourceLines, symbols, node, dartVariableKind(children, i, node), parent);
+        }
+        else if (node.type === 'enum_constant') {
+            const name = extractName(node);
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, 'const', node, { parent });
+            }
+        }
+    }
+}
+function extractDartSymbols(tree) {
+    const symbols = [];
+    const children = tree.rootNode.namedChildren;
+    const sourceLines = tree.rootNode.text.split('\n');
+    for (let i = 0; i < children.length; i++) {
+        const node = children[i];
+        const start = dartAnnotatedStart(children, i);
+        if (node.type === 'function_signature' ||
+            node.type === 'getter_signature' ||
+            node.type === 'setter_signature') {
+            const name = dartCallableName(node);
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, 'function', node, {
+                    start,
+                    end: dartDefinitionEnd(children, i),
+                });
+            }
+        }
+        else if (node.type === 'class_definition') {
+            const name = dartDefinitionName(node);
+            if (!name)
+                continue;
+            pushDartSymbol(sourceLines, symbols, name, 'class', node, { start });
+            const body = node.childForFieldName('body');
+            if (body)
+                extractDartMembers(sourceLines, body, name, symbols);
+        }
+        else if (node.type === 'mixin_declaration') {
+            const name = dartDefinitionName(node);
+            if (!name)
+                continue;
+            pushDartSymbol(sourceLines, symbols, name, 'interface', node, { start });
+            const body = dartDirectChild(node, 'class_body');
+            if (body)
+                extractDartMembers(sourceLines, body, name, symbols);
+        }
+        else if (node.type === 'extension_declaration') {
+            const name = extractName(node);
+            if (!name)
+                continue;
+            pushDartSymbol(sourceLines, symbols, name, 'class', node, { start });
+            const body = node.childForFieldName('body');
+            if (body)
+                extractDartMembers(sourceLines, body, name, symbols);
+        }
+        else if (node.type === 'enum_declaration') {
+            const name = extractName(node);
+            if (!name)
+                continue;
+            pushDartSymbol(sourceLines, symbols, name, 'class', node, { start });
+            const body = node.childForFieldName('body');
+            if (body)
+                extractDartMembers(sourceLines, body, name, symbols);
+        }
+        else if (node.type === 'extension_type_declaration') {
+            const name = extractName(node);
+            if (!name)
+                continue;
+            pushDartSymbol(sourceLines, symbols, name, 'type', node, { start });
+            const representation = node.childForFieldName('representation');
+            const representationName = representation?.childForFieldName('name');
+            if (representation && representationName) {
+                pushDartSymbol(sourceLines, symbols, representationName.text, 'variable', representation, { parent: name });
+            }
+            const body = node.childForFieldName('body');
+            if (body)
+                extractDartMembers(sourceLines, body, name, symbols);
+        }
+        else if (node.type === 'type_alias') {
+            const name = dartDefinitionName(node);
+            if (name) {
+                pushDartSymbol(sourceLines, symbols, name, 'type', node, { start });
+            }
+        }
+        else if (node.type === 'initialized_identifier_list' ||
+            node.type === 'static_final_declaration_list') {
+            extractDartVariables(sourceLines, symbols, node, dartVariableKind(children, i));
+        }
+    }
+    return symbols;
+}
+const javaTypeKinds = {
+    annotation_type_declaration: 'interface',
+    class_declaration: 'class',
+    enum_declaration: 'class',
+    interface_declaration: 'interface',
+    record_declaration: 'class',
+};
+function javaSignature(sourceLines, node) {
+    const declarator = node.namedChildren.find((child) => child.type === 'variable_declarator');
+    const name = node.childForFieldName('name') ?? declarator?.childForFieldName('name');
+    return sourceLines[(name ?? node).startPosition.row]?.trim() ?? '';
+}
+function pushJavaSymbol(sourceLines, symbols, name, kind, node, parent) {
+    const symbol = {
+        name,
+        kind,
+        ...(parent ? { parent } : {}),
+        startLine: node.startPosition.row + 1,
+        endLine: node.endPosition.row + 1,
+        signature: javaSignature(sourceLines, node),
+    };
+    symbols.push(symbol);
+    return symbol;
+}
+function extractJavaVariables(sourceLines, symbols, node, parent) {
+    const kind = node.type === 'constant_declaration' ? 'const' : 'variable';
+    for (const declarator of node.namedChildren) {
+        if (declarator.type !== 'variable_declarator')
+            continue;
+        const name = extractName(declarator);
+        if (name) {
+            pushJavaSymbol(sourceLines, symbols, name, kind, node, parent);
+        }
+    }
+}
+function collectJavaScope(sourceLines, scope, parent, symbols) {
+    for (const node of scope.namedChildren) {
+        const typeKind = javaTypeKinds[node.type];
+        if (typeKind) {
+            const name = extractName(node);
+            if (!name)
+                continue;
+            const symbol = pushJavaSymbol(sourceLines, symbols, name, typeKind, node);
+            if (parent)
+                symbols.push({ ...symbol, parent });
+            if (node.type === 'record_declaration') {
+                const parameters = node.childForFieldName('parameters');
+                for (const parameter of parameters?.namedChildren ?? []) {
+                    const component = extractName(parameter);
+                    if (component) {
+                        pushJavaSymbol(sourceLines, symbols, component, 'variable', parameter, name);
+                    }
+                }
+            }
+            const body = node.childForFieldName('body');
+            if (body)
+                collectJavaScope(sourceLines, body, name, symbols);
+            continue;
+        }
+        if (!parent)
+            continue;
+        if (node.type === 'method_declaration' ||
+            node.type === 'constructor_declaration' ||
+            node.type === 'compact_constructor_declaration' ||
+            node.type === 'annotation_type_element_declaration') {
+            const name = extractName(node);
+            if (name) {
+                pushJavaSymbol(sourceLines, symbols, name, 'method', node, parent);
+            }
+        }
+        else if (node.type === 'field_declaration' ||
+            node.type === 'constant_declaration') {
+            extractJavaVariables(sourceLines, symbols, node, parent);
+        }
+        else if (node.type === 'enum_constant') {
+            const name = extractName(node);
+            if (name) {
+                pushJavaSymbol(sourceLines, symbols, name, 'const', node, parent);
+            }
+        }
+        else if (node.type === 'enum_body_declarations') {
+            collectJavaScope(sourceLines, node, parent, symbols);
+        }
+    }
+}
+function extractJavaSymbols(tree) {
+    const symbols = [];
+    const sourceLines = tree.rootNode.text.split('\n');
+    collectJavaScope(sourceLines, tree.rootNode, undefined, symbols);
+    return symbols;
+}
 function extractRustSymbols(tree) {
     const symbols = [];
     const root = tree.rootNode;
@@ -784,72 +1092,200 @@ function firstLine(text) {
     const nl = text.indexOf('\n');
     return nl === -1 ? text : text.slice(0, nl);
 }
+/** Every supported source extension must declare a symbol extractor. */
+const symbolExtractors = {
+    '.c': extractCSymbols,
+    '.dart': extractDartSymbols,
+    '.go': extractGoSymbols,
+    '.h': extractCSymbols,
+    '.java': extractJavaSymbols,
+    '.js': extractTsSymbols,
+    '.jsx': extractTsSymbols,
+    '.py': extractPySymbols,
+    '.rs': extractRustSymbols,
+    '.ts': extractTsSymbols,
+    '.tsx': extractTsSymbols,
+};
 export async function parseSourceSymbols(filePath, content) {
-    const ext = filePath.match(/\.[^.]+$/)?.[0] ?? '';
-    const lang = await getLanguage(ext);
-    if (!lang)
+    const ext = sourceFileExtension(filePath);
+    if (!ext)
         return [];
+    const lang = await getLanguage(ext);
     const p = await ensureParser();
     p.setLanguage(lang);
     const tree = p.parse(content);
     if (!tree)
         return [];
     try {
-        if (ext === '.py') {
-            return extractPySymbols(tree);
-        }
-        if (ext === '.rs') {
-            return extractRustSymbols(tree);
-        }
-        if (ext === '.go') {
-            return extractGoSymbols(tree);
-        }
-        if (ext === '.c' || ext === '.h') {
-            return extractCSymbols(tree);
-        }
-        return extractTsSymbols(tree);
+        return symbolExtractors[ext](tree);
     }
     finally {
         tree.delete();
     }
 }
-// Per-invocation cache for parsed source symbols, keyed by absolute file path.
-// Prevents re-parsing the same file when multiple wiki links reference it
-// (e.g. 20+ links to quickjs.c would otherwise parse a 60K-line file 20 times).
-const symbolCache = new Map();
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+const sourceSymbolKinds = new Set([
+    'function',
+    'class',
+    'const',
+    'type',
+    'interface',
+    'method',
+    'variable',
+]);
+function isSourceSymbol(value) {
+    if (!isRecord(value))
+        return false;
+    return (typeof value.name === 'string' &&
+        typeof value.kind === 'string' &&
+        sourceSymbolKinds.has(value.kind) &&
+        (value.parent === undefined || typeof value.parent === 'string') &&
+        Number.isInteger(value.startLine) &&
+        value.startLine > 0 &&
+        Number.isInteger(value.endLine) &&
+        value.endLine >= value.startLine &&
+        typeof value.signature === 'string');
+}
+function cachedSourceAnalysis(entry, contentHash, identity, timings) {
+    if (!entry ||
+        entry.version !== PARSER_CACHE_VERSION ||
+        entry.contentHash !== contentHash ||
+        !isRecord(entry.value) ||
+        entry.value.path !== identity ||
+        !Array.isArray(entry.value.symbols) ||
+        !entry.value.symbols.every(isSourceSymbol)) {
+        return null;
+    }
+    return {
+        path: identity,
+        symbols: entry.value.symbols,
+        timings: {
+            ...timings,
+            cacheStatus: 'hit',
+            cacheWriteMs: 0,
+            parseMs: 0,
+        },
+    };
+}
+/** Request-scoped owner for in-flight and completed source parser work. */
+export class SourceParserRuntime {
+    analyses = new Map();
+    symbols = new Map();
+    clear() {
+        this.analyses.clear();
+        this.symbols.clear();
+    }
+}
+const defaultSourceParserRuntime = new SourceParserRuntime();
+/** Return the parser cache path for one project source file. */
+export function sourceAnalysisCachePath(latDir, projectRoot, absolutePath) {
+    return parsedCachePath(latDir, parserCacheIdentity(absolutePath, projectRoot));
+}
+/** Analyze source content through the shared versioned persistent cache. */
+export async function analyzeSourceSymbols(filePath, content, latDir, options = {}) {
+    const identity = (options.identity ?? filePath)
+        .replaceAll('\\', '/')
+        .normalize('NFC');
+    const hashStarted = performance.now();
+    const contentHash = hashParserContent(content);
+    const hashMs = performance.now() - hashStarted;
+    const cache = options.cache !== false;
+    const cachePath = parsedCachePath(latDir, identity);
+    const promiseKey = `${cache ? 'cache' : 'direct'}\0${cachePath}\0${contentHash}`;
+    const runtime = options.runtime ?? defaultSourceParserRuntime;
+    let analysis = runtime.analyses.get(promiseKey);
+    if (!analysis) {
+        analysis = (async () => {
+            const cacheStarted = performance.now();
+            const entry = cache ? await readParsedCache(cachePath) : null;
+            const cacheReadMs = cache ? performance.now() - cacheStarted : 0;
+            const timings = {
+                readMs: options.readMs ?? 0,
+                hashMs,
+                cacheReadMs,
+                cacheStatus: cache ? 'miss' : 'disabled',
+            };
+            const cached = cache
+                ? cachedSourceAnalysis(entry, contentHash, identity, timings)
+                : null;
+            if (cached)
+                return cached;
+            const parseStarted = performance.now();
+            const symbols = await parseSourceSymbols(filePath, content);
+            const result = {
+                path: identity,
+                symbols,
+                timings: {
+                    ...timings,
+                    cacheWriteMs: 0,
+                    parseMs: performance.now() - parseStarted,
+                },
+            };
+            if (!cache)
+                return result;
+            const writeStarted = performance.now();
+            try {
+                await writeParsedCache(cachePath, contentHash, result);
+            }
+            catch {
+                // The cache is a disposable optimization; parsing must work read-only.
+            }
+            result.timings.cacheWriteMs = performance.now() - writeStarted;
+            return result;
+        })();
+        runtime.analyses.set(promiseKey, analysis);
+    }
+    return analysis;
+}
 /** Clear the symbol cache. Call between top-level operations. */
 export function clearSymbolCache() {
-    symbolCache.clear();
+    defaultSourceParserRuntime.clear();
 }
 /**
  * Check whether a source file path (relative to projectRoot) has a given symbol.
  * Used by lat check to validate source code wiki links lazily.
  */
-export async function resolveSourceSymbol(filePath, symbolPath, projectRoot) {
+export async function resolveSourceSymbol(filePath, symbolPath, projectRoot, options = {}) {
     const absPath = join(projectRoot, filePath);
-    let cached = symbolCache.get(absPath);
-    if (!cached) {
-        let content;
-        try {
-            content = readFileSync(absPath, 'utf-8');
-        }
-        catch {
-            cached = { symbols: [] };
-            symbolCache.set(absPath, cached);
-            return { found: false, symbols: [] };
-        }
-        try {
-            const symbols = await parseSourceSymbols(filePath, content);
-            cached = { symbols };
-        }
-        catch (err) {
-            cached = {
-                symbols: [],
-                error: `failed to parse "${filePath}": ${err instanceof Error ? err.message : String(err)}`,
-            };
-        }
-        symbolCache.set(absPath, cached);
+    const latDir = options.latDir ?? join(projectRoot, 'lat.md');
+    const cacheKey = `${latDir}\0${absPath}`;
+    const runtime = options.runtime ?? defaultSourceParserRuntime;
+    let cachedPromise = runtime.symbols.get(cacheKey);
+    const created = !cachedPromise;
+    if (!cachedPromise) {
+        cachedPromise = (async () => {
+            const readStarted = performance.now();
+            let content;
+            try {
+                content = await readFile(absPath, 'utf-8');
+            }
+            catch {
+                return { symbols: [] };
+            }
+            const readMs = performance.now() - readStarted;
+            try {
+                const analysis = await analyzeSourceSymbols(filePath, content, latDir, {
+                    cache: options.cache,
+                    identity: parserCacheIdentity(absPath, projectRoot),
+                    readMs,
+                    runtime,
+                });
+                return { symbols: analysis.symbols, analysis };
+            }
+            catch (err) {
+                return {
+                    symbols: [],
+                    error: `failed to parse "${filePath}": ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        })();
+        runtime.symbols.set(cacheKey, cachedPromise);
     }
+    const cached = await cachedPromise;
+    if (created && cached.analysis)
+        options.onFileAnalyzed?.(cached.analysis);
     if (cached.error) {
         return { found: false, symbols: cached.symbols, error: cached.error };
     }
@@ -873,6 +1309,7 @@ export async function resolveSourceSymbol(filePath, symbolPath, projectRoot) {
 export function sourceSymbolsToSections(symbols, filePath) {
     const sections = [];
     const classMap = new Map();
+    const parentNames = new Set(symbols.flatMap((symbol) => (symbol.parent ? [symbol.parent] : [])));
     for (const sym of symbols) {
         if (sym.parent)
             continue; // Handle methods after their class
@@ -888,7 +1325,7 @@ export function sourceSymbolsToSections(symbols, filePath) {
             firstParagraph: sym.signature,
         };
         sections.push(section);
-        if (sym.kind === 'class') {
+        if (parentNames.has(sym.name)) {
             classMap.set(sym.name, section);
         }
     }