@selvakumaresra/specship 0.1.3 → 0.4.0

package/dist/extraction/grammars.js

@@ -0,0 +1,426 @@
+"use strict";
+/**
+ * Grammar Loading and Caching
+ *
+ * Uses web-tree-sitter (WASM) for universal cross-platform support.
+ * Grammars are loaded lazily — only languages actually present in the project
+ * are compiled, keeping V8 WASM memory pressure low on large codebases.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EXTENSION_MAP = void 0;
+exports.isSourceFile = isSourceFile;
+exports.isPlayRoutesFile = isPlayRoutesFile;
+exports.initGrammars = initGrammars;
+exports.loadGrammarsForLanguages = loadGrammarsForLanguages;
+exports.loadAllGrammars = loadAllGrammars;
+exports.isGrammarsInitialized = isGrammarsInitialized;
+exports.getParser = getParser;
+exports.detectLanguage = detectLanguage;
+exports.isLanguageSupported = isLanguageSupported;
+exports.isGrammarLoaded = isGrammarLoaded;
+exports.isFileLevelOnlyLanguage = isFileLevelOnlyLanguage;
+exports.getSupportedLanguages = getSupportedLanguages;
+exports.resetParser = resetParser;
+exports.clearParserCache = clearParserCache;
+exports.getUnavailableGrammarErrors = getUnavailableGrammarErrors;
+exports.getLanguageDisplayName = getLanguageDisplayName;
+const path = __importStar(require("path"));
+const web_tree_sitter_1 = require("web-tree-sitter");
+/**
+ * WASM filename map — maps each language to its .wasm grammar file
+ * in the tree-sitter-wasms package.
+ */
+const WASM_GRAMMAR_FILES = {
+    typescript: 'tree-sitter-typescript.wasm',
+    tsx: 'tree-sitter-tsx.wasm',
+    javascript: 'tree-sitter-javascript.wasm',
+    jsx: 'tree-sitter-javascript.wasm',
+    python: 'tree-sitter-python.wasm',
+    go: 'tree-sitter-go.wasm',
+    rust: 'tree-sitter-rust.wasm',
+    java: 'tree-sitter-java.wasm',
+    c: 'tree-sitter-c.wasm',
+    cpp: 'tree-sitter-cpp.wasm',
+    csharp: 'tree-sitter-c_sharp.wasm',
+    php: 'tree-sitter-php.wasm',
+    ruby: 'tree-sitter-ruby.wasm',
+    swift: 'tree-sitter-swift.wasm',
+    kotlin: 'tree-sitter-kotlin.wasm',
+    dart: 'tree-sitter-dart.wasm',
+    pascal: 'tree-sitter-pascal.wasm',
+    scala: 'tree-sitter-scala.wasm',
+    lua: 'tree-sitter-lua.wasm',
+    luau: 'tree-sitter-luau.wasm',
+    objc: 'tree-sitter-objc.wasm',
+};
+/**
+ * File extension to Language mapping
+ */
+exports.EXTENSION_MAP = {
+    '.ts': 'typescript',
+    '.tsx': 'tsx',
+    // ESM/CJS TypeScript module extensions — parsed as TS (no JSX). (#366)
+    '.mts': 'typescript',
+    '.cts': 'typescript',
+    '.js': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+    // SAP HANA XS Classic server-side JavaScript. (#556)
+    '.xsjs': 'javascript',
+    '.xsjslib': 'javascript',
+    '.jsx': 'jsx',
+    '.py': 'python',
+    '.pyw': 'python',
+    '.go': 'go',
+    '.rs': 'rust',
+    '.java': 'java',
+    '.c': 'c',
+    '.h': 'c', // Could also be C++, defaulting to C
+    '.cpp': 'cpp',
+    '.cc': 'cpp',
+    '.cxx': 'cpp',
+    '.hpp': 'cpp',
+    '.hxx': 'cpp',
+    '.cs': 'csharp',
+    '.php': 'php',
+    // Drupal-specific PHP file extensions
+    '.module': 'php',
+    '.install': 'php',
+    '.theme': 'php',
+    '.inc': 'php',
+    // YAML (used for Drupal routing files; no symbol extraction, file-level tracking only)
+    '.yml': 'yaml',
+    '.yaml': 'yaml',
+    // Twig templates (file-level tracking only, no symbol extraction)
+    '.twig': 'twig',
+    '.rb': 'ruby',
+    '.rake': 'ruby',
+    '.swift': 'swift',
+    '.kt': 'kotlin',
+    '.kts': 'kotlin',
+    '.dart': 'dart',
+    '.liquid': 'liquid',
+    '.svelte': 'svelte',
+    '.vue': 'vue',
+    '.pas': 'pascal',
+    '.dpr': 'pascal',
+    '.dpk': 'pascal',
+    '.lpr': 'pascal',
+    '.dfm': 'pascal',
+    '.fmx': 'pascal',
+    '.scala': 'scala',
+    '.sc': 'scala',
+    '.lua': 'lua',
+    '.luau': 'luau',
+    '.m': 'objc',
+    '.mm': 'objc',
+    // XML: file-level tracking; the MyBatis extractor matches `<mapper namespace="...">`
+    // shape and emits SQL-statement nodes (other XML returns empty).
+    '.xml': 'xml',
+    // Spring config: `application.properties` / `application-*.properties`. Same
+    // shape as the `.yml` variants — the YAML/properties extractor emits one node
+    // per leaf key, and the Spring resolver links `@Value("${k}")` references.
+    '.properties': 'properties',
+};
+/**
+ * Whether a file is one SpecShip can parse, based purely on its extension.
+ * This is the single source of truth for "should we index this file" — derived
+ * from EXTENSION_MAP so parser support and indexing selection never drift.
+ */
+function isSourceFile(filePath) {
+    if (isPlayRoutesFile(filePath))
+        return true; // Play `conf/routes` is extensionless
+    const dot = filePath.lastIndexOf('.');
+    if (dot < 0)
+        return false;
+    return filePath.slice(dot).toLowerCase() in exports.EXTENSION_MAP;
+}
+/**
+ * Play Framework routes file: the extensionless `conf/routes` (and included
+ * `conf/*.routes`). No grammar — route extraction is done by the Play framework
+ * resolver, so it's processed through the no-grammar (`yaml`-style) path.
+ */
+function isPlayRoutesFile(filePath) {
+    return (filePath === 'conf/routes' ||
+        filePath.endsWith('/conf/routes') ||
+        filePath.endsWith('.routes'));
+}
+/**
+ * Caches for loaded grammars and parsers
+ */
+const parserCache = new Map();
+const languageCache = new Map();
+const unavailableGrammarErrors = new Map();
+let parserInitialized = false;
+/**
+ * Initialize the tree-sitter WASM runtime. Must be called before loading grammars.
+ * Does NOT load any grammar WASM files — use loadGrammarsForLanguages() for that.
+ * Idempotent — safe to call multiple times.
+ */
+async function initGrammars() {
+    if (parserInitialized)
+        return;
+    await web_tree_sitter_1.Parser.init();
+    parserInitialized = true;
+}
+/**
+ * Load grammar WASM files for specific languages only.
+ * Skips languages that are already loaded or have no WASM grammar.
+ * Must be called after initGrammars().
+ */
+async function loadGrammarsForLanguages(languages) {
+    if (!parserInitialized) {
+        await initGrammars();
+    }
+    // Deduplicate and filter to languages that have WASM grammars and aren't already loaded
+    const toLoad = [...new Set(languages)].filter((lang) => lang in WASM_GRAMMAR_FILES &&
+        !languageCache.has(lang) &&
+        !unavailableGrammarErrors.has(lang));
+    // Load grammars sequentially to avoid web-tree-sitter WASM race condition on Node 20+
+    // See: https://github.com/tree-sitter/tree-sitter/issues/2338
+    for (const lang of toLoad) {
+        const wasmFile = WASM_GRAMMAR_FILES[lang];
+        try {
+            // Some grammars ship their own WASMs (not in tree-sitter-wasms, or the
+            // tree-sitter-wasms build is too old). Lua: tree-sitter-wasms ships an
+            // ABI-13 build that corrupts the shared WASM heap under web-tree-sitter
+            // 0.25 (drops nested calls/imports on every file after the first); we
+            // vendor the upstream ABI-15 wasm instead.
+            const wasmPath = (lang === 'pascal' || lang === 'scala' || lang === 'lua' || lang === 'luau')
+                ? path.join(__dirname, 'wasm', wasmFile)
+                : require.resolve(`tree-sitter-wasms/out/${wasmFile}`);
+            const language = await web_tree_sitter_1.Language.load(wasmPath);
+            languageCache.set(lang, language);
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            console.warn(`[SpecShip] Failed to load ${lang} grammar — parsing will be unavailable: ${message}`);
+            unavailableGrammarErrors.set(lang, message);
+        }
+    }
+}
+/**
+ * Load ALL grammar WASM files. Convenience function for tests and
+ * backward compatibility. Prefer loadGrammarsForLanguages() in production.
+ */
+async function loadAllGrammars() {
+    const allLanguages = Object.keys(WASM_GRAMMAR_FILES);
+    await loadGrammarsForLanguages(allLanguages);
+}
+/**
+ * Check if grammars have been initialized
+ */
+function isGrammarsInitialized() {
+    return parserInitialized;
+}
+/**
+ * Get a parser for the specified language.
+ * Returns synchronously from pre-loaded cache.
+ */
+function getParser(language) {
+    if (parserCache.has(language)) {
+        return parserCache.get(language);
+    }
+    const lang = languageCache.get(language);
+    if (!lang) {
+        return null;
+    }
+    const parser = new web_tree_sitter_1.Parser();
+    parser.setLanguage(lang);
+    parserCache.set(language, parser);
+    return parser;
+}
+/**
+ * Detect language from file extension
+ */
+function detectLanguage(filePath, source) {
+    // Play `conf/routes` has no grammar — route through the no-symbol path; the
+    // Play framework resolver extracts route nodes from it.
+    if (isPlayRoutesFile(filePath))
+        return 'yaml';
+    const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase();
+    const lang = exports.EXTENSION_MAP[ext] || 'unknown';
+    // .h files could be C, C++, or Objective-C — check source content
+    if (lang === 'c' && ext === '.h' && source) {
+        if (looksLikeCpp(source))
+            return 'cpp';
+        if (looksLikeObjc(source))
+            return 'objc';
+    }
+    return lang;
+}
+/**
+ * Heuristic: does a .h file contain C++ constructs?
+ * Checks the first ~8KB for patterns that are unique to C++ and never valid C.
+ */
+function looksLikeCpp(source) {
+    const sample = source.substring(0, 8192);
+    return /\bnamespace\b|\bclass\s+\w+\s*[:{]|\btemplate\s*<|\b(?:public|private|protected)\s*:|\bvirtual\b|\busing\s+(?:namespace\b|\w+\s*=)/.test(sample);
+}
+/**
+ * Heuristic: does a .h file contain Objective-C constructs?
+ */
+function looksLikeObjc(source) {
+    const sample = source.substring(0, 8192);
+    return /@(?:interface|implementation|protocol|synthesize)\b/.test(sample);
+}
+/**
+ * Check if a language is supported (has a grammar defined).
+ * Returns true if the grammar exists, even if not yet loaded.
+ */
+function isLanguageSupported(language) {
+    if (language === 'svelte')
+        return true; // custom extractor (script block delegation)
+    if (language === 'vue')
+        return true; // custom extractor (script block delegation)
+    if (language === 'liquid')
+        return true; // custom regex extractor
+    if (language === 'yaml')
+        return true; // file-level tracking only; Drupal routing extraction via framework resolver
+    if (language === 'twig')
+        return true; // file-level tracking only
+    if (language === 'xml')
+        return true; // MyBatis mapper extractor
+    if (language === 'properties')
+        return true; // Spring config keys
+    if (language === 'unknown')
+        return false;
+    return language in WASM_GRAMMAR_FILES;
+}
+/**
+ * Check if a grammar has been loaded and is ready for parsing.
+ */
+function isGrammarLoaded(language) {
+    if (language === 'svelte' || language === 'vue' || language === 'liquid')
+        return true;
+    if (language === 'yaml' || language === 'twig')
+        return true; // no WASM grammar needed
+    if (language === 'xml' || language === 'properties')
+        return true; // no WASM grammar needed
+    return languageCache.has(language);
+}
+/**
+ * Languages tracked at the file-record level only: parsing emits zero symbol
+ * nodes, but the file is still stored (and framework resolvers may add per-file
+ * references later, e.g. Drupal routing yml, Spring `@Value` against
+ * application.properties). This is the canonical set behind the no-symbol
+ * branch in `tree-sitter.ts`; `xml` is intentionally excluded because its
+ * MyBatis extractor emits a file node. Callers use this to count such files as
+ * indexed rather than skipped, so it must stay in sync with that branch.
+ */
+function isFileLevelOnlyLanguage(language) {
+    return language === 'yaml' || language === 'twig' || language === 'properties';
+}
+/**
+ * Get all supported languages (those with grammar definitions).
+ */
+function getSupportedLanguages() {
+    return [...Object.keys(WASM_GRAMMAR_FILES), 'svelte', 'vue', 'liquid'];
+}
+/**
+ * Reset the cached parser for a language to reclaim WASM heap memory.
+ * The tree-sitter WASM runtime accumulates fragmented memory over thousands
+ * of parses. Deleting and recreating the Parser instance forces the WASM
+ * heap to reset, preventing "memory access out of bounds" crashes in
+ * large repos.
+ */
+function resetParser(language) {
+    const old = parserCache.get(language);
+    if (old) {
+        old.delete();
+        parserCache.delete(language);
+    }
+}
+/**
+ * Clear parser/grammar caches (useful for testing)
+ */
+function clearParserCache() {
+    for (const parser of parserCache.values()) {
+        parser.delete();
+    }
+    parserCache.clear();
+    // Note: languageCache is NOT cleared — WASM languages persist.
+    // To fully re-init, set parserInitialized = false and call initGrammars() again.
+    unavailableGrammarErrors.clear();
+}
+/**
+ * Report grammars that failed to load.
+ */
+function getUnavailableGrammarErrors() {
+    const out = {};
+    for (const [language, message] of unavailableGrammarErrors.entries()) {
+        out[language] = message;
+    }
+    return out;
+}
+/**
+ * Get language display name
+ */
+function getLanguageDisplayName(language) {
+    const names = {
+        typescript: 'TypeScript',
+        javascript: 'JavaScript',
+        tsx: 'TypeScript (TSX)',
+        jsx: 'JavaScript (JSX)',
+        python: 'Python',
+        go: 'Go',
+        rust: 'Rust',
+        java: 'Java',
+        c: 'C',
+        cpp: 'C++',
+        csharp: 'C#',
+        php: 'PHP',
+        ruby: 'Ruby',
+        swift: 'Swift',
+        kotlin: 'Kotlin',
+        dart: 'Dart',
+        svelte: 'Svelte',
+        vue: 'Vue',
+        liquid: 'Liquid',
+        pascal: 'Pascal / Delphi',
+        scala: 'Scala',
+        lua: 'Lua',
+        luau: 'Luau',
+        objc: 'Objective-C',
+        yaml: 'YAML',
+        twig: 'Twig',
+        xml: 'XML',
+        properties: 'Java properties',
+        unknown: 'Unknown',
+    };
+    return names[language] || language;
+}
+//# sourceMappingURL=grammars.js.map

package/dist/extraction/grammars.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grammars.js","sourceRoot":"","sources":["../../src/extraction/grammars.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+GH,oCAKC;AAOD,4CAMC;AAgBD,oCAMC;AAOD,4DAkCC;AAMD,0CAGC;AAKD,sDAEC;AAMD,8BAcC;AAKD,wCAcC;AAuBD,kDAUC;AAKD,0CAKC;AAWD,0DAEC;AAKD,sDAEC;AASD,kCAMC;AAKD,4CAQC;AAKD,kEAMC;AAKD,wDAiCC;AAjYD,2CAA6B;AAC7B,qDAAmE;AAKnE;;;GAGG;AACH,MAAM,kBAAkB,GAAoC;IAC1D,UAAU,EAAE,6BAA6B;IACzC,GAAG,EAAE,sBAAsB;IAC3B,UAAU,EAAE,6BAA6B;IACzC,GAAG,EAAE,6BAA6B;IAClC,MAAM,EAAE,yBAAyB;IACjC,EAAE,EAAE,qBAAqB;IACzB,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,uBAAuB;IAC7B,CAAC,EAAE,oBAAoB;IACvB,GAAG,EAAE,sBAAsB;IAC3B,MAAM,EAAE,0BAA0B;IAClC,GAAG,EAAE,sBAAsB;IAC3B,IAAI,EAAE,uBAAuB;IAC7B,KAAK,EAAE,wBAAwB;IAC/B,MAAM,EAAE,yBAAyB;IACjC,IAAI,EAAE,uBAAuB;IAC7B,MAAM,EAAE,yBAAyB;IACjC,KAAK,EAAE,wBAAwB;IAC/B,GAAG,EAAE,sBAAsB;IAC3B,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,uBAAuB;CAC9B,CAAC;AAEF;;GAEG;AACU,QAAA,aAAa,GAA6B;IACrD,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,uEAAuE;IACvE,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,qDAAqD;IACrD,OAAO,EAAE,YAAY;IACrB,UAAU,EAAE,YAAY;IACxB,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG,EAAE,qCAAqC;IAChD,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,KAAK;IACb,sCAAsC;IACtC,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,QAAQ,EAAE,KAAK;IACf,MAAM,EAAE,KAAK;IACb,uFAAuF;IACvF,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,kEAAkE;IAClE,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,OAAO,EAAE,MAAM;IACf,SAAS,EAAE,QAAQ;IACnB,SAAS,EAAE,QAAQ;IACnB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,MAAM,EAAE,QAAQ;IAChB,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,OAAO;IACd,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,IAAI,EAAE,MAAM;IACZ,KAAK,EAAE,MAAM;IACb,qFAAqF;IACrF,iEAAiE;IACjE,MAAM,EAAE,KAAK;IACb,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,aAAa,EAAE,YAAY;CAC5B,CAAC;AAEF;;;;GAIG;AACH,SAAgB,YAAY,CAAC,QAAgB;IAC3C,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sCAAsC;IACnF,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1B,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,IAAI,qBAAa,CAAC;AAC5D,CAAC;AAED;;;;GAIG;AACH,SAAgB,gBAAgB,CAAC,QAAgB;IAC/C,OAAO,CACL,QAAQ,KAAK,aAAa;QAC1B,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC;QACjC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAC7B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAoB,CAAC;AAChD,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0B,CAAC;AACxD,MAAM,wBAAwB,GAAG,IAAI,GAAG,EAAoB,CAAC;AAE7D,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAE9B;;;;GAIG;AACI,KAAK,UAAU,YAAY;IAChC,IAAI,iBAAiB;QAAE,OAAO;IAE9B,MAAM,wBAAM,CAAC,IAAI,EAAE,CAAC;IAEpB,iBAAiB,GAAG,IAAI,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACI,KAAK,UAAU,wBAAwB,CAAC,SAAqB;IAClE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,YAAY,EAAE,CAAC;IACvB,CAAC;IAED,wFAAwF;IACxF,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAC3C,CAAC,IAAI,EAA2B,EAAE,CAChC,IAAI,IAAI,kBAAkB;QAC1B,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;QACxB,CAAC,wBAAwB,CAAC,GAAG,CAAC,IAAI,CAAC,CACtC,CAAC;IAEF,sFAAsF;IACtF,8DAA8D;IAC9D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC;YACH,uEAAuE;YACvE,uEAAuE;YACvE,wEAAwE;YACxE,sEAAsE;YACtE,2CAA2C;YAC3C,MAAM,QAAQ,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,MAAM,CAAC;gBAC3F,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC;gBACxC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAyB,QAAQ,EAAE,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,0BAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACnD,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,6BAA6B,IAAI,2CAA2C,OAAO,EAAE,CAAC,CAAC;YACpG,wBAAwB,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;GAGG;AACI,KAAK,UAAU,eAAe;IACnC,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAsB,CAAC;IAC1E,MAAM,wBAAwB,CAAC,YAAY,CAAC,CAAC;AAC/C,CAAC;AAED;;GAEG;AACH,SAAgB,qBAAqB;IACnC,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS,CAAC,QAAkB;IAC1C,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;IACpC,CAAC;IAED,MAAM,IAAI,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,wBAAM,EAAE,CAAC;IAC5B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACzB,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,QAAgB,EAAE,MAAe;IAC9D,4EAA4E;IAC5E,wDAAwD;IACxD,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9C,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACxE,MAAM,IAAI,GAAG,qBAAa,CAAC,GAAG,CAAC,IAAI,SAAS,CAAC;IAE7C,kEAAkE;IAClE,IAAI,IAAI,KAAK,GAAG,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,EAAE,CAAC;QAC3C,IAAI,YAAY,CAAC,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;QACvC,IAAI,aAAa,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;IAC3C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,MAAc;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACzC,OAAO,oIAAoI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3J,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,MAAc;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACzC,OAAO,qDAAqD,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5E,CAAC;AAED;;;GAGG;AACH,SAAgB,mBAAmB,CAAC,QAAkB;IACpD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,CAAC,6CAA6C;IACrF,IAAI,QAAQ,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC,CAAC,6CAA6C;IAClF,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IACjE,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,6EAA6E;IACnH,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,2BAA2B;IACjE,IAAI,QAAQ,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC,CAAC,2BAA2B;IAChE,IAAI,QAAQ,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IACjE,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACzC,OAAO,QAAQ,IAAI,kBAAkB,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,SAAgB,eAAe,CAAC,QAAkB;IAChD,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IACtF,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IACtF,IAAI,QAAQ,KAAK,KAAK,IAAI,QAAQ,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IAC3F,OAAO,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,uBAAuB,CAAC,QAAkB;IACxD,OAAO,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,MAAM,IAAI,QAAQ,KAAK,YAAY,CAAC;AACjF,CAAC;AAED;;GAEG;AACH,SAAgB,qBAAqB;IACnC,OAAO,CAAC,GAAI,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAuB,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;AAChG,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,QAAkB;IAC5C,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,GAAG,EAAE,CAAC;QACR,GAAG,CAAC,MAAM,EAAE,CAAC;QACb,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB;IAC9B,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;IACD,WAAW,CAAC,KAAK,EAAE,CAAC;IACpB,+DAA+D;IAC/D,iFAAiF;IACjF,wBAAwB,CAAC,KAAK,EAAE,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,SAAgB,2BAA2B;IACzC,MAAM,GAAG,GAAsC,EAAE,CAAC;IAClD,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,wBAAwB,CAAC,OAAO,EAAE,EAAE,CAAC;QACrE,GAAG,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC;IAC1B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,SAAgB,sBAAsB,CAAC,QAAkB;IACvD,MAAM,KAAK,GAA6B;QACtC,UAAU,EAAE,YAAY;QACxB,UAAU,EAAE,YAAY;QACxB,GAAG,EAAE,kBAAkB;QACvB,GAAG,EAAE,kBAAkB;QACvB,MAAM,EAAE,QAAQ;QAChB,EAAE,EAAE,IAAI;QACR,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,CAAC,EAAE,GAAG;QACN,GAAG,EAAE,KAAK;QACV,MAAM,EAAE,IAAI;QACZ,GAAG,EAAE,KAAK;QACV,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,OAAO;QACd,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,MAAM;QACZ,MAAM,EAAE,QAAQ;QAChB,GAAG,EAAE,KAAK;QACV,MAAM,EAAE,QAAQ;QAChB,MAAM,EAAE,iBAAiB;QACzB,KAAK,EAAE,OAAO;QACd,GAAG,EAAE,KAAK;QACV,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,KAAK;QACV,UAAU,EAAE,iBAAiB;QAC7B,OAAO,EAAE,SAAS;KACnB,CAAC;IACF,OAAO,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC;AACrC,CAAC"}

package/dist/extraction/index.d.ts

@@ -0,0 +1,138 @@
+/**
+ * Extraction Orchestrator
+ *
+ * Coordinates file scanning, parsing, and database storage.
+ */
+import * as fs from 'fs';
+import { ExtractionResult, ExtractionError } from '../types';
+import { QueryBuilder } from '../db/queries';
+import { Ignore } from 'ignore';
+/**
+ * Progress callback for indexing operations
+ */
+export interface IndexProgress {
+    phase: 'scanning' | 'parsing' | 'storing' | 'resolving';
+    current: number;
+    total: number;
+    currentFile?: string;
+}
+/**
+ * Result of an indexing operation
+ */
+export interface IndexResult {
+    success: boolean;
+    filesIndexed: number;
+    filesSkipped: number;
+    filesErrored: number;
+    nodesCreated: number;
+    edgesCreated: number;
+    errors: ExtractionError[];
+    durationMs: number;
+}
+/**
+ * Result of a sync operation
+ */
+export interface SyncResult {
+    filesChecked: number;
+    filesAdded: number;
+    filesModified: number;
+    filesRemoved: number;
+    nodesUpdated: number;
+    durationMs: number;
+    changedFilePaths?: string[];
+}
+/**
+ * Calculate SHA256 hash of file contents
+ */
+export declare function hashContent(content: string): string;
+/**
+ * An `ignore` matcher seeded with the built-in defaults, merged with the project's
+ * root .gitignore so a negation there (e.g. `!vendor/`) overrides a default. Shared
+ * by both enumeration paths so behavior is identical with or without git — and so
+ * the defaults apply to tracked files too (committing a dependency dir doesn't make
+ * it project code; the explicit `.gitignore` negation is the only opt-in).
+ */
+export declare function buildDefaultIgnore(rootDir: string): Ignore;
+/**
+ * Recursively scan a directory for source files.
+ *
+ * In git repos, uses `git ls-files` (inherently respects .gitignore at all
+ * levels), then keeps files with a supported source extension. For non-git
+ * projects, falls back to a filesystem walk that parses .gitignore itself.
+ */
+export declare function scanDirectory(rootDir: string, onProgress?: (current: number, file: string) => void): string[];
+/**
+ * Async variant of scanDirectory that yields to the event loop periodically,
+ * allowing worker threads to receive and render progress messages.
+ */
+export declare function scanDirectoryAsync(rootDir: string, onProgress?: (current: number, file: string) => void): Promise<string[]>;
+/**
+ * Extraction orchestrator
+ */
+export declare class ExtractionOrchestrator {
+    private rootDir;
+    private queries;
+    /**
+     * Names of frameworks detected for this project, populated by indexAll().
+     * Passed to extractFromSource so framework-specific extractors (route nodes,
+     * middleware, etc.) run after the tree-sitter pass. Cleared if detection
+     * hasn't run yet so single-file re-index paths can detect on the spot.
+     */
+    private detectedFrameworkNames;
+    constructor(rootDir: string, queries: QueryBuilder);
+    /**
+     * Build a filesystem-backed ResolutionContext sufficient for framework
+     * detection. Graph-query methods (getNodesByName etc.) return empty because
+     * the DB hasn't been populated yet, but detect() only uses readFile,
+     * fileExists, and getAllFiles, so that's fine.
+     */
+    private buildDetectionContext;
+    /**
+     * Detect frameworks on demand using the current scanned files (or a fresh
+     * scan if none are provided). Cached on the orchestrator so repeat calls
+     * inside a single run don't re-scan.
+     */
+    private ensureDetectedFrameworks;
+    /**
+     * Index all files in the project
+     */
+    indexAll(onProgress?: (progress: IndexProgress) => void, signal?: AbortSignal, verbose?: boolean): Promise<IndexResult>;
+    /**
+     * Index specific files
+     */
+    indexFiles(filePaths: string[]): Promise<IndexResult>;
+    /**
+     * Index a single file
+     */
+    indexFile(relativePath: string): Promise<ExtractionResult>;
+    /**
+     * Index a single file with pre-read content and stats.
+     * Used by the parallel batch reader to avoid redundant file I/O.
+     */
+    indexFileWithContent(relativePath: string, content: string, stats: fs.Stats): Promise<ExtractionResult>;
+    /**
+     * Store extraction result in database
+     */
+    private storeExtractionResult;
+    /**
+     * Sync the index with the current file state.
+     *
+     * Change detection is filesystem-based, never git: a (size, mtime) stat
+     * pre-filter skips unchanged files, then a content-hash compare confirms real
+     * changes. This works in non-git projects and catches committed changes from
+     * `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
+     */
+    sync(onProgress?: (progress: IndexProgress) => void): Promise<SyncResult>;
+    /**
+     * Get files that have changed since last index.
+     * Uses git status as a fast path when available, falling back to full scan.
+     */
+    getChangedFiles(): {
+        added: string[];
+        modified: string[];
+        removed: string[];
+    };
+}
+export { extractFromSource } from './tree-sitter';
+export { detectLanguage, isSourceFile, isLanguageSupported, isGrammarLoaded, getSupportedLanguages, initGrammars, loadGrammarsForLanguages, loadAllGrammars } from './grammars';
+//# sourceMappingURL=index.d.ts.map

package/dist/extraction/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/extraction/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AAKzB,OAAO,EAGL,gBAAgB,EAChB,eAAe,EAChB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAK7C,OAAe,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AA4BxC;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,WAAW,CAAC;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEnD;AAqED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAS1D;AAoJD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GACnD,MAAM,EAAE,CAkBV;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GACnD,OAAO,CAAC,MAAM,EAAE,CAAC,CAoBnB;AAyHD;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAe;IAC9B;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB,CAAyB;gBAE3C,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY;IAKlD;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAkD7B;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;OAEG;IACG,QAAQ,CACZ,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,EAC9C,MAAM,CAAC,EAAE,WAAW,EACpB,OAAO,CAAC,EAAE,OAAO,GAChB,OAAO,CAAC,WAAW,CAAC;IA8evB;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,WAAW,CAAC;IA4C3D;;OAEG;IACG,SAAS,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuChE;;;OAGG;IACG,oBAAoB,CACxB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,EAAE,CAAC,KAAK,GACd,OAAO,CAAC,gBAAgB,CAAC;IA0D5B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsE7B;;;;;;;OAOG;IACG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC;IA6H/E;;;OAGG;IACH,eAAe,IAAI;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE;CAwF9E;AAGD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,qBAAqB,EAAE,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC"}