@aroman22/codegraph-vba-darwin-arm64 1.3.3

package/lib/dist/extraction/mybatis-extractor.js

@@ -0,0 +1,198 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MyBatisExtractor = void 0;
+const tree_sitter_helpers_1 = require("./tree-sitter-helpers");
+/**
+ * MyBatisExtractor — parses MyBatis mapper XML files.
+ *
+ * MyBatis splits a DAO interface across two files: a Java interface (parsed by
+ * tree-sitter) declares the method, and an XML mapper file holds the SQL keyed
+ * by `<namespace>` (the fully-qualified Java type name) and `id` (the method
+ * name). Without the XML side in the graph, `trace(Controller, ...DAO.method)`
+ * dead-ends at the interface method — the SQL it actually runs is invisible,
+ * and "what does this query touch" / "where is this column written" can't be
+ * answered.
+ *
+ * This extractor emits one method-shaped node per `<select|insert|update|
+ * delete>` and per `<sql>` fragment, qualified as `<namespace>::<id>` so the
+ * MyBatis framework synthesizer (`src/resolution/frameworks/mybatis.ts`) can
+ * link the matching Java method → XML statement by suffix-matching qualified
+ * names. `<include refid="...">` inside a statement yields an unresolved
+ * reference to the SQL fragment, also keyed by `<namespace>::<refid>`.
+ *
+ * Non-mapper XML (Maven `pom.xml`, Spring beans XML, `web.xml`, log4j config,
+ * etc.) is detected by the absence of a `<mapper namespace="...">` root and
+ * returns just a file node — we still need the file row so the watcher can
+ * track it, but we emit no symbols.
+ */
+class MyBatisExtractor {
+    filePath;
+    source;
+    nodes = [];
+    edges = [];
+    unresolvedReferences = [];
+    errors = [];
+    lineStarts = [];
+    constructor(filePath, source) {
+        this.filePath = filePath;
+        this.source = source;
+        this.computeLineStarts();
+    }
+    extract() {
+        const startTime = Date.now();
+        const fileNode = this.createFileNode();
+        try {
+            const mapperMatch = this.findMapperRoot();
+            if (mapperMatch) {
+                this.extractMapper(fileNode.id, mapperMatch.namespace, mapperMatch.bodyStart, mapperMatch.bodyEnd);
+            }
+        }
+        catch (error) {
+            this.errors.push({
+                message: `MyBatis extraction error: ${error instanceof Error ? error.message : String(error)}`,
+                severity: 'error',
+                code: 'parse_error',
+            });
+        }
+        return {
+            nodes: this.nodes,
+            edges: this.edges,
+            unresolvedReferences: this.unresolvedReferences,
+            errors: this.errors,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    createFileNode() {
+        const lines = this.source.split('\n');
+        const id = (0, tree_sitter_helpers_1.generateNodeId)(this.filePath, 'file', this.filePath, 1);
+        const node = {
+            id,
+            kind: 'file',
+            name: this.filePath.split('/').pop() || this.filePath,
+            qualifiedName: this.filePath,
+            filePath: this.filePath,
+            language: 'xml',
+            startLine: 1,
+            endLine: lines.length || 1,
+            startColumn: 0,
+            endColumn: lines[lines.length - 1]?.length ?? 0,
+            updatedAt: Date.now(),
+        };
+        this.nodes.push(node);
+        return node;
+    }
+    /**
+     * Find the `<mapper namespace="X">` opening tag. Returns the namespace and
+     * the byte offsets of the body (between the opening and closing tag) so
+     * statement extraction can be scoped to mapper contents.
+     */
+    findMapperRoot() {
+        const open = /<mapper\b([^>]*)>/.exec(this.source);
+        if (!open)
+            return null;
+        const attrs = open[1] ?? '';
+        const nsMatch = /\bnamespace\s*=\s*"([^"]+)"/.exec(attrs);
+        if (!nsMatch)
+            return null;
+        const bodyStart = open.index + open[0].length;
+        const closeIdx = this.source.indexOf('</mapper>', bodyStart);
+        const bodyEnd = closeIdx >= 0 ? closeIdx : this.source.length;
+        return { namespace: nsMatch[1], bodyStart, bodyEnd };
+    }
+    extractMapper(fileNodeId, namespace, bodyStart, bodyEnd) {
+        const body = this.source.slice(bodyStart, bodyEnd);
+        // Match each top-level statement-shaped element. The body may have nested
+        // tags (`<if>`, `<foreach>`, `<include>`), so we scan with a regex that
+        // pairs an opening tag to its matching close — the simple form below works
+        // because MyBatis statement elements are not themselves nested.
+        const stmtRegex = /<(select|insert|update|delete|sql)\b([^>]*)>([\s\S]*?)<\/\1>/g;
+        let m;
+        while ((m = stmtRegex.exec(body)) !== null) {
+            const elemType = m[1];
+            const attrs = m[2] ?? '';
+            const elemBody = m[3] ?? '';
+            const idMatch = /\bid\s*=\s*"([^"]+)"/.exec(attrs);
+            if (!idMatch)
+                continue;
+            const id = idMatch[1];
+            const absoluteIndex = bodyStart + m.index;
+            const startLine = this.getLineNumber(absoluteIndex);
+            const endLine = this.getLineNumber(absoluteIndex + m[0].length);
+            const qualified = `${namespace}::${id}`;
+            const isSqlFragment = elemType === 'sql';
+            const nodeId = (0, tree_sitter_helpers_1.generateNodeId)(this.filePath, 'method', qualified, startLine);
+            const node = {
+                id: nodeId,
+                kind: 'method',
+                name: id,
+                qualifiedName: qualified,
+                filePath: this.filePath,
+                language: 'xml',
+                signature: this.buildSignature(elemType, attrs, isSqlFragment),
+                startLine,
+                endLine,
+                startColumn: 0,
+                endColumn: 0,
+                docstring: this.previewSql(elemBody),
+                updatedAt: Date.now(),
+            };
+            this.nodes.push(node);
+            this.edges.push({ source: fileNodeId, target: nodeId, kind: 'contains' });
+            // <include refid="X"/> → reference to the SQL fragment in this mapper
+            // (or in another mapper, when the refid is qualified — `ns.X`).
+            const includeRegex = /<include\b[^>]*\brefid\s*=\s*"([^"]+)"/g;
+            let inc;
+            while ((inc = includeRegex.exec(elemBody)) !== null) {
+                const refid = inc[1];
+                const refQualified = refid.includes('.') ? refid.replace(/\./g, '::') : `${namespace}::${refid}`;
+                const includeOffset = absoluteIndex + (m[0].length - m[3].length - `</${elemType}>`.length) + inc.index;
+                const line = this.getLineNumber(includeOffset);
+                this.unresolvedReferences.push({
+                    fromNodeId: nodeId,
+                    referenceName: refQualified,
+                    referenceKind: 'references',
+                    line,
+                    column: 0,
+                });
+            }
+        }
+    }
+    buildSignature(elemType, attrs, isSqlFragment) {
+        if (isSqlFragment)
+            return '<sql>';
+        const verb = elemType.toUpperCase();
+        const result = /\bresultType\s*=\s*"([^"]+)"/.exec(attrs)?.[1];
+        const param = /\bparameterType\s*=\s*"([^"]+)"/.exec(attrs)?.[1];
+        const parts = [verb];
+        if (param)
+            parts.push(`param=${param}`);
+        if (result)
+            parts.push(`result=${result}`);
+        return parts.join(' ');
+    }
+    previewSql(body) {
+        return body.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 200);
+    }
+    computeLineStarts() {
+        this.lineStarts = [0];
+        for (let i = 0; i < this.source.length; i++) {
+            if (this.source.charCodeAt(i) === 10)
+                this.lineStarts.push(i + 1);
+        }
+    }
+    getLineNumber(offset) {
+        // Binary search
+        let lo = 0;
+        let hi = this.lineStarts.length - 1;
+        while (lo < hi) {
+            const mid = (lo + hi + 1) >>> 1;
+            if (this.lineStarts[mid] <= offset)
+                lo = mid;
+            else
+                hi = mid - 1;
+        }
+        return lo + 1;
+    }
+}
+exports.MyBatisExtractor = MyBatisExtractor;
+//# sourceMappingURL=mybatis-extractor.js.map

package/lib/dist/extraction/mybatis-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mybatis-extractor.js","sourceRoot":"","sources":["../../src/extraction/mybatis-extractor.ts"],"names":[],"mappings":";;;AACA,+DAAuD;AAEvD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAa,gBAAgB;IACnB,QAAQ,CAAS;IACjB,MAAM,CAAS;IACf,KAAK,GAAW,EAAE,CAAC;IACnB,KAAK,GAAW,EAAE,CAAC;IACnB,oBAAoB,GAA0B,EAAE,CAAC;IACjD,MAAM,GAAsB,EAAE,CAAC;IAC/B,UAAU,GAAa,EAAE,CAAC;IAElC,YAAY,QAAgB,EAAE,MAAc;QAC1C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO;QACL,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEvC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YAC1C,IAAI,WAAW,EAAE,CAAC;gBAChB,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,SAAS,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;YACrG,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACf,OAAO,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBAC9F,QAAQ,EAAE,OAAO;gBACjB,IAAI,EAAE,aAAa;aACpB,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,IAAA,oCAAc,EAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAS;YACjB,EAAE;YACF,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ;YACrD,aAAa,EAAE,IAAI,CAAC,QAAQ;YAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;YAC1B,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC;YAC/C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACK,cAAc;QACpB,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC9D,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IACxD,CAAC;IAEO,aAAa,CAAC,UAAkB,EAAE,SAAiB,EAAE,SAAiB,EAAE,OAAe;QAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACnD,0EAA0E;QAC1E,wEAAwE;QACxE,2EAA2E;QAC3E,gEAAgE;QAChE,MAAM,SAAS,GAAG,+DAA+D,CAAC;QAClF,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnD,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,aAAa,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC;YAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,SAAS,GAAG,GAAG,SAAS,KAAK,EAAE,EAAE,CAAC;YACxC,MAAM,aAAa,GAAG,QAAQ,KAAK,KAAK,CAAC;YACzC,MAAM,MAAM,GAAG,IAAA,oCAAc,EAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAC7E,MAAM,IAAI,GAAS;gBACjB,EAAE,EAAE,MAAM;gBACV,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,EAAE;gBACR,aAAa,EAAE,SAAS;gBACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC;gBAC9D,SAAS;gBACT,OAAO;gBACP,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACpC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE1E,sEAAsE;YACtE,gEAAgE;YAChE,MAAM,YAAY,GAAG,yCAAyC,CAAC;YAC/D,IAAI,GAA2B,CAAC;YAChC,OAAO,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,KAAK,KAAK,EAAE,CAAC;gBACjG,MAAM,aAAa,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;gBACzG,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC/C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;oBAC7B,UAAU,EAAE,MAAM;oBAClB,aAAa,EAAE,YAAY;oBAC3B,aAAa,EAAE,YAAY;oBAC3B,IAAI;oBACJ,MAAM,EAAE,CAAC;iBACV,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,KAAa,EAAE,aAAsB;QAC5E,IAAI,aAAa;YAAE,OAAO,OAAO,CAAC;QAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,iCAAiC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC;QACxC,IAAI,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC;QAC3C,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAEO,UAAU,CAAC,IAAY;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjF,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE;gBAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,MAAc;QAClC,gBAAgB;QAChB,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAE,IAAI,MAAM;gBAAE,EAAE,GAAG,GAAG,CAAC;;gBACzC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,EAAE,GAAG,CAAC,CAAC;IAChB,CAAC;CACF;AA3KD,4CA2KC"}

package/lib/dist/extraction/parse-pool.d.ts

@@ -0,0 +1,126 @@
+/**
+ * Parse worker pool — runs tree-sitter parsing across N worker threads so a full
+ * `codegraph index` uses every core instead of pinning one.
+ *
+ * Why this exists: `ExtractionOrchestrator.indexAll()` already reads files in
+ * parallel, but it parsed them through a SINGLE worker thread, so on an
+ * N-core machine indexing a large repo used one core and left the rest idle
+ * (issue #1015, the parse-time half of #320). Spreading the parse calls across a
+ * pool of workers — each its own tree-sitter WASM heap — restores multi-core
+ * throughput. SQLite storage stays on the main thread (it isn't thread-safe), so
+ * only the CPU-bound parse step is parallelised; results are stored as they
+ * arrive, in whatever order they finish.
+ *
+ * Design mirrors {@link ../mcp/query-pool} (idle-list dispatch, lazy growth,
+ * throttled cold-starts, crash recovery), with parse-specific behaviour:
+ *   - per-worker recycle: WASM linear memory grows but never shrinks, so each
+ *     worker is torn down and replaced after `recycleInterval` parses to reclaim
+ *     its heap — the same reason the old single worker recycled.
+ *   - reject, don't retry: a parse that crashes or times out its worker REJECTS
+ *     (with a message the orchestrator's retry pass recognises) rather than being
+ *     silently requeued — the orchestrator owns the smarter two-stage retry
+ *     (fresh worker, then comment-stripped) on a clean WASM heap.
+ *   - a size-1 pool reproduces the old single-worker path exactly, which is the
+ *     conservative rollback: set `CODEGRAPH_PARSE_WORKERS=1`.
+ *
+ * Memory: peak scales with pool size (≈ size × a worker's pre-recycle heap), so
+ * the default is capped and the env var lets constrained machines dial it down.
+ */
+import type { Language, ExtractionResult } from '../types';
+/**
+ * Minimal worker surface the pool drives — satisfied by a real `worker_threads`
+ * Worker. Abstracted so tests can inject a fake worker and exercise the pool's
+ * queue / growth / recycle / crash-recovery logic without spawning threads or a
+ * built `dist/`.
+ */
+export interface ParsePoolWorker {
+    postMessage(msg: unknown): void;
+    terminate(): Promise<number> | void;
+    on(event: 'message', cb: (m: unknown) => void): void;
+    on(event: 'error', cb: (e: Error) => void): void;
+    on(event: 'exit', cb: (code: number) => void): void;
+}
+/** A single file to parse. `language` is resolved on the main thread (it holds
+ *  the project's codegraph.json extension overrides) and handed to the worker. */
+export interface ParseTask {
+    filePath: string;
+    content: string;
+    language: Language;
+    frameworkNames?: string[];
+}
+/**
+ * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the
+ * machine's core count.
+ *   - explicit `0` or `1` → 1 worker (the old single-worker path; the rollback).
+ *   - explicit `N` → N, clamped to [1, 16].
+ *   - unset / blank / non-numeric → `clamp(cores - 1, 1, 8)` (leave a core for
+ *     the main thread + UI; never zero — parsing always needs a worker).
+ */
+export declare function resolveParsePoolSize(envVal: string | undefined, cpuCount: number): number;
+export interface ParseWorkerPoolOptions {
+    /** Languages to load grammars for in every worker at spawn. */
+    languages: Language[];
+    /** Number of worker threads (≥1). Clamp the resolved value before passing. */
+    size: number;
+    /** Compiled `parse-worker.js` path. Required unless `createWorker` is given. */
+    workerScriptPath?: string;
+    /** Parses per worker before recycle. Default 250. */
+    recycleInterval?: number;
+    /** Base per-parse timeout (ms); scaled by file size per parse. Default 10s. */
+    parseTimeoutMs?: number;
+    /** Worker factory (tests inject a fake). Defaults to a real `worker_threads` Worker. */
+    createWorker?: () => ParsePoolWorker;
+    /** Optional verbose logger (the orchestrator's `[worker] …` logger). */
+    log?: (msg: string) => void;
+}
+export declare class ParseWorkerPool {
+    private idle;
+    private queue;
+    private inflight;
+    private workers;
+    private pending;
+    private parseCounts;
+    private nextId;
+    private totalCrashes;
+    private destroyed;
+    private readonly languages;
+    private readonly maxSize;
+    private readonly recycleInterval;
+    private readonly parseTimeoutMs;
+    private readonly createWorker;
+    private readonly log;
+    constructor(opts: ParseWorkerPoolOptions);
+    /** Pool size cap (for logging). */
+    get size(): number;
+    /** Live worker count (for tests). */
+    get liveWorkers(): number;
+    /** False once the crash budget is exhausted (or after destroy). */
+    get healthy(): boolean;
+    /**
+     * Parse one file on the pool. Resolves with the extraction result, or REJECTS
+     * if the parse times out or its worker crashes — the caller records the error
+     * and (for worker-exit/OOM rejections) re-attempts in its retry pass.
+     */
+    requestParse(task: ParseTask): Promise<ExtractionResult>;
+    private spawnOne;
+    private onMessage;
+    /** A worker died (crash hook / OOM exit / spawn error). Reject its in-flight
+     *  parse so the caller's retry pass can re-attempt it, then respawn. */
+    private onWorkerGone;
+    /** Tear down a worker that has hit its recycle threshold and replace it. Not a
+     *  crash, so it doesn't count against the budget. */
+    private recycle;
+    private removeWorker;
+    private dispatch;
+    private onTimeout;
+    private drain;
+    private settle;
+    /**
+     * Recycle every idle worker now (fresh WASM heaps). The orchestrator calls
+     * this before its retry pass so crash-on-memory files get the cleanest heap.
+     */
+    recycleAll(): void;
+    /** Terminate all workers and reject any outstanding parses. */
+    destroy(): Promise<void>;
+}
+//# sourceMappingURL=parse-pool.d.ts.map

package/lib/dist/extraction/parse-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-pool.d.ts","sourceRoot":"","sources":["../../src/extraction/parse-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IACpC,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,GAAG,IAAI,CAAC;IACrD,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACjD,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAAC;CACrD;AAED;kFACkF;AAClF,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAyBD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CASzF;AAkBD,MAAM,WAAW,sBAAsB;IACrC,+DAA+D;IAC/D,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,eAAe,CAAC;IACrC,wEAAwE;IACxE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,QAAQ,CAAwC;IACxD,OAAO,CAAC,OAAO,CAA8B;IAG7C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAwB;gBAEhC,IAAI,EAAE,sBAAsB;IAiBxC,mCAAmC;IACnC,IAAI,IAAI,IAAI,MAAM,CAAyB;IAE3C,qCAAqC;IACrC,IAAI,WAAW,IAAI,MAAM,CAA8B;IAEvD,mEAAmE;IACnE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;OAIG;IACH,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAQxD,OAAO,CAAC,QAAQ;IAmBhB,OAAO,CAAC,SAAS;IAwBjB;4EACwE;IACxE,OAAO,CAAC,YAAY;IAYpB;yDACqD;IACrD,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,SAAS;IAcjB,OAAO,CAAC,KAAK;IA8Bb,OAAO,CAAC,MAAM;IAQd;;;OAGG;IACH,UAAU,IAAI,IAAI;IAIlB,+DAA+D;IACzD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAe/B"}

package/lib/dist/extraction/parse-pool.js

@@ -0,0 +1,319 @@
+"use strict";
+/**
+ * Parse worker pool — runs tree-sitter parsing across N worker threads so a full
+ * `codegraph index` uses every core instead of pinning one.
+ *
+ * Why this exists: `ExtractionOrchestrator.indexAll()` already reads files in
+ * parallel, but it parsed them through a SINGLE worker thread, so on an
+ * N-core machine indexing a large repo used one core and left the rest idle
+ * (issue #1015, the parse-time half of #320). Spreading the parse calls across a
+ * pool of workers — each its own tree-sitter WASM heap — restores multi-core
+ * throughput. SQLite storage stays on the main thread (it isn't thread-safe), so
+ * only the CPU-bound parse step is parallelised; results are stored as they
+ * arrive, in whatever order they finish.
+ *
+ * Design mirrors {@link ../mcp/query-pool} (idle-list dispatch, lazy growth,
+ * throttled cold-starts, crash recovery), with parse-specific behaviour:
+ *   - per-worker recycle: WASM linear memory grows but never shrinks, so each
+ *     worker is torn down and replaced after `recycleInterval` parses to reclaim
+ *     its heap — the same reason the old single worker recycled.
+ *   - reject, don't retry: a parse that crashes or times out its worker REJECTS
+ *     (with a message the orchestrator's retry pass recognises) rather than being
+ *     silently requeued — the orchestrator owns the smarter two-stage retry
+ *     (fresh worker, then comment-stripped) on a clean WASM heap.
+ *   - a size-1 pool reproduces the old single-worker path exactly, which is the
+ *     conservative rollback: set `CODEGRAPH_PARSE_WORKERS=1`.
+ *
+ * Memory: peak scales with pool size (≈ size × a worker's pre-recycle heap), so
+ * the default is capped and the env var lets constrained machines dial it down.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParseWorkerPool = void 0;
+exports.resolveParsePoolSize = resolveParsePoolSize;
+const worker_threads_1 = require("worker_threads");
+/** Default upper bound on the pool size derived from the core count. */
+const DEFAULT_PARSE_POOL_CAP = 8;
+/** Hard ceiling on pool size regardless of an explicit env override. */
+const MAX_PARSE_POOL_SIZE = 16;
+/** Parses a worker performs before it's recycled to reclaim WASM heap. */
+const DEFAULT_RECYCLE_INTERVAL = 250;
+/** Base per-parse timeout; scaled up for large files by the caller's formula. */
+const DEFAULT_PARSE_TIMEOUT_MS = 10_000;
+/**
+ * Max workers cold-starting at once. A worker's cold start is heavy (module load
+ * + grammar WASM compile); starting the whole pool simultaneously thrashes CPU.
+ * Warming a couple at a time keeps each start fast while the pool still reaches
+ * full size within a few parses of a large run.
+ */
+const MAX_CONCURRENT_SPAWN = 2;
+/**
+ * Total worker deaths before the pool stops respawning and fails outstanding
+ * work, so a systematically-broken worker platform degrades instead of
+ * respawning forever. Set high: normal per-file WASM crashes are cleared by the
+ * orchestrator's retry pass and shouldn't trip this on a merely-crashy repo.
+ */
+const CRASH_BUDGET = 100;
+/**
+ * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the
+ * machine's core count.
+ *   - explicit `0` or `1` → 1 worker (the old single-worker path; the rollback).
+ *   - explicit `N` → N, clamped to [1, 16].
+ *   - unset / blank / non-numeric → `clamp(cores - 1, 1, 8)` (leave a core for
+ *     the main thread + UI; never zero — parsing always needs a worker).
+ */
+function resolveParsePoolSize(envVal, cpuCount) {
+    if (envVal !== undefined && envVal !== '') {
+        const n = Number(envVal);
+        if (Number.isFinite(n) && n >= 0) {
+            return Math.max(1, Math.min(Math.floor(n), MAX_PARSE_POOL_SIZE));
+        }
+        // non-numeric / negative → fall through to the default
+    }
+    return Math.max(1, Math.min(cpuCount - 1, DEFAULT_PARSE_POOL_CAP));
+}
+class ParseWorkerPool {
+    idle = [];
+    queue = [];
+    inflight = new Map();
+    workers = new Set();
+    // Spawned but not yet 'grammars-loaded'. Growth counts these so a single first
+    // parse doesn't spawn the whole pool before the eager worker reports ready.
+    pending = new Set();
+    parseCounts = new Map();
+    nextId = 1;
+    totalCrashes = 0;
+    destroyed = false;
+    languages;
+    maxSize;
+    recycleInterval;
+    parseTimeoutMs;
+    createWorker;
+    log;
+    constructor(opts) {
+        this.languages = opts.languages;
+        this.maxSize = Math.max(1, Math.min(opts.size, MAX_PARSE_POOL_SIZE));
+        this.recycleInterval = opts.recycleInterval ?? DEFAULT_RECYCLE_INTERVAL;
+        this.parseTimeoutMs = opts.parseTimeoutMs ?? DEFAULT_PARSE_TIMEOUT_MS;
+        this.log = opts.log ?? (() => { });
+        if (opts.createWorker) {
+            this.createWorker = opts.createWorker;
+        }
+        else if (opts.workerScriptPath) {
+            const scriptPath = opts.workerScriptPath;
+            this.createWorker = () => new worker_threads_1.Worker(scriptPath);
+        }
+        else {
+            throw new Error('ParseWorkerPool requires workerScriptPath or createWorker');
+        }
+        this.spawnOne(); // one eager warm worker, ready for the first parse
+    }
+    /** Pool size cap (for logging). */
+    get size() { return this.maxSize; }
+    /** Live worker count (for tests). */
+    get liveWorkers() { return this.workers.size; }
+    /** False once the crash budget is exhausted (or after destroy). */
+    get healthy() {
+        return !this.destroyed && this.totalCrashes < CRASH_BUDGET;
+    }
+    /**
+     * Parse one file on the pool. Resolves with the extraction result, or REJECTS
+     * if the parse times out or its worker crashes — the caller records the error
+     * and (for worker-exit/OOM rejections) re-attempts in its retry pass.
+     */
+    requestParse(task) {
+        if (this.destroyed)
+            return Promise.reject(new Error('Parse pool destroyed'));
+        return new Promise((resolve, reject) => {
+            this.queue.push({ id: this.nextId++, task, resolve, reject, settled: false });
+            this.drain();
+        });
+    }
+    spawnOne() {
+        if (this.destroyed || this.workers.size >= this.maxSize || !this.healthy)
+            return;
+        let w;
+        try {
+            w = this.createWorker();
+        }
+        catch {
+            this.totalCrashes++; // counts toward the circuit breaker
+            return;
+        }
+        this.workers.add(w);
+        this.pending.add(w);
+        this.parseCounts.set(w, 0);
+        w.on('message', (m) => this.onMessage(w, (m ?? {})));
+        w.on('error', (e) => this.onWorkerGone(w, `Worker error: ${e?.message ?? 'unknown'}`));
+        w.on('exit', (code) => { if (code !== 0)
+            this.onWorkerGone(w, `Worker exited with code ${code}`); });
+        // Load grammars; the worker replies 'grammars-loaded' and only then is idle.
+        w.postMessage({ type: 'load-grammars', languages: this.languages });
+    }
+    onMessage(w, m) {
+        if (m.type === 'grammars-loaded') {
+            if (!this.workers.has(w))
+                return; // recycled/destroyed before ready
+            this.pending.delete(w);
+            this.idle.push(w);
+            this.drain();
+            return;
+        }
+        if (m.type === 'parse-result') {
+            const job = this.inflight.get(w);
+            if (!job || (m.id !== undefined && m.id !== job.id))
+                return; // stale (post-recycle)
+            this.inflight.delete(w);
+            // Recycle the worker once it's done enough parses to have grown its WASM
+            // heap; otherwise return it to the idle set for the next job.
+            if ((this.parseCounts.get(w) ?? 0) >= this.recycleInterval) {
+                this.recycle(w);
+            }
+            else {
+                this.idle.push(w);
+            }
+            this.settle(job, m.result);
+            this.drain();
+        }
+    }
+    /** A worker died (crash hook / OOM exit / spawn error). Reject its in-flight
+     *  parse so the caller's retry pass can re-attempt it, then respawn. */
+    onWorkerGone(w, message) {
+        if (!this.workers.has(w))
+            return; // already handled (error+exit both fire), or recycled
+        this.removeWorker(w);
+        this.totalCrashes++;
+        const job = this.inflight.get(w);
+        this.inflight.delete(w);
+        try {
+            void w.terminate();
+        }
+        catch { /* already gone */ }
+        if (job)
+            this.settle(job, undefined, new Error(message));
+        if (this.healthy)
+            this.spawnOne(); // keep capacity
+        this.drain();
+    }
+    /** Tear down a worker that has hit its recycle threshold and replace it. Not a
+     *  crash, so it doesn't count against the budget. */
+    recycle(w) {
+        this.log(`Recycling worker after ${this.parseCounts.get(w)} parses (heap: ${Math.round(process.memoryUsage().rss / 1024 / 1024)}MB RSS)`);
+        this.removeWorker(w);
+        // Fire-and-forget: worker.terminate() can hang if WASM is wedged.
+        try {
+            void w.terminate();
+        }
+        catch { /* already gone */ }
+        if (this.healthy && !this.destroyed)
+            this.spawnOne();
+    }
+    removeWorker(w) {
+        this.workers.delete(w);
+        this.pending.delete(w);
+        this.parseCounts.delete(w);
+        this.idle = this.idle.filter((x) => x !== w);
+    }
+    dispatch(w, job) {
+        this.inflight.set(w, job);
+        this.parseCounts.set(w, (this.parseCounts.get(w) ?? 0) + 1);
+        // Scale the timeout for large files: base + 10s per 100KB (matches the
+        // original single-worker formula so pathological-file behaviour is unchanged).
+        const timeoutMs = this.parseTimeoutMs + Math.floor(job.task.content.length / 100_000) * 10_000;
+        job.timer = setTimeout(() => this.onTimeout(w, job, timeoutMs), timeoutMs);
+        job.timer.unref?.();
+        w.postMessage({
+            type: 'parse',
+            id: job.id,
+            filePath: job.task.filePath,
+            content: job.task.content,
+            frameworkNames: job.task.frameworkNames,
+            language: job.task.language,
+        });
+    }
+    onTimeout(w, job, ms) {
+        if (job.settled || !this.workers.has(w))
+            return;
+        this.log(`TIMEOUT: ${job.task.filePath} exceeded ${ms}ms — killing worker`);
+        // Kill the (possibly WASM-wedged) worker and reject this parse. A timeout
+        // isn't a crash — don't charge the budget — but the worker is gone, so spawn
+        // a replacement to keep capacity.
+        this.removeWorker(w);
+        this.inflight.delete(w);
+        try {
+            void w.terminate();
+        }
+        catch { /* already gone */ }
+        this.settle(job, undefined, new Error(`Parse timed out after ${ms}ms`));
+        if (this.healthy)
+            this.spawnOne();
+        this.drain();
+    }
+    drain() {
+        // Grow toward maxSize while queued work outstrips workers that are idle OR
+        // already on their way up — throttled so we never cold-start the whole pool
+        // at once.
+        while (this.queue.length > this.idle.length + this.pending.size &&
+            this.workers.size < this.maxSize &&
+            this.pending.size < MAX_CONCURRENT_SPAWN &&
+            !this.destroyed &&
+            this.healthy) {
+            this.spawnOne();
+        }
+        // Dispatch queued jobs to idle workers.
+        while (this.idle.length && this.queue.length) {
+            let job;
+            while (this.queue.length && (job = this.queue.shift()) && job.settled)
+                job = undefined;
+            if (!job || job.settled)
+                break;
+            const w = this.idle.pop();
+            this.dispatch(w, job);
+        }
+        // Hang-prevention: if there's queued work but nothing can ever run it (no
+        // idle workers, none spawning, none alive), fail it instead of hanging
+        // forever. Reached only when the crash budget is exhausted or after destroy.
+        if (this.queue.length && this.idle.length === 0 && this.pending.size === 0 && this.workers.size === 0) {
+            const reason = this.destroyed ? 'parse pool destroyed' : 'parse pool exhausted its worker crash budget';
+            for (const job of this.queue.splice(0))
+                this.settle(job, undefined, new Error(reason));
+        }
+    }
+    settle(job, result, err) {
+        if (job.settled)
+            return;
+        job.settled = true;
+        if (job.timer)
+            clearTimeout(job.timer);
+        if (err)
+            job.reject(err);
+        else
+            job.resolve(result);
+    }
+    /**
+     * Recycle every idle worker now (fresh WASM heaps). The orchestrator calls
+     * this before its retry pass so crash-on-memory files get the cleanest heap.
+     */
+    recycleAll() {
+        for (const w of [...this.idle])
+            this.recycle(w);
+    }
+    /** Terminate all workers and reject any outstanding parses. */
+    async destroy() {
+        if (this.destroyed)
+            return;
+        this.destroyed = true;
+        const ws = [...this.workers];
+        this.workers.clear();
+        this.pending.clear();
+        this.parseCounts.clear();
+        this.idle = [];
+        for (const job of [...this.inflight.values(), ...this.queue]) {
+            this.settle(job, undefined, new Error('parse pool destroyed'));
+        }
+        this.inflight.clear();
+        this.queue = [];
+        await Promise.all(ws.map((w) => Promise.resolve(w.terminate()).catch(() => { })));
+    }
+}
+exports.ParseWorkerPool = ParseWorkerPool;
+//# sourceMappingURL=parse-pool.js.map