@saluzi/codegraph 0.2.0 → 0.2.1

package/dist/extraction/mybatis-extractor.js

@@ -0,0 +1,300 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MyBatisExtractor = void 0;
+const tree_sitter_helpers_1 = require("./tree-sitter-helpers");
+/**
+ * MyBatisExtractor — parses MyBatis mapper XML files.
+ *
+ * MyBatis splits a DAO interface across two files: a Java interface (parsed by
+ * tree-sitter) declares the method, and an XML mapper file holds the SQL keyed
+ * by `<namespace>` (the fully-qualified Java type name) and `id` (the method
+ * name). Without the XML side in the graph, `trace(Controller, ...DAO.method)`
+ * dead-ends at the interface method — the SQL it actually runs is invisible,
+ * and "what does this query touch" / "where is this column written" can't be
+ * answered.
+ *
+ * This extractor emits one method-shaped node per `<select|insert|update|
+ * delete>` and per `<sql>` fragment, qualified as `<namespace>::<id>` so the
+ * MyBatis framework synthesizer can link the matching Java method → XML
+ * statement by suffix-matching qualified names. `<include refid="...">` inside
+ * a statement yields an unresolved reference to the SQL fragment, also keyed
+ * by `<namespace>::<refid>`.
+ *
+ * Both dialects are covered: MyBatis 3 `<mapper namespace="...">` and the
+ * legacy iBatis 2 `<sqlMap>` (namespaced, or namespace-less with `Map.stmt`
+ * ids, plus its extra `<statement>`/`<procedure>` verbs). Attribute values may
+ * use either quote style, and statements commented out with `<!-- ... -->` are
+ * ignored (see the constructor's comment-stripping pre-pass).
+ *
+ * Non-mapper XML (Maven `pom.xml`, Spring beans XML, `web.xml`, log4j config,
+ * etc.) is detected by the absence of a `<mapper namespace="...">` /
+ * `<sqlMap>` root and returns just a file node — we still need the file row so
+ * the watcher can track it, but we emit no symbols.
+ */
+class MyBatisExtractor {
+    filePath;
+    source;
+    nodes = [];
+    edges = [];
+    unresolvedReferences = [];
+    errors = [];
+    lineStarts = [];
+    constructor(filePath, source) {
+        this.filePath = filePath;
+        // Blank out XML comments up front so commented-out statements and includes
+        // aren't matched by the scans below (a `<!-- <select id="old">…</select> -->`
+        // block must not produce a phantom node). Length-preserving — comment bytes
+        // become spaces, newlines are kept — so the offsets and line numbers
+        // computed afterwards still map to the original source. Text inside
+        // `<![CDATA[ … ]]>` is left intact: a literal `<!--` there is SQL data, not
+        // an XML comment.
+        this.source = MyBatisExtractor.stripXmlComments(source);
+        this.computeLineStarts();
+    }
+    static stripXmlComments(source) {
+        const out = source.split('');
+        const n = source.length;
+        let i = 0;
+        while (i < n) {
+            if (source.startsWith('<![CDATA[', i)) {
+                const end = source.indexOf(']]>', i + 9);
+                i = end >= 0 ? end + 3 : n;
+                continue;
+            }
+            if (source.startsWith('<!--', i)) {
+                const end = source.indexOf('-->', i + 4);
+                const stop = end >= 0 ? end + 3 : n;
+                for (let j = i; j < stop; j++) {
+                    if (source.charCodeAt(j) !== 10)
+                        out[j] = ' ';
+                }
+                i = stop;
+                continue;
+            }
+            i++;
+        }
+        return out.join('');
+    }
+    extract() {
+        const startTime = Date.now();
+        const fileNode = this.createFileNode();
+        try {
+            const root = this.findMapperRoot();
+            if (root) {
+                this.extractMapper(fileNode.id, root.namespace, root.dialect, root.bodyStart, root.bodyEnd);
+            }
+        }
+        catch (error) {
+            this.errors.push({
+                message: `MyBatis extraction error: ${error instanceof Error ? error.message : String(error)}`,
+                severity: 'error',
+                code: 'parse_error',
+            });
+        }
+        return {
+            nodes: this.nodes,
+            edges: this.edges,
+            unresolvedReferences: this.unresolvedReferences,
+            errors: this.errors,
+            durationMs: Date.now() - startTime,
+        };
+    }
+    createFileNode() {
+        const lines = this.source.split('\n');
+        const id = (0, tree_sitter_helpers_1.generateNodeId)(this.filePath, 'file', this.filePath, 1);
+        const node = {
+            id,
+            kind: 'file',
+            name: this.filePath.split('/').pop() || this.filePath,
+            qualifiedName: this.filePath,
+            filePath: this.filePath,
+            language: 'xml',
+            startLine: 1,
+            endLine: lines.length || 1,
+            startColumn: 0,
+            endColumn: lines[lines.length - 1]?.length ?? 0,
+            updatedAt: Date.now(),
+        };
+        this.nodes.push(node);
+        return node;
+    }
+    /**
+     * Find the mapper root and its dialect. Two shapes are recognized:
+     *   - MyBatis 3: `<mapper namespace="com.foo.Bar">` — namespace required.
+     *   - iBatis 2:  `<sqlMap namespace="Account">`, or a namespace-less
+     *     `<sqlMap>` whose statement ids carry the qualifier as `Map.statement`.
+     * Returns the namespace, the dialect, and the byte offsets of the body
+     * (between the opening and closing tag) so statement extraction is scoped to
+     * the root's contents. Either quote style is accepted for the namespace
+     * (`namespace='X'` is legal XML and common in older mappers).
+     */
+    findMapperRoot() {
+        const mapper = /<mapper\b([^>]*)>/.exec(this.source);
+        if (mapper) {
+            const nsMatch = /\bnamespace\s*=\s*(["'])([^"']+)\1/.exec(mapper[1] ?? '');
+            if (nsMatch) {
+                const bodyStart = mapper.index + mapper[0].length;
+                const closeIdx = this.source.indexOf('</mapper>', bodyStart);
+                return {
+                    namespace: nsMatch[2],
+                    dialect: 'mybatis',
+                    bodyStart,
+                    bodyEnd: closeIdx >= 0 ? closeIdx : this.source.length,
+                };
+            }
+        }
+        // iBatis 2 SqlMap. `\b` keeps `<sqlMapConfig>` (the iBatis config root,
+        // which holds no statements) from matching here. namespace is optional.
+        const sqlMap = /<sqlMap\b([^>]*)>/.exec(this.source);
+        if (sqlMap) {
+            const nsMatch = /\bnamespace\s*=\s*(["'])([^"']+)\1/.exec(sqlMap[1] ?? '');
+            const bodyStart = sqlMap.index + sqlMap[0].length;
+            const closeIdx = this.source.indexOf('</sqlMap>', bodyStart);
+            return {
+                namespace: nsMatch?.[2] ?? '',
+                dialect: 'ibatis',
+                bodyStart,
+                bodyEnd: closeIdx >= 0 ? closeIdx : this.source.length,
+            };
+        }
+        return null;
+    }
+    extractMapper(fileNodeId, namespace, dialect, bodyStart, bodyEnd) {
+        const body = this.source.slice(bodyStart, bodyEnd);
+        // Match each top-level statement-shaped element. The body may have nested
+        // tags (`<if>`, `<foreach>`, `<include>`), so we scan with a regex that
+        // pairs an opening tag to its matching close — the simple form below works
+        // because MyBatis/iBatis statement elements are not themselves nested.
+        // iBatis 2 adds the generic `<statement>` and `<procedure>` on top of the
+        // MyBatis 3 verbs; gating by dialect keeps MyBatis extraction unchanged.
+        const verbs = dialect === 'ibatis'
+            ? 'select|insert|update|delete|sql|statement|procedure'
+            : 'select|insert|update|delete|sql';
+        const stmtRegex = new RegExp(`<(${verbs})\\b([^>]*)>([\\s\\S]*?)</\\1>`, 'g');
+        let m;
+        while ((m = stmtRegex.exec(body)) !== null) {
+            const elemType = m[1];
+            const attrs = m[2] ?? '';
+            const elemBody = m[3] ?? '';
+            // Accept either quote style (`(["'])…\1`). The identifier-shaped MyBatis
+            // attributes matched here and below (namespace/id/refid/resultType/
+            // parameterType) are Java FQNs, method names, or type aliases and never
+            // contain a quote character, so excluding both quotes from the value is safe.
+            const idMatch = /\bid\s*=\s*(["'])([^"']+)\1/.exec(attrs);
+            if (!idMatch)
+                continue;
+            const id = idMatch[2];
+            const absoluteIndex = bodyStart + m.index;
+            const startLine = this.getLineNumber(absoluteIndex);
+            const endLine = this.getLineNumber(absoluteIndex + m[0].length);
+            const { qualifiedName: qualified, name } = this.qualifyStatement(namespace, id);
+            const isSqlFragment = elemType === 'sql';
+            // The id-hash folds in the statement's byte offset (unique per statement
+            // in the file), not just the start line: two statements sharing a
+            // qualifiedName AND a start line — e.g. a vendor-split `databaseId` pair
+            // (`<select id="x" databaseId="oracle">…</select><select id="x"
+            // databaseId="mysql">…`) written on one line — would otherwise hash to
+            // the same node id, and `INSERT OR REPLACE INTO nodes` (id is the PRIMARY
+            // KEY) would silently drop one. qualifiedName and startLine are stored
+            // unchanged, so the Java↔XML suffix-match bridge is untouched.
+            const nodeId = (0, tree_sitter_helpers_1.generateNodeId)(this.filePath, 'method', qualified, absoluteIndex);
+            const node = {
+                id: nodeId,
+                kind: 'method',
+                name,
+                qualifiedName: qualified,
+                filePath: this.filePath,
+                language: 'xml',
+                signature: this.buildSignature(elemType, attrs, isSqlFragment),
+                startLine,
+                endLine,
+                startColumn: 0,
+                endColumn: 0,
+                docstring: this.previewSql(elemBody),
+                updatedAt: Date.now(),
+            };
+            this.nodes.push(node);
+            this.edges.push({ source: fileNodeId, target: nodeId, kind: 'contains' });
+            // <include refid="X"/> → reference to the SQL fragment in this mapper
+            // (or in another mapper, when the refid is qualified — `ns.X`).
+            const includeRegex = /<include\b[^>]*\brefid\s*=\s*(["'])([^"']+)\1/g;
+            let inc;
+            while ((inc = includeRegex.exec(elemBody)) !== null) {
+                const refid = inc[2];
+                const refQualified = refid.includes('.')
+                    ? refid.replace(/\./g, '::')
+                    : namespace
+                        ? `${namespace}::${refid}`
+                        : refid;
+                const includeOffset = absoluteIndex + (m[0].length - m[3].length - `</${elemType}>`.length) + inc.index;
+                const line = this.getLineNumber(includeOffset);
+                this.unresolvedReferences.push({
+                    fromNodeId: nodeId,
+                    referenceName: refQualified,
+                    referenceKind: 'references',
+                    line,
+                    column: 0,
+                });
+            }
+        }
+    }
+    buildSignature(elemType, attrs, isSqlFragment) {
+        if (isSqlFragment)
+            return '<sql>';
+        const verb = elemType.toUpperCase();
+        const result = /\bresultType\s*=\s*(["'])([^"']+)\1/.exec(attrs)?.[2];
+        const param = /\bparameterType\s*=\s*(["'])([^"']+)\1/.exec(attrs)?.[2];
+        // A vendor-split statement carries `databaseId`; surface it so the two
+        // otherwise-identical `<namespace>::<id>` nodes are distinguishable.
+        const dbId = /\bdatabaseId\s*=\s*(["'])([^"']+)\1/.exec(attrs)?.[2];
+        const parts = [verb];
+        if (param)
+            parts.push(`param=${param}`);
+        if (result)
+            parts.push(`result=${result}`);
+        if (dbId)
+            parts.push(`databaseId=${dbId}`);
+        return parts.join(' ');
+    }
+    /**
+     * Build the `<namespace>::<id>` qualified name the MyBatis synthesizer
+     * suffix-matches against a Java `<Class>::<method>`, and the display name.
+     * For a namespace-less iBatis `<sqlMap>`, the statement id carries the
+     * qualifier as `Map.statement`, so split on the last dot to reach the same
+     * shape (`Account.getById` → `Account::getById`, name `getById`).
+     */
+    qualifyStatement(namespace, id) {
+        if (namespace)
+            return { qualifiedName: `${namespace}::${id}`, name: id };
+        const dot = id.lastIndexOf('.');
+        if (dot >= 0) {
+            return { qualifiedName: `${id.slice(0, dot)}::${id.slice(dot + 1)}`, name: id.slice(dot + 1) };
+        }
+        return { qualifiedName: id, name: id };
+    }
+    previewSql(body) {
+        return body.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 200);
+    }
+    computeLineStarts() {
+        this.lineStarts = [0];
+        for (let i = 0; i < this.source.length; i++) {
+            if (this.source.charCodeAt(i) === 10)
+                this.lineStarts.push(i + 1);
+        }
+    }
+    getLineNumber(offset) {
+        // Binary search
+        let lo = 0;
+        let hi = this.lineStarts.length - 1;
+        while (lo < hi) {
+            const mid = (lo + hi + 1) >>> 1;
+            if (this.lineStarts[mid] <= offset)
+                lo = mid;
+            else
+                hi = mid - 1;
+        }
+        return lo + 1;
+    }
+}
+exports.MyBatisExtractor = MyBatisExtractor;
+//# sourceMappingURL=mybatis-extractor.js.map

package/dist/extraction/mybatis-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mybatis-extractor.js","sourceRoot":"","sources":["../../src/extraction/mybatis-extractor.ts"],"names":[],"mappings":";;;AACA,+DAAuD;AAEvD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAa,gBAAgB;IACnB,QAAQ,CAAS;IACjB,MAAM,CAAS;IACf,KAAK,GAAW,EAAE,CAAC;IACnB,KAAK,GAAW,EAAE,CAAC;IACnB,oBAAoB,GAA0B,EAAE,CAAC;IACjD,MAAM,GAAsB,EAAE,CAAC;IAC/B,UAAU,GAAa,EAAE,CAAC;IAElC,YAAY,QAAgB,EAAE,MAAc;QAC1C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,2EAA2E;QAC3E,8EAA8E;QAC9E,4EAA4E;QAC5E,qEAAqE;QACrE,oEAAoE;QACpE,4EAA4E;QAC5E,kBAAkB;QAClB,IAAI,CAAC,MAAM,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,MAAM,CAAC,gBAAgB,CAAC,MAAc;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3B,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC9B,IAAI,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE;wBAAE,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;gBAChD,CAAC;gBACD,CAAC,GAAG,IAAI,CAAC;gBACT,SAAS;YACX,CAAC;YACD,CAAC,EAAE,CAAC;QACN,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,CAAC;IAED,OAAO;QACL,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,MAAM,QAAQ,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QAEvC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;YACnC,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9F,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACf,OAAO,EAAE,6BAA6B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBAC9F,QAAQ,EAAE,OAAO;gBACjB,IAAI,EAAE,aAAa;aACpB,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SACnC,CAAC;IACJ,CAAC;IAEO,cAAc;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,IAAA,oCAAc,EAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAS;YACjB,EAAE;YACF,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ;YACrD,aAAa,EAAE,IAAI,CAAC,QAAQ;YAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;YAC1B,WAAW,EAAE,CAAC;YACd,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC;YAC/C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;OASG;IACK,cAAc;QAGpB,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,oCAAoC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3E,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;gBAC7D,OAAO;oBACL,SAAS,EAAE,OAAO,CAAC,CAAC,CAAE;oBACtB,OAAO,EAAE,SAAS;oBAClB,SAAS;oBACT,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;iBACvD,CAAC;YACJ,CAAC;QACH,CAAC;QACD,wEAAwE;QACxE,wEAAwE;QACxE,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,OAAO,GAAG,oCAAoC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3E,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC7D,OAAO;gBACL,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;gBAC7B,OAAO,EAAE,QAAQ;gBACjB,SAAS;gBACT,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;aACvD,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,aAAa,CACnB,UAAkB,EAClB,SAAiB,EACjB,OAA6B,EAC7B,SAAiB,EACjB,OAAe;QAEf,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACnD,0EAA0E;QAC1E,wEAAwE;QACxE,2EAA2E;QAC3E,uEAAuE;QACvE,0EAA0E;QAC1E,yEAAyE;QACzE,MAAM,KAAK,GACT,OAAO,KAAK,QAAQ;YAClB,CAAC,CAAC,qDAAqD;YACvD,CAAC,CAAC,iCAAiC,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,KAAK,KAAK,gCAAgC,EAAE,GAAG,CAAC,CAAC;QAC9E,IAAI,CAAyB,CAAC;QAC9B,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,yEAAyE;YACzE,oEAAoE;YACpE,wEAAwE;YACxE,8EAA8E;YAC9E,MAAM,OAAO,GAAG,6BAA6B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1D,IAAI,CAAC,OAAO;gBAAE,SAAS;YACvB,MAAM,EAAE,GAAG,OAAO,CAAC,CAAC,CAAE,CAAC;YACvB,MAAM,aAAa,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC;YAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAChF,MAAM,aAAa,GAAG,QAAQ,KAAK,KAAK,CAAC;YACzC,yEAAyE;YACzE,kEAAkE;YAClE,yEAAyE;YACzE,gEAAgE;YAChE,uEAAuE;YACvE,0EAA0E;YAC1E,uEAAuE;YACvE,+DAA+D;YAC/D,MAAM,MAAM,GAAG,IAAA,oCAAc,EAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YACjF,MAAM,IAAI,GAAS;gBACjB,EAAE,EAAE,MAAM;gBACV,IAAI,EAAE,QAAQ;gBACd,IAAI;gBACJ,aAAa,EAAE,SAAS;gBACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,QAAQ,EAAE,KAAK;gBACf,SAAS,EAAE,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,EAAE,aAAa,CAAC;gBAC9D,SAAS;gBACT,OAAO;gBACP,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACpC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAE1E,sEAAsE;YACtE,gEAAgE;YAChE,MAAM,YAAY,GAAG,gDAAgD,CAAC;YACtE,IAAI,GAA2B,CAAC;YAChC,OAAO,CAAC,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;gBACtB,MAAM,YAAY,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;oBACtC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;oBAC5B,CAAC,CAAC,SAAS;wBACT,CAAC,CAAC,GAAG,SAAS,KAAK,KAAK,EAAE;wBAC1B,CAAC,CAAC,KAAK,CAAC;gBACZ,MAAM,aAAa,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC;gBACzG,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;gBAC/C,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC;oBAC7B,UAAU,EAAE,MAAM;oBAClB,aAAa,EAAE,YAAY;oBAC3B,aAAa,EAAE,YAAY;oBAC3B,IAAI;oBACJ,MAAM,EAAE,CAAC;iBACV,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,KAAa,EAAE,aAAsB;QAC5E,IAAI,aAAa;YAAE,OAAO,OAAO,CAAC;QAClC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,wCAAwC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,uEAAuE;QACvE,qEAAqE;QACrE,MAAM,IAAI,GAAG,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC;QACxC,IAAI,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;QAC3C,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,SAAiB,EAAE,EAAU;QACpD,IAAI,SAAS;YAAE,OAAO,EAAE,aAAa,EAAE,GAAG,SAAS,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QACzE,MAAM,GAAG,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC;QACjG,CAAC;QACD,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACzC,CAAC;IAEO,UAAU,CAAC,IAAY;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjF,CAAC;IAEO,iBAAiB;QACvB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,EAAE;gBAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,aAAa,CAAC,MAAc;QAClC,gBAAgB;QAChB,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC;QACpC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAE,IAAI,MAAM;gBAAE,EAAE,GAAG,GAAG,CAAC;;gBACzC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,EAAE,GAAG,CAAC,CAAC;IAChB,CAAC;CACF;AArRD,4CAqRC"}

package/dist/extraction/parse-pool.d.ts

@@ -0,0 +1,152 @@
+/**
+ * Parse worker pool — runs tree-sitter parsing across N worker threads so a full
+ * `codegraph index` uses every core instead of pinning one.
+ *
+ * Why this exists: `ExtractionOrchestrator.indexAll()` already reads files in
+ * parallel, but it parsed them through a SINGLE worker thread, so on an
+ * N-core machine indexing a large repo used one core and left the rest idle
+ * (issue #1015, the parse-time half of #320). Spreading the parse calls across a
+ * pool of workers — each its own tree-sitter WASM heap — restores multi-core
+ * throughput. SQLite storage stays on the main thread (it isn't thread-safe), so
+ * only the CPU-bound parse step is parallelised; results are stored as they
+ * arrive, in whatever order they finish.
+ *
+ * Design mirrors {@link ../mcp/query-pool} (idle-list dispatch, lazy growth,
+ * throttled cold-starts, crash recovery), with parse-specific behaviour:
+ *   - per-worker recycle: WASM linear memory grows but never shrinks, so each
+ *     worker is torn down and replaced after `recycleInterval` parses to reclaim
+ *     its heap — the same reason the old single worker recycled.
+ *   - reject, don't retry: a parse that crashes or times out its worker REJECTS
+ *     (with a message the orchestrator's retry pass recognises) rather than being
+ *     silently requeued — the orchestrator owns the smarter two-stage retry
+ *     (fresh worker, then comment-stripped) on a clean WASM heap.
+ *   - a size-1 pool reproduces the old single-worker path exactly, which is the
+ *     conservative rollback: set `CODEGRAPH_PARSE_WORKERS=1`.
+ *
+ * Memory: peak scales with pool size (≈ size × a worker's pre-recycle heap), so
+ * the default is capped and the env var lets constrained machines dial it down.
+ */
+import type { Language, ExtractionResult } from '../types';
+/**
+ * Minimal worker surface the pool drives — satisfied by a real `worker_threads`
+ * Worker. Abstracted so tests can inject a fake worker and exercise the pool's
+ * queue / growth / recycle / crash-recovery logic without spawning threads or a
+ * built `dist/`.
+ */
+export interface ParsePoolWorker {
+    postMessage(msg: unknown): void;
+    terminate(): Promise<number> | void;
+    on(event: 'message', cb: (m: unknown) => void): void;
+    on(event: 'error', cb: (e: Error) => void): void;
+    on(event: 'exit', cb: (code: number) => void): void;
+}
+/** A single file to parse. `language` is resolved on the main thread (it holds
+ *  the project's codegraph.json extension overrides) and handed to the worker. */
+export interface ParseTask {
+    filePath: string;
+    content: string;
+    language: Language;
+    frameworkNames?: string[];
+}
+/**
+ * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the
+ * machine's core count.
+ *   - explicit `0` or `1` → 1 worker (the old single-worker path; the rollback).
+ *   - explicit `N` → N, clamped to [1, 16].
+ *   - unset / blank / non-numeric → `clamp(cores - 1, 1, 8)` (leave a core for
+ *     the main thread + UI; never zero — parsing always needs a worker).
+ */
+/**
+ * Resolve the base per-parse timeout from the `CODEGRAPH_PARSE_TIMEOUT_MS`
+ * override. Slow storage (HDD, network folders) can need a larger budget; a
+ * non-numeric / non-positive value falls back to the default (10s).
+ */
+export declare function resolveParseTimeoutMs(envVal: string | undefined): number;
+export declare function resolveParsePoolSize(envVal: string | undefined, cpuCount: number): number;
+export interface ParseWorkerPoolOptions {
+    /** Languages to load grammars for in every worker at spawn. */
+    languages: Language[];
+    /** Number of worker threads (≥1). Clamp the resolved value before passing. */
+    size: number;
+    /** Compiled `parse-worker.js` path. Required unless `createWorker` is given. */
+    workerScriptPath?: string;
+    /** Parses per worker before recycle. Default 250. */
+    recycleInterval?: number;
+    /** Base per-parse timeout (ms); scaled by file size per parse. Default 10s. */
+    parseTimeoutMs?: number;
+    /** Worker factory (tests inject a fake). Defaults to a real `worker_threads` Worker. */
+    createWorker?: () => ParsePoolWorker;
+    /** Optional verbose logger (the orchestrator's `[worker] …` logger). */
+    log?: (msg: string) => void;
+    /**
+     * Pre-read grammar WASM bytes keyed by language, forwarded to every worker's
+     * `load-grammars` message so a spawn/respawn loads grammars from memory
+     * instead of re-reading them from disk — on slow storage each respawn's
+     * grammar re-read otherwise amplifies the very I/O contention that caused
+     * the respawn (issue #1231). Best-effort: a missing language falls back to
+     * the worker's own disk read.
+     */
+    grammarBuffers?: Record<string, Uint8Array>;
+}
+export declare class ParseWorkerPool {
+    private idle;
+    private queue;
+    private inflight;
+    private workers;
+    private pending;
+    private parseCounts;
+    private nextId;
+    private totalCrashes;
+    private destroyed;
+    private readonly languages;
+    private readonly maxSize;
+    private readonly recycleInterval;
+    private readonly parseTimeoutMs;
+    private readonly createWorker;
+    private readonly log;
+    private readonly grammarBuffers?;
+    constructor(opts: ParseWorkerPoolOptions);
+    /** Pool size cap (for logging). */
+    get size(): number;
+    /** Live worker count (for tests). */
+    get liveWorkers(): number;
+    /** False once the crash budget is exhausted (or after destroy). */
+    get healthy(): boolean;
+    /**
+     * Parse one file on the pool. Resolves with the extraction result, or REJECTS
+     * if the parse times out or its worker crashes — the caller records the error
+     * and (for worker-exit/OOM/timeout rejections) re-attempts in its retry pass.
+     */
+    requestParse(task: ParseTask): Promise<ExtractionResult>;
+    private spawnOne;
+    private onMessage;
+    /** A worker died (crash hook / OOM exit / spawn error). Reject its in-flight
+     *  parse so the caller's retry pass can re-attempt it, then respawn. */
+    private onWorkerGone;
+    /** Tear down a worker that has hit its recycle threshold and replace it. Not a
+     *  crash, so it doesn't count against the budget. */
+    private recycle;
+    private removeWorker;
+    private dispatch;
+    /**
+     * The base timer fired with no result processed yet. Do NOT kill or settle:
+     * the timer firing doesn't prove the parse is still running — after a long
+     * synchronous main-thread stretch Node services the timers phase before the
+     * poll phase, so an already-delivered `parse-result` is still queued behind
+     * this callback. Mark the job late (onMessage accepts a result that shows up)
+     * and arm the hard-kill backstop for workers that are genuinely hung.
+     */
+    private onTimeout;
+    /** No result after the full hard-kill window — the worker really is hung. */
+    private onHardTimeout;
+    private drain;
+    private settle;
+    /**
+     * Recycle every idle worker now (fresh WASM heaps). The orchestrator calls
+     * this before its retry pass so crash-on-memory files get the cleanest heap.
+     */
+    recycleAll(): void;
+    /** Terminate all workers and reject any outstanding parses. */
+    destroy(): Promise<void>;
+}
+//# sourceMappingURL=parse-pool.d.ts.map

package/dist/extraction/parse-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-pool.d.ts","sourceRoot":"","sources":["../../src/extraction/parse-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAE3D;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;IACpC,EAAE,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,GAAG,IAAI,CAAC;IACrD,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,GAAG,IAAI,CAAC;IACjD,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAAC;CACrD;AAED;kFACkF;AAClF,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,QAAQ,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAqCD;;;;;;;GAOG;AACH;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAMxE;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CASzF;AAyBD,MAAM,WAAW,sBAAsB;IACrC,+DAA+D;IAC/D,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,8EAA8E;IAC9E,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,eAAe,CAAC;IACrC,wEAAwE;IACxE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;CAC7C;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,IAAI,CAAyB;IACrC,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,QAAQ,CAAwC;IACxD,OAAO,CAAC,OAAO,CAA8B;IAG7C,OAAO,CAAC,OAAO,CAA8B;IAC7C,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,SAAS,CAAS;IAE1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwB;IACrD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAwB;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAA6B;gBAEjD,IAAI,EAAE,sBAAsB;IAkBxC,mCAAmC;IACnC,IAAI,IAAI,IAAI,MAAM,CAAyB;IAE3C,qCAAqC;IACrC,IAAI,WAAW,IAAI,MAAM,CAA8B;IAEvD,mEAAmE;IACnE,IAAI,OAAO,IAAI,OAAO,CAErB;IAED;;;;OAIG;IACH,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAQxD,OAAO,CAAC,QAAQ;IAqBhB,OAAO,CAAC,SAAS;IAwCjB;4EACwE;IACxE,OAAO,CAAC,YAAY;IAYpB;yDACqD;IACrD,OAAO,CAAC,OAAO;IAQf,OAAO,CAAC,YAAY;IAOpB,OAAO,CAAC,QAAQ;IAmBhB;;;;;;;OAOG;IACH,OAAO,CAAC,SAAS;IASjB,6EAA6E;IAC7E,OAAO,CAAC,aAAa;IAerB,OAAO,CAAC,KAAK;IA8Bb,OAAO,CAAC,MAAM;IASd;;;OAGG;IACH,UAAU,IAAI,IAAI;IAIlB,+DAA+D;IACzD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAe/B"}