codevault 1.6.1 → 1.7.0

package/dist/core/indexing/chunk-pipeline.d.ts

@@ -0,0 +1,39 @@
+import type { LanguageRule } from '../../languages/rules.js';
+import type { ModelProfile } from '../../providers/base.js';
+type SizeLimits = {
+    optimal: number;
+    min: number;
+    max: number;
+    overlap: number;
+    unit: string;
+};
+interface ExistingChunks {
+    staleChunkIds: Set<string>;
+    existingChunks: Map<string, any>;
+}
+interface EmbedStoreParams {
+    code: string;
+    enhancedEmbeddingText: string;
+    chunkId: string;
+    sha: string;
+    lang: string;
+    rel: string;
+    symbol: string;
+    chunkType: string;
+    codevaultMetadata: any;
+    importantVariables: any[];
+    docComments: string | null;
+    contextInfo: any;
+    symbolData: any;
+}
+export declare class ChunkPipeline {
+    private parser;
+    private processedNodes;
+    constructor();
+    collectNodesForFile(source: string, rule: LanguageRule): Promise<import("tree-sitter").SyntaxNode[]>;
+    processGroups(nodeGroups: any[], source: string, rule: LanguageRule, limits: SizeLimits, modelProfile: ModelProfile, rel: string, existing: ExistingChunks, chunkMerkleHashes: string[], onProgress: any, embedAndStore: (params: EmbedStoreParams) => Promise<void>, chunkingStats: any): Promise<void>;
+    private yieldChunk;
+    private processChunk;
+}
+export {};
+//# sourceMappingURL=chunk-pipeline.d.ts.map

package/dist/core/indexing/chunk-pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAE5D,KAAK,UAAU,GAAG;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,UAAU,cAAc;IACtB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC3B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClC;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,qBAAqB,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,GAAG,CAAC;IACvB,kBAAkB,EAAE,GAAG,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,GAAG,CAAC;IACjB,UAAU,EAAE,GAAG,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,cAAc,CAAqB;;IAMrC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY;IA6DtD,aAAa,CACjB,UAAU,EAAE,GAAG,EAAE,EACjB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,cAAc,EACxB,iBAAiB,EAAE,MAAM,EAAE,EAC3B,UAAU,EAAE,GAAG,EACf,aAAa,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,OAAO,CAAC,IAAI,CAAC,EAC1D,aAAa,EAAE,GAAG,GACjB,OAAO,CAAC,IAAI,CAAC;YAgCF,UAAU;YAqGV,YAAY;CAmF3B"}

package/dist/core/indexing/chunk-pipeline.js

@@ -0,0 +1,210 @@
+import Parser from 'tree-sitter';
+import crypto from 'crypto';
+import { analyzeNodeForChunking, batchAnalyzeNodes, yieldStatementChunks } from '../../chunking/semantic-chunker.js';
+import { createCombinedChunk } from '../../chunking/file-grouper.js';
+import { extractSymbolMetadata } from '../../symbols/extract.js';
+import { extractSymbolName } from '../symbol-extractor.js';
+import { extractCodevaultMetadata, extractSemanticTags, extractImportantVariables, extractDocComments, generateEnhancedEmbeddingText } from '../metadata.js';
+import { computeFastHash } from '../../indexer/merkle.js';
+import { SIZE_THRESHOLD, CHUNK_SIZE } from '../../config/constants.js';
+export class ChunkPipeline {
+    parser;
+    processedNodes = new Set();
+    constructor() {
+        this.parser = new Parser();
+    }
+    async collectNodesForFile(source, rule) {
+        this.parser.setLanguage(rule.ts);
+        let tree;
+        if (source.length > SIZE_THRESHOLD) {
+            tree = this.parser.parse((index) => {
+                if (index < source.length) {
+                    return source.slice(index, Math.min(index + CHUNK_SIZE, source.length));
+                }
+                return null;
+            });
+        }
+        else {
+            tree = this.parser.parse(source);
+        }
+        if (!tree || !tree.rootNode) {
+            throw new Error('Failed to create syntax tree');
+        }
+        const collectedNodes = [];
+        const collectNodes = (node) => {
+            if (node.type === 'export_statement') {
+                let hasDeclaration = false;
+                for (let i = 0; i < node.childCount; i++) {
+                    const child = node.child(i);
+                    if (child && ['function_declaration', 'class_declaration', 'method_definition'].includes(child.type)) {
+                        hasDeclaration = true;
+                        break;
+                    }
+                }
+                if (!hasDeclaration && rule.nodeTypes.includes(node.type)) {
+                    collectedNodes.push(node);
+                    return;
+                }
+                if (hasDeclaration) {
+                    for (let i = 0; i < node.childCount; i++) {
+                        const child = node.child(i);
+                        if (child) {
+                            collectNodes(child);
+                        }
+                    }
+                    return;
+                }
+            }
+            if (rule.nodeTypes.includes(node.type)) {
+                collectedNodes.push(node);
+            }
+            for (let i = 0; i < node.childCount; i++) {
+                const child = node.child(i);
+                if (child) {
+                    collectNodes(child);
+                }
+            }
+        };
+        collectNodes(tree.rootNode);
+        return collectedNodes;
+    }
+    async processGroups(nodeGroups, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats) {
+        this.processedNodes = new Set();
+        for (const nodeGroup of nodeGroups) {
+            if (nodeGroup.nodes.length === 1) {
+                await this.yieldChunk(nodeGroup.nodes[0], source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+            }
+            else {
+                const combinedChunk = createCombinedChunk(nodeGroup, source);
+                if (combinedChunk) {
+                    chunkingStats.totalNodes += nodeGroup.nodes.length;
+                    chunkingStats.fileGrouped = (chunkingStats.fileGrouped || 0) + 1;
+                    chunkingStats.functionsGrouped = (chunkingStats.functionsGrouped || 0) + nodeGroup.nodes.length;
+                    await this.processChunk(combinedChunk.node, combinedChunk.code, `group_${nodeGroup.nodes.length}funcs`, null, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+                }
+            }
+        }
+    }
+    async yieldChunk(node, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats, parentNode = null) {
+        chunkingStats.totalNodes++;
+        const analysis = await analyzeNodeForChunking(node, source, rule, modelProfile);
+        if (analysis.size < limits.min && parentNode !== null) {
+            chunkingStats.skippedSmall++;
+            return;
+        }
+        if (analysis.needsSubdivision && analysis.subdivisionCandidates.length > 0) {
+            chunkingStats.subdivided++;
+            const subAnalyses = await batchAnalyzeNodes(analysis.subdivisionCandidates, source, rule, modelProfile, true);
+            const smallChunks = [];
+            for (let i = 0; i < subAnalyses.length; i++) {
+                const subAnalysis = subAnalyses[i];
+                const subNode = subAnalysis.node;
+                if (subAnalysis.size < limits.min) {
+                    const subCode = source.slice(subNode.startIndex, subNode.endIndex);
+                    smallChunks.push({
+                        node: subNode,
+                        code: subCode,
+                        size: subAnalysis.size
+                    });
+                    if (subNode.id !== undefined) {
+                        this.processedNodes.add(subNode.id);
+                    }
+                }
+                else {
+                    if (subNode.id !== undefined) {
+                        this.processedNodes.add(subNode.id);
+                    }
+                    await this.yieldChunk(subNode, source, rule, limits, modelProfile, rel, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats, node);
+                }
+            }
+            if (smallChunks.length > 0) {
+                const totalSmallSize = smallChunks.reduce((sum, c) => sum + c.size, 0);
+                if (totalSmallSize >= limits.min || smallChunks.length >= 3) {
+                    const mergedCode = smallChunks.map((c) => c.code).join('\n\n');
+                    const mergedNode = {
+                        ...node,
+                        type: `${node.type}_merged`,
+                        startIndex: smallChunks[0].node.startIndex,
+                        endIndex: smallChunks[smallChunks.length - 1].node.endIndex
+                    };
+                    const suffix = `small_methods_${smallChunks.length}`;
+                    chunkingStats.mergedSmall++;
+                    await this.processChunk(mergedNode, mergedCode, suffix, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+                }
+                else {
+                    chunkingStats.skippedSmall += smallChunks.length;
+                }
+            }
+            return;
+        }
+        else if (analysis.size > limits.max) {
+            chunkingStats.statementFallback++;
+            const statementChunks = await yieldStatementChunks(node, source, limits.max, limits.overlap, modelProfile);
+            for (let i = 0; i < statementChunks.length; i++) {
+                const stmtChunk = statementChunks[i];
+                await this.processChunk(node, stmtChunk.code, `${i + 1}`, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+            }
+            return;
+        }
+        chunkingStats.normalChunks++;
+        const code = source.slice(node.startIndex, node.endIndex);
+        await this.processChunk(node, code, null, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats);
+    }
+    async processChunk(node, code, suffix, parentNode, source, rel, rule, existing, chunkMerkleHashes, onProgress, embedAndStore, chunkingStats) {
+        let symbol = extractSymbolName(node, source);
+        if (!symbol)
+            return;
+        if (suffix) {
+            symbol = `${symbol}_part${suffix}`;
+        }
+        const docComments = extractDocComments(source, node, rule);
+        const codevaultMetadata = extractCodevaultMetadata(docComments);
+        const automaticTags = extractSemanticTags(rel, symbol, code);
+        const allTags = [...new Set([...codevaultMetadata.tags, ...automaticTags])];
+        codevaultMetadata.tags = allTags;
+        const importantVariables = extractImportantVariables(node, source, rule);
+        const symbolData = extractSymbolMetadata({ node, source, symbol });
+        const enhancedEmbeddingText = generateEnhancedEmbeddingText(code, codevaultMetadata, importantVariables, docComments);
+        const chunkType = node.type.includes('class') ? 'class' :
+            node.type.includes('method') ? 'method' : 'function';
+        const contextInfo = {
+            nodeType: node.type,
+            startLine: source.slice(0, node.startIndex).split('\n').length,
+            endLine: source.slice(0, node.endIndex).split('\n').length,
+            codeLength: code.length,
+            hasDocumentation: !!docComments,
+            variableCount: importantVariables.length,
+            isSubdivision: !!suffix,
+            hasParentContext: !!parentNode
+        };
+        const sha = crypto.createHash('sha1').update(code).digest('hex');
+        const chunkId = `${rel}:${symbol}:${sha.substring(0, 8)}`;
+        const chunkMerkleHash = await computeFastHash(code);
+        if (existing.existingChunks.has(chunkId)) {
+            existing.staleChunkIds.delete(chunkId);
+            chunkMerkleHashes.push(chunkMerkleHash);
+            return;
+        }
+        await embedAndStore({
+            code,
+            enhancedEmbeddingText,
+            chunkId,
+            sha,
+            lang: rule.lang,
+            rel,
+            symbol,
+            chunkType,
+            codevaultMetadata,
+            importantVariables,
+            docComments,
+            contextInfo,
+            symbolData
+        });
+        existing.staleChunkIds.delete(chunkId);
+        chunkMerkleHashes.push(chunkMerkleHash);
+        if (onProgress) {
+            onProgress({ type: 'chunk_processed', file: rel, symbol, chunkId });
+        }
+    }
+}
+//# sourceMappingURL=chunk-pipeline.js.map

package/dist/core/indexing/chunk-pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-pipeline.js","sourceRoot":"","sources":["../../../src/core/indexing/chunk-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oCAAoC,CAAC;AACrH,OAAO,EAAyB,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AAC5F,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,EACL,wBAAwB,EACxB,mBAAmB,EACnB,yBAAyB,EACzB,kBAAkB,EAClB,6BAA6B,EAC9B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAkCvE,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IAE3C;QACE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,MAAc,EAAE,IAAkB;QAC1D,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC;QACT,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YACnC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAa,EAAE,EAAE;gBACzC,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAC1B,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAC1E,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,CAAC,IAAoB,EAAE,EAAE;YAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,kBAAkB,EAAE,CAAC;gBACtC,IAAI,cAAc,GAAG,KAAK,CAAC;gBAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC5B,IAAI,KAAK,IAAI,CAAC,sBAAsB,EAAE,mBAAmB,EAAE,mBAAmB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;wBACrG,cAAc,GAAG,IAAI,CAAC;wBACtB,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1D,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBAED,IAAI,cAAc,EAAE,CAAC;oBACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;wBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBAC5B,IAAI,KAAK,EAAE,CAAC;4BACV,YAAY,CAAC,KAAK,CAAC,CAAC;wBACtB,CAAC;oBACH,CAAC;oBACD,OAAO;gBACT,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5B,IAAI,KAAK,EAAE,CAAC;oBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,UAAiB,EACjB,MAAc,EACd,IAAkB,EAClB,MAAkB,EAClB,YAA0B,EAC1B,GAAW,EACX,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB;QAElB,IAAI,CAAC,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAExC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YAC5J,CAAC;iBAAM,CAAC;gBACN,MAAM,aAAa,GAAG,mBAAmB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;gBAC7D,IAAI,aAAa,EAAE,CAAC;oBAClB,aAAa,CAAC,UAAU,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;oBACnD,aAAa,CAAC,WAAW,GAAG,CAAC,aAAa,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBACjE,aAAa,CAAC,gBAAgB,GAAG,CAAC,aAAa,CAAC,gBAAgB,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC;oBAEhG,MAAM,IAAI,CAAC,YAAY,CACrB,aAAa,CAAC,IAAI,EAClB,aAAa,CAAC,IAAI,EAClB,SAAS,SAAS,CAAC,KAAK,CAAC,MAAM,OAAO,EACtC,IAAI,EACJ,MAAM,EACN,GAAG,EACH,IAAI,EACJ,QAAQ,EACR,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,aAAa,CACd,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,IAAoB,EACpB,MAAc,EACd,IAAkB,EAClB,MAAkB,EAClB,YAA0B,EAC1B,GAAW,EACX,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB,EAClB,aAAoC,IAAI;QAE1C,aAAa,CAAC,UAAU,EAAE,CAAC;QAE3B,MAAM,QAAQ,GAAG,MAAM,sBAAsB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;QAEhF,IAAI,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACtD,aAAa,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,qBAAqB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3E,aAAa,CAAC,UAAU,EAAE,CAAC;YAE3B,MAAM,WAAW,GAAG,MAAM,iBAAiB,CACzC,QAAQ,CAAC,qBAAqB,EAC9B,MAAM,EACN,IAAI,EACJ,YAAY,EACZ,IAAI,CACL,CAAC;YAEF,MAAM,WAAW,GAAU,EAAE,CAAC;YAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,MAAM,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAEjC,IAAI,WAAW,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;oBAClC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;oBACnE,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,WAAW,CAAC,IAAI;qBACvB,CAAC,CAAC;oBACH,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACtC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,OAAO,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;wBAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBACtC,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,EAAE,IAAI,CAAC,CAAC;gBACvJ,CAAC;YACH,CAAC;YAED,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAW,EAAE,CAAM,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAEpF,IAAI,cAAc,IAAI,MAAM,CAAC,GAAG,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;oBAC5D,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,UAAU,GAAmB;wBACjC,GAAG,IAAI;wBACP,IAAI,EAAE,GAAG,IAAI,CAAC,IAAI,SAAS;wBAC3B,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU;wBAC1C,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ;qBAC5D,CAAC;oBACF,MAAM,MAAM,GAAG,iBAAiB,WAAW,CAAC,MAAM,EAAE,CAAC;oBAErD,aAAa,CAAC,WAAW,EAAE,CAAC;oBAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;gBAChK,CAAC;qBAAM,CAAC;oBACN,aAAa,CAAC,YAAY,IAAI,WAAW,CAAC,MAAM,CAAC;gBACnD,CAAC;YACH,CAAC;YAED,OAAO;QACT,CAAC;aAAM,IAAI,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YACtC,aAAa,CAAC,iBAAiB,EAAE,CAAC;YAClC,MAAM,eAAe,GAAG,MAAM,oBAAoB,CAChD,IAAI,EACJ,MAAM,EACN,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,OAAO,EACd,YAAY,CACb,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,MAAM,SAAS,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACrC,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YAClK,CAAC;YACD,OAAO;QACT,CAAC;QAED,aAAa,CAAC,YAAY,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1D,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;IAClJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACtB,IAAoB,EACpB,IAAY,EACZ,MAAqB,EACrB,UAAiC,EACjC,MAAc,EACd,GAAW,EACX,IAAkB,EAClB,QAAwB,EACxB,iBAA2B,EAC3B,UAAe,EACf,aAA0D,EAC1D,aAAkB;QAEpB,IAAI,MAAM,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,MAAM;YAAE,OAAO;QAEpB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,MAAM,QAAQ,MAAM,EAAE,CAAC;QACrC,CAAC;QAED,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;QAChE,MAAM,aAAa,GAAG,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAC7D,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,iBAAiB,CAAC,IAAI,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC5E,iBAAiB,CAAC,IAAI,GAAG,OAAO,CAAC;QAEjC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,qBAAqB,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnE,MAAM,qBAAqB,GAAG,6BAA6B,CACzD,IAAI,EACJ,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,CACZ,CAAC;QAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC;QAEvD,MAAM,WAAW,GAAG;YAClB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;YAC9D,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;YAC1D,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,gBAAgB,EAAE,CAAC,CAAC,WAAW;YAC/B,aAAa,EAAE,kBAAkB,CAAC,MAAM;YACxC,aAAa,EAAE,CAAC,CAAC,MAAM;YACvB,gBAAgB,EAAE,CAAC,CAAC,UAAU;SAC/B,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1D,MAAM,eAAe,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,CAAC;QAEpD,IAAI,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,OAAO;QACT,CAAC;QAED,MAAM,aAAa,CAAC;YAClB,IAAI;YACJ,qBAAqB;YACrB,OAAO;YACP,GAAG;YACH,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,GAAG;YACH,MAAM;YACN,SAAS;YACT,iBAAiB;YACjB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;SACX,CAAC,CAAC;QAEH,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvC,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACxC,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;CACF"}

package/dist/core/indexing/file-scanner.d.ts

@@ -0,0 +1,11 @@
+export interface ScanResult {
+    files: string[];
+    toDelete: string[];
+}
+/**
+ * Lightweight helper responsible only for discovering files eligible for indexing.
+ */
+export declare class FileScanner {
+    scan(repo: string, normalizedChanged: string[] | null): Promise<ScanResult>;
+}
+//# sourceMappingURL=file-scanner.d.ts.map

package/dist/core/indexing/file-scanner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-scanner.d.ts","sourceRoot":"","sources":["../../../src/core/indexing/file-scanner.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;GAEG;AACH,qBAAa,WAAW;IAChB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC;CAyClF"}

package/dist/core/indexing/file-scanner.js

@@ -0,0 +1,49 @@
+import fg from 'fast-glob';
+import path from 'path';
+import fs from 'fs';
+import { LANG_RULES, getSupportedLanguageExtensions } from '../../languages/rules.js';
+import { DEFAULT_SCAN_IGNORES } from '../../utils/scan-patterns.js';
+/**
+ * Lightweight helper responsible only for discovering files eligible for indexing.
+ */
+export class FileScanner {
+    async scan(repo, normalizedChanged) {
+        const languagePatterns = getSupportedLanguageExtensions().map(ext => `**/*${ext}`);
+        let files = [];
+        if (normalizedChanged === null) {
+            files = await fg(languagePatterns, {
+                cwd: repo,
+                absolute: false,
+                followSymbolicLinks: false,
+                ignore: DEFAULT_SCAN_IGNORES,
+                onlyFiles: true,
+                dot: false
+            });
+        }
+        else {
+            files = normalizedChanged.filter(rel => {
+                const ext = path.extname(rel).toLowerCase();
+                return !!LANG_RULES[ext];
+            });
+        }
+        const uniqueFiles = [];
+        const toDelete = [];
+        const seenFiles = new Set();
+        for (const rel of files) {
+            if (!rel || seenFiles.has(rel)) {
+                continue;
+            }
+            const absPath = path.join(repo, rel);
+            try {
+                await fs.promises.access(absPath);
+                seenFiles.add(rel);
+                uniqueFiles.push(rel);
+            }
+            catch {
+                toDelete.push(rel);
+            }
+        }
+        return { files: uniqueFiles, toDelete };
+    }
+}
+//# sourceMappingURL=file-scanner.js.map

package/dist/core/indexing/file-scanner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-scanner.js","sourceRoot":"","sources":["../../../src/core/indexing/file-scanner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,UAAU,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAOpE;;GAEG;AACH,MAAM,OAAO,WAAW;IACtB,KAAK,CAAC,IAAI,CAAC,IAAY,EAAE,iBAAkC;QACzD,MAAM,gBAAgB,GAAG,8BAA8B,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;QACnF,IAAI,KAAK,GAAa,EAAE,CAAC;QAEzB,IAAI,iBAAiB,KAAK,IAAI,EAAE,CAAC;YAC/B,KAAK,GAAG,MAAM,EAAE,CAAC,gBAAgB,EAAE;gBACjC,GAAG,EAAE,IAAI;gBACT,QAAQ,EAAE,KAAK;gBACf,mBAAmB,EAAE,KAAK;gBAC1B,MAAM,EAAE,oBAAoB;gBAC5B,SAAS,EAAE,IAAI;gBACf,GAAG,EAAE,KAAK;aACX,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBACrC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;gBAC5C,OAAO,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QAEpC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACnB,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;IAC1C,CAAC;CACF"}

package/dist/core/search.d.ts

@@ -1,7 +1,7 @@
 import type { ScopeFilters } from '../types/search.js';
 import type { SearchCodeResult, GetChunkResult } from './types.js';
-export declare function clearSearchCaches(): void;
 export declare function searchCode(query: string, limit?: number, provider?: string, workingPath?: string, scopeOptions?: ScopeFilters): Promise<SearchCodeResult>;
 export declare function getOverview(limit?: number, workingPath?: string): Promise<SearchCodeResult>;
 export declare function getChunk(sha: string, workingPath?: string): Promise<GetChunkResult>;
+export declare function clearSearchCaches(): void;
 //# sourceMappingURL=search.d.ts.map

package/dist/core/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/core/search.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAgB,cAAc,EAAE,MAAM,YAAY,CAAC;AA0EjF,wBAAgB,iBAAiB,IAAI,IAAI,CAGxC;AA4HD,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,KAAK,SAAK,EACV,QAAQ,SAAS,EACjB,WAAW,SAAM,EACjB,YAAY,GAAE,YAAiB,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CA0X3B;AAED,wBAAsB,WAAW,CAAC,KAAK,SAAK,EAAE,WAAW,SAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA+C1F;AAED,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,SAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqBtF"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/core/search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKnE,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EACpB,YAAY,CAAC,EAAE,YAAY,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CAE3B;AAED,wBAAsB,WAAW,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAEjG;AAED,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAEzF;AAED,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}