agentic-qe 2.5.10 → 2.6.0

package/dist/code-intelligence/chunking/ASTChunker.d.ts

@@ -0,0 +1,64 @@
+/**
+ * AST-aware code chunker with semantic preservation
+ * Splits code into 256-512 token chunks while preserving function/class boundaries
+ */
+import { ChunkingResult, ChunkingConfig, TokenCounter } from './types.js';
+/**
+ * Simple word-based token counter (approximation)
+ * ~4 characters per token for code
+ */
+export declare class SimpleTokenCounter implements TokenCounter {
+    count(text: string): number;
+}
+export declare class ASTChunker {
+    private parser;
+    private tokenCounter;
+    private config;
+    private splitter;
+    constructor(config?: Partial<ChunkingConfig>, tokenCounter?: TokenCounter);
+    /**
+     * Chunk a file using AST-aware semantic boundaries
+     */
+    chunkFile(filePath: string, content: string, language?: string): ChunkingResult;
+    /**
+     * Process lines not covered by parsed entities
+     */
+    private processUncoveredLines;
+    /**
+     * Create a module-level chunk (imports, constants, etc.)
+     */
+    private createModuleChunk;
+    /**
+     * Add overlap between consecutive chunks
+     */
+    private addOverlap;
+    /**
+     * Extract overlap content from chunk
+     */
+    private extractOverlap;
+    /**
+     * Fallback to line-based chunking if AST parsing fails
+     */
+    private fallbackChunking;
+    /**
+     * Create a fragment chunk (fallback)
+     */
+    private createFragmentChunk;
+    /**
+     * Calculate chunking statistics
+     */
+    private calculateStats;
+    /**
+     * Generate unique chunk ID
+     */
+    private generateChunkId;
+    /**
+     * Get configuration
+     */
+    getConfig(): ChunkingConfig;
+    /**
+     * Update configuration
+     */
+    setConfig(config: Partial<ChunkingConfig>): void;
+}
+//# sourceMappingURL=ASTChunker.d.ts.map

package/dist/code-intelligence/chunking/ASTChunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ASTChunker.d.ts","sourceRoot":"","sources":["../../../src/code-intelligence/chunking/ASTChunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAEL,cAAc,EACd,cAAc,EACd,YAAY,EAEb,MAAM,YAAY,CAAC;AAGpB;;;GAGG;AACH,qBAAa,kBAAmB,YAAW,YAAY;IACrD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAO5B;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAgB;gBAEpB,MAAM,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,EAAE,YAAY;IAwBzE;;OAEG;IACH,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,cAAc;IA0D/E;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA+D7B;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;OAEG;IACH,OAAO,CAAC,UAAU;IA8ClB;;OAEG;IACH,OAAO,CAAC,cAAc;IA0CtB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA8CxB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+B3B;;OAEG;IACH,OAAO,CAAC,cAAc;IAuCtB;;OAEG;IACH,OAAO,CAAC,eAAe;IAKvB;;OAEG;IACH,SAAS,IAAI,cAAc;IAI3B;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,IAAI;CAajD"}

package/dist/code-intelligence/chunking/ASTChunker.js

@@ -0,0 +1,344 @@
+"use strict";
+/**
+ * AST-aware code chunker with semantic preservation
+ * Splits code into 256-512 token chunks while preserving function/class boundaries
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ASTChunker = exports.SimpleTokenCounter = void 0;
+const TreeSitterParser_js_1 = require("../parser/TreeSitterParser.js");
+const ChunkSplitter_js_1 = require("./ChunkSplitter.js");
+const crypto_1 = require("crypto");
+/**
+ * Simple word-based token counter (approximation)
+ * ~4 characters per token for code
+ */
+class SimpleTokenCounter {
+    count(text) {
+        // Approximate: 4 chars per token
+        // This is a rough approximation; for production, use tiktoken
+        const chars = text.length;
+        const tokens = Math.ceil(chars / 4);
+        return tokens;
+    }
+}
+exports.SimpleTokenCounter = SimpleTokenCounter;
+class ASTChunker {
+    constructor(config, tokenCounter) {
+        this.parser = new TreeSitterParser_js_1.TreeSitterParser();
+        this.tokenCounter = tokenCounter || new SimpleTokenCounter();
+        // Default configuration
+        this.config = {
+            minTokens: config?.minTokens ?? 256,
+            maxTokens: config?.maxTokens ?? 512,
+            overlapPercent: config?.overlapPercent ?? 15,
+            overlapTokens: config?.overlapTokens ?? 50,
+            preserveSemanticBoundaries: config?.preserveSemanticBoundaries ?? true,
+            splitLargeEntities: config?.splitLargeEntities ?? true,
+        };
+        // Calculate overlap tokens if not provided
+        if (!config?.overlapTokens && config?.overlapPercent) {
+            this.config.overlapTokens = Math.round(this.config.maxTokens * (this.config.overlapPercent / 100));
+        }
+        this.splitter = new ChunkSplitter_js_1.ChunkSplitter(this.config, this.tokenCounter);
+    }
+    /**
+     * Chunk a file using AST-aware semantic boundaries
+     */
+    chunkFile(filePath, content, language) {
+        // Parse file to extract entities
+        const parseResult = this.parser.parseFile(filePath, content, language);
+        if (parseResult.errors.length > 0) {
+            // Fallback to line-based chunking if parsing fails
+            return this.fallbackChunking(filePath, content, language || 'unknown');
+        }
+        // Sort entities by line number
+        const sortedEntities = parseResult.entities.sort((a, b) => a.lineStart - b.lineStart);
+        const chunks = [];
+        let processedLines = new Set();
+        // Process each entity
+        for (const entity of sortedEntities) {
+            // Skip if already processed (e.g., method inside class)
+            if (processedLines.has(entity.lineStart)) {
+                continue;
+            }
+            // Split entity into chunks
+            const entityChunks = this.splitter.splitEntity(entity);
+            chunks.push(...entityChunks);
+            // Mark lines as processed
+            for (let i = entity.lineStart; i <= entity.lineEnd; i++) {
+                processedLines.add(i);
+            }
+        }
+        // Handle lines not covered by entities (module-level code, imports, etc.)
+        const uncoveredChunks = this.processUncoveredLines(filePath, content, language || 'unknown', processedLines);
+        chunks.push(...uncoveredChunks);
+        // Sort chunks by line number
+        chunks.sort((a, b) => a.lineStart - b.lineStart);
+        // Add overlap between chunks
+        const chunksWithOverlap = this.addOverlap(chunks, content);
+        // Calculate statistics
+        const stats = this.calculateStats(chunksWithOverlap);
+        return {
+            chunks: chunksWithOverlap,
+            stats,
+        };
+    }
+    /**
+     * Process lines not covered by parsed entities
+     */
+    processUncoveredLines(filePath, content, language, processedLines) {
+        const lines = content.split('\n');
+        const chunks = [];
+        let currentLines = [];
+        let currentLineStart = 1;
+        for (let i = 0; i < lines.length; i++) {
+            const lineNumber = i + 1;
+            if (processedLines.has(lineNumber)) {
+                // Flush current chunk if any
+                if (currentLines.length > 0) {
+                    const chunkContent = currentLines.join('\n').trim();
+                    if (chunkContent.length > 0) {
+                        chunks.push(this.createModuleChunk(filePath, currentLines, currentLineStart, language));
+                    }
+                    currentLines = [];
+                }
+                continue;
+            }
+            currentLines.push(lines[i]);
+            // Check if we should create a chunk
+            const currentContent = currentLines.join('\n');
+            const tokenCount = this.tokenCounter.count(currentContent);
+            if (tokenCount >= this.config.maxTokens) {
+                chunks.push(this.createModuleChunk(filePath, currentLines, currentLineStart, language));
+                currentLines = [];
+                currentLineStart = i + 2;
+            }
+            else if (currentLines.length === 1) {
+                currentLineStart = lineNumber;
+            }
+        }
+        // Flush remaining lines
+        if (currentLines.length > 0) {
+            const chunkContent = currentLines.join('\n').trim();
+            if (chunkContent.length > 0) {
+                chunks.push(this.createModuleChunk(filePath, currentLines, currentLineStart, language));
+            }
+        }
+        return chunks;
+    }
+    /**
+     * Create a module-level chunk (imports, constants, etc.)
+     */
+    createModuleChunk(filePath, lines, lineStart, language) {
+        const content = lines.join('\n');
+        const tokenCount = this.tokenCounter.count(content);
+        const lineEnd = lineStart + lines.length - 1;
+        return {
+            id: this.generateChunkId(filePath, lineStart),
+            content,
+            filePath,
+            language,
+            lineStart,
+            lineEnd,
+            tokenCount,
+            entityType: 'module',
+            metadata: {
+                filePath,
+                language,
+                lineStart,
+                lineEnd,
+                entityType: 'module',
+                isComplete: true,
+            },
+        };
+    }
+    /**
+     * Add overlap between consecutive chunks
+     */
+    addOverlap(chunks, fileContent) {
+        if (chunks.length <= 1) {
+            return chunks;
+        }
+        const overlapTokens = this.config.overlapTokens || 50;
+        const chunksWithOverlap = [];
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            let content = chunk.content;
+            let originalTokenCount = chunk.tokenCount;
+            // Add overlap from previous chunk
+            if (i > 0) {
+                const prevChunk = chunks[i - 1];
+                const overlapContent = this.extractOverlap(prevChunk.content, overlapTokens, 'end');
+                if (overlapContent && overlapContent.trim()) {
+                    content = overlapContent + '\n' + content;
+                    chunk.metadata.hasOverlapStart = true;
+                    chunk.metadata.overlapTokens = this.tokenCounter.count(overlapContent);
+                }
+            }
+            // Add overlap to next chunk (will be prepended when processing next chunk)
+            // This is already handled in the previous chunk's loop
+            chunksWithOverlap.push({
+                ...chunk,
+                content,
+                tokenCount: this.tokenCounter.count(content),
+                metadata: {
+                    ...chunk.metadata,
+                    originalTokenCount,
+                },
+            });
+        }
+        return chunksWithOverlap;
+    }
+    /**
+     * Extract overlap content from chunk
+     */
+    extractOverlap(content, targetTokens, position) {
+        const lines = content.split('\n');
+        if (position === 'end') {
+            // Extract from end
+            let overlapLines = [];
+            let tokens = 0;
+            for (let i = lines.length - 1; i >= 0; i--) {
+                overlapLines.unshift(lines[i]);
+                const overlapContent = overlapLines.join('\n');
+                tokens = this.tokenCounter.count(overlapContent);
+                if (tokens >= targetTokens) {
+                    break;
+                }
+            }
+            return overlapLines.length > 0 ? overlapLines.join('\n') : null;
+        }
+        else {
+            // Extract from start
+            let overlapLines = [];
+            let tokens = 0;
+            for (let i = 0; i < lines.length; i++) {
+                overlapLines.push(lines[i]);
+                const overlapContent = overlapLines.join('\n');
+                tokens = this.tokenCounter.count(overlapContent);
+                if (tokens >= targetTokens) {
+                    break;
+                }
+            }
+            return overlapLines.length > 0 ? overlapLines.join('\n') : null;
+        }
+    }
+    /**
+     * Fallback to line-based chunking if AST parsing fails
+     */
+    fallbackChunking(filePath, content, language) {
+        const lines = content.split('\n');
+        const chunks = [];
+        let currentLines = [];
+        let currentLineStart = 1;
+        for (let i = 0; i < lines.length; i++) {
+            currentLines.push(lines[i]);
+            const currentContent = currentLines.join('\n');
+            const tokenCount = this.tokenCounter.count(currentContent);
+            if (tokenCount >= this.config.maxTokens) {
+                chunks.push(this.createFragmentChunk(filePath, currentLines, currentLineStart, language));
+                currentLines = [];
+                currentLineStart = i + 2;
+            }
+            else if (currentLines.length === 1) {
+                currentLineStart = i + 1;
+            }
+        }
+        // Flush remaining lines
+        if (currentLines.length > 0) {
+            chunks.push(this.createFragmentChunk(filePath, currentLines, currentLineStart, language));
+        }
+        const stats = this.calculateStats(chunks);
+        return {
+            chunks,
+            stats,
+        };
+    }
+    /**
+     * Create a fragment chunk (fallback)
+     */
+    createFragmentChunk(filePath, lines, lineStart, language) {
+        const content = lines.join('\n');
+        const tokenCount = this.tokenCounter.count(content);
+        const lineEnd = lineStart + lines.length - 1;
+        return {
+            id: this.generateChunkId(filePath, lineStart),
+            content,
+            filePath,
+            language,
+            lineStart,
+            lineEnd,
+            tokenCount,
+            entityType: 'fragment',
+            metadata: {
+                filePath,
+                language,
+                lineStart,
+                lineEnd,
+                entityType: 'fragment',
+                isComplete: false,
+                isFallback: true,
+            },
+        };
+    }
+    /**
+     * Calculate chunking statistics
+     */
+    calculateStats(chunks) {
+        const totalChunks = chunks.length;
+        if (totalChunks === 0) {
+            return {
+                totalChunks: 0,
+                avgTokens: 0,
+                minTokens: 0,
+                maxTokens: 0,
+                semanticPreservation: 0,
+                totalTokens: 0,
+            };
+        }
+        // Use original token count if available (before overlap)
+        const tokenCounts = chunks.map((c) => c.metadata.originalTokenCount || c.tokenCount);
+        const totalTokens = tokenCounts.reduce((sum, t) => sum + t, 0);
+        const avgTokens = totalTokens / totalChunks;
+        const minTokens = Math.min(...tokenCounts);
+        const maxTokens = Math.max(...tokenCounts);
+        // Calculate semantic preservation (% of chunks with complete semantic units)
+        const completeChunks = chunks.filter((c) => c.metadata.isComplete === true).length;
+        const semanticPreservation = (completeChunks / totalChunks) * 100;
+        return {
+            totalChunks,
+            avgTokens: Math.round(avgTokens),
+            minTokens,
+            maxTokens,
+            semanticPreservation: Math.round(semanticPreservation * 100) / 100,
+            totalTokens,
+        };
+    }
+    /**
+     * Generate unique chunk ID
+     */
+    generateChunkId(filePath, lineStart) {
+        const content = `${filePath}:${lineStart}:${Date.now()}`;
+        return (0, crypto_1.createHash)('sha256').update(content).digest('hex').substring(0, 16);
+    }
+    /**
+     * Get configuration
+     */
+    getConfig() {
+        return { ...this.config };
+    }
+    /**
+     * Update configuration
+     */
+    setConfig(config) {
+        this.config = { ...this.config, ...config };
+        // Recalculate overlap tokens if needed
+        if (config.overlapPercent && !config.overlapTokens) {
+            this.config.overlapTokens = Math.round(this.config.maxTokens * (this.config.overlapPercent / 100));
+        }
+        // Update splitter config
+        this.splitter = new ChunkSplitter_js_1.ChunkSplitter(this.config, this.tokenCounter);
+    }
+}
+exports.ASTChunker = ASTChunker;
+//# sourceMappingURL=ASTChunker.js.map

package/dist/code-intelligence/chunking/ASTChunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ASTChunker.js","sourceRoot":"","sources":["../../../src/code-intelligence/chunking/ASTChunker.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,uEAAiE;AAEjE,yDAAmD;AAQnD,mCAAoC;AAEpC;;;GAGG;AACH,MAAa,kBAAkB;IAC7B,KAAK,CAAC,IAAY;QAChB,iCAAiC;QACjC,8DAA8D;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AARD,gDAQC;AAED,MAAa,UAAU;IAMrB,YAAY,MAAgC,EAAE,YAA2B;QACvE,IAAI,CAAC,MAAM,GAAG,IAAI,sCAAgB,EAAE,CAAC;QACrC,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,IAAI,kBAAkB,EAAE,CAAC;QAE7D,wBAAwB;QACxB,IAAI,CAAC,MAAM,GAAG;YACZ,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,GAAG;YACnC,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,GAAG;YACnC,cAAc,EAAE,MAAM,EAAE,cAAc,IAAI,EAAE;YAC5C,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE;YAC1C,0BAA0B,EAAE,MAAM,EAAE,0BAA0B,IAAI,IAAI;YACtE,kBAAkB,EAAE,MAAM,EAAE,kBAAkB,IAAI,IAAI;SACvD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,MAAM,EAAE,cAAc,EAAE,CAAC;YACrD,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,CACpC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC,CAC3D,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,gCAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACpE,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,QAAgB,EAAE,OAAe,EAAE,QAAiB;QAC5D,iCAAiC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAEvE,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,mDAAmD;YACnD,OAAO,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,SAAS,CAAC,CAAC;QACzE,CAAC;QAED,+BAA+B;QAC/B,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAC9C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CACpC,CAAC;QAEF,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEvC,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,wDAAwD;YACxD,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,2BAA2B;YAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;YAE7B,0BAA0B;YAC1B,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxD,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,0EAA0E;QAC1E,MAAM,eAAe,GAAG,IAAI,CAAC,qBAAqB,CAChD,QAAQ,EACR,OAAO,EACP,QAAQ,IAAI,SAAS,EACrB,cAAc,CACf,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;QAEhC,6BAA6B;QAC7B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QAEjD,6BAA6B;QAC7B,MAAM,iBAAiB,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAE3D,uBAAuB;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAErD,OAAO;YACL,MAAM,EAAE,iBAAiB;YACzB,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,qBAAqB,CAC3B,QAAgB,EAChB,OAAe,EACf,QAAgB,EAChB,cAA2B;QAE3B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;YAEzB,IAAI,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACnC,6BAA6B;gBAC7B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;oBACpD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CACpB,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,QAAQ,CACT,CACF,CAAC;oBACJ,CAAC;oBACD,YAAY,GAAG,EAAE,CAAC;gBACpB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAE5B,oCAAoC;YACpC,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAE3D,IAAI,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAC3E,CAAC;gBACF,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,gBAAgB,GAAG,UAAU,CAAC;YAChC,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACpD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAC3E,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,iBAAiB,CACvB,QAAgB,EAChB,KAAe,EACf,SAAiB,EACjB,QAAgB;QAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAE7C,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC;YAC7C,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,OAAO;YACP,UAAU;YACV,UAAU,EAAE,QAAQ;YACpB,QAAQ,EAAE;gBACR,QAAQ;gBACR,QAAQ;gBACR,SAAS;gBACT,OAAO;gBACP,UAAU,EAAE,QAAQ;gBACpB,UAAU,EAAE,IAAI;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,MAAmB,EAAE,WAAmB;QACzD,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,EAAE,CAAC;QACtD,MAAM,iBAAiB,GAAgB,EAAE,CAAC;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,IAAI,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YAC5B,IAAI,kBAAkB,GAAG,KAAK,CAAC,UAAU,CAAC;YAE1C,kCAAkC;YAClC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChC,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,CACxC,SAAS,CAAC,OAAO,EACjB,aAAa,EACb,KAAK,CACN,CAAC;gBAEF,IAAI,cAAc,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;oBAC5C,OAAO,GAAG,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC;oBAC1C,KAAK,CAAC,QAAQ,CAAC,eAAe,GAAG,IAAI,CAAC;oBACtC,KAAK,CAAC,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBACzE,CAAC;YACH,CAAC;YAED,2EAA2E;YAC3E,uDAAuD;YAEvD,iBAAiB,CAAC,IAAI,CAAC;gBACrB,GAAG,KAAK;gBACR,OAAO;gBACP,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC;gBAC5C,QAAQ,EAAE;oBACR,GAAG,KAAK,CAAC,QAAQ;oBACjB,kBAAkB;iBACnB;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED;;OAEG;IACK,cAAc,CACpB,OAAe,EACf,YAAoB,EACpB,QAAyB;QAEzB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAElC,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;YACvB,mBAAmB;YACnB,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/C,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBAEjD,IAAI,MAAM,IAAI,YAAY,EAAE,CAAC;oBAC3B,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,CAAC;aAAM,CAAC;YACN,qBAAqB;YACrB,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,MAAM,GAAG,CAAC,CAAC;YAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC/C,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;gBAEjD,IAAI,MAAM,IAAI,YAAY,EAAE,CAAC;oBAC3B,MAAM;gBACR,CAAC;YACH,CAAC;YAED,OAAO,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,QAAgB,EAChB,OAAe,EACf,QAAgB;QAEhB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,IAAI,YAAY,GAAa,EAAE,CAAC;QAChC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAE3D,IAAI,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,mBAAmB,CACtB,QAAQ,EACR,YAAY,EACZ,gBAAgB,EAChB,QAAQ,CACT,CACF,CAAC;gBACF,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrC,gBAAgB,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,QAAQ,CAAC,CAC7E,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAE1C,OAAO;YACL,MAAM;YACN,KAAK;SACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,mBAAmB,CACzB,QAAgB,EAChB,KAAe,EACf,SAAiB,EACjB,QAAgB;QAEhB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAE7C,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC;YAC7C,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,OAAO;YACP,UAAU;YACV,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE;gBACR,QAAQ;gBACR,QAAQ;gBACR,SAAS;gBACT,OAAO;gBACP,UAAU,EAAE,UAAU;gBACtB,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,IAAI;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,MAAmB;QACxC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QAElC,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO;gBACL,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,CAAC;gBACZ,oBAAoB,EAAE,CAAC;gBACvB,WAAW,EAAE,CAAC;aACf,CAAC;QACJ,CAAC;QAED,yDAAyD;QACzD,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClC,CAAC,CAAC,QAAQ,CAAC,kBAA6B,IAAI,CAAC,CAAC,UAAU,CAC1D,CAAC;QACF,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,MAAM,SAAS,GAAG,WAAW,GAAG,WAAW,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;QAE3C,6EAA6E;QAC7E,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,KAAK,IAAI,CACtC,CAAC,MAAM,CAAC;QACT,MAAM,oBAAoB,GAAG,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC;QAElE,OAAO;YACL,WAAW;YACX,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YAChC,SAAS;YACT,SAAS;YACT,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,oBAAoB,GAAG,GAAG,CAAC,GAAG,GAAG;YAClE,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,QAAgB,EAAE,SAAiB;QACzD,MAAM,OAAO,GAAG,GAAG,QAAQ,IAAI,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QACzD,OAAO,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,MAA+B;QACvC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QAE5C,uCAAuC;QACvC,IAAI,MAAM,CAAC,cAAc,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,KAAK,CACpC,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC,CAC3D,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,gCAAa,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;IACpE,CAAC;CACF;AAxbD,gCAwbC"}

package/dist/code-intelligence/chunking/ChunkSplitter.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Recursive splitter for large code entities
+ * Splits entities >512 tokens while preserving semantic boundaries
+ */
+import { CodeEntity } from '../parser/types.js';
+import { CodeChunk, ChunkingConfig, TokenCounter } from './types.js';
+export declare class ChunkSplitter {
+    private config;
+    private tokenCounter;
+    constructor(config: ChunkingConfig, tokenCounter: TokenCounter);
+    /**
+     * Split a large entity into smaller chunks recursively
+     */
+    splitEntity(entity: CodeEntity): CodeChunk[];
+    /**
+     * Recursively split entity at semantic boundaries
+     */
+    private recursiveSplit;
+    /**
+     * Find semantic boundaries in code (function/class boundaries, blank lines, etc.)
+     */
+    private findSemanticBoundaries;
+    /**
+     * Get language-specific regex patterns for boundary detection
+     */
+    private getLanguagePatterns;
+    /**
+     * Split entity by line count when no semantic boundaries work
+     */
+    private splitByLines;
+    /**
+     * Create a chunk from entity
+     */
+    private createChunk;
+    /**
+     * Create chunk from lines
+     */
+    private createChunkFromLines;
+    /**
+     * Map parser entity type to chunk entity type
+     */
+    private mapEntityType;
+    /**
+     * Generate unique chunk ID
+     */
+    private generateChunkId;
+}
+//# sourceMappingURL=ChunkSplitter.d.ts.map

package/dist/code-intelligence/chunking/ChunkSplitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ChunkSplitter.d.ts","sourceRoot":"","sources":["../../../src/code-intelligence/chunking/ChunkSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,YAAY,EAAc,MAAM,YAAY,CAAC;AAGjF,qBAAa,aAAa;IAEtB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,YAAY;gBADZ,MAAM,EAAE,cAAc,EACtB,YAAY,EAAE,YAAY;IAGpC;;OAEG;IACH,WAAW,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,EAAE;IAiB5C;;OAEG;IACH,OAAO,CAAC,cAAc;IA4EtB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAwC9B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoD3B;;OAEG;IACH,OAAO,CAAC,YAAY;IA4DpB;;OAEG;IACH,OAAO,CAAC,WAAW;IAmCnB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAsC5B;;OAEG;IACH,OAAO,CAAC,aAAa;IAYrB;;OAEG;IACH,OAAO,CAAC,eAAe;CAIxB"}