codevault 1.4.1 → 1.6.0

package/dist/chunking/file-grouper.js

@@ -1,181 +0,0 @@
-import { batchAnalyzeCodeSize } from './token-counter.js';
-function getSizeLimits(profile) {
-    if (profile.useTokens && profile.tokenCounter) {
-        return {
-            optimal: profile.optimalTokens,
-            min: profile.minChunkTokens,
-            max: profile.maxChunkTokens,
-            overlap: profile.overlapTokens,
-            unit: 'tokens'
-        };
-    }
-    return {
-        optimal: profile.optimalChars,
-        min: profile.minChunkChars,
-        max: profile.maxChunkChars,
-        overlap: profile.overlapChars,
-        unit: 'characters'
-    };
-}
-async function batchAnalyzeNodesInternal(nodes, source, profile) {
-    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
-    const limits = getSizeLimits(profile);
-    if (profile.useTokens && profile.tokenCounter) {
-        const analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, false);
-        return nodes.map((node, i) => ({
-            node,
-            size: analyses[i].size,
-            code: codes[i]
-        }));
-    }
-    return nodes.map((node, i) => ({
-        node,
-        size: codes[i].length,
-        code: codes[i]
-    }));
-}
-function isContainerNode(node, rule) {
-    const containerTypes = [
-        'class_declaration',
-        'class_definition',
-        'interface_declaration',
-        'module_declaration',
-        'namespace_declaration',
-        'trait_declaration',
-        'enum_declaration'
-    ];
-    return containerTypes.includes(node.type);
-}
-function identifySemanticGroups(nodes, source, nodeAnalyses, rule) {
-    const groups = [];
-    let currentGroup = {
-        type: 'file_section',
-        nodes: [],
-        analyses: [],
-        parentNode: null
-    };
-    for (let i = 0; i < nodes.length; i++) {
-        const node = nodes[i];
-        const analysis = nodeAnalyses[i];
-        if (isContainerNode(node, rule)) {
-            if (currentGroup.nodes.length > 0) {
-                groups.push(currentGroup);
-            }
-            currentGroup = {
-                type: 'container',
-                containerType: node.type,
-                nodes: [node],
-                analyses: [analysis],
-                parentNode: node
-            };
-            groups.push(currentGroup);
-            currentGroup = {
-                type: 'file_section',
-                nodes: [],
-                analyses: [],
-                parentNode: null
-            };
-        }
-        else {
-            currentGroup.nodes.push(node);
-            currentGroup.analyses.push(analysis);
-        }
-    }
-    if (currentGroup.nodes.length > 0) {
-        groups.push(currentGroup);
-    }
-    return groups;
-}
-async function combineGroupsToOptimalSize(semanticGroups, source, profile, limits) {
-    const optimalGroups = [];
-    let currentCombinedGroup = {
-        nodes: [],
-        totalSize: 0,
-        groupInfo: []
-    };
-    for (const group of semanticGroups) {
-        const groupTotalSize = group.analyses.reduce((sum, a) => sum + a.size, 0);
-        if (groupTotalSize > limits.optimal) {
-            if (currentCombinedGroup.nodes.length > 0) {
-                optimalGroups.push(currentCombinedGroup);
-            }
-            optimalGroups.push({
-                nodes: group.nodes,
-                totalSize: groupTotalSize,
-                groupInfo: [group]
-            });
-            currentCombinedGroup = {
-                nodes: [],
-                totalSize: 0,
-                groupInfo: []
-            };
-            continue;
-        }
-        if (currentCombinedGroup.totalSize + groupTotalSize > limits.max) {
-            if (currentCombinedGroup.nodes.length > 0) {
-                optimalGroups.push(currentCombinedGroup);
-            }
-            currentCombinedGroup = {
-                nodes: group.nodes,
-                totalSize: groupTotalSize,
-                groupInfo: [group]
-            };
-            continue;
-        }
-        currentCombinedGroup.nodes.push(...group.nodes);
-        currentCombinedGroup.totalSize += groupTotalSize;
-        currentCombinedGroup.groupInfo.push(group);
-        if (currentCombinedGroup.totalSize >= limits.optimal * 0.9) {
-            optimalGroups.push(currentCombinedGroup);
-            currentCombinedGroup = {
-                nodes: [],
-                totalSize: 0,
-                groupInfo: []
-            };
-        }
-    }
-    if (currentCombinedGroup.nodes.length > 0) {
-        optimalGroups.push(currentCombinedGroup);
-    }
-    return optimalGroups;
-}
-export async function groupNodesForChunking(nodes, source, profile, rule) {
-    if (!nodes || nodes.length === 0)
-        return [];
-    const limits = getSizeLimits(profile);
-    if (nodes.length <= 10) {
-        return nodes.map(node => ({
-            nodes: [node],
-            totalSize: 0,
-            groupInfo: []
-        }));
-    }
-    const nodeAnalyses = await batchAnalyzeNodesInternal(nodes, source, profile);
-    const semanticGroups = identifySemanticGroups(nodes, source, nodeAnalyses, rule);
-    const optimalGroups = await combineGroupsToOptimalSize(semanticGroups, source, profile, limits);
-    return optimalGroups;
-}
-export function createCombinedChunk(nodeGroup, source, filerel) {
-    if (!nodeGroup.nodes || nodeGroup.nodes.length === 0) {
-        return null;
-    }
-    const codes = nodeGroup.nodes.map(node => source.slice(node.startIndex, node.endIndex));
-    const combinedCode = codes.join('\n\n');
-    const firstNode = nodeGroup.nodes[0];
-    const lastNode = nodeGroup.nodes[nodeGroup.nodes.length - 1];
-    return {
-        code: combinedCode,
-        node: {
-            ...firstNode,
-            type: `${firstNode.type}_group_${nodeGroup.nodes.length}`,
-            endIndex: lastNode.endIndex
-        },
-        metadata: {
-            isGroup: true,
-            nodeCount: nodeGroup.nodes.length,
-            totalSize: nodeGroup.totalSize,
-            groupTypes: nodeGroup.groupInfo?.map(g => g.type) || ['combined']
-        }
-    };
-}
-//# sourceMappingURL=file-grouper.js.map

package/dist/chunking/file-grouper.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-grouper.js","sourceRoot":"","sources":["../../src/chunking/file-grouper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,oBAAoB,EAAyB,MAAM,oBAAoB,CAAC;AAqClG,SAAS,aAAa,CAAC,OAAqB;IAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,IAAI,EAAE,QAAQ;SACf,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,KAAuB,EAAE,MAAc,EAAE,OAAqB;IACrG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QACxF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI;YACJ,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;YACtB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI;QACJ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;QACrB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;KACf,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CAAC,IAAoB,EAAE,IAAkB;IAC/D,MAAM,cAAc,GAAG;QACrB,mBAAmB;QACnB,kBAAkB;QAClB,uBAAuB;QACvB,oBAAoB;QACpB,uBAAuB;QACvB,mBAAmB;QACnB,kBAAkB;KACnB,CAAC;IAEF,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAuB,EAAE,MAAc,EAAE,YAA4B,EAAE,IAAkB;IACvH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,YAAY,GAAkB;QAChC,IAAI,EAAE,cAAc;QACpB,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,IAAI;KACjB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAChC,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC5B,CAAC;YAED,YAAY,GAAG;gBACb,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,IAAI,CAAC,IAAI;gBACxB,KAAK,EAAE,CAAC,IAAI,CAAC;gBACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,UAAU,EAAE,IAAI;aACjB,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE1B,YAAY,GAAG;gBACb,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;gBACZ,UAAU,EAAE,IAAI;aACjB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,cAA+B,EAC/B,MAAc,EACd,OAAqB,EACrB,MAAkB;IAElB,MAAM,aAAa,GAAgB,EAAE,CAAC;IACtC,IAAI,oBAAoB,GAAc;QACpC,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,EAAE;KACd,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE1E,IAAI,cAAc,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,aAAa,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC,CAAC;YAEH,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,GAAG,cAAc,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YACjE,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,oBAAoB,GAAG;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC;YACF,SAAS;QACX,CAAC;QAED,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAChD,oBAAoB,CAAC,SAAS,IAAI,cAAc,CAAC;QACjD,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,oBAAoB,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;YAC3D,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACzC,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAuB,EACvB,MAAc,EACd,OAAqB,EACrB,IAAkB;IAElB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,IAAI,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7E,MAAM,cAAc,GAAG,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACjF,MAAM,aAAa,GAAG,MAAM,0BAA0B,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAEhG,OAAO,aAAa,CAAC;AACvB,CAAC;AAaD,MAAM,UAAU,mBAAmB,CAAC,SAAoB,EAAE,MAAc,EAAE,OAAe;IACvF,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAC7C,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAExC,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE;YACJ,GAAG,SAAS;YACZ,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,UAAU,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;YACzD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,UAAU,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;SAClE;KACF,CAAC;AACJ,CAAC"}

package/dist/chunking/semantic-chunker.d.ts

@@ -1,37 +0,0 @@
-import type { ModelProfile } from '../providers/base.js';
-import type { TreeSitterNode } from '../types/ast.js';
-interface LanguageRule {
-    subdivisionTypes?: Record<string, string[]>;
-    [key: string]: any;
-}
-export declare function findSemanticSubdivisions(node: TreeSitterNode, rule: LanguageRule): TreeSitterNode[];
-export declare function findLastCompleteBoundary(code: string, maxSize: number): number;
-export declare function extractSignature(node: TreeSitterNode, source: string): string;
-export declare function extractLinesBeforeNode(node: TreeSitterNode, source: string, numLines: number): string;
-export declare function extractParentContext(node: TreeSitterNode, source: string): {
-    signature: string;
-    startLine: number;
-    endLine: number;
-};
-export declare function getLineNumber(byteOffset: number, source: string): number;
-export interface NodeAnalysis {
-    isSingleChunk: boolean;
-    needsSubdivision: boolean;
-    subdivisionCandidates: TreeSitterNode[];
-    size: number;
-    unit: string;
-    method: string;
-    estimatedSubchunks: number;
-}
-export declare function analyzeNodeForChunking(node: TreeSitterNode, source: string, rule: LanguageRule, profile: ModelProfile): Promise<NodeAnalysis>;
-export declare function batchAnalyzeNodes(nodes: TreeSitterNode[], source: string, rule: LanguageRule, profile: ModelProfile, isSubdivision?: boolean): Promise<Array<NodeAnalysis & {
-    node: TreeSitterNode;
-}>>;
-export interface StatementChunk {
-    code: string;
-    size: number;
-    unit: string;
-}
-export declare function yieldStatementChunks(node: TreeSitterNode, source: string, maxSize: number, overlapSize: number, profile: ModelProfile): Promise<StatementChunk[]>;
-export {};
-//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunking/semantic-chunker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunking/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,UAAU,YAAY;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAUD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,EAAE,CAwBnG;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAgB9E;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAO7E;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIrG;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAGxE;AAqBD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,qBAAqB,EAAE,cAAc,EAAE,CAAC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,YAAY,CAAC,CA2BvB;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,EACrB,aAAa,UAAQ,GACpB,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC,CAAC,CAgCzD;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,cAAc,EAAE,CAAC,CAwC3B"}

package/dist/chunking/semantic-chunker.js

@@ -1,172 +0,0 @@
-import { analyzeCodeSize, batchAnalyzeCodeSize } from './token-counter.js';
-export function findSemanticSubdivisions(node, rule) {
-    if (!node || !rule)
-        return [];
-    const subdivisionTypes = rule.subdivisionTypes?.[node.type] || [];
-    if (subdivisionTypes.length === 0)
-        return [];
-    const candidates = [];
-    function walk(n, depth = 0) {
-        if (depth > 0 && subdivisionTypes.includes(n.type)) {
-            candidates.push(n);
-            return;
-        }
-        for (let i = 0; i < n.childCount; i++) {
-            const child = n.child(i);
-            if (child) {
-                walk(child, depth + 1);
-            }
-        }
-    }
-    walk(node);
-    return candidates;
-}
-export function findLastCompleteBoundary(code, maxSize) {
-    const boundaries = [
-        { pattern: /\n\s*}\s*$/gm, priority: 1 },
-        { pattern: /;\s*$/gm, priority: 2 },
-        { pattern: /\n\s*$/gm, priority: 3 }
-    ];
-    for (const boundary of boundaries) {
-        const matches = [...code.substring(0, maxSize).matchAll(boundary.pattern)];
-        if (matches.length > 0) {
-            const lastMatch = matches[matches.length - 1];
-            return lastMatch.index + lastMatch[0].length;
-        }
-    }
-    return maxSize;
-}
-export function extractSignature(node, source) {
-    const code = source.slice(node.startIndex, node.endIndex);
-    const firstBrace = code.indexOf('{');
-    if (firstBrace !== -1) {
-        return code.substring(0, firstBrace).trim() + ' {';
-    }
-    return code.split('\n')[0];
-}
-export function extractLinesBeforeNode(node, source, numLines) {
-    const beforeCode = source.substring(0, node.startIndex);
-    const lines = beforeCode.split('\n');
-    return lines.slice(-numLines).join('\n');
-}
-export function extractParentContext(node, source) {
-    return {
-        signature: extractSignature(node, source),
-        startLine: getLineNumber(node.startIndex, source),
-        endLine: getLineNumber(node.endIndex, source)
-    };
-}
-export function getLineNumber(byteOffset, source) {
-    const before = source.substring(0, byteOffset);
-    return before.split('\n').length;
-}
-function getSizeLimits(profile) {
-    if (profile.useTokens && profile.tokenCounter) {
-        return {
-            optimal: profile.optimalTokens,
-            min: profile.minChunkTokens,
-            max: profile.maxChunkTokens,
-            overlap: profile.overlapTokens,
-            unit: 'tokens'
-        };
-    }
-    return {
-        optimal: profile.optimalChars,
-        min: profile.minChunkChars,
-        max: profile.maxChunkChars,
-        overlap: profile.overlapChars,
-        unit: 'characters'
-    };
-}
-export async function analyzeNodeForChunking(node, source, rule, profile) {
-    const code = source.slice(node.startIndex, node.endIndex);
-    const limits = getSizeLimits(profile);
-    let actualSize;
-    let method;
-    if (profile.useTokens && profile.tokenCounter) {
-        const analysis = await analyzeCodeSize(code, limits, profile.tokenCounter);
-        actualSize = analysis.size;
-        method = analysis.method;
-    }
-    else {
-        actualSize = code.length;
-        method = 'chars';
-    }
-    const subdivisionThreshold = limits.max;
-    return {
-        isSingleChunk: actualSize <= subdivisionThreshold,
-        needsSubdivision: actualSize > subdivisionThreshold,
-        subdivisionCandidates: findSemanticSubdivisions(node, rule),
-        size: actualSize,
-        unit: limits.unit,
-        method,
-        estimatedSubchunks: Math.ceil(actualSize / limits.optimal)
-    };
-}
-export async function batchAnalyzeNodes(nodes, source, rule, profile, isSubdivision = false) {
-    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
-    const limits = getSizeLimits(profile);
-    let analyses;
-    if (profile.useTokens && profile.tokenCounter) {
-        analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, isSubdivision);
-    }
-    else {
-        analyses = codes.map(code => ({
-            size: code.length,
-            decision: code.length < limits.min ? 'too_small'
-                : code.length > limits.max ? 'too_large'
-                    : code.length <= limits.optimal ? 'optimal'
-                        : 'needs_tokenization',
-            method: 'chars'
-        }));
-    }
-    return nodes.map((node, i) => {
-        const analysis = analyses[i];
-        const subdivisionThreshold = limits.max;
-        return {
-            node,
-            isSingleChunk: analysis.size <= subdivisionThreshold,
-            needsSubdivision: analysis.size > subdivisionThreshold,
-            subdivisionCandidates: findSemanticSubdivisions(node, rule),
-            size: analysis.size,
-            unit: limits.unit,
-            method: analysis.method,
-            estimatedSubchunks: Math.ceil(analysis.size / limits.optimal)
-        };
-    });
-}
-export async function yieldStatementChunks(node, source, maxSize, overlapSize, profile) {
-    const code = source.slice(node.startIndex, node.endIndex);
-    const lines = code.split('\n');
-    const chunks = [];
-    let currentChunk = [];
-    let currentSize = 0;
-    for (const line of lines) {
-        const lineSize = profile.useTokens && profile.tokenCounter
-            ? await profile.tokenCounter(line)
-            : line.length;
-        if (currentSize + lineSize > maxSize && currentChunk.length > 0) {
-            chunks.push({
-                code: currentChunk.join('\n'),
-                size: currentSize,
-                unit: profile.useTokens ? 'tokens' : 'characters'
-            });
-            const overlapLines = Math.floor(currentChunk.length * 0.2);
-            currentChunk = currentChunk.slice(-overlapLines);
-            currentSize = profile.useTokens && profile.tokenCounter
-                ? await profile.tokenCounter(currentChunk.join('\n'))
-                : currentChunk.join('\n').length;
-        }
-        currentChunk.push(line);
-        currentSize += lineSize;
-    }
-    if (currentChunk.length > 0) {
-        chunks.push({
-            code: currentChunk.join('\n'),
-            size: currentSize,
-            unit: profile.useTokens ? 'tokens' : 'characters'
-        });
-    }
-    return chunks;
-}
-//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunking/semantic-chunker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunking/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAyB,MAAM,oBAAoB,CAAC;AAiBlG,MAAM,UAAU,wBAAwB,CAAC,IAAoB,EAAE,IAAkB;IAC/E,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7C,MAAM,UAAU,GAAqB,EAAE,CAAC;IAExC,SAAS,IAAI,CAAC,CAAiB,EAAE,KAAK,GAAG,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC;IACX,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAY,EAAE,OAAe;IACpE,MAAM,UAAU,GAAG;QACjB,EAAE,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE;QACxC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;QACnC,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE;KACrC,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9C,OAAO,SAAS,CAAC,KAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAoB,EAAE,MAAc;IACnE,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAAoB,EAAE,MAAc,EAAE,QAAgB;IAC3F,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAoB,EAAE,MAAc;IAKvE,OAAO;QACL,SAAS,EAAE,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;QACzC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QACjD,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,UAAkB,EAAE,MAAc;IAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,OAAqB;IAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,IAAI,EAAE,QAAQ;SACf,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAYD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAoB,EACpB,MAAc,EACd,IAAkB,EAClB,OAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAc,CAAC;IAEnB,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QAC3E,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QACzB,MAAM,GAAG,OAAO,CAAC;IACnB,CAAC;IAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAC;IAExC,OAAO;QACL,aAAa,EAAE,UAAU,IAAI,oBAAoB;QACjD,gBAAgB,EAAE,UAAU,GAAG,oBAAoB;QACnD,qBAAqB,EAAE,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC;QAC3D,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM;QACN,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAuB,EACvB,MAAc,EACd,IAAkB,EAClB,OAAqB,EACrB,aAAa,GAAG,KAAK;IAErB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,QAA4B,CAAC;IACjC,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC5F,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAoB;gBACnD,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAoB;oBACjD,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAkB;wBACpD,CAAC,CAAC,oBAA6B;YACrC,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAC;QACxC,OAAO;YACL,IAAI;YACJ,aAAa,EAAE,QAAQ,CAAC,IAAI,IAAI,oBAAoB;YACpD,gBAAgB,EAAE,QAAQ,CAAC,IAAI,GAAG,oBAAoB;YACtD,qBAAqB,EAAE,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC;YAC3D,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;SAC9D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAoB,EACpB,MAAc,EACd,OAAe,EACf,WAAmB,EACnB,OAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY;YACxD,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;YAClC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAEhB,IAAI,WAAW,GAAG,QAAQ,GAAG,OAAO,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC7B,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;aAClD,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YAC3D,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YACjD,WAAW,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY;gBACrD,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrD,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACrC,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7B,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/chunking/token-counter.d.ts

@@ -1,28 +0,0 @@
-interface TokenCountStats {
-    totalRequests: number;
-    cacheHits: number;
-    charFilterSkips: number;
-    actualTokenizations: number;
-    batchTokenizations: number;
-    cacheHitRate: string;
-    charFilterRate: string;
-    tokenizationRate: string;
-}
-interface SizeLimits {
-    min: number;
-    max: number;
-    optimal: number;
-}
-type Decision = 'too_small' | 'too_large' | 'needs_tokenization' | 'optimal';
-export interface CodeSizeAnalysis {
-    size: number;
-    decision: Decision;
-    method: string;
-}
-export declare function analyzeCodeSize(code: string, limits: SizeLimits, tokenCounter: (text: string) => number | Promise<number>, allowEstimateForSkip?: boolean): Promise<CodeSizeAnalysis>;
-export declare function batchAnalyzeCodeSize(codeSnippets: string[], limits: SizeLimits, tokenCounter: (text: string) => number | Promise<number>, allowEstimateForSkip?: boolean): Promise<CodeSizeAnalysis[]>;
-export declare function getTokenCountStats(): TokenCountStats;
-export declare function resetTokenCountStats(): void;
-export declare function clearTokenCache(): void;
-export {};
-//# sourceMappingURL=token-counter.d.ts.map

package/dist/chunking/token-counter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"token-counter.d.ts","sourceRoot":"","sources":["../../src/chunking/token-counter.ts"],"names":[],"mappings":"AAqCA,UAAU,eAAe;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAeD,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GAAG,WAAW,GAAG,WAAW,GAAG,oBAAoB,GAAG,SAAS,CAAC;AAwF7E,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACxD,oBAAoB,UAAQ,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAgC3B;AAED,wBAAsB,oBAAoB,CACxC,YAAY,EAAE,MAAM,EAAE,EACtB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACxD,oBAAoB,UAAQ,GAC3B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAiD7B;AAED,wBAAgB,kBAAkB,IAAI,eAAe,CAapD;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAM3C;AAED,wBAAgB,eAAe,IAAI,IAAI,CAEtC"}

package/dist/chunking/token-counter.js

@@ -1,207 +0,0 @@
-class LRUCache {
-    maxSize;
-    cache = new Map();
-    constructor(maxSize = 1000) {
-        this.maxSize = maxSize;
-    }
-    get(key) {
-        if (!this.cache.has(key))
-            return undefined;
-        const value = this.cache.get(key);
-        this.cache.delete(key);
-        this.cache.set(key, value);
-        return value;
-    }
-    set(key, value) {
-        if (this.cache.has(key)) {
-            this.cache.delete(key);
-        }
-        this.cache.set(key, value);
-        if (this.cache.size > this.maxSize) {
-            const firstKey = this.cache.keys().next().value;
-            if (firstKey !== undefined) {
-                this.cache.delete(firstKey);
-            }
-        }
-    }
-    clear() {
-        this.cache.clear();
-    }
-}
-const tokenCountCache = new LRUCache(1000);
-const stats = {
-    totalRequests: 0,
-    cacheHits: 0,
-    charFilterSkips: 0,
-    actualTokenizations: 0,
-    batchTokenizations: 0
-};
-function estimateTokensFromChars(charCount) {
-    return Math.ceil(charCount / 4);
-}
-function preFilterByChars(code, limits) {
-    const charCount = code.length;
-    const estimatedTokens = estimateTokensFromChars(charCount);
-    const minEstimate = limits.min * 0.8;
-    const maxEstimate = limits.max * 1.2;
-    const optimalLow = limits.optimal * 0.8;
-    const optimalHigh = limits.optimal * 1.2;
-    if (estimatedTokens < minEstimate) {
-        return { decision: 'too_small', estimate: estimatedTokens };
-    }
-    if (estimatedTokens > maxEstimate) {
-        return { decision: 'too_large', estimate: estimatedTokens };
-    }
-    if (estimatedTokens >= optimalLow && estimatedTokens <= optimalHigh) {
-        return { decision: 'optimal', estimate: estimatedTokens };
-    }
-    return { decision: 'needs_tokenization', estimate: estimatedTokens };
-}
-async function countTokensWithCache(code, tokenCounter) {
-    stats.totalRequests++;
-    const cached = tokenCountCache.get(code);
-    if (cached !== undefined) {
-        stats.cacheHits++;
-        return cached;
-    }
-    stats.actualTokenizations++;
-    const result = tokenCounter(code);
-    const count = result instanceof Promise ? await result : result;
-    tokenCountCache.set(code, count);
-    return count;
-}
-async function batchCountTokens(codeSnippets, tokenCounter) {
-    stats.batchTokenizations++;
-    const results = [];
-    const uncached = [];
-    const uncachedIndices = [];
-    for (let i = 0; i < codeSnippets.length; i++) {
-        const code = codeSnippets[i];
-        const cached = tokenCountCache.get(code);
-        if (cached !== undefined) {
-            stats.cacheHits++;
-            results[i] = cached;
-        }
-        else {
-            uncached.push(code);
-            uncachedIndices.push(i);
-        }
-    }
-    if (uncached.length > 0) {
-        stats.actualTokenizations += uncached.length;
-        const counts = await Promise.all(uncached.map(async (code) => {
-            const result = tokenCounter(code);
-            return result instanceof Promise ? await result : result;
-        }));
-        for (let i = 0; i < counts.length; i++) {
-            const code = uncached[i];
-            const count = counts[i];
-            tokenCountCache.set(code, count);
-            results[uncachedIndices[i]] = count;
-        }
-    }
-    return results;
-}
-export async function analyzeCodeSize(code, limits, tokenCounter, allowEstimateForSkip = false) {
-    stats.totalRequests++;
-    const preFilter = preFilterByChars(code, limits);
-    if (allowEstimateForSkip && preFilter.decision === 'too_large') {
-        stats.charFilterSkips++;
-        return {
-            size: preFilter.estimate,
-            decision: preFilter.decision,
-            method: 'char_estimate'
-        };
-    }
-    const actualSize = await countTokensWithCache(code, tokenCounter);
-    let decision;
-    if (actualSize < limits.min) {
-        decision = 'too_small';
-    }
-    else if (actualSize > limits.max) {
-        decision = 'too_large';
-    }
-    else if (actualSize <= limits.optimal) {
-        decision = 'optimal';
-    }
-    else {
-        decision = 'needs_tokenization';
-    }
-    return {
-        size: actualSize,
-        decision,
-        method: 'tokenized'
-    };
-}
-export async function batchAnalyzeCodeSize(codeSnippets, limits, tokenCounter, allowEstimateForSkip = false) {
-    const results = [];
-    const needsTokenization = [];
-    const needsTokenizationIndices = [];
-    for (let i = 0; i < codeSnippets.length; i++) {
-        const code = codeSnippets[i];
-        const preFilter = preFilterByChars(code, limits);
-        if (allowEstimateForSkip && preFilter.decision === 'too_large') {
-            stats.charFilterSkips++;
-            results[i] = {
-                size: preFilter.estimate,
-                decision: preFilter.decision,
-                method: 'char_estimate'
-            };
-        }
-        else {
-            needsTokenization.push(code);
-            needsTokenizationIndices.push(i);
-        }
-    }
-    if (needsTokenization.length > 0) {
-        const tokenCounts = await batchCountTokens(needsTokenization, tokenCounter);
-        for (let i = 0; i < tokenCounts.length; i++) {
-            const actualSize = tokenCounts[i];
-            const idx = needsTokenizationIndices[i];
-            let decision;
-            if (actualSize < limits.min) {
-                decision = 'too_small';
-            }
-            else if (actualSize > limits.max) {
-                decision = 'too_large';
-            }
-            else if (actualSize <= limits.optimal) {
-                decision = 'optimal';
-            }
-            else {
-                decision = 'needs_tokenization';
-            }
-            results[idx] = {
-                size: actualSize,
-                decision,
-                method: 'tokenized'
-            };
-        }
-    }
-    return results;
-}
-export function getTokenCountStats() {
-    return {
-        ...stats,
-        cacheHitRate: stats.totalRequests > 0
-            ? (stats.cacheHits / stats.totalRequests * 100).toFixed(1) + '%'
-            : '0%',
-        charFilterRate: stats.totalRequests > 0
-            ? (stats.charFilterSkips / stats.totalRequests * 100).toFixed(1) + '%'
-            : '0%',
-        tokenizationRate: stats.totalRequests > 0
-            ? (stats.actualTokenizations / stats.totalRequests * 100).toFixed(1) + '%'
-            : '0%'
-    };
-}
-export function resetTokenCountStats() {
-    stats.totalRequests = 0;
-    stats.cacheHits = 0;
-    stats.charFilterSkips = 0;
-    stats.actualTokenizations = 0;
-    stats.batchTokenizations = 0;
-}
-export function clearTokenCache() {
-    tokenCountCache.clear();
-}
-//# sourceMappingURL=token-counter.js.map