codevault 1.6.0 → 1.6.1

package/dist/chunking/file-grouper.d.ts

@@ -0,0 +1,39 @@
+import type { ModelProfile } from '../providers/base.js';
+import type { TreeSitterNode } from '../types/ast.js';
+interface LanguageRule {
+    subdivisionTypes?: Record<string, string[]>;
+    [key: string]: any;
+}
+interface NodeAnalysis {
+    node: TreeSitterNode;
+    size: number;
+    code: string;
+}
+interface SemanticGroup {
+    type: string;
+    containerType?: string;
+    nodes: TreeSitterNode[];
+    analyses: NodeAnalysis[];
+    parentNode: TreeSitterNode | null;
+}
+export interface NodeGroup {
+    nodes: TreeSitterNode[];
+    totalSize: number;
+    groupInfo: SemanticGroup[];
+}
+export declare function groupNodesForChunking(nodes: TreeSitterNode[], source: string, profile: ModelProfile, rule: LanguageRule): Promise<NodeGroup[]>;
+export interface CombinedChunk {
+    code: string;
+    node: TreeSitterNode & {
+        type: string;
+    };
+    metadata: {
+        isGroup: boolean;
+        nodeCount: number;
+        totalSize: number;
+        groupTypes: string[];
+    };
+}
+export declare function createCombinedChunk(nodeGroup: NodeGroup, source: string): CombinedChunk | null;
+export {};
+//# sourceMappingURL=file-grouper.d.ts.map

package/dist/chunking/file-grouper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-grouper.d.ts","sourceRoot":"","sources":["../../src/chunking/file-grouper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,UAAU,YAAY;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,UAAU,YAAY;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,UAAU,EAAE,cAAc,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AAwJD,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,SAAS,EAAE,CAAC,CAkBtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,QAAQ,EAAE;QACR,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CA4B9F"}

package/dist/chunking/file-grouper.js

@@ -0,0 +1,164 @@
+import { batchAnalyzeCodeSize } from './token-counter.js';
+import { getSizeLimits } from '../providers/base.js';
+async function batchAnalyzeNodesInternal(nodes, source, profile) {
+    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const limits = getSizeLimits(profile);
+    if (profile.useTokens && profile.tokenCounter) {
+        const analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, false);
+        return nodes.map((node, i) => ({
+            node,
+            size: analyses[i].size,
+            code: codes[i]
+        }));
+    }
+    return nodes.map((node, i) => ({
+        node,
+        size: codes[i].length,
+        code: codes[i]
+    }));
+}
+function isContainerNode(node, rule) {
+    const containerTypes = [
+        'class_declaration',
+        'class_definition',
+        'interface_declaration',
+        'module_declaration',
+        'namespace_declaration',
+        'trait_declaration',
+        'enum_declaration'
+    ];
+    return containerTypes.includes(node.type);
+}
+function identifySemanticGroups(nodes, source, nodeAnalyses, rule) {
+    const groups = [];
+    let currentGroup = {
+        type: 'file_section',
+        nodes: [],
+        analyses: [],
+        parentNode: null
+    };
+    for (let i = 0; i < nodes.length; i++) {
+        const node = nodes[i];
+        const analysis = nodeAnalyses[i];
+        if (isContainerNode(node, rule)) {
+            if (currentGroup.nodes.length > 0) {
+                groups.push(currentGroup);
+            }
+            currentGroup = {
+                type: 'container',
+                containerType: node.type,
+                nodes: [node],
+                analyses: [analysis],
+                parentNode: node
+            };
+            groups.push(currentGroup);
+            currentGroup = {
+                type: 'file_section',
+                nodes: [],
+                analyses: [],
+                parentNode: null
+            };
+        }
+        else {
+            currentGroup.nodes.push(node);
+            currentGroup.analyses.push(analysis);
+        }
+    }
+    if (currentGroup.nodes.length > 0) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
+async function combineGroupsToOptimalSize(semanticGroups, source, profile, limits) {
+    const optimalGroups = [];
+    let currentCombinedGroup = {
+        nodes: [],
+        totalSize: 0,
+        groupInfo: []
+    };
+    for (const group of semanticGroups) {
+        const groupTotalSize = group.analyses.reduce((sum, a) => sum + a.size, 0);
+        if (groupTotalSize > limits.optimal) {
+            if (currentCombinedGroup.nodes.length > 0) {
+                optimalGroups.push(currentCombinedGroup);
+            }
+            optimalGroups.push({
+                nodes: group.nodes,
+                totalSize: groupTotalSize,
+                groupInfo: [group]
+            });
+            currentCombinedGroup = {
+                nodes: [],
+                totalSize: 0,
+                groupInfo: []
+            };
+            continue;
+        }
+        if (currentCombinedGroup.totalSize + groupTotalSize > limits.max) {
+            if (currentCombinedGroup.nodes.length > 0) {
+                optimalGroups.push(currentCombinedGroup);
+            }
+            currentCombinedGroup = {
+                nodes: group.nodes,
+                totalSize: groupTotalSize,
+                groupInfo: [group]
+            };
+            continue;
+        }
+        currentCombinedGroup.nodes.push(...group.nodes);
+        currentCombinedGroup.totalSize += groupTotalSize;
+        currentCombinedGroup.groupInfo.push(group);
+        if (currentCombinedGroup.totalSize >= limits.optimal * 0.9) {
+            optimalGroups.push(currentCombinedGroup);
+            currentCombinedGroup = {
+                nodes: [],
+                totalSize: 0,
+                groupInfo: []
+            };
+        }
+    }
+    if (currentCombinedGroup.nodes.length > 0) {
+        optimalGroups.push(currentCombinedGroup);
+    }
+    return optimalGroups;
+}
+export async function groupNodesForChunking(nodes, source, profile, rule) {
+    if (!nodes || nodes.length === 0)
+        return [];
+    const limits = getSizeLimits(profile);
+    if (nodes.length <= 10) {
+        return nodes.map(node => ({
+            nodes: [node],
+            totalSize: 0,
+            groupInfo: []
+        }));
+    }
+    const nodeAnalyses = await batchAnalyzeNodesInternal(nodes, source, profile);
+    const semanticGroups = identifySemanticGroups(nodes, source, nodeAnalyses, rule);
+    const optimalGroups = await combineGroupsToOptimalSize(semanticGroups, source, profile, limits);
+    return optimalGroups;
+}
+export function createCombinedChunk(nodeGroup, source) {
+    if (!nodeGroup.nodes || nodeGroup.nodes.length === 0) {
+        return null;
+    }
+    const codes = nodeGroup.nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const combinedCode = codes.join('\n\n');
+    const firstNode = nodeGroup.nodes[0];
+    const lastNode = nodeGroup.nodes[nodeGroup.nodes.length - 1];
+    return {
+        code: combinedCode,
+        node: {
+            ...firstNode,
+            type: `${firstNode.type}_group_${nodeGroup.nodes.length}`,
+            endIndex: lastNode.endIndex
+        },
+        metadata: {
+            isGroup: true,
+            nodeCount: nodeGroup.nodes.length,
+            totalSize: nodeGroup.totalSize,
+            groupTypes: nodeGroup.groupInfo?.map(g => g.type) || ['combined']
+        }
+    };
+}
+//# sourceMappingURL=file-grouper.js.map

package/dist/chunking/file-grouper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-grouper.js","sourceRoot":"","sources":["../../src/chunking/file-grouper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,oBAAoB,EAAyB,MAAM,oBAAoB,CAAC;AAElG,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AA4BrD,KAAK,UAAU,yBAAyB,CAAC,KAAuB,EAAE,MAAc,EAAE,OAAqB;IACrG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QACxF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI;YACJ,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;YACtB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI;QACJ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;QACrB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;KACf,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CAAC,IAAoB,EAAE,IAAkB;IAC/D,MAAM,cAAc,GAAG;QACrB,mBAAmB;QACnB,kBAAkB;QAClB,uBAAuB;QACvB,oBAAoB;QACpB,uBAAuB;QACvB,mBAAmB;QACnB,kBAAkB;KACnB,CAAC;IAEF,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAuB,EAAE,MAAc,EAAE,YAA4B,EAAE,IAAkB;IACvH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,YAAY,GAAkB;QAChC,IAAI,EAAE,cAAc;QACpB,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,IAAI;KACjB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAChC,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC5B,CAAC;YAED,YAAY,GAAG;gBACb,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,IAAI,CAAC,IAAI;gBACxB,KAAK,EAAE,CAAC,IAAI,CAAC;gBACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,UAAU,EAAE,IAAI;aACjB,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE1B,YAAY,GAAG;gBACb,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;gBACZ,UAAU,EAAE,IAAI;aACjB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,cAA+B,EAC/B,MAAc,EACd,OAAqB,EACrB,MAAwC;IAExC,MAAM,aAAa,GAAgB,EAAE,CAAC;IACtC,IAAI,oBAAoB,GAAc;QACpC,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,EAAE;KACd,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE1E,IAAI,cAAc,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,aAAa,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC,CAAC;YAEH,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,GAAG,cAAc,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YACjE,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,oBAAoB,GAAG;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC;YACF,SAAS;QACX,CAAC;QAED,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAChD,oBAAoB,CAAC,SAAS,IAAI,cAAc,CAAC;QACjD,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,oBAAoB,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;YAC3D,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACzC,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAuB,EACvB,MAAc,EACd,OAAqB,EACrB,IAAkB;IAElB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,IAAI,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7E,MAAM,cAAc,GAAG,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACjF,MAAM,aAAa,GAAG,MAAM,0BAA0B,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAEhG,OAAO,aAAa,CAAC;AACvB,CAAC;AAaD,MAAM,UAAU,mBAAmB,CAAC,SAAoB,EAAE,MAAc;IACtE,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAC7C,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAExC,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE;YACJ,GAAG,SAAS;YACZ,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,UAAU,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;YACzD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,UAAU,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;SAClE;KACF,CAAC;AACJ,CAAC"}

package/dist/chunking/semantic-chunker.d.ts

@@ -0,0 +1,37 @@
+import type { ModelProfile } from '../providers/base.js';
+import type { TreeSitterNode } from '../types/ast.js';
+interface LanguageRule {
+    subdivisionTypes?: Record<string, string[]>;
+    [key: string]: any;
+}
+export declare function findSemanticSubdivisions(node: TreeSitterNode, rule: LanguageRule): TreeSitterNode[];
+export declare function findLastCompleteBoundary(code: string, maxSize: number): number;
+export declare function extractSignature(node: TreeSitterNode, source: string): string;
+export declare function extractLinesBeforeNode(node: TreeSitterNode, source: string, numLines: number): string;
+export declare function extractParentContext(node: TreeSitterNode, source: string): {
+    signature: string;
+    startLine: number;
+    endLine: number;
+};
+export declare function getLineNumber(byteOffset: number, source: string): number;
+export interface NodeAnalysis {
+    isSingleChunk: boolean;
+    needsSubdivision: boolean;
+    subdivisionCandidates: TreeSitterNode[];
+    size: number;
+    unit: string;
+    method: string;
+    estimatedSubchunks: number;
+}
+export declare function analyzeNodeForChunking(node: TreeSitterNode, source: string, rule: LanguageRule, profile: ModelProfile): Promise<NodeAnalysis>;
+export declare function batchAnalyzeNodes(nodes: TreeSitterNode[], source: string, rule: LanguageRule, profile: ModelProfile, isSubdivision?: boolean): Promise<Array<NodeAnalysis & {
+    node: TreeSitterNode;
+}>>;
+export interface StatementChunk {
+    code: string;
+    size: number;
+    unit: string;
+}
+export declare function yieldStatementChunks(node: TreeSitterNode, source: string, maxSize: number, overlapSize: number, profile: ModelProfile): Promise<StatementChunk[]>;
+export {};
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunking/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunking/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,UAAU,YAAY;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,EAAE,CAwBnG;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAgB9E;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAO7E;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIrG;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAGxE;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,qBAAqB,EAAE,cAAc,EAAE,CAAC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,YAAY,CAAC,CA2BvB;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,EACrB,aAAa,UAAQ,GACpB,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC,CAAC,CAgCzD;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,cAAc,EAAE,CAAC,CA0C3B"}

package/dist/chunking/semantic-chunker.js

@@ -0,0 +1,157 @@
+import { analyzeCodeSize, batchAnalyzeCodeSize } from './token-counter.js';
+import { getSizeLimits } from '../providers/base.js';
+export function findSemanticSubdivisions(node, rule) {
+    if (!node || !rule)
+        return [];
+    const subdivisionTypes = rule.subdivisionTypes?.[node.type] || [];
+    if (subdivisionTypes.length === 0)
+        return [];
+    const candidates = [];
+    function walk(n, depth = 0) {
+        if (depth > 0 && subdivisionTypes.includes(n.type)) {
+            candidates.push(n);
+            return;
+        }
+        for (let i = 0; i < n.childCount; i++) {
+            const child = n.child(i);
+            if (child) {
+                walk(child, depth + 1);
+            }
+        }
+    }
+    walk(node);
+    return candidates;
+}
+export function findLastCompleteBoundary(code, maxSize) {
+    const boundaries = [
+        { pattern: /\n\s*}\s*$/gm, priority: 1 },
+        { pattern: /;\s*$/gm, priority: 2 },
+        { pattern: /\n\s*$/gm, priority: 3 }
+    ];
+    for (const boundary of boundaries) {
+        const matches = [...code.substring(0, maxSize).matchAll(boundary.pattern)];
+        if (matches.length > 0) {
+            const lastMatch = matches[matches.length - 1];
+            return lastMatch.index + lastMatch[0].length;
+        }
+    }
+    return maxSize;
+}
+export function extractSignature(node, source) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const firstBrace = code.indexOf('{');
+    if (firstBrace !== -1) {
+        return code.substring(0, firstBrace).trim() + ' {';
+    }
+    return code.split('\n')[0];
+}
+export function extractLinesBeforeNode(node, source, numLines) {
+    const beforeCode = source.substring(0, node.startIndex);
+    const lines = beforeCode.split('\n');
+    return lines.slice(-numLines).join('\n');
+}
+export function extractParentContext(node, source) {
+    return {
+        signature: extractSignature(node, source),
+        startLine: getLineNumber(node.startIndex, source),
+        endLine: getLineNumber(node.endIndex, source)
+    };
+}
+export function getLineNumber(byteOffset, source) {
+    const before = source.substring(0, byteOffset);
+    return before.split('\n').length;
+}
+export async function analyzeNodeForChunking(node, source, rule, profile) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const limits = getSizeLimits(profile);
+    let actualSize;
+    let method;
+    if (profile.useTokens && profile.tokenCounter) {
+        const analysis = await analyzeCodeSize(code, limits, profile.tokenCounter);
+        actualSize = analysis.size;
+        method = analysis.method;
+    }
+    else {
+        actualSize = code.length;
+        method = 'chars';
+    }
+    const subdivisionThreshold = limits.max;
+    return {
+        isSingleChunk: actualSize <= subdivisionThreshold,
+        needsSubdivision: actualSize > subdivisionThreshold,
+        subdivisionCandidates: findSemanticSubdivisions(node, rule),
+        size: actualSize,
+        unit: limits.unit,
+        method,
+        estimatedSubchunks: Math.ceil(actualSize / limits.optimal)
+    };
+}
+export async function batchAnalyzeNodes(nodes, source, rule, profile, isSubdivision = false) {
+    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const limits = getSizeLimits(profile);
+    let analyses;
+    if (profile.useTokens && profile.tokenCounter) {
+        analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, isSubdivision);
+    }
+    else {
+        analyses = codes.map(code => ({
+            size: code.length,
+            decision: code.length < limits.min ? 'too_small'
+                : code.length > limits.max ? 'too_large'
+                    : code.length <= limits.optimal ? 'optimal'
+                        : 'needs_tokenization',
+            method: 'chars'
+        }));
+    }
+    return nodes.map((node, i) => {
+        const analysis = analyses[i];
+        const subdivisionThreshold = limits.max;
+        return {
+            node,
+            isSingleChunk: analysis.size <= subdivisionThreshold,
+            needsSubdivision: analysis.size > subdivisionThreshold,
+            subdivisionCandidates: findSemanticSubdivisions(node, rule),
+            size: analysis.size,
+            unit: limits.unit,
+            method: analysis.method,
+            estimatedSubchunks: Math.ceil(analysis.size / limits.optimal)
+        };
+    });
+}
+export async function yieldStatementChunks(node, source, maxSize, overlapSize, profile) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const lines = code.split('\n');
+    const chunks = [];
+    let currentChunk = [];
+    let currentSize = 0;
+    for (const line of lines) {
+        const lineSize = profile.useTokens && profile.tokenCounter
+            ? await profile.tokenCounter(line)
+            : line.length;
+        if (currentSize + lineSize > maxSize && currentChunk.length > 0) {
+            chunks.push({
+                code: currentChunk.join('\n'),
+                size: currentSize,
+                unit: profile.useTokens ? 'tokens' : 'characters'
+            });
+            // Use the provided overlapSize parameter instead of hardcoded 20%
+            const overlapRatio = Math.min(1, Math.max(0, overlapSize / maxSize));
+            const overlapLines = Math.max(1, Math.floor(currentChunk.length * overlapRatio));
+            currentChunk = currentChunk.slice(-overlapLines);
+            currentSize = profile.useTokens && profile.tokenCounter
+                ? await profile.tokenCounter(currentChunk.join('\n'))
+                : currentChunk.join('\n').length;
+        }
+        currentChunk.push(line);
+        currentSize += lineSize;
+    }
+    if (currentChunk.length > 0) {
+        chunks.push({
+            code: currentChunk.join('\n'),
+            size: currentSize,
+            unit: profile.useTokens ? 'tokens' : 'characters'
+        });
+    }
+    return chunks;
+}
+//# sourceMappingURL=semantic-chunker.js.map

package/dist/chunking/semantic-chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunking/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAyB,MAAM,oBAAoB,CAAC;AAElG,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAQrD,MAAM,UAAU,wBAAwB,CAAC,IAAoB,EAAE,IAAkB;IAC/E,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAE9B,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClE,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7C,MAAM,UAAU,GAAqB,EAAE,CAAC;IAExC,SAAS,IAAI,CAAC,CAAiB,EAAE,KAAK,GAAG,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,OAAO;QACT,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC;IACX,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAY,EAAE,OAAe;IACpE,MAAM,UAAU,GAAG;QACjB,EAAE,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,EAAE;QACxC,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,EAAE;QACnC,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE;KACrC,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9C,OAAO,SAAS,CAAC,KAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAoB,EAAE,MAAc;IACnE,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,IAAoB,EAAE,MAAc,EAAE,QAAgB;IAC3F,MAAM,UAAU,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,IAAoB,EAAE,MAAc;IAKvE,OAAO;QACL,SAAS,EAAE,gBAAgB,CAAC,IAAI,EAAE,MAAM,CAAC;QACzC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC;QACjD,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,UAAkB,EAAE,MAAc;IAC9D,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/C,OAAO,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACnC,CAAC;AAYD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,IAAoB,EACpB,MAAc,EACd,IAAkB,EAClB,OAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAc,CAAC;IAEnB,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;QAC3E,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3B,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC3B,CAAC;SAAM,CAAC;QACN,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;QACzB,MAAM,GAAG,OAAO,CAAC;IACnB,CAAC;IAED,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAC;IAExC,OAAO;QACL,aAAa,EAAE,UAAU,IAAI,oBAAoB;QACjD,gBAAgB,EAAE,UAAU,GAAG,oBAAoB;QACnD,qBAAqB,EAAE,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC;QAC3D,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,MAAM;QACN,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC;KAC3D,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAuB,EACvB,MAAc,EACd,IAAkB,EAClB,OAAqB,EACrB,aAAa,GAAG,KAAK;IAErB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,QAA4B,CAAC;IACjC,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC;IAC5F,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,QAAQ,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAoB;gBACnD,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,WAAoB;oBACjD,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAkB;wBACpD,CAAC,CAAC,oBAA6B;YACrC,MAAM,EAAE,OAAO;SAChB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,oBAAoB,GAAG,MAAM,CAAC,GAAG,CAAC;QACxC,OAAO;YACL,IAAI;YACJ,aAAa,EAAE,QAAQ,CAAC,IAAI,IAAI,oBAAoB;YACpD,gBAAgB,EAAE,QAAQ,CAAC,IAAI,GAAG,oBAAoB;YACtD,qBAAqB,EAAE,wBAAwB,CAAC,IAAI,EAAE,IAAI,CAAC;YAC3D,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;SAC9D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAoB,EACpB,MAAc,EACd,OAAe,EACf,WAAmB,EACnB,OAAqB;IAErB,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,YAAY,GAAa,EAAE,CAAC;IAChC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY;YACxD,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC;YAClC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAEhB,IAAI,WAAW,GAAG,QAAQ,GAAG,OAAO,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC7B,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;aAClD,CAAC,CAAC;YAEH,kEAAkE;YAClE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC;YACjF,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;YACjD,WAAW,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY;gBACrD,CAAC,CAAC,MAAM,OAAO,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrD,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACrC,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAC7B,IAAI,EAAE,WAAW;YACjB,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY;SAClD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/chunking/token-counter.d.ts

@@ -0,0 +1,28 @@
+interface TokenCountStats {
+    totalRequests: number;
+    cacheHits: number;
+    charFilterSkips: number;
+    actualTokenizations: number;
+    batchTokenizations: number;
+    cacheHitRate: string;
+    charFilterRate: string;
+    tokenizationRate: string;
+}
+interface SizeLimits {
+    min: number;
+    max: number;
+    optimal: number;
+}
+type Decision = 'too_small' | 'too_large' | 'needs_tokenization' | 'optimal';
+export interface CodeSizeAnalysis {
+    size: number;
+    decision: Decision;
+    method: string;
+}
+export declare function analyzeCodeSize(code: string, limits: SizeLimits, tokenCounter: (text: string) => number | Promise<number>, allowEstimateForSkip?: boolean): Promise<CodeSizeAnalysis>;
+export declare function batchAnalyzeCodeSize(codeSnippets: string[], limits: SizeLimits, tokenCounter: (text: string) => number | Promise<number>, allowEstimateForSkip?: boolean): Promise<CodeSizeAnalysis[]>;
+export declare function getTokenCountStats(): TokenCountStats;
+export declare function resetTokenCountStats(): void;
+export declare function clearTokenCache(): void;
+export {};
+//# sourceMappingURL=token-counter.d.ts.map

package/dist/chunking/token-counter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-counter.d.ts","sourceRoot":"","sources":["../../src/chunking/token-counter.ts"],"names":[],"mappings":"AAqCA,UAAU,eAAe;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAeD,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,KAAK,QAAQ,GAAG,WAAW,GAAG,WAAW,GAAG,oBAAoB,GAAG,SAAS,CAAC;AAwF7E,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,wBAAsB,eAAe,CACnC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACxD,oBAAoB,UAAQ,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAgC3B;AAED,wBAAsB,oBAAoB,CACxC,YAAY,EAAE,MAAM,EAAE,EACtB,MAAM,EAAE,UAAU,EAClB,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,EACxD,oBAAoB,UAAQ,GAC3B,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAiD7B;AAED,wBAAgB,kBAAkB,IAAI,eAAe,CAapD;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAM3C;AAED,wBAAgB,eAAe,IAAI,IAAI,CAEtC"}

package/dist/chunking/token-counter.js

@@ -0,0 +1,207 @@
+class LRUCache {
+    maxSize;
+    cache = new Map();
+    constructor(maxSize = 1000) {
+        this.maxSize = maxSize;
+    }
+    get(key) {
+        if (!this.cache.has(key))
+            return undefined;
+        const value = this.cache.get(key);
+        this.cache.delete(key);
+        this.cache.set(key, value);
+        return value;
+    }
+    set(key, value) {
+        if (this.cache.has(key)) {
+            this.cache.delete(key);
+        }
+        this.cache.set(key, value);
+        if (this.cache.size > this.maxSize) {
+            const firstKey = this.cache.keys().next().value;
+            if (firstKey !== undefined) {
+                this.cache.delete(firstKey);
+            }
+        }
+    }
+    clear() {
+        this.cache.clear();
+    }
+}
+const tokenCountCache = new LRUCache(1000);
+const stats = {
+    totalRequests: 0,
+    cacheHits: 0,
+    charFilterSkips: 0,
+    actualTokenizations: 0,
+    batchTokenizations: 0
+};
+function estimateTokensFromChars(charCount) {
+    return Math.ceil(charCount / 4);
+}
+function preFilterByChars(code, limits) {
+    const charCount = code.length;
+    const estimatedTokens = estimateTokensFromChars(charCount);
+    const minEstimate = limits.min * 0.8;
+    const maxEstimate = limits.max * 1.2;
+    const optimalLow = limits.optimal * 0.8;
+    const optimalHigh = limits.optimal * 1.2;
+    if (estimatedTokens < minEstimate) {
+        return { decision: 'too_small', estimate: estimatedTokens };
+    }
+    if (estimatedTokens > maxEstimate) {
+        return { decision: 'too_large', estimate: estimatedTokens };
+    }
+    if (estimatedTokens >= optimalLow && estimatedTokens <= optimalHigh) {
+        return { decision: 'optimal', estimate: estimatedTokens };
+    }
+    return { decision: 'needs_tokenization', estimate: estimatedTokens };
+}
+async function countTokensWithCache(code, tokenCounter) {
+    stats.totalRequests++;
+    const cached = tokenCountCache.get(code);
+    if (cached !== undefined) {
+        stats.cacheHits++;
+        return cached;
+    }
+    stats.actualTokenizations++;
+    const result = tokenCounter(code);
+    const count = result instanceof Promise ? await result : result;
+    tokenCountCache.set(code, count);
+    return count;
+}
+async function batchCountTokens(codeSnippets, tokenCounter) {
+    stats.batchTokenizations++;
+    const results = [];
+    const uncached = [];
+    const uncachedIndices = [];
+    for (let i = 0; i < codeSnippets.length; i++) {
+        const code = codeSnippets[i];
+        const cached = tokenCountCache.get(code);
+        if (cached !== undefined) {
+            stats.cacheHits++;
+            results[i] = cached;
+        }
+        else {
+            uncached.push(code);
+            uncachedIndices.push(i);
+        }
+    }
+    if (uncached.length > 0) {
+        stats.actualTokenizations += uncached.length;
+        const counts = await Promise.all(uncached.map(async (code) => {
+            const result = tokenCounter(code);
+            return result instanceof Promise ? await result : result;
+        }));
+        for (let i = 0; i < counts.length; i++) {
+            const code = uncached[i];
+            const count = counts[i];
+            tokenCountCache.set(code, count);
+            results[uncachedIndices[i]] = count;
+        }
+    }
+    return results;
+}
+export async function analyzeCodeSize(code, limits, tokenCounter, allowEstimateForSkip = false) {
+    stats.totalRequests++;
+    const preFilter = preFilterByChars(code, limits);
+    if (allowEstimateForSkip && preFilter.decision === 'too_large') {
+        stats.charFilterSkips++;
+        return {
+            size: preFilter.estimate,
+            decision: preFilter.decision,
+            method: 'char_estimate'
+        };
+    }
+    const actualSize = await countTokensWithCache(code, tokenCounter);
+    let decision;
+    if (actualSize < limits.min) {
+        decision = 'too_small';
+    }
+    else if (actualSize > limits.max) {
+        decision = 'too_large';
+    }
+    else if (actualSize <= limits.optimal) {
+        decision = 'optimal';
+    }
+    else {
+        decision = 'needs_tokenization';
+    }
+    return {
+        size: actualSize,
+        decision,
+        method: 'tokenized'
+    };
+}
+export async function batchAnalyzeCodeSize(codeSnippets, limits, tokenCounter, allowEstimateForSkip = false) {
+    const results = [];
+    const needsTokenization = [];
+    const needsTokenizationIndices = [];
+    for (let i = 0; i < codeSnippets.length; i++) {
+        const code = codeSnippets[i];
+        const preFilter = preFilterByChars(code, limits);
+        if (allowEstimateForSkip && preFilter.decision === 'too_large') {
+            stats.charFilterSkips++;
+            results[i] = {
+                size: preFilter.estimate,
+                decision: preFilter.decision,
+                method: 'char_estimate'
+            };
+        }
+        else {
+            needsTokenization.push(code);
+            needsTokenizationIndices.push(i);
+        }
+    }
+    if (needsTokenization.length > 0) {
+        const tokenCounts = await batchCountTokens(needsTokenization, tokenCounter);
+        for (let i = 0; i < tokenCounts.length; i++) {
+            const actualSize = tokenCounts[i];
+            const idx = needsTokenizationIndices[i];
+            let decision;
+            if (actualSize < limits.min) {
+                decision = 'too_small';
+            }
+            else if (actualSize > limits.max) {
+                decision = 'too_large';
+            }
+            else if (actualSize <= limits.optimal) {
+                decision = 'optimal';
+            }
+            else {
+                decision = 'needs_tokenization';
+            }
+            results[idx] = {
+                size: actualSize,
+                decision,
+                method: 'tokenized'
+            };
+        }
+    }
+    return results;
+}
+export function getTokenCountStats() {
+    return {
+        ...stats,
+        cacheHitRate: stats.totalRequests > 0
+            ? (stats.cacheHits / stats.totalRequests * 100).toFixed(1) + '%'
+            : '0%',
+        charFilterRate: stats.totalRequests > 0
+            ? (stats.charFilterSkips / stats.totalRequests * 100).toFixed(1) + '%'
+            : '0%',
+        tokenizationRate: stats.totalRequests > 0
+            ? (stats.actualTokenizations / stats.totalRequests * 100).toFixed(1) + '%'
+            : '0%'
+    };
+}
+export function resetTokenCountStats() {
+    stats.totalRequests = 0;
+    stats.cacheHits = 0;
+    stats.charFilterSkips = 0;
+    stats.actualTokenizations = 0;
+    stats.batchTokenizations = 0;
+}
+export function clearTokenCache() {
+    tokenCountCache.clear();
+}
+//# sourceMappingURL=token-counter.js.map