codevault 1.0.0

package/.env.example

@@ -0,0 +1,37 @@
+# OpenAI / Nebius API Configuration
+# For Nebius AI (Qwen embeddings), use your Nebius JWT token and base URL
+OPENAI_API_KEY=your-api-key-here
+OPENAI_BASE_URL=https://api.openai.com/v1
+
+# Embedding Model Configuration
+# For standard OpenAI: text-embedding-3-large
+# For Nebius/Qwen: Qwen/Qwen3-Embedding-8B
+CODEVAULT_OPENAI_EMBEDDING_MODEL=text-embedding-3-large
+
+# For Ollama (local embeddings)
+CODEVAULT_OLLAMA_MODEL=nomic-embed-text
+
+# Chunking Configuration
+# Standard: 8192 tokens, Qwen: 32000 tokens
+CODEVAULT_MAX_TOKENS=8192
+
+# Embedding Dimensions
+# OpenAI text-embedding-3-large: 3072
+# Qwen3-Embedding-8B: 4096
+# Ollama nomic-embed-text: 768
+CODEVAULT_DIMENSIONS=3072
+
+# Rate Limiting
+# Adjust based on your API tier
+CODEVAULT_RATE_LIMIT_RPM=10000
+CODEVAULT_RATE_LIMIT_TPM=600000
+
+# Reranking API (Optional)
+# For NVIDIA reranking service
+CODEVAULT_RERANK_API_URL=https://ai.api.nvidia.com/v1/retrieval/nvidia/nv-rerankqa-mistral-4b-v3/reranking
+CODEVAULT_RERANK_API_KEY=your-nvidia-api-key-here
+CODEVAULT_RERANK_MODEL=nvidia/nv-rerankqa-mistral-4b-v3
+
+# Encryption (Optional)
+# 32-byte key encoded as base64 or hex
+# CODEVAULT_ENCRYPTION_KEY=your-encryption-key-here

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Shariq Riaz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,174 @@
+# CodeVault
+
+> AI-powered semantic code search via Model Context Protocol (MCP)
+
+CodeVault is an intelligent code indexing and search system that enables AI assistants to understand and navigate your codebase using semantic search, symbol-aware ranking, and hybrid retrieval techniques.
+
+## 🌟 Features
+
+- **🔍 Semantic Code Search**: Find code by meaning, not just keywords
+- **🤖 MCP Integration**: Native support for Claude Desktop and other MCP clients
+- **🎯 Symbol-Aware Ranking**: Boost results based on function signatures and relationships
+- **⚡ Hybrid Search**: Combines vector embeddings with BM25 keyword matching
+- **🔄 Context Packs**: Save and reuse search scopes for different projects
+- **📊 Multi-Language Support**: 25+ programming languages via Tree-sitter
+- **🏠 Local-First**: Works with local models (Ollama) or cloud APIs
+- **🔐 Optional Encryption**: Secure your indexed code chunks
+- **📈 Smart Chunking**: Token-aware code splitting for optimal context
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+npm install
+npm run build
+npm link
+```
+
+### Index Your Project
+
+```bash
+# Navigate to your project
+cd /path/to/your/project
+
+# Using Ollama (local, no API key required)
+codevault index --provider ollama
+
+# Using OpenAI
+export OPENAI_API_KEY=your-key-here
+codevault index --provider openai
+
+# Using Qwen (via Nebius AI Studio)
+export OPENAI_API_KEY=your-nebius-api-key
+export OPENAI_BASE_URL=https://api.studio.nebius.com/v1/
+export CODEVAULT_OPENAI_EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B
+codevault index --provider openai
+```
+
+### Search Your Code
+
+```bash
+codevault search "authentication function"
+codevault search "stripe checkout" --tags stripe --lang php
+```
+
+### Use with Claude Desktop
+
+Add to your `claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "codevault": {
+      "command": "node",
+      "args": ["/path/to/codevault-v2/dist/mcp-server.js"],
+      "env": {
+        "OPENAI_API_KEY": "your-api-key-here"
+      }
+    }
+  }
+}
+```
+
+## 📖 Documentation
+
+### Supported Languages
+
+- **Web**: JavaScript, TypeScript, TSX, HTML, CSS, JSON, Markdown
+- **Backend**: Python, PHP, Go, Java, Kotlin, C#, Ruby, Scala, Swift
+- **Systems**: C, C++, Rust
+- **Functional**: Haskell, OCaml, Elixir
+- **Scripting**: Bash, Lua
+
+### Embedding Providers
+
+| Provider | Model | Dimensions | Context | Best For | API Key Required |
+|----------|-------|------------|---------|----------|------------------|
+| **ollama** | nomic-embed-text | 768 | 8K | Local, no API costs | ❌ No |
+| **openai** | text-embedding-3-large | 3072 | 8K | Highest quality | ✅ Yes |
+| **openai** | Qwen/Qwen3-Embedding-8B | 4096 | 32K | Large context, high quality | ✅ Yes (Nebius AI) |
+
+### CLI Commands
+
+```bash
+# Indexing
+codevault index [path]              # Index project
+codevault update [path]             # Update existing index
+codevault watch [path]              # Watch for changes
+
+# Searching
+codevault search <query> [path]     # Search code
+  --limit <num>                     # Max results
+  --provider <name>                 # Embedding provider
+  --path_glob <pattern>             # Filter by file pattern
+  --tags <tag...>                   # Filter by tags
+  --lang <language...>              # Filter by language
+
+# Context Packs
+codevault context list              # List saved contexts
+codevault context show <name>       # Show context pack
+codevault context use <name>        # Activate context pack
+
+# Utilities
+codevault info                      # Project statistics
+codevault mcp                       # Start MCP server
+```
+
+### MCP Tools
+
+- **`search_code`**: Semantic code search with filters
+- **`search_code_with_chunks`**: Search + retrieve full code
+- **`get_code_chunk`**: Get specific code by SHA
+- **`index_project`**: Index a new project
+- **`update_project`**: Update existing index
+- **`get_project_stats`**: Project overview
+- **`use_context_pack`**: Apply saved search context
+
+### Environment Variables
+
+```bash
+# OpenAI Configuration
+OPENAI_API_KEY=sk-...
+OPENAI_BASE_URL=https://api.openai.com/v1
+CODEVAULT_OPENAI_EMBEDDING_MODEL=text-embedding-3-large
+
+# Ollama Configuration
+CODEVAULT_OLLAMA_MODEL=nomic-embed-text
+
+# Chunking
+CODEVAULT_MAX_TOKENS=8192
+CODEVAULT_DIMENSIONS=3072
+
+# Rate Limiting
+CODEVAULT_RATE_LIMIT_RPM=10000    # Requests per minute
+CODEVAULT_RATE_LIMIT_TPM=600000   # Tokens per minute
+
+# Reranking
+CODEVAULT_RERANK_API_URL=...
+CODEVAULT_RERANK_API_KEY=...
+CODEVAULT_RERANK_MODEL=...
+
+# Encryption
+CODEVAULT_ENCRYPTION_KEY=...
+```
+
+## 🏗️ Architecture
+
+### Project Structure
+
+```
+.codevault/
+├── codevault.db          # SQLite database
+├── chunks/               # Compressed code chunks
+└── contextpacks/         # Saved search contexts
+codevault.codemap.json    # Lightweight index
+```
+
+## 📄 License
+
+MIT License
+
+---
+
+**Built by Shariq Riaz**

package/dist/chunking/file-grouper.d.ts

@@ -0,0 +1,39 @@
+import type { ModelProfile } from '../providers/base.js';
+import type { TreeSitterNode } from '../types/ast.js';
+interface LanguageRule {
+    subdivisionTypes?: Record<string, string[]>;
+    [key: string]: any;
+}
+interface NodeAnalysis {
+    node: TreeSitterNode;
+    size: number;
+    code: string;
+}
+interface SemanticGroup {
+    type: string;
+    containerType?: string;
+    nodes: TreeSitterNode[];
+    analyses: NodeAnalysis[];
+    parentNode: TreeSitterNode | null;
+}
+export interface NodeGroup {
+    nodes: TreeSitterNode[];
+    totalSize: number;
+    groupInfo: SemanticGroup[];
+}
+export declare function groupNodesForChunking(nodes: TreeSitterNode[], source: string, profile: ModelProfile, rule: LanguageRule): Promise<NodeGroup[]>;
+export interface CombinedChunk {
+    code: string;
+    node: TreeSitterNode & {
+        type: string;
+    };
+    metadata: {
+        isGroup: boolean;
+        nodeCount: number;
+        totalSize: number;
+        groupTypes: string[];
+    };
+}
+export declare function createCombinedChunk(nodeGroup: NodeGroup, source: string, filerel: string): CombinedChunk | null;
+export {};
+//# sourceMappingURL=file-grouper.d.ts.map

package/dist/chunking/file-grouper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-grouper.d.ts","sourceRoot":"","sources":["../../src/chunking/file-grouper.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,UAAU,YAAY;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAUD,UAAU,YAAY;IACpB,IAAI,EAAE,cAAc,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,UAAU,aAAa;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,UAAU,EAAE,cAAc,GAAG,IAAI,CAAC;CACnC;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AA2KD,wBAAsB,qBAAqB,CACzC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,SAAS,EAAE,CAAC,CAkBtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,cAAc,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACxC,QAAQ,EAAE;QACR,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,CAAC;CACH;AAED,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CA4B/G"}

package/dist/chunking/file-grouper.js

@@ -0,0 +1,181 @@
+import { batchAnalyzeCodeSize } from './token-counter.js';
+function getSizeLimits(profile) {
+    if (profile.useTokens && profile.tokenCounter) {
+        return {
+            optimal: profile.optimalTokens,
+            min: profile.minChunkTokens,
+            max: profile.maxChunkTokens,
+            overlap: profile.overlapTokens,
+            unit: 'tokens'
+        };
+    }
+    return {
+        optimal: profile.optimalChars,
+        min: profile.minChunkChars,
+        max: profile.maxChunkChars,
+        overlap: profile.overlapChars,
+        unit: 'characters'
+    };
+}
+async function batchAnalyzeNodesInternal(nodes, source, profile) {
+    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const limits = getSizeLimits(profile);
+    if (profile.useTokens && profile.tokenCounter) {
+        const analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, false);
+        return nodes.map((node, i) => ({
+            node,
+            size: analyses[i].size,
+            code: codes[i]
+        }));
+    }
+    return nodes.map((node, i) => ({
+        node,
+        size: codes[i].length,
+        code: codes[i]
+    }));
+}
+function isContainerNode(node, rule) {
+    const containerTypes = [
+        'class_declaration',
+        'class_definition',
+        'interface_declaration',
+        'module_declaration',
+        'namespace_declaration',
+        'trait_declaration',
+        'enum_declaration'
+    ];
+    return containerTypes.includes(node.type);
+}
+function identifySemanticGroups(nodes, source, nodeAnalyses, rule) {
+    const groups = [];
+    let currentGroup = {
+        type: 'file_section',
+        nodes: [],
+        analyses: [],
+        parentNode: null
+    };
+    for (let i = 0; i < nodes.length; i++) {
+        const node = nodes[i];
+        const analysis = nodeAnalyses[i];
+        if (isContainerNode(node, rule)) {
+            if (currentGroup.nodes.length > 0) {
+                groups.push(currentGroup);
+            }
+            currentGroup = {
+                type: 'container',
+                containerType: node.type,
+                nodes: [node],
+                analyses: [analysis],
+                parentNode: node
+            };
+            groups.push(currentGroup);
+            currentGroup = {
+                type: 'file_section',
+                nodes: [],
+                analyses: [],
+                parentNode: null
+            };
+        }
+        else {
+            currentGroup.nodes.push(node);
+            currentGroup.analyses.push(analysis);
+        }
+    }
+    if (currentGroup.nodes.length > 0) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
+async function combineGroupsToOptimalSize(semanticGroups, source, profile, limits) {
+    const optimalGroups = [];
+    let currentCombinedGroup = {
+        nodes: [],
+        totalSize: 0,
+        groupInfo: []
+    };
+    for (const group of semanticGroups) {
+        const groupTotalSize = group.analyses.reduce((sum, a) => sum + a.size, 0);
+        if (groupTotalSize > limits.optimal) {
+            if (currentCombinedGroup.nodes.length > 0) {
+                optimalGroups.push(currentCombinedGroup);
+            }
+            optimalGroups.push({
+                nodes: group.nodes,
+                totalSize: groupTotalSize,
+                groupInfo: [group]
+            });
+            currentCombinedGroup = {
+                nodes: [],
+                totalSize: 0,
+                groupInfo: []
+            };
+            continue;
+        }
+        if (currentCombinedGroup.totalSize + groupTotalSize > limits.max) {
+            if (currentCombinedGroup.nodes.length > 0) {
+                optimalGroups.push(currentCombinedGroup);
+            }
+            currentCombinedGroup = {
+                nodes: group.nodes,
+                totalSize: groupTotalSize,
+                groupInfo: [group]
+            };
+            continue;
+        }
+        currentCombinedGroup.nodes.push(...group.nodes);
+        currentCombinedGroup.totalSize += groupTotalSize;
+        currentCombinedGroup.groupInfo.push(group);
+        if (currentCombinedGroup.totalSize >= limits.optimal * 0.9) {
+            optimalGroups.push(currentCombinedGroup);
+            currentCombinedGroup = {
+                nodes: [],
+                totalSize: 0,
+                groupInfo: []
+            };
+        }
+    }
+    if (currentCombinedGroup.nodes.length > 0) {
+        optimalGroups.push(currentCombinedGroup);
+    }
+    return optimalGroups;
+}
+export async function groupNodesForChunking(nodes, source, profile, rule) {
+    if (!nodes || nodes.length === 0)
+        return [];
+    const limits = getSizeLimits(profile);
+    if (nodes.length <= 10) {
+        return nodes.map(node => ({
+            nodes: [node],
+            totalSize: 0,
+            groupInfo: []
+        }));
+    }
+    const nodeAnalyses = await batchAnalyzeNodesInternal(nodes, source, profile);
+    const semanticGroups = identifySemanticGroups(nodes, source, nodeAnalyses, rule);
+    const optimalGroups = await combineGroupsToOptimalSize(semanticGroups, source, profile, limits);
+    return optimalGroups;
+}
+export function createCombinedChunk(nodeGroup, source, filerel) {
+    if (!nodeGroup.nodes || nodeGroup.nodes.length === 0) {
+        return null;
+    }
+    const codes = nodeGroup.nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const combinedCode = codes.join('\n\n');
+    const firstNode = nodeGroup.nodes[0];
+    const lastNode = nodeGroup.nodes[nodeGroup.nodes.length - 1];
+    return {
+        code: combinedCode,
+        node: {
+            ...firstNode,
+            type: `${firstNode.type}_group_${nodeGroup.nodes.length}`,
+            endIndex: lastNode.endIndex
+        },
+        metadata: {
+            isGroup: true,
+            nodeCount: nodeGroup.nodes.length,
+            totalSize: nodeGroup.totalSize,
+            groupTypes: nodeGroup.groupInfo?.map(g => g.type) || ['combined']
+        }
+    };
+}
+//# sourceMappingURL=file-grouper.js.map

package/dist/chunking/file-grouper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-grouper.js","sourceRoot":"","sources":["../../src/chunking/file-grouper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,oBAAoB,EAAyB,MAAM,oBAAoB,CAAC;AAqClG,SAAS,aAAa,CAAC,OAAqB;IAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,OAAO;YACL,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,GAAG,EAAE,OAAO,CAAC,cAAc;YAC3B,OAAO,EAAE,OAAO,CAAC,aAAa;YAC9B,IAAI,EAAE,QAAQ;SACf,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,GAAG,EAAE,OAAO,CAAC,aAAa;QAC1B,OAAO,EAAE,OAAO,CAAC,YAAY;QAC7B,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,yBAAyB,CAAC,KAAuB,EAAE,MAAc,EAAE,OAAqB;IACrG,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QACxF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,IAAI;YACJ,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;YACtB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;SACf,CAAC,CAAC,CAAC;IACN,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI;QACJ,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;QACrB,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;KACf,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,eAAe,CAAC,IAAoB,EAAE,IAAkB;IAC/D,MAAM,cAAc,GAAG;QACrB,mBAAmB;QACnB,kBAAkB;QAClB,uBAAuB;QACvB,oBAAoB;QACpB,uBAAuB;QACvB,mBAAmB;QACnB,kBAAkB;KACnB,CAAC;IAEF,OAAO,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAuB,EAAE,MAAc,EAAE,YAA4B,EAAE,IAAkB;IACvH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,YAAY,GAAkB;QAChC,IAAI,EAAE,cAAc;QACpB,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,EAAE;QACZ,UAAU,EAAE,IAAI;KACjB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;YAChC,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC5B,CAAC;YAED,YAAY,GAAG;gBACb,IAAI,EAAE,WAAW;gBACjB,aAAa,EAAE,IAAI,CAAC,IAAI;gBACxB,KAAK,EAAE,CAAC,IAAI,CAAC;gBACb,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,UAAU,EAAE,IAAI;aACjB,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAE1B,YAAY,GAAG;gBACb,IAAI,EAAE,cAAc;gBACpB,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;gBACZ,UAAU,EAAE,IAAI;aACjB,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,cAA+B,EAC/B,MAAc,EACd,OAAqB,EACrB,MAAkB;IAElB,MAAM,aAAa,GAAgB,EAAE,CAAC;IACtC,IAAI,oBAAoB,GAAc;QACpC,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,EAAE;KACd,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAE1E,IAAI,cAAc,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,aAAa,CAAC,IAAI,CAAC;gBACjB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC,CAAC;YAEH,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;YACF,SAAS;QACX,CAAC;QAED,IAAI,oBAAoB,CAAC,SAAS,GAAG,cAAc,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC;YACjE,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC3C,CAAC;YAED,oBAAoB,GAAG;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,cAAc;gBACzB,SAAS,EAAE,CAAC,KAAK,CAAC;aACnB,CAAC;YACF,SAAS;QACX,CAAC;QAED,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAChD,oBAAoB,CAAC,SAAS,IAAI,cAAc,CAAC;QACjD,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3C,IAAI,oBAAoB,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;YAC3D,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;YACzC,oBAAoB,GAAG;gBACrB,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,CAAC;gBACZ,SAAS,EAAE,EAAE;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,IAAI,oBAAoB,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,aAAa,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,KAAuB,EACvB,MAAc,EACd,OAAqB,EACrB,IAAkB;IAElB,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAEtC,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,IAAI,CAAC;YACb,SAAS,EAAE,CAAC;YACZ,SAAS,EAAE,EAAE;SACd,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7E,MAAM,cAAc,GAAG,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;IACjF,MAAM,aAAa,GAAG,MAAM,0BAA0B,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAEhG,OAAO,aAAa,CAAC;AACvB,CAAC;AAaD,MAAM,UAAU,mBAAmB,CAAC,SAAoB,EAAE,MAAc,EAAE,OAAe;IACvF,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAC7C,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAExC,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE7D,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE;YACJ,GAAG,SAAS;YACZ,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,UAAU,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE;YACzD,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B;QACD,QAAQ,EAAE;YACR,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;YACjC,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,UAAU,EAAE,SAAS,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC;SAClE;KACF,CAAC;AACJ,CAAC"}

package/dist/chunking/semantic-chunker.d.ts

@@ -0,0 +1,37 @@
+import type { ModelProfile } from '../providers/base.js';
+import type { TreeSitterNode } from '../types/ast.js';
+interface LanguageRule {
+    subdivisionTypes?: Record<string, string[]>;
+    [key: string]: any;
+}
+export declare function findSemanticSubdivisions(node: TreeSitterNode, rule: LanguageRule): TreeSitterNode[];
+export declare function findLastCompleteBoundary(code: string, maxSize: number): number;
+export declare function extractSignature(node: TreeSitterNode, source: string): string;
+export declare function extractLinesBeforeNode(node: TreeSitterNode, source: string, numLines: number): string;
+export declare function extractParentContext(node: TreeSitterNode, source: string): {
+    signature: string;
+    startLine: number;
+    endLine: number;
+};
+export declare function getLineNumber(byteOffset: number, source: string): number;
+export interface NodeAnalysis {
+    isSingleChunk: boolean;
+    needsSubdivision: boolean;
+    subdivisionCandidates: TreeSitterNode[];
+    size: number;
+    unit: string;
+    method: string;
+    estimatedSubchunks: number;
+}
+export declare function analyzeNodeForChunking(node: TreeSitterNode, source: string, rule: LanguageRule, profile: ModelProfile): Promise<NodeAnalysis>;
+export declare function batchAnalyzeNodes(nodes: TreeSitterNode[], source: string, rule: LanguageRule, profile: ModelProfile, isSubdivision?: boolean): Promise<Array<NodeAnalysis & {
+    node: TreeSitterNode;
+}>>;
+export interface StatementChunk {
+    code: string;
+    size: number;
+    unit: string;
+}
+export declare function yieldStatementChunks(node: TreeSitterNode, source: string, maxSize: number, overlapSize: number, profile: ModelProfile): Promise<StatementChunk[]>;
+export {};
+//# sourceMappingURL=semantic-chunker.d.ts.map

package/dist/chunking/semantic-chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunking/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEtD,UAAU,YAAY;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5C,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAUD,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,GAAG,cAAc,EAAE,CAwBnG;AAED,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAgB9E;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAO7E;AAED,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIrG;AAED,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG;IAC1E,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB,CAMA;AAED,wBAAgB,aAAa,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAGxE;AAqBD,MAAM,WAAW,YAAY;IAC3B,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,qBAAqB,EAAE,cAAc,EAAE,CAAC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,wBAAsB,sBAAsB,CAC1C,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,YAAY,CAAC,CA2BvB;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,cAAc,EAAE,EACvB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,YAAY,EACrB,aAAa,UAAQ,GACpB,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG;IAAE,IAAI,EAAE,cAAc,CAAA;CAAE,CAAC,CAAC,CAgCzD;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,YAAY,GACpB,OAAO,CAAC,cAAc,EAAE,CAAC,CAwC3B"}

package/dist/chunking/semantic-chunker.js

@@ -0,0 +1,172 @@
+import { analyzeCodeSize, batchAnalyzeCodeSize } from './token-counter.js';
+export function findSemanticSubdivisions(node, rule) {
+    if (!node || !rule)
+        return [];
+    const subdivisionTypes = rule.subdivisionTypes?.[node.type] || [];
+    if (subdivisionTypes.length === 0)
+        return [];
+    const candidates = [];
+    function walk(n, depth = 0) {
+        if (depth > 0 && subdivisionTypes.includes(n.type)) {
+            candidates.push(n);
+            return;
+        }
+        for (let i = 0; i < n.childCount; i++) {
+            const child = n.child(i);
+            if (child) {
+                walk(child, depth + 1);
+            }
+        }
+    }
+    walk(node);
+    return candidates;
+}
+export function findLastCompleteBoundary(code, maxSize) {
+    const boundaries = [
+        { pattern: /\n\s*}\s*$/gm, priority: 1 },
+        { pattern: /;\s*$/gm, priority: 2 },
+        { pattern: /\n\s*$/gm, priority: 3 }
+    ];
+    for (const boundary of boundaries) {
+        const matches = [...code.substring(0, maxSize).matchAll(boundary.pattern)];
+        if (matches.length > 0) {
+            const lastMatch = matches[matches.length - 1];
+            return lastMatch.index + lastMatch[0].length;
+        }
+    }
+    return maxSize;
+}
+export function extractSignature(node, source) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const firstBrace = code.indexOf('{');
+    if (firstBrace !== -1) {
+        return code.substring(0, firstBrace).trim() + ' {';
+    }
+    return code.split('\n')[0];
+}
+export function extractLinesBeforeNode(node, source, numLines) {
+    const beforeCode = source.substring(0, node.startIndex);
+    const lines = beforeCode.split('\n');
+    return lines.slice(-numLines).join('\n');
+}
+export function extractParentContext(node, source) {
+    return {
+        signature: extractSignature(node, source),
+        startLine: getLineNumber(node.startIndex, source),
+        endLine: getLineNumber(node.endIndex, source)
+    };
+}
+export function getLineNumber(byteOffset, source) {
+    const before = source.substring(0, byteOffset);
+    return before.split('\n').length;
+}
+function getSizeLimits(profile) {
+    if (profile.useTokens && profile.tokenCounter) {
+        return {
+            optimal: profile.optimalTokens,
+            min: profile.minChunkTokens,
+            max: profile.maxChunkTokens,
+            overlap: profile.overlapTokens,
+            unit: 'tokens'
+        };
+    }
+    return {
+        optimal: profile.optimalChars,
+        min: profile.minChunkChars,
+        max: profile.maxChunkChars,
+        overlap: profile.overlapChars,
+        unit: 'characters'
+    };
+}
+export async function analyzeNodeForChunking(node, source, rule, profile) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const limits = getSizeLimits(profile);
+    let actualSize;
+    let method;
+    if (profile.useTokens && profile.tokenCounter) {
+        const analysis = await analyzeCodeSize(code, limits, profile.tokenCounter);
+        actualSize = analysis.size;
+        method = analysis.method;
+    }
+    else {
+        actualSize = code.length;
+        method = 'chars';
+    }
+    const subdivisionThreshold = limits.max;
+    return {
+        isSingleChunk: actualSize <= subdivisionThreshold,
+        needsSubdivision: actualSize > subdivisionThreshold,
+        subdivisionCandidates: findSemanticSubdivisions(node, rule),
+        size: actualSize,
+        unit: limits.unit,
+        method,
+        estimatedSubchunks: Math.ceil(actualSize / limits.optimal)
+    };
+}
+export async function batchAnalyzeNodes(nodes, source, rule, profile, isSubdivision = false) {
+    const codes = nodes.map(node => source.slice(node.startIndex, node.endIndex));
+    const limits = getSizeLimits(profile);
+    let analyses;
+    if (profile.useTokens && profile.tokenCounter) {
+        analyses = await batchAnalyzeCodeSize(codes, limits, profile.tokenCounter, isSubdivision);
+    }
+    else {
+        analyses = codes.map(code => ({
+            size: code.length,
+            decision: code.length < limits.min ? 'too_small'
+                : code.length > limits.max ? 'too_large'
+                    : code.length <= limits.optimal ? 'optimal'
+                        : 'needs_tokenization',
+            method: 'chars'
+        }));
+    }
+    return nodes.map((node, i) => {
+        const analysis = analyses[i];
+        const subdivisionThreshold = limits.max;
+        return {
+            node,
+            isSingleChunk: analysis.size <= subdivisionThreshold,
+            needsSubdivision: analysis.size > subdivisionThreshold,
+            subdivisionCandidates: findSemanticSubdivisions(node, rule),
+            size: analysis.size,
+            unit: limits.unit,
+            method: analysis.method,
+            estimatedSubchunks: Math.ceil(analysis.size / limits.optimal)
+        };
+    });
+}
+export async function yieldStatementChunks(node, source, maxSize, overlapSize, profile) {
+    const code = source.slice(node.startIndex, node.endIndex);
+    const lines = code.split('\n');
+    const chunks = [];
+    let currentChunk = [];
+    let currentSize = 0;
+    for (const line of lines) {
+        const lineSize = profile.useTokens && profile.tokenCounter
+            ? await profile.tokenCounter(line)
+            : line.length;
+        if (currentSize + lineSize > maxSize && currentChunk.length > 0) {
+            chunks.push({
+                code: currentChunk.join('\n'),
+                size: currentSize,
+                unit: profile.useTokens ? 'tokens' : 'characters'
+            });
+            const overlapLines = Math.floor(currentChunk.length * 0.2);
+            currentChunk = currentChunk.slice(-overlapLines);
+            currentSize = profile.useTokens && profile.tokenCounter
+                ? await profile.tokenCounter(currentChunk.join('\n'))
+                : currentChunk.join('\n').length;
+        }
+        currentChunk.push(line);
+        currentSize += lineSize;
+    }
+    if (currentChunk.length > 0) {
+        chunks.push({
+            code: currentChunk.join('\n'),
+            size: currentSize,
+            unit: profile.useTokens ? 'tokens' : 'characters'
+        });
+    }
+    return chunks;
+}
+//# sourceMappingURL=semantic-chunker.js.map