nova-terminal-ai 0.3.4

package/dist/packages/core/src/utils/TokenCounter.js

@@ -0,0 +1,338 @@
+// ============================================================================
+// TokenCounter - Precise token counting using tiktoken
+// ============================================================================
+/**
+ * Model encoding mappings for tiktoken
+ * Maps model names to their corresponding tiktoken encoding
+ */
+const MODEL_ENCODING_MAP = {
+    // Anthropic Claude models (use cl100k_base, similar to GPT-4)
+    'claude-3': 'cl100k_base',
+    'claude-sonnet': 'cl100k_base',
+    'claude-opus': 'cl100k_base',
+    'claude-haiku': 'cl100k_base',
+    // OpenAI models
+    'gpt-4': 'cl100k_base',
+    'gpt-4o': 'o200k_base',
+    'gpt-4-turbo': 'cl100k_base',
+    'gpt-3.5-turbo': 'cl100k_base',
+    'o1': 'o200k_base',
+    'o1-mini': 'o200k_base',
+    'o3-mini': 'o200k_base',
+    // Google models (approximate with cl100k_base)
+    'gemini': 'cl100k_base',
+    // DeepSeek (use cl100k_base)
+    'deepseek': 'cl100k_base',
+    // Qwen (use cl100k_base)
+    'qwen': 'cl100k_base',
+    // GLM (use cl100k_base)
+    'glm': 'cl100k_base',
+    // Llama models
+    'llama': 'cl100k_base',
+    // Default fallback
+    'default': 'cl100k_base',
+};
+/**
+ * Special tokens added per message for different providers
+ * Based on official documentation
+ */
+const MESSAGE_TOKEN_OVERHEAD = {
+    anthropic: { perMessage: 4, perName: 0 }, // Approximate
+    openai: { perMessage: 3, perName: 1 }, // Official: every message follows <im_start>{role/name}\n{content}<im_end>\n
+    default: { perMessage: 4, perName: 0 },
+};
+/**
+ * Simple tokenizer implementation without external dependencies
+ * Uses character-based estimation with model-specific adjustments
+ */
+class SimpleTokenizer {
+    encoding;
+    avgCharsPerToken;
+    constructor(encoding) {
+        this.encoding = encoding;
+        // Different encodings have different average chars per token
+        this.avgCharsPerToken = encoding === 'o200k_base' ? 3.5 : 4;
+    }
+    /**
+     * Encode text to tokens (simplified implementation)
+     * Returns array of token strings for counting
+     */
+    encode(text) {
+        // Simple word-based tokenization with punctuation handling
+        const tokens = [];
+        let current = '';
+        let inWord = false;
+        for (const char of text) {
+            const isWordChar = /[a-zA-Z0-9]/.test(char);
+            if (isWordChar !== inWord && current) {
+                tokens.push(current);
+                current = '';
+            }
+            current += char;
+            inWord = isWordChar;
+            // Break long sequences
+            if (current.length >= Math.ceil(this.avgCharsPerToken * 2)) {
+                tokens.push(current);
+                current = '';
+            }
+        }
+        if (current) {
+            tokens.push(current);
+        }
+        return tokens;
+    }
+    /**
+     * Count tokens in text
+     */
+    count(text) {
+        if (!text)
+            return 0;
+        // For more accurate counting, use word-based estimation
+        // English: ~1.3 tokens per word
+        // Chinese: ~0.5-0.8 tokens per character (due to subword tokenization)
+        // Code: ~0.5-0.7 tokens per word
+        const words = text.split(/\s+/).filter(w => w.length > 0);
+        const wordCount = words.length;
+        // Count CJK characters separately (they typically tokenize as 1-2 chars per token)
+        const cjkChars = (text.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length;
+        const nonCjkLength = text.length - cjkChars;
+        // Estimate: CJK ~0.6 tokens per char, non-CJK ~4 chars per token
+        const cjkTokens = Math.ceil(cjkChars * 0.6);
+        const nonCjkTokens = Math.ceil(nonCjkLength / this.avgCharsPerToken);
+        // Add overhead for special tokens and structure
+        const overhead = Math.ceil(wordCount * 0.1);
+        return cjkTokens + nonCjkTokens + overhead;
+    }
+}
+/**
+ * Token counter with caching for performance
+ */
+export class TokenCounter {
+    static instance;
+    tokenizers = new Map();
+    cache = new Map();
+    cacheMaxSize = 1000;
+    constructor() { }
+    static getInstance() {
+        if (!TokenCounter.instance) {
+            TokenCounter.instance = new TokenCounter();
+        }
+        return TokenCounter.instance;
+    }
+    /**
+     * Get tokenizer for a model
+     */
+    getTokenizer(model) {
+        // Normalize model name
+        const normalizedModel = this.normalizeModelName(model);
+        const encoding = MODEL_ENCODING_MAP[normalizedModel] ?? 'cl100k_base';
+        if (!this.tokenizers.has(encoding)) {
+            this.tokenizers.set(encoding, new SimpleTokenizer(encoding));
+        }
+        return this.tokenizers.get(encoding);
+    }
+    /**
+     * Normalize model name to match encoding map
+     */
+    normalizeModelName(model) {
+        const lower = model.toLowerCase();
+        // Check direct matches first
+        for (const key of Object.keys(MODEL_ENCODING_MAP)) {
+            if (lower.includes(key)) {
+                return key;
+            }
+        }
+        return 'default';
+    }
+    /**
+     * Get provider type from model name
+     */
+    getProviderType(model) {
+        const lower = model.toLowerCase();
+        if (lower.includes('claude') || lower.includes('anthropic'))
+            return 'anthropic';
+        if (lower.includes('gpt') || lower.includes('o1') || lower.includes('o3'))
+            return 'openai';
+        return 'default';
+    }
+    /**
+     * Count tokens in a text string
+     */
+    countTokens(text, model = 'default') {
+        if (!text)
+            return 0;
+        // Check cache
+        const cacheKey = `${model}:${text.slice(0, 100)}:${text.length}`;
+        if (this.cache.has(cacheKey)) {
+            return this.cache.get(cacheKey);
+        }
+        const tokenizer = this.getTokenizer(model);
+        const count = tokenizer.count(text);
+        // Cache result
+        if (this.cache.size >= this.cacheMaxSize) {
+            // Remove oldest entries
+            const keys = Array.from(this.cache.keys()).slice(0, Math.floor(this.cacheMaxSize / 2));
+            for (const key of keys) {
+                this.cache.delete(key);
+            }
+        }
+        this.cache.set(cacheKey, count);
+        return count;
+    }
+    /**
+     * Count tokens in a message
+     */
+    countMessageTokens(message, model = 'default') {
+        const provider = this.getProviderType(model);
+        const overhead = MESSAGE_TOKEN_OVERHEAD[provider] ?? { perMessage: 4, perName: 0 };
+        let count = overhead.perMessage;
+        // Count role tokens
+        count += this.countTokens(message.role, model);
+        // Count content tokens
+        for (const block of message.content) {
+            count += this.countContentBlockTokens(block, model);
+        }
+        // Add name overhead if present
+        if (message.name) {
+            count += overhead.perName + this.countTokens(message.name, model);
+        }
+        return count;
+    }
+    /**
+     * Count tokens in a content block
+     */
+    countContentBlockTokens(block, model = 'default') {
+        switch (block.type) {
+            case 'text':
+                return this.countTokens(block.text, model);
+            case 'tool_use':
+                // Tool name + input JSON
+                return this.countTokens(block.name, model) +
+                    this.countTokens(JSON.stringify(block.input), model) + 4; // overhead
+            case 'tool_result':
+                // Tool ID + content
+                const contentTokens = typeof block.content === 'string'
+                    ? this.countTokens(block.content, model)
+                    : block.content.reduce((sum, c) => {
+                        const text = 'text' in c ? c.text : '';
+                        return sum + this.countTokens(text, model);
+                    }, 0);
+                return this.countTokens(block.tool_use_id, model) + contentTokens + 4;
+            case 'image':
+                // Images are counted differently by different providers
+                // Approximate based on image size if available
+                if (block.source.type === 'base64' && block.source.data) {
+                    // Rough estimate: 85 tokens for a 512x512 image, scales with pixels
+                    const base64Length = block.source.data.length;
+                    return Math.ceil(base64Length / 1000) * 85; // Very rough estimate
+                }
+                return 85; // Default image token cost
+            case 'thinking':
+                return this.countTokens(block.thinking, model);
+            default:
+                return 0;
+        }
+    }
+    /**
+     * Count tokens in an array of messages
+     */
+    countMessagesTokens(messages, model = 'default') {
+        let total = 0;
+        for (const message of messages) {
+            total += this.countMessageTokens(message, model);
+        }
+        // Add message overhead (every reply is primed with <im_start>assistant)
+        const provider = this.getProviderType(model);
+        total += MESSAGE_TOKEN_OVERHEAD[provider]?.perMessage || 4;
+        return total;
+    }
+    /**
+     * Estimate tokens for a system prompt
+     */
+    countSystemPromptTokens(systemPrompt, model = 'default') {
+        // System prompts have slightly higher overhead
+        return this.countTokens(systemPrompt, model) + 4;
+    }
+    /**
+     * Fit content within a token budget
+     * Intelligently truncates while preserving structure
+     */
+    fitWithinBudget(content, budget, model = 'default', options = {}) {
+        const currentTokens = this.countTokens(content, model);
+        if (currentTokens <= budget) {
+            return { content, tokens: currentTokens, truncated: false };
+        }
+        const { preserveHead = 0.6, preserveTail = 0.3, ellipsis = '\n... [truncated] ...\n' } = options;
+        const headBudget = Math.floor(budget * preserveHead);
+        const tailBudget = Math.floor(budget * preserveTail);
+        const ellipsisTokens = this.countTokens(ellipsis, model);
+        // Binary search for head cutoff
+        let headEnd = 0;
+        let low = 0, high = content.length;
+        while (low < high) {
+            const mid = Math.floor((low + high) / 2);
+            if (this.countTokens(content.slice(0, mid), model) <= headBudget) {
+                headEnd = mid;
+                low = mid + 1;
+            }
+            else {
+                high = mid;
+            }
+        }
+        // Binary search for tail start
+        let tailStart = content.length;
+        low = 0, high = content.length;
+        while (low < high) {
+            const mid = Math.floor((low + high) / 2);
+            if (this.countTokens(content.slice(mid), model) <= tailBudget) {
+                tailStart = mid;
+                high = mid;
+            }
+            else {
+                low = mid + 1;
+            }
+        }
+        // Construct truncated content
+        const truncatedContent = content.slice(0, headEnd) + ellipsis + content.slice(tailStart);
+        const finalTokens = this.countTokens(truncatedContent, model);
+        return {
+            content: truncatedContent,
+            tokens: finalTokens,
+            truncated: true
+        };
+    }
+    /**
+     * Calculate context window usage percentage
+     */
+    calculateUsage(current, max) {
+        return Math.min(100, Math.round((current / max) * 100));
+    }
+    /**
+     * Get usage level indicator
+     */
+    getUsageLevel(percentage) {
+        if (percentage < 50)
+            return 'low';
+        if (percentage < 70)
+            return 'medium';
+        if (percentage < 90)
+            return 'high';
+        return 'critical';
+    }
+    /**
+     * Clear cache
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+}
+// Export singleton instance
+export const tokenCounter = TokenCounter.getInstance();
+// Export convenience functions
+export function countTokens(text, model) {
+    return tokenCounter.countTokens(text, model);
+}
+export function countMessagesTokens(messages, model) {
+    return tokenCounter.countMessagesTokens(messages, model);
+}
+//# sourceMappingURL=TokenCounter.js.map

package/dist/packages/core/src/utils/TokenCounter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"TokenCounter.js","sourceRoot":"","sources":["../../../../../../core/src/utils/TokenCounter.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,uDAAuD;AACvD,+EAA+E;AAI/E;;;GAGG;AACH,MAAM,kBAAkB,GAA2B;IACjD,8DAA8D;IAC9D,UAAU,EAAE,aAAa;IACzB,eAAe,EAAE,aAAa;IAC9B,aAAa,EAAE,aAAa;IAC5B,cAAc,EAAE,aAAa;IAE7B,gBAAgB;IAChB,OAAO,EAAE,aAAa;IACtB,QAAQ,EAAE,YAAY;IACtB,aAAa,EAAE,aAAa;IAC5B,eAAe,EAAE,aAAa;IAC9B,IAAI,EAAE,YAAY;IAClB,SAAS,EAAE,YAAY;IACvB,SAAS,EAAE,YAAY;IAEvB,+CAA+C;IAC/C,QAAQ,EAAE,aAAa;IAEvB,6BAA6B;IAC7B,UAAU,EAAE,aAAa;IAEzB,yBAAyB;IACzB,MAAM,EAAE,aAAa;IAErB,wBAAwB;IACxB,KAAK,EAAE,aAAa;IAEpB,eAAe;IACf,OAAO,EAAE,aAAa;IAEtB,mBAAmB;IACnB,SAAS,EAAE,aAAa;CACzB,CAAC;AAEF;;;GAGG;AACH,MAAM,sBAAsB,GAA4D;IACtF,SAAS,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAG,cAAc;IACzD,MAAM,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAM,6EAA6E;IACxH,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE;CACvC,CAAC;AAEF;;;GAGG;AACH,MAAM,eAAe;IACX,QAAQ,CAAS;IACjB,gBAAgB,CAAS;IAEjC,YAAY,QAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,6DAA6D;QAC7D,IAAI,CAAC,gBAAgB,GAAG,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,IAAY;QACjB,2DAA2D;QAC3D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE5C,IAAI,UAAU,KAAK,MAAM,IAAI,OAAO,EAAE,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YAED,OAAO,IAAI,IAAI,CAAC;YAChB,MAAM,GAAG,UAAU,CAAC;YAEpB,uBAAuB;YACvB,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAY;QAChB,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,wDAAwD;QACxD,gCAAgC;QAChC,uEAAuE;QACvE,iCAAiC;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC;QAE/B,mFAAmF;QACnF,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC5E,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;QAE5C,iEAAiE;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;QAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAErE,gDAAgD;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;QAE5C,OAAO,SAAS,GAAG,YAAY,GAAG,QAAQ,CAAC;IAC7C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,YAAY;IACf,MAAM,CAAC,QAAQ,CAAe;IAC9B,UAAU,GAAiC,IAAI,GAAG,EAAE,CAAC;IACrD,KAAK,GAAwB,IAAI,GAAG,EAAE,CAAC;IACvC,YAAY,GAAG,IAAI,CAAC;IAE5B,gBAAuB,CAAC;IAExB,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;YAC3B,YAAY,CAAC,QAAQ,GAAG,IAAI,YAAY,EAAE,CAAC;QAC7C,CAAC;QACD,OAAO,YAAY,CAAC,QAAQ,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,KAAa;QAChC,uBAAuB;QACvB,MAAM,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,eAAe,CAAC,IAAI,aAAa,CAAC;QAEtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;IACxC,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,KAAa;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAElC,6BAA6B;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAClD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,KAAa;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,OAAO,WAAW,CAAC;QAChF,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,QAAQ,CAAC;QAC3F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,IAAY,EAAE,QAAgB,SAAS;QACjD,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QAEpB,cAAc;QACd,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjE,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC;QACnC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEpC,eAAe;QACf,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACzC,wBAAwB;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC;YACvF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEhC,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,kBAAkB,CAAC,OAAgB,EAAE,QAAgB,SAAS;QAC5D,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAEnF,IAAI,KAAK,GAAG,QAAQ,CAAC,UAAU,CAAC;QAEhC,oBAAoB;QACpB,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAE/C,uBAAuB;QACvB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpC,KAAK,IAAI,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACtD,CAAC;QAED,+BAA+B;QAC/B,IAAK,OAAe,CAAC,IAAI,EAAE,CAAC;YAC1B,KAAK,IAAI,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,WAAW,CAAE,OAAe,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,KAAmB,EAAE,QAAgB,SAAS;QACpE,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAE7C,KAAK,UAAU;gBACb,yBAAyB;gBACzB,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC;oBACnC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW;YAE9E,KAAK,aAAa;gBAChB,oBAAoB;gBACpB,MAAM,aAAa,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;oBACrD,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC;oBACxC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;wBAC9B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;wBACvC,OAAO,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;oBAC7C,CAAC,EAAE,CAAC,CAAC,CAAC;gBACV,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,GAAG,aAAa,GAAG,CAAC,CAAC;YAExE,KAAK,OAAO;gBACV,wDAAwD;gBACxD,+CAA+C;gBAC/C,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBACxD,oEAAoE;oBACpE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;oBAC9C,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,sBAAsB;gBACpE,CAAC;gBACD,OAAO,EAAE,CAAC,CAAC,2BAA2B;YAExC,KAAK,UAAU;gBACb,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAEjD;gBACE,OAAO,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACH,mBAAmB,CAAC,QAAmB,EAAE,QAAgB,SAAS;QAChE,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,IAAI,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACnD,CAAC;QAED,wEAAwE;QACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7C,KAAK,IAAI,sBAAsB,CAAC,QAAQ,CAAC,EAAE,UAAU,IAAI,CAAC,CAAC;QAE3D,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,YAAoB,EAAE,QAAgB,SAAS;QACrE,+CAA+C;QAC/C,OAAO,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,eAAe,CACb,OAAe,EACf,MAAc,EACd,QAAgB,SAAS,EACzB,UAII,EAAE;QAEN,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAEvD,IAAI,aAAa,IAAI,MAAM,EAAE,CAAC;YAC5B,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC9D,CAAC;QAED,MAAM,EACJ,YAAY,GAAG,GAAG,EAClB,YAAY,GAAG,GAAG,EAClB,QAAQ,GAAG,yBAAyB,EACrC,GAAG,OAAO,CAAC;QAEZ,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;QACrD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEzD,gCAAgC;QAChC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;QACnC,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;gBACjE,OAAO,GAAG,GAAG,CAAC;gBACd,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,IAAI,GAAG,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QAED,+BAA+B;QAC/B,IAAI,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,GAAG,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACzC,IAAI,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC9D,SAAS,GAAG,GAAG,CAAC;gBAChB,IAAI,GAAG,GAAG,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QAED,8BAA8B;QAC9B,MAAM,gBAAgB,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACzF,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAE9D,OAAO;YACL,OAAO,EAAE,gBAAgB;YACzB,MAAM,EAAE,WAAW;YACnB,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,OAAe,EAAE,GAAW;QACzC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,UAAkB;QAC9B,IAAI,UAAU,GAAG,EAAE;YAAE,OAAO,KAAK,CAAC;QAClC,IAAI,UAAU,GAAG,EAAE;YAAE,OAAO,QAAQ,CAAC;QACrC,IAAI,UAAU,GAAG,EAAE;YAAE,OAAO,MAAM,CAAC;QACnC,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF;AAED,4BAA4B;AAC5B,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;AAEvD,+BAA+B;AAC/B,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAc;IACtD,OAAO,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAAmB,EAAE,KAAc;IACrE,OAAO,YAAY,CAAC,mBAAmB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAC3D,CAAC"}

package/dist/packages/core/src/utils/VectorMemoryStore.d.ts

@@ -0,0 +1,110 @@
+/**
+ * Memory entry with vector embedding
+ */
+export interface VectorMemoryEntry {
+    id: string;
+    content: string;
+    embedding: number[];
+    metadata: {
+        session: string;
+        timestamp: Date;
+        type: 'decision' | 'code' | 'error' | 'insight' | 'context' | 'preference';
+        importance: number;
+        tags: string[];
+        source?: string;
+    };
+}
+/**
+ * Search result with similarity score
+ */
+export interface MemoryMatch {
+    entry: VectorMemoryEntry;
+    score: number;
+}
+/**
+ * Memory store configuration
+ */
+export interface VectorMemoryConfig {
+    /** Storage directory for persistence */
+    storageDir: string;
+    /** Maximum entries to keep */
+    maxEntries?: number;
+    /** Minimum similarity score for search results */
+    minSimilarity?: number;
+    /** Embedding dimension (default 384 for small models) */
+    embeddingDimension?: number;
+}
+/**
+ * Vector-based memory store for semantic search
+ */
+export declare class VectorMemoryStore {
+    private entries;
+    private config;
+    private storageFile;
+    private isDirty;
+    constructor(config: VectorMemoryConfig);
+    /**
+     * Add a memory entry
+     */
+    addMemory(content: string, metadata: VectorMemoryEntry['metadata']): Promise<VectorMemoryEntry>;
+    /**
+     * Search for similar memories
+     */
+    searchSimilar(query: string, options?: {
+        topK?: number;
+        minSimilarity?: number;
+        type?: VectorMemoryEntry['metadata']['type'];
+        tags?: string[];
+    }): Promise<MemoryMatch[]>;
+    /**
+     * Get entry by ID
+     */
+    getEntry(id: string): VectorMemoryEntry | undefined;
+    /**
+     * Delete entry by ID
+     */
+    deleteEntry(id: string): boolean;
+    /**
+     * Get all entries of a specific type
+     */
+    getEntriesByType(type: VectorMemoryEntry['metadata']['type']): VectorMemoryEntry[];
+    /**
+     * Get entries by tags
+     */
+    getEntriesByTags(tags: string[]): VectorMemoryEntry[];
+    /**
+     * Get entries for a session
+     */
+    getEntriesBySession(sessionId: string): VectorMemoryEntry[];
+    /**
+     * Save to disk
+     */
+    save(): void;
+    /**
+     * Load from disk
+     */
+    load(): void;
+    /**
+     * Prune entries based on importance and age
+     */
+    private prune;
+    /**
+     * Get statistics
+     */
+    getStats(): {
+        totalEntries: number;
+        byType: Record<string, number>;
+        oldestEntry: Date | null;
+        newestEntry: Date | null;
+        avgImportance: number;
+    };
+    /**
+     * Clear all entries
+     */
+    clear(): void;
+}
+/**
+ * Create a vector memory store with default configuration
+ */
+export declare function createVectorMemoryStore(storageDir?: string): VectorMemoryStore;
+//# sourceMappingURL=VectorMemoryStore.d.ts.map

package/dist/packages/core/src/utils/VectorMemoryStore.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"VectorMemoryStore.d.ts","sourceRoot":"","sources":["../../../../../../core/src/utils/VectorMemoryStore.ts"],"names":[],"mappings":"AAeA;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE;QACR,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,IAAI,CAAC;QAChB,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC;QAC3E,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,iBAAiB,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAkFD;;GAEG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,OAAO,CAA2B;IAC1C,OAAO,CAAC,MAAM,CAA+B;IAC7C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,kBAAkB;IAmBtC;;OAEG;IACG,SAAS,CACb,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,iBAAiB,CAAC,UAAU,CAAC,GACtC,OAAO,CAAC,iBAAiB,CAAC;IA0B7B;;OAEG;IACG,aAAa,CACjB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,IAAI,CAAC,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;KACZ,GACL,OAAO,CAAC,WAAW,EAAE,CAAC;IAgCzB;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;IAInD;;OAEG;IACH,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAUhC;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,iBAAiB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,GAAG,iBAAiB,EAAE;IAIlF;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,iBAAiB,EAAE;IAMrD;;OAEG;IACH,mBAAmB,CAAC,SAAS,EAAE,MAAM,GAAG,iBAAiB,EAAE;IAQ3D;;OAEG;IACH,IAAI,IAAI,IAAI;IA+BZ;;OAEG;IACH,IAAI,IAAI,IAAI;IA8BZ;;OAEG;IACH,OAAO,CAAC,KAAK;IA2Bb;;OAEG;IACH,QAAQ,IAAI;QACV,YAAY,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/B,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;QACzB,WAAW,EAAE,IAAI,GAAG,IAAI,CAAC;QACzB,aAAa,EAAE,MAAM,CAAC;KACvB;IA4BD;;OAEG;IACH,KAAK,IAAI,IAAI;CAId;AAMD;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,CAAC,EAAE,MAAM,GAClB,iBAAiB,CAGnB"}