@nanocollective/nanocoder 1.15.1 → 1.16.1

package/dist/tokenization/tokenizers/llama-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,qBAAa,cAAe,YAAW,SAAS;IAC/C,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAU5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/llama-tokenizer.js

@@ -0,0 +1,33 @@
+/**
+ * Llama tokenizer for local models
+ * Uses llama-tokenizer-js package
+ */
+import llamaTokenizer from 'llama-tokenizer-js';
+export class LlamaTokenizer {
+    modelName;
+    constructor(modelId) {
+        this.modelName = modelId || 'llama';
+    }
+    encode(text) {
+        try {
+            const tokens = llamaTokenizer.encode(text);
+            return tokens.length;
+        }
+        catch {
+            // Fallback to character-based estimation if tokenization fails
+            return Math.ceil(text.length / 4);
+        }
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // Llama format: <|start_header_id|>role<|end_header_id|>content<|eot_id|>
+        // Approximate overhead for message formatting
+        const messageOverhead = 6;
+        return this.encode(content) + this.encode(role) + messageOverhead;
+    }
+    getName() {
+        return `llama-${this.modelName}`;
+    }
+}
+//# sourceMappingURL=llama-tokenizer.js.map

package/dist/tokenization/tokenizers/llama-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,cAAc,MAAM,oBAAoB,CAAC;AAIhD,MAAM,OAAO,cAAc;IAClB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0EAA0E;QAC1E,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IAClC,CAAC;CACD"}

package/dist/tokenization/tokenizers/llama-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for llama-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=llama-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/llama-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js

@@ -0,0 +1,170 @@
+/**
+ * Tests for llama-tokenizer.ts
+ */
+import test from 'ava';
+import { LlamaTokenizer } from './llama-tokenizer.js';
+console.log(`\nllama-tokenizer.spec.ts`);
+test('LlamaTokenizer encodes simple text', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const count = tokenizer.encode('Hello, world!');
+    // Should return a positive token count
+    t.true(count > 0);
+    t.true(count < 10);
+});
+test('LlamaTokenizer encodes empty string', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('LlamaTokenizer encodes longer text', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // Should have significantly more tokens
+    t.true(count > 10);
+    t.true(count < 50);
+});
+test('LlamaTokenizer defaults to llama when no model specified', t => {
+    const tokenizer = new LlamaTokenizer();
+    t.is(tokenizer.getName(), 'llama-llama');
+});
+test('LlamaTokenizer getName returns correct format', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-70b');
+    t.is(tokenizer.getName(), 'llama-llama-3-70b');
+});
+test('LlamaTokenizer countTokens for user message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should include content tokens + role tokens + overhead
+    t.true(count > 5);
+    t.true(count < 25);
+});
+test('LlamaTokenizer countTokens for assistant message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('LlamaTokenizer countTokens for system message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('LlamaTokenizer countTokens handles empty content', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should still have overhead for role and message structure
+    t.true(count >= 6);
+});
+test('LlamaTokenizer countTokens handles missing content', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully
+    t.true(count >= 0);
+});
+test('LlamaTokenizer countTokens includes message overhead', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const shortMessage = {
+        role: 'user',
+        content: 'Hi',
+    };
+    const count = tokenizer.countTokens(shortMessage);
+    const contentOnly = tokenizer.encode('Hi');
+    const roleOnly = tokenizer.encode('user');
+    // Total should be more than just content + role due to overhead
+    t.true(count > contentOnly + roleOnly);
+});
+test('LlamaTokenizer handles special characters', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    t.true(count > 0);
+});
+test('LlamaTokenizer handles code snippets', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 10);
+});
+test('LlamaTokenizer works with mistral model', t => {
+    const tokenizer = new LlamaTokenizer('mistral-7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-mistral-7b');
+});
+test('LlamaTokenizer works with qwen model', t => {
+    const tokenizer = new LlamaTokenizer('qwen-2.5');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-qwen-2.5');
+});
+test('LlamaTokenizer works with codellama model', t => {
+    const tokenizer = new LlamaTokenizer('codellama-7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-codellama-7b');
+});
+test('LlamaTokenizer handles long messages', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+});
+test('LlamaTokenizer uses fallback on encoding error', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    // The fallback should kick in for any edge cases
+    // Testing with normal text should still work
+    const count = tokenizer.encode('Normal text');
+    t.true(count > 0);
+});
+test('LlamaTokenizer countTokens with tool message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'tool',
+        content: 'Tool result here',
+        tool_call_id: '123',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle tool messages
+    t.true(count > 0);
+});
+test('LlamaTokenizer handles deepseek model', t => {
+    const tokenizer = new LlamaTokenizer('deepseek-coder-33b');
+    const count = tokenizer.encode('const x = 42;');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-deepseek-coder-33b');
+});
+test('LlamaTokenizer handles mixtral model', t => {
+    const tokenizer = new LlamaTokenizer('mixtral-8x7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-mixtral-8x7b');
+});
+//# sourceMappingURL=llama-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAGpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAEzC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;IAEvC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,mBAAmB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,CAAC,CAAC,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IAEnD,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts

@@ -0,0 +1,22 @@
+/**
+ * OpenAI tokenizer using tiktoken
+ * Supports GPT-3.5, GPT-4, and other OpenAI models
+ */
+import type { Tokenizer } from '../../types/tokenization.js';
+import type { Message } from '../../types/core.js';
+/**
+ * OpenAI tokenizer using tiktoken for accurate token counting
+ */
+export declare class OpenAITokenizer implements Tokenizer {
+    private encoding;
+    private modelName;
+    constructor(modelId?: string);
+    encode(text: string): number;
+    countTokens(message: Message): number;
+    getName(): string;
+    /**
+     * Clean up encoding resources
+     */
+    free(): void;
+}
+//# sourceMappingURL=openai-tokenizer.d.ts.map

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C;;GAEG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAU5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;IAIjB;;OAEG;IACH,IAAI,IAAI,IAAI;CAGZ"}

package/dist/tokenization/tokenizers/openai-tokenizer.js

@@ -0,0 +1,48 @@
+/**
+ * OpenAI tokenizer using tiktoken
+ * Supports GPT-3.5, GPT-4, and other OpenAI models
+ */
+import { encoding_for_model, get_encoding } from 'tiktoken';
+/**
+ * OpenAI tokenizer using tiktoken for accurate token counting
+ */
+export class OpenAITokenizer {
+    encoding;
+    modelName;
+    constructor(modelId) {
+        this.modelName = modelId || 'gpt-4';
+        try {
+            this.encoding = encoding_for_model(modelId);
+        }
+        catch {
+            this.encoding = get_encoding('cl100k_base');
+        }
+    }
+    encode(text) {
+        try {
+            const tokens = this.encoding.encode(text);
+            return tokens.length;
+        }
+        catch {
+            return Math.ceil(text.length / 4);
+        }
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // OpenAI format: each message has overhead for role markers
+        // <|im_start|>role\ncontent<|im_end|>
+        const messageOverhead = 4; // Approximate overhead per message
+        return this.encode(content) + this.encode(role) + messageOverhead;
+    }
+    getName() {
+        return `openai-${this.modelName}`;
+    }
+    /**
+     * Clean up encoding resources
+     */
+    free() {
+        this.encoding.free();
+    }
+}
+//# sourceMappingURL=openai-tokenizer.js.map

package/dist/tokenization/tokenizers/openai-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAC,kBAAkB,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAI9E;;GAEG;AACH,MAAM,OAAO,eAAe;IACnB,QAAQ,CAAkC;IAC1C,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;QAEpC,IAAI,CAAC;YACJ,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAwB,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,4DAA4D;QAC5D,sCAAsC;QACtC,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,mCAAmC;QAE9D,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;CACD"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for openai-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=openai-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js

@@ -0,0 +1,140 @@
+/**
+ * Tests for openai-tokenizer.ts
+ */
+import test from 'ava';
+import { OpenAITokenizer } from './openai-tokenizer.js';
+console.log(`\nopenai-tokenizer.spec.ts`);
+test('OpenAITokenizer encodes simple text', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const count = tokenizer.encode('Hello, world!');
+    // "Hello, world!" should tokenize to around 4 tokens
+    t.true(count > 0);
+    t.true(count < 10);
+});
+test('OpenAITokenizer encodes empty string', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('OpenAITokenizer encodes longer text', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // Should have significantly more tokens
+    t.true(count > 10);
+    t.true(count < 50);
+});
+test('OpenAITokenizer uses fallback encoding for unsupported model', t => {
+    const tokenizer = new OpenAITokenizer('unknown-model-xyz');
+    const count = tokenizer.encode('Hello, world!');
+    // Should still return a count using fallback
+    t.true(count > 0);
+});
+test('OpenAITokenizer defaults to gpt-4 when no model specified', t => {
+    const tokenizer = new OpenAITokenizer();
+    t.is(tokenizer.getName(), 'openai-gpt-4');
+});
+test('OpenAITokenizer getName returns correct format', t => {
+    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+test('OpenAITokenizer countTokens for user message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should include content tokens + role tokens + overhead
+    t.true(count > 5);
+    t.true(count < 20);
+});
+test('OpenAITokenizer countTokens for assistant message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('OpenAITokenizer countTokens for system message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('OpenAITokenizer countTokens handles empty content', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should still have overhead for role and message structure
+    t.true(count >= 4);
+});
+test('OpenAITokenizer countTokens handles missing content', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully
+    t.true(count >= 0);
+});
+test('OpenAITokenizer countTokens includes message overhead', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const shortMessage = {
+        role: 'user',
+        content: 'Hi',
+    };
+    const count = tokenizer.countTokens(shortMessage);
+    const contentOnly = tokenizer.encode('Hi');
+    const roleOnly = tokenizer.encode('user');
+    // Total should be more than just content + role due to overhead
+    t.true(count > contentOnly + roleOnly);
+});
+test('OpenAITokenizer free method exists', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    t.notThrows(() => {
+        tokenizer.free();
+    });
+});
+test('OpenAITokenizer handles special characters', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    t.true(count > 0);
+});
+test('OpenAITokenizer handles code snippets', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 10);
+});
+test('OpenAITokenizer works with gpt-3.5-turbo model', t => {
+    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+test('OpenAITokenizer handles long messages', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+});
+//# sourceMappingURL=openai-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAGtD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;AAE1C,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,qDAAqD;IACrD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8DAA8D,EAAE,CAAC,CAAC,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,6CAA6C;IAC7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2DAA2D,EAAE,CAAC,CAAC,EAAE;IACrE,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IAExC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IAEvD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;QAChB,SAAS,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC"}

package/dist/types/commands.d.ts

@@ -6,6 +6,7 @@ export interface Command<T = React.ReactElement> {
         provider: string;
         model: string;
         tokens: number;
+        getMessageTokens: (message: Message) => number;
     }) => Promise<T>;
 }
 export interface ParsedCommand {

package/dist/types/commands.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../source/types/commands.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAErC,MAAM,WAAW,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,YAAY;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,CACR,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,OAAO,EAAE,EACnB,QAAQ,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAC,KACvD,OAAO,CAAC,CAAC,CAAC,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;CAChB"}
+{"version":3,"file":"commands.d.ts","sourceRoot":"","sources":["../../source/types/commands.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAErC,MAAM,WAAW,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,YAAY;IAC9C,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,CACR,IAAI,EAAE,MAAM,EAAE,EACd,QAAQ,EAAE,OAAO,EAAE,EACnB,QAAQ,EAAE;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,MAAM,CAAC;KAC/C,KACG,OAAO,CAAC,CAAC,CAAC,CAAC;CAChB;AAED,MAAM,WAAW,aAAa;IAC7B,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,EAAE,qBAAqB,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;CAChB"}

package/dist/types/tokenization.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Tokenizer interface and types
+ */
+import type { Message } from '../types/core.js';
+/**
+ * Tokenizer interface for encoding text and counting tokens
+ */
+export interface Tokenizer {
+    /**
+     * Encode text and return token count
+     */
+    encode(text: string): number;
+    /**
+     * Count tokens in a message (content + role)
+     */
+    countTokens(message: Message): number;
+    /**
+     * Get the tokenizer name/type
+     */
+    getName(): string;
+    /**
+     * Optional cleanup method for releasing resources
+     * Should be called when the tokenizer is no longer needed
+     */
+    free?(): void;
+}
+/**
+ * Provider types for tokenizer selection
+ */
+export type TokenizerProvider = 'openai' | 'anthropic' | 'llama' | 'fallback' | 'auto';
+//# sourceMappingURL=tokenization.d.ts.map

package/dist/types/tokenization.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenization.d.ts","sourceRoot":"","sources":["../../source/types/tokenization.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB;;OAEG;IACH,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAE7B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAAC;IAEtC;;OAEG;IACH,OAAO,IAAI,MAAM,CAAC;IAElB;;;OAGG;IACH,IAAI,CAAC,IAAI,IAAI,CAAC;CACd;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAC1B,QAAQ,GACR,WAAW,GACX,OAAO,GACP,UAAU,GACV,MAAM,CAAC"}

package/dist/types/tokenization.js

@@ -0,0 +1,5 @@
+/**
+ * Tokenizer interface and types
+ */
+export {};
+//# sourceMappingURL=tokenization.js.map

package/dist/types/tokenization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenization.js","sourceRoot":"","sources":["../../source/types/tokenization.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/types/usage.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Token breakdown by category
+ */
+export interface TokenBreakdown {
+    system: number;
+    userMessages: number;
+    assistantMessages: number;
+    toolDefinitions: number;
+    toolResults: number;
+    total: number;
+}
+/**
+ * Session usage data
+ */
+export interface SessionUsage {
+    id: string;
+    timestamp: number;
+    provider: string;
+    model: string;
+    tokens: TokenBreakdown;
+    messageCount: number;
+    duration?: number;
+}
+/**
+ * Daily aggregate usage
+ */
+export interface DailyAggregate {
+    date: string;
+    sessions: number;
+    totalTokens: number;
+    providers: Record<string, number>;
+    models: Record<string, number>;
+}
+/**
+ * Persistent usage data structure
+ */
+export interface UsageData {
+    sessions: SessionUsage[];
+    dailyAggregates: DailyAggregate[];
+    totalLifetime: number;
+    lastUpdated: number;
+}
+/**
+ * Current session statistics
+ */
+export interface CurrentSessionStats {
+    tokens: TokenBreakdown;
+    messageCount: number;
+    startTime: number;
+    provider: string;
+    model: string;
+    contextLimit: number | null;
+    percentUsed: number;
+}
+//# sourceMappingURL=usage.d.ts.map

package/dist/types/usage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../../source/types/usage.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,cAAc,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACzB,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,eAAe,EAAE,cAAc,EAAE,CAAC;IAClC,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,MAAM,EAAE,cAAc,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;CACpB"}

package/dist/types/usage.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=usage.js.map

package/dist/types/usage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"usage.js","sourceRoot":"","sources":["../../source/types/usage.ts"],"names":[],"mappings":""}

package/dist/usage/calculator.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Usage calculator
+ * Calculates token breakdown by category
+ */
+import type { Message } from '../types/core.js';
+import type { Tokenizer } from '../types/tokenization.js';
+import type { TokenBreakdown } from '../types/usage.js';
+/**
+ * Calculate token breakdown from messages
+ * @param messages - Messages to calculate breakdown for
+ * @param tokenizer - Tokenizer instance (used as fallback if getTokens not provided)
+ * @param getTokens - Optional cached token counting function for performance
+ */
+export declare function calculateTokenBreakdown(messages: Message[], tokenizer: Tokenizer, getTokens?: (message: Message) => number): TokenBreakdown;
+/**
+ * Calculate tool definitions token count
+ * This estimates the tokens used by tool definitions sent to the model
+ */
+export declare function calculateToolDefinitionsTokens(toolCount: number): number;
+/**
+ * Get status color based on percentage used
+ */
+export declare function getUsageStatusColor(percentUsed: number): 'success' | 'warning' | 'error';
+/**
+ * Format token count with thousands separator
+ */
+export declare function formatTokenCount(tokens: number): string;
+//# sourceMappingURL=calculator.d.ts.map

package/dist/usage/calculator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calculator.d.ts","sourceRoot":"","sources":["../../source/usage/calculator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAC1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AACpD,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,gBAAgB,CAAC;AAEnD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACtC,QAAQ,EAAE,OAAO,EAAE,EACnB,SAAS,EAAE,SAAS,EACpB,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,MAAM,GACtC,cAAc,CA+ChB;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAMxE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAClC,WAAW,EAAE,MAAM,GACjB,SAAS,GAAG,SAAS,GAAG,OAAO,CAQjC;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD"}