@nanocollective/nanocoder 1.19.2 → 1.20.0

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js

@@ -1,170 +0,0 @@
-/**
- * Tests for llama-tokenizer.ts
- */
-import test from 'ava';
-import { LlamaTokenizer } from './llama-tokenizer.js';
-console.log(`\nllama-tokenizer.spec.ts`);
-test('LlamaTokenizer encodes simple text', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const count = tokenizer.encode('Hello, world!');
-    // Should return a positive token count
-    t.true(count > 0);
-    t.true(count < 10);
-});
-test('LlamaTokenizer encodes empty string', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const count = tokenizer.encode('');
-    t.is(count, 0);
-});
-test('LlamaTokenizer encodes longer text', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
-    const count = tokenizer.encode(text);
-    // Should have significantly more tokens
-    t.true(count > 10);
-    t.true(count < 50);
-});
-test('LlamaTokenizer defaults to llama when no model specified', t => {
-    const tokenizer = new LlamaTokenizer();
-    t.is(tokenizer.getName(), 'llama-llama');
-});
-test('LlamaTokenizer getName returns correct format', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-70b');
-    t.is(tokenizer.getName(), 'llama-llama-3-70b');
-});
-test('LlamaTokenizer countTokens for user message', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'user',
-        content: 'Hello, how are you?',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should include content tokens + role tokens + overhead
-    t.true(count > 5);
-    t.true(count < 25);
-});
-test('LlamaTokenizer countTokens for assistant message', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'assistant',
-        content: 'I am doing well, thank you!',
-    };
-    const count = tokenizer.countTokens(message);
-    t.true(count > 5);
-});
-test('LlamaTokenizer countTokens for system message', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'system',
-        content: 'You are a helpful assistant.',
-    };
-    const count = tokenizer.countTokens(message);
-    t.true(count > 5);
-});
-test('LlamaTokenizer countTokens handles empty content', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'user',
-        content: '',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should still have overhead for role and message structure
-    t.true(count >= 6);
-});
-test('LlamaTokenizer countTokens handles missing content', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'user',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should handle gracefully
-    t.true(count >= 0);
-});
-test('LlamaTokenizer countTokens includes message overhead', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const shortMessage = {
-        role: 'user',
-        content: 'Hi',
-    };
-    const count = tokenizer.countTokens(shortMessage);
-    const contentOnly = tokenizer.encode('Hi');
-    const roleOnly = tokenizer.encode('user');
-    // Total should be more than just content + role due to overhead
-    t.true(count > contentOnly + roleOnly);
-});
-test('LlamaTokenizer handles special characters', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const text = '你好世界 🌍 Привет мир';
-    const count = tokenizer.encode(text);
-    t.true(count > 0);
-});
-test('LlamaTokenizer handles code snippets', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-    const count = tokenizer.encode(code);
-    t.true(count > 10);
-});
-test('LlamaTokenizer works with mistral model', t => {
-    const tokenizer = new LlamaTokenizer('mistral-7b');
-    const count = tokenizer.encode('Hello, world!');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'llama-mistral-7b');
-});
-test('LlamaTokenizer works with qwen model', t => {
-    const tokenizer = new LlamaTokenizer('qwen-2.5');
-    const count = tokenizer.encode('Hello, world!');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'llama-qwen-2.5');
-});
-test('LlamaTokenizer works with codellama model', t => {
-    const tokenizer = new LlamaTokenizer('codellama-7b');
-    const count = tokenizer.encode('Hello, world!');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'llama-codellama-7b');
-});
-test('LlamaTokenizer handles long messages', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const longText = 'Hello '.repeat(1000);
-    const message = {
-        role: 'user',
-        content: longText,
-    };
-    const count = tokenizer.countTokens(message);
-    // Should handle long text without crashing
-    t.true(count > 1000);
-});
-test('LlamaTokenizer uses fallback on encoding error', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    // The fallback should kick in for any edge cases
-    // Testing with normal text should still work
-    const count = tokenizer.encode('Normal text');
-    t.true(count > 0);
-});
-test('LlamaTokenizer countTokens with tool message', t => {
-    const tokenizer = new LlamaTokenizer('llama-3-8b');
-    const message = {
-        role: 'tool',
-        content: 'Tool result here',
-        tool_call_id: '123',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should handle tool messages
-    t.true(count > 0);
-});
-test('LlamaTokenizer handles deepseek model', t => {
-    const tokenizer = new LlamaTokenizer('deepseek-coder-33b');
-    const count = tokenizer.encode('const x = 42;');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'llama-deepseek-coder-33b');
-});
-test('LlamaTokenizer handles mixtral model', t => {
-    const tokenizer = new LlamaTokenizer('mixtral-8x7b');
-    const count = tokenizer.encode('Hello, world!');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'llama-mixtral-8x7b');
-});
-//# sourceMappingURL=llama-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"llama-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAEpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAEzC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;IAEvC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,mBAAmB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,CAAC,CAAC,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IAEnD,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts

@@ -1,5 +0,0 @@
-/**
- * Tests for openai-tokenizer.ts
- */
-export {};
-//# sourceMappingURL=openai-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js

@@ -1,140 +0,0 @@
-/**
- * Tests for openai-tokenizer.ts
- */
-import test from 'ava';
-import { OpenAITokenizer } from './openai-tokenizer.js';
-console.log(`\nopenai-tokenizer.spec.ts`);
-test('OpenAITokenizer encodes simple text', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const count = tokenizer.encode('Hello, world!');
-    // "Hello, world!" should tokenize to around 4 tokens
-    t.true(count > 0);
-    t.true(count < 10);
-});
-test('OpenAITokenizer encodes empty string', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const count = tokenizer.encode('');
-    t.is(count, 0);
-});
-test('OpenAITokenizer encodes longer text', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
-    const count = tokenizer.encode(text);
-    // Should have significantly more tokens
-    t.true(count > 10);
-    t.true(count < 50);
-});
-test('OpenAITokenizer uses fallback encoding for unsupported model', t => {
-    const tokenizer = new OpenAITokenizer('unknown-model-xyz');
-    const count = tokenizer.encode('Hello, world!');
-    // Should still return a count using fallback
-    t.true(count > 0);
-});
-test('OpenAITokenizer defaults to gpt-4 when no model specified', t => {
-    const tokenizer = new OpenAITokenizer();
-    t.is(tokenizer.getName(), 'openai-gpt-4');
-});
-test('OpenAITokenizer getName returns correct format', t => {
-    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
-    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
-});
-test('OpenAITokenizer countTokens for user message', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const message = {
-        role: 'user',
-        content: 'Hello, how are you?',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should include content tokens + role tokens + overhead
-    t.true(count > 5);
-    t.true(count < 20);
-});
-test('OpenAITokenizer countTokens for assistant message', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const message = {
-        role: 'assistant',
-        content: 'I am doing well, thank you!',
-    };
-    const count = tokenizer.countTokens(message);
-    t.true(count > 5);
-});
-test('OpenAITokenizer countTokens for system message', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const message = {
-        role: 'system',
-        content: 'You are a helpful assistant.',
-    };
-    const count = tokenizer.countTokens(message);
-    t.true(count > 5);
-});
-test('OpenAITokenizer countTokens handles empty content', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const message = {
-        role: 'user',
-        content: '',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should still have overhead for role and message structure
-    t.true(count >= 4);
-});
-test('OpenAITokenizer countTokens handles missing content', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const message = {
-        role: 'user',
-    };
-    const count = tokenizer.countTokens(message);
-    // Should handle gracefully
-    t.true(count >= 0);
-});
-test('OpenAITokenizer countTokens includes message overhead', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const shortMessage = {
-        role: 'user',
-        content: 'Hi',
-    };
-    const count = tokenizer.countTokens(shortMessage);
-    const contentOnly = tokenizer.encode('Hi');
-    const roleOnly = tokenizer.encode('user');
-    // Total should be more than just content + role due to overhead
-    t.true(count > contentOnly + roleOnly);
-});
-test('OpenAITokenizer free method exists', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    t.notThrows(() => {
-        tokenizer.free();
-    });
-});
-test('OpenAITokenizer handles special characters', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const text = '你好世界 🌍 Привет мир';
-    const count = tokenizer.encode(text);
-    t.true(count > 0);
-});
-test('OpenAITokenizer handles code snippets', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-    const count = tokenizer.encode(code);
-    t.true(count > 10);
-});
-test('OpenAITokenizer works with gpt-3.5-turbo model', t => {
-    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
-    const count = tokenizer.encode('Hello, world!');
-    t.true(count > 0);
-    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
-});
-test('OpenAITokenizer handles long messages', t => {
-    const tokenizer = new OpenAITokenizer('gpt-4');
-    const longText = 'Hello '.repeat(1000);
-    const message = {
-        role: 'user',
-        content: longText,
-    };
-    const count = tokenizer.countTokens(message);
-    // Should handle long text without crashing
-    t.true(count > 1000);
-});
-//# sourceMappingURL=openai-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"openai-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAEtD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;AAE1C,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,qDAAqD;IACrD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8DAA8D,EAAE,CAAC,CAAC,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,6CAA6C;IAC7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2DAA2D,EAAE,CAAC,CAAC,EAAE;IACrE,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IAExC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IAEvD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;QAChB,SAAS,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC"}

package/dist/tool-calling/json-parser.spec.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=json-parser.spec.d.ts.map

package/dist/tool-calling/json-parser.spec.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"json-parser.spec.d.ts","sourceRoot":"","sources":["../../source/tool-calling/json-parser.spec.ts"],"names":[],"mappings":""}