@gguf/coder 0.0.1

package/dist/tokenization/tokenizers/llama-tokenizer.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Llama tokenizer for local models
+ * Uses llama-tokenizer-js package
+ */
+import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
+export declare class LlamaTokenizer implements Tokenizer {
+    private modelName;
+    constructor(modelId?: string);
+    encode(text: string): number;
+    countTokens(message: Message): number;
+    getName(): string;
+}
+//# sourceMappingURL=llama-tokenizer.d.ts.map

package/dist/tokenization/tokenizers/llama-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD,qBAAa,cAAe,YAAW,SAAS;IAC/C,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAU5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/llama-tokenizer.js

@@ -0,0 +1,33 @@
+/**
+ * Llama tokenizer for local models
+ * Uses llama-tokenizer-js package
+ */
+import llamaTokenizer from 'llama-tokenizer-js';
+export class LlamaTokenizer {
+    modelName;
+    constructor(modelId) {
+        this.modelName = modelId || 'llama';
+    }
+    encode(text) {
+        try {
+            const tokens = llamaTokenizer.encode(text);
+            return tokens.length;
+        }
+        catch {
+            // Fallback to character-based estimation if tokenization fails
+            return Math.ceil(text.length / 4);
+        }
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // Llama format: <|start_header_id|>role<|end_header_id|>content<|eot_id|>
+        // Approximate overhead for message formatting
+        const messageOverhead = 6;
+        return this.encode(content) + this.encode(role) + messageOverhead;
+    }
+    getName() {
+        return `llama-${this.modelName}`;
+    }
+}
+//# sourceMappingURL=llama-tokenizer.js.map

package/dist/tokenization/tokenizers/llama-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,cAAc,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,cAAc;IAClB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0EAA0E;QAC1E,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IAClC,CAAC;CACD"}

package/dist/tokenization/tokenizers/llama-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for llama-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=llama-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/llama-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js

@@ -0,0 +1,170 @@
+/**
+ * Tests for llama-tokenizer.ts
+ */
+import test from 'ava';
+import { LlamaTokenizer } from './llama-tokenizer.js';
+console.log(`\nllama-tokenizer.spec.ts`);
+test('LlamaTokenizer encodes simple text', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const count = tokenizer.encode('Hello, world!');
+    // Should return a positive token count
+    t.true(count > 0);
+    t.true(count < 10);
+});
+test('LlamaTokenizer encodes empty string', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('LlamaTokenizer encodes longer text', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // Should have significantly more tokens
+    t.true(count > 10);
+    t.true(count < 50);
+});
+test('LlamaTokenizer defaults to llama when no model specified', t => {
+    const tokenizer = new LlamaTokenizer();
+    t.is(tokenizer.getName(), 'llama-llama');
+});
+test('LlamaTokenizer getName returns correct format', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-70b');
+    t.is(tokenizer.getName(), 'llama-llama-3-70b');
+});
+test('LlamaTokenizer countTokens for user message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should include content tokens + role tokens + overhead
+    t.true(count > 5);
+    t.true(count < 25);
+});
+test('LlamaTokenizer countTokens for assistant message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('LlamaTokenizer countTokens for system message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('LlamaTokenizer countTokens handles empty content', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should still have overhead for role and message structure
+    t.true(count >= 6);
+});
+test('LlamaTokenizer countTokens handles missing content', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully
+    t.true(count >= 0);
+});
+test('LlamaTokenizer countTokens includes message overhead', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const shortMessage = {
+        role: 'user',
+        content: 'Hi',
+    };
+    const count = tokenizer.countTokens(shortMessage);
+    const contentOnly = tokenizer.encode('Hi');
+    const roleOnly = tokenizer.encode('user');
+    // Total should be more than just content + role due to overhead
+    t.true(count > contentOnly + roleOnly);
+});
+test('LlamaTokenizer handles special characters', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    t.true(count > 0);
+});
+test('LlamaTokenizer handles code snippets', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 10);
+});
+test('LlamaTokenizer works with mistral model', t => {
+    const tokenizer = new LlamaTokenizer('mistral-7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-mistral-7b');
+});
+test('LlamaTokenizer works with qwen model', t => {
+    const tokenizer = new LlamaTokenizer('qwen-2.5');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-qwen-2.5');
+});
+test('LlamaTokenizer works with codellama model', t => {
+    const tokenizer = new LlamaTokenizer('codellama-7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-codellama-7b');
+});
+test('LlamaTokenizer handles long messages', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+});
+test('LlamaTokenizer uses fallback on encoding error', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    // The fallback should kick in for any edge cases
+    // Testing with normal text should still work
+    const count = tokenizer.encode('Normal text');
+    t.true(count > 0);
+});
+test('LlamaTokenizer countTokens with tool message', t => {
+    const tokenizer = new LlamaTokenizer('llama-3-8b');
+    const message = {
+        role: 'tool',
+        content: 'Tool result here',
+        tool_call_id: '123',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle tool messages
+    t.true(count > 0);
+});
+test('LlamaTokenizer handles deepseek model', t => {
+    const tokenizer = new LlamaTokenizer('deepseek-coder-33b');
+    const count = tokenizer.encode('const x = 42;');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-deepseek-coder-33b');
+});
+test('LlamaTokenizer handles mixtral model', t => {
+    const tokenizer = new LlamaTokenizer('mixtral-8x7b');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'llama-mixtral-8x7b');
+});
+//# sourceMappingURL=llama-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAEpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAEzC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;IAEvC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,mBAAmB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,CAAC,CAAC,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IAEnD,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts

@@ -0,0 +1,22 @@
+/**
+ * OpenAI tokenizer using tiktoken
+ * Supports GPT-3.5, GPT-4, and other OpenAI models
+ */
+import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
+/**
+ * OpenAI tokenizer using tiktoken for accurate token counting
+ */
+export declare class OpenAITokenizer implements Tokenizer {
+    private encoding;
+    private modelName;
+    constructor(modelId?: string);
+    encode(text: string): number;
+    countTokens(message: Message): number;
+    getName(): string;
+    /**
+     * Clean up encoding resources
+     */
+    free(): void;
+}
+//# sourceMappingURL=openai-tokenizer.d.ts.map

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;GAEG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAU5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;IAIjB;;OAEG;IACH,IAAI,IAAI,IAAI;CAGZ"}

package/dist/tokenization/tokenizers/openai-tokenizer.js

@@ -0,0 +1,48 @@
+/**
+ * OpenAI tokenizer using tiktoken
+ * Supports GPT-3.5, GPT-4, and other OpenAI models
+ */
+import { encoding_for_model, get_encoding } from 'tiktoken';
+/**
+ * OpenAI tokenizer using tiktoken for accurate token counting
+ */
+export class OpenAITokenizer {
+    encoding;
+    modelName;
+    constructor(modelId) {
+        this.modelName = modelId || 'gpt-4';
+        try {
+            this.encoding = encoding_for_model(modelId);
+        }
+        catch {
+            this.encoding = get_encoding('cl100k_base');
+        }
+    }
+    encode(text) {
+        try {
+            const tokens = this.encoding.encode(text);
+            return tokens.length;
+        }
+        catch {
+            return Math.ceil(text.length / 4);
+        }
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // OpenAI format: each message has overhead for role markers
+        // <|im_start|>role\ncontent<|im_end|>
+        const messageOverhead = 4; // Approximate overhead per message
+        return this.encode(content) + this.encode(role) + messageOverhead;
+    }
+    getName() {
+        return `openai-${this.modelName}`;
+    }
+    /**
+     * Clean up encoding resources
+     */
+    free() {
+        this.encoding.free();
+    }
+}
+//# sourceMappingURL=openai-tokenizer.js.map

package/dist/tokenization/tokenizers/openai-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAqB,kBAAkB,EAAE,YAAY,EAAC,MAAM,UAAU,CAAC;AAG9E;;GAEG;AACH,MAAM,OAAO,eAAe;IACnB,QAAQ,CAAkC;IAC1C,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;QAEpC,IAAI,CAAC;YACJ,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAwB,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,4DAA4D;QAC5D,sCAAsC;QACtC,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,mCAAmC;QAE9D,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;CACD"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for openai-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=openai-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js

@@ -0,0 +1,140 @@
+/**
+ * Tests for openai-tokenizer.ts
+ */
+import test from 'ava';
+import { OpenAITokenizer } from './openai-tokenizer.js';
+console.log(`\nopenai-tokenizer.spec.ts`);
+test('OpenAITokenizer encodes simple text', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const count = tokenizer.encode('Hello, world!');
+    // "Hello, world!" should tokenize to around 4 tokens
+    t.true(count > 0);
+    t.true(count < 10);
+});
+test('OpenAITokenizer encodes empty string', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('OpenAITokenizer encodes longer text', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // Should have significantly more tokens
+    t.true(count > 10);
+    t.true(count < 50);
+});
+test('OpenAITokenizer uses fallback encoding for unsupported model', t => {
+    const tokenizer = new OpenAITokenizer('unknown-model-xyz');
+    const count = tokenizer.encode('Hello, world!');
+    // Should still return a count using fallback
+    t.true(count > 0);
+});
+test('OpenAITokenizer defaults to gpt-4 when no model specified', t => {
+    const tokenizer = new OpenAITokenizer();
+    t.is(tokenizer.getName(), 'openai-gpt-4');
+});
+test('OpenAITokenizer getName returns correct format', t => {
+    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+test('OpenAITokenizer countTokens for user message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should include content tokens + role tokens + overhead
+    t.true(count > 5);
+    t.true(count < 20);
+});
+test('OpenAITokenizer countTokens for assistant message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('OpenAITokenizer countTokens for system message', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('OpenAITokenizer countTokens handles empty content', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should still have overhead for role and message structure
+    t.true(count >= 4);
+});
+test('OpenAITokenizer countTokens handles missing content', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully
+    t.true(count >= 0);
+});
+test('OpenAITokenizer countTokens includes message overhead', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const shortMessage = {
+        role: 'user',
+        content: 'Hi',
+    };
+    const count = tokenizer.countTokens(shortMessage);
+    const contentOnly = tokenizer.encode('Hi');
+    const roleOnly = tokenizer.encode('user');
+    // Total should be more than just content + role due to overhead
+    t.true(count > contentOnly + roleOnly);
+});
+test('OpenAITokenizer free method exists', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    t.notThrows(() => {
+        tokenizer.free();
+    });
+});
+test('OpenAITokenizer handles special characters', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    t.true(count > 0);
+});
+test('OpenAITokenizer handles code snippets', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 10);
+});
+test('OpenAITokenizer works with gpt-3.5-turbo model', t => {
+    const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+test('OpenAITokenizer handles long messages', t => {
+    const tokenizer = new OpenAITokenizer('gpt-4');
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+});
+//# sourceMappingURL=openai-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/openai-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,eAAe,EAAC,MAAM,uBAAuB,CAAC;AAEtD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;AAE1C,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,qDAAqD;IACrD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8DAA8D,EAAE,CAAC,CAAC,EAAE;IACxE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,mBAAmB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,6CAA6C;IAC7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2DAA2D,EAAE,CAAC,CAAC,EAAE;IACrE,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IAExC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,cAAc,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IAEvD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAE/C,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE;QAChB,SAAS,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,eAAe,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,sBAAsB,CAAC,CAAC;AACnD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC"}

package/dist/tool-calling/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tool calling utilities - main exports
+ */
+export { parseToolCalls } from '../tool-calling/tool-parser.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/tool-calling/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../source/tool-calling/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC"}

package/dist/tool-calling/index.js

@@ -0,0 +1,5 @@
+/**
+ * Tool calling utilities - main exports
+ */
+export { parseToolCalls } from '../tool-calling/tool-parser.js';
+//# sourceMappingURL=index.js.map

package/dist/tool-calling/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../source/tool-calling/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAC,cAAc,EAAC,MAAM,4BAA4B,CAAC"}

package/dist/tool-calling/json-parser.d.ts

@@ -0,0 +1,24 @@
+import type { ToolCall } from '../types/index.js';
+/**
+ * Internal JSON tool call parser
+ * Note: This is now an internal utility. Use tool-parser.ts for public API.
+ */
+/**
+ * Detects malformed JSON tool call attempts and returns error details
+ * Returns null if no malformed tool calls detected
+ */
+export declare function detectMalformedJSONToolCall(content: string): {
+    error: string;
+    examples: string;
+} | null;
+/**
+ * Parses JSON-formatted tool calls from content
+ * This is an internal function - use tool-parser.ts for public API
+ */
+export declare function parseJSONToolCalls(content: string): ToolCall[];
+/**
+ * Cleans content by removing tool call JSON blocks
+ * This is an internal function - use tool-parser.ts for public API
+ */
+export declare function cleanJSONToolCalls(content: string, toolCalls: ToolCall[]): string;
+//# sourceMappingURL=json-parser.d.ts.map

package/dist/tool-calling/json-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-parser.d.ts","sourceRoot":"","sources":["../../source/tool-calling/json-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,eAAe,CAAC;AAE5C;;;GAGG;AAEH;;;GAGG;AACH,wBAAgB,2BAA2B,CAC1C,OAAO,EAAE,MAAM,GACb;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAC,GAAG,IAAI,CAkC1C;AASD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,EAAE,CAqG9D;AAsBD;;;GAGG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,QAAQ,EAAE,GACnB,MAAM,CAwDR"}