@gguf/coder 0.0.1

package/dist/tokenization/tokenizer-factory.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for tokenizer-factory.ts
+ */
+export {};
+//# sourceMappingURL=tokenizer-factory.spec.d.ts.map

package/dist/tokenization/tokenizer-factory.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer-factory.spec.d.ts","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizer-factory.spec.js

@@ -0,0 +1,137 @@
+/**
+ * Tests for tokenizer-factory.ts
+ */
+import test from 'ava';
+import { createTokenizer, createTokenizerForProvider, } from './tokenizer-factory.js';
+import { AnthropicTokenizer } from './tokenizers/anthropic-tokenizer.js';
+import { FallbackTokenizer } from './tokenizers/fallback-tokenizer.js';
+import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
+import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
+console.log(`\ntokenizer-factory.spec.ts`);
+// Test createTokenizer with OpenAI provider detection
+test('createTokenizer detects OpenAI from provider name', t => {
+    const tokenizer = createTokenizer('openai', 'custom-model');
+    t.true(tokenizer instanceof OpenAITokenizer);
+    t.is(tokenizer.getName(), 'openai-custom-model');
+});
+test('createTokenizer detects OpenAI from GPT model name', t => {
+    const tokenizer = createTokenizer('custom', 'gpt-4-turbo');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+test('createTokenizer detects OpenAI from model name with openai keyword', t => {
+    const tokenizer = createTokenizer('custom', 'openai-model');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+// Test createTokenizer with Anthropic provider detection
+test('createTokenizer detects Anthropic from provider name', t => {
+    const tokenizer = createTokenizer('anthropic', 'custom-model');
+    t.true(tokenizer instanceof AnthropicTokenizer);
+});
+test('createTokenizer detects Anthropic from provider name with claude keyword', t => {
+    const tokenizer = createTokenizer('claude-provider', 'model');
+    t.true(tokenizer instanceof AnthropicTokenizer);
+});
+test('createTokenizer detects Anthropic from claude model name', t => {
+    const tokenizer = createTokenizer('custom', 'claude-3-opus');
+    t.true(tokenizer instanceof AnthropicTokenizer);
+});
+// Test createTokenizer with Llama provider detection
+test('createTokenizer detects Llama from llama model name', t => {
+    const tokenizer = createTokenizer('custom', 'llama-3-8b');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from mistral model name', t => {
+    const tokenizer = createTokenizer('custom', 'mistral-7b');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from qwen model name', t => {
+    const tokenizer = createTokenizer('custom', 'qwen-2.5');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from gemma model name', t => {
+    const tokenizer = createTokenizer('custom', 'gemma-2b');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from phi model name', t => {
+    const tokenizer = createTokenizer('custom', 'phi-3');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from codellama model name', t => {
+    const tokenizer = createTokenizer('custom', 'codellama-7b');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from deepseek model name', t => {
+    const tokenizer = createTokenizer('custom', 'deepseek-coder');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from mixtral model name', t => {
+    const tokenizer = createTokenizer('custom', 'mixtral-8x7b');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from ollama provider', t => {
+    const tokenizer = createTokenizer('ollama', 'custom-model');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from llama.cpp provider', t => {
+    const tokenizer = createTokenizer('llama.cpp', 'custom-model');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizer detects Llama from local provider', t => {
+    const tokenizer = createTokenizer('local', 'custom-model');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+// Test createTokenizer with fallback
+test('createTokenizer returns FallbackTokenizer for unknown provider', t => {
+    const tokenizer = createTokenizer('unknown', 'unknown-model');
+    t.true(tokenizer instanceof FallbackTokenizer);
+});
+// Test cloud suffix stripping
+test('createTokenizer strips :cloud suffix from model name', t => {
+    const tokenizer = createTokenizer('ollama', 'llama-3:cloud');
+    t.true(tokenizer instanceof LlamaTokenizer);
+    t.is(tokenizer.getName(), 'llama-llama-3');
+});
+test('createTokenizer strips -cloud suffix from model name', t => {
+    const tokenizer = createTokenizer('ollama', 'llama-3-cloud');
+    t.true(tokenizer instanceof LlamaTokenizer);
+    t.is(tokenizer.getName(), 'llama-llama-3');
+});
+// Test case insensitivity
+test('createTokenizer is case insensitive for provider detection', t => {
+    const tokenizer = createTokenizer('OPENAI', 'model');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+test('createTokenizer is case insensitive for model detection', t => {
+    const tokenizer = createTokenizer('custom', 'GPT-4-TURBO');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+// Test createTokenizerForProvider
+test('createTokenizerForProvider creates OpenAI tokenizer', t => {
+    const tokenizer = createTokenizerForProvider('openai', 'gpt-4');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+test('createTokenizerForProvider creates Anthropic tokenizer', t => {
+    const tokenizer = createTokenizerForProvider('anthropic', 'claude-3');
+    t.true(tokenizer instanceof AnthropicTokenizer);
+});
+test('createTokenizerForProvider creates Llama tokenizer', t => {
+    const tokenizer = createTokenizerForProvider('llama', 'llama-3');
+    t.true(tokenizer instanceof LlamaTokenizer);
+});
+test('createTokenizerForProvider creates FallbackTokenizer', t => {
+    const tokenizer = createTokenizerForProvider('fallback');
+    t.true(tokenizer instanceof FallbackTokenizer);
+});
+test('createTokenizerForProvider handles auto mode with model ID', t => {
+    const tokenizer = createTokenizerForProvider('auto', 'gpt-4');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+test('createTokenizerForProvider handles auto mode without model ID', t => {
+    const tokenizer = createTokenizerForProvider('auto');
+    t.true(tokenizer instanceof FallbackTokenizer);
+});
+test('createTokenizerForProvider works without model ID', t => {
+    const tokenizer = createTokenizerForProvider('openai');
+    t.true(tokenizer instanceof OpenAITokenizer);
+});
+//# sourceMappingURL=tokenizer-factory.spec.js.map

package/dist/tokenization/tokenizer-factory.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer-factory.spec.js","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EACN,eAAe,EACf,0BAA0B,GAC1B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,kBAAkB,EAAC,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AACrE,OAAO,EAAC,cAAc,EAAC,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAC,eAAe,EAAC,MAAM,kCAAkC,CAAC;AAEjE,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;AAE3C,sDAAsD;AACtD,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,qBAAqB,CAAC,CAAC;AAClD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oEAAoE,EAAE,CAAC,CAAC,EAAE;IAC9E,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,yDAAyD;AACzD,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,CAAC,CAAC,EAAE;IACpF,MAAM,SAAS,GAAG,eAAe,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,qDAAqD;AACrD,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,qCAAqC;AACrC,IAAI,CAAC,gEAAgE,EAAE,CAAC,CAAC,EAAE;IAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,8BAA8B;AAC9B,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,0BAA0B;AAC1B,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,kCAAkC;AAClC,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,0BAA0B,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACtE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACjE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+DAA+D,EAAE,CAAC,CAAC,EAAE;IACzE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Anthropic tokenizer for Claude models
+ * Uses @anthropic-ai/tokenizer package
+ */
+import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
+/**
+ * Anthropic tokenizer for Claude models
+ */
+export declare class AnthropicTokenizer implements Tokenizer {
+    private modelName;
+    constructor(modelId?: string);
+    encode(text: string): number;
+    countTokens(message: Message): number;
+    getName(): string;
+}
+//# sourceMappingURL=anthropic-tokenizer.d.ts.map

package/dist/tokenization/tokenizers/anthropic-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;GAEG;AACH,qBAAa,kBAAmB,YAAW,SAAS;IACnD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.js

@@ -0,0 +1,35 @@
+/**
+ * Anthropic tokenizer for Claude models
+ * Uses @anthropic-ai/tokenizer package
+ */
+import { countTokens as anthropicCountTokens } from '@anthropic-ai/tokenizer';
+/**
+ * Anthropic tokenizer for Claude models
+ */
+export class AnthropicTokenizer {
+    modelName;
+    constructor(modelId) {
+        this.modelName = modelId || 'claude-3';
+    }
+    encode(text) {
+        try {
+            return anthropicCountTokens(text);
+        }
+        catch {
+            // Fallback to character-based estimation if tokenization fails
+            return Math.ceil(text.length / 4);
+        }
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // Anthropic format includes role in the message structure
+        // Approximate overhead for message formatting
+        const messageOverhead = 3;
+        return this.encode(content) + this.encode(role) + messageOverhead;
+    }
+    getName() {
+        return `anthropic-${this.modelName}`;
+    }
+}
+//# sourceMappingURL=anthropic-tokenizer.js.map

package/dist/tokenization/tokenizers/anthropic-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAC,WAAW,IAAI,oBAAoB,EAAC,MAAM,yBAAyB,CAAC;AAG5E;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACtB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,UAAU,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0DAA0D;QAC1D,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC;IACtC,CAAC;CACD"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for anthropic-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=anthropic-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/anthropic-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.spec.js

@@ -0,0 +1,152 @@
+/**
+ * Tests for anthropic-tokenizer.ts
+ */
+import test from 'ava';
+import { AnthropicTokenizer } from './anthropic-tokenizer.js';
+console.log(`\nantrhopic-tokenizer.spec.ts`);
+test('AnthropicTokenizer encodes simple text', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const count = tokenizer.encode('Hello, world!');
+    // Should return a positive token count
+    t.true(count > 0);
+    t.true(count < 10);
+});
+test('AnthropicTokenizer encodes empty string', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('AnthropicTokenizer encodes longer text', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // Should have significantly more tokens
+    t.true(count > 10);
+    t.true(count < 50);
+});
+test('AnthropicTokenizer defaults to claude-3 when no model specified', t => {
+    const tokenizer = new AnthropicTokenizer();
+    t.is(tokenizer.getName(), 'anthropic-claude-3');
+});
+test('AnthropicTokenizer getName returns correct format', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
+    t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
+});
+test('AnthropicTokenizer countTokens for user message', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should include content tokens + role tokens + overhead
+    t.true(count > 5);
+    t.true(count < 20);
+});
+test('AnthropicTokenizer countTokens for assistant message', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('AnthropicTokenizer countTokens for system message', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    t.true(count > 5);
+});
+test('AnthropicTokenizer countTokens handles empty content', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should still have overhead for role and message structure
+    t.true(count >= 3);
+});
+test('AnthropicTokenizer countTokens handles missing content', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully
+    t.true(count >= 0);
+});
+test('AnthropicTokenizer countTokens includes message overhead', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const shortMessage = {
+        role: 'user',
+        content: 'Hi',
+    };
+    const count = tokenizer.countTokens(shortMessage);
+    const contentOnly = tokenizer.encode('Hi');
+    const roleOnly = tokenizer.encode('user');
+    // Total should be more than just content + role due to overhead
+    t.true(count > contentOnly + roleOnly);
+});
+test('AnthropicTokenizer handles special characters', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    t.true(count > 0);
+});
+test('AnthropicTokenizer handles code snippets', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 10);
+});
+test('AnthropicTokenizer works with claude-3-sonnet model', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
+});
+test('AnthropicTokenizer works with claude-3-haiku model', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-haiku');
+    const count = tokenizer.encode('Hello, world!');
+    t.true(count > 0);
+    t.is(tokenizer.getName(), 'anthropic-claude-3-haiku');
+});
+test('AnthropicTokenizer handles long messages', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+});
+test('AnthropicTokenizer uses fallback on encoding error', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    // The fallback should kick in for any edge cases
+    // Testing with normal text should still work
+    const count = tokenizer.encode('Normal text');
+    t.true(count > 0);
+});
+test('AnthropicTokenizer countTokens with tool message', t => {
+    const tokenizer = new AnthropicTokenizer('claude-3-opus');
+    const message = {
+        role: 'tool',
+        content: 'Tool result here',
+        tool_call_id: '123',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle tool messages
+    t.true(count > 0);
+});
+//# sourceMappingURL=anthropic-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/anthropic-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAE5D,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AAE7C,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iEAAiE,EAAE,CAAC,CAAC,EAAE;IAC3E,MAAM,SAAS,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAE3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAE5D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAE1D,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/fallback-tokenizer.d.ts

@@ -0,0 +1,13 @@
+import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
+/**
+ * Fallback tokenizer for unsupported models
+ * Uses a simple character-based estimation (4 chars per token)
+ */
+export declare class FallbackTokenizer implements Tokenizer {
+    private readonly CHARS_PER_TOKEN;
+    encode(text: string): number;
+    countTokens(message: Message): number;
+    getName(): string;
+}
+//# sourceMappingURL=fallback-tokenizer.d.ts.map

package/dist/tokenization/tokenizers/fallback-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fallback-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAC1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,SAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAK;IAErC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/fallback-tokenizer.js

@@ -0,0 +1,20 @@
+/**
+ * Fallback tokenizer for unsupported models
+ * Uses a simple character-based estimation (4 chars per token)
+ */
+export class FallbackTokenizer {
+    CHARS_PER_TOKEN = 4;
+    encode(text) {
+        return Math.ceil(text.length / this.CHARS_PER_TOKEN);
+    }
+    countTokens(message) {
+        const content = message.content || '';
+        const role = message.role || '';
+        // Count tokens for content + a small overhead for role and formatting
+        return this.encode(content) + Math.ceil(role.length / this.CHARS_PER_TOKEN);
+    }
+    getName() {
+        return 'fallback';
+    }
+}
+//# sourceMappingURL=fallback-tokenizer.js.map

package/dist/tokenization/tokenizers/fallback-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fallback-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACZ,eAAe,GAAG,CAAC,CAAC;IAErC,MAAM,CAAC,IAAY;QAClB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;IACtD,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,sEAAsE;QACtE,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,CAAC;IAC7E,CAAC;IAED,OAAO;QACN,OAAO,UAAU,CAAC;IACnB,CAAC;CACD"}

package/dist/tokenization/tokenizers/fallback-tokenizer.spec.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for fallback-tokenizer.ts
+ */
+export {};
+//# sourceMappingURL=fallback-tokenizer.spec.d.ts.map

package/dist/tokenization/tokenizers/fallback-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fallback-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/tokenization/tokenizers/fallback-tokenizer.spec.js

@@ -0,0 +1,183 @@
+/**
+ * Tests for fallback-tokenizer.ts
+ */
+import test from 'ava';
+import { FallbackTokenizer } from './fallback-tokenizer.js';
+console.log(`\nfallback-tokenizer.spec.ts`);
+test('FallbackTokenizer encodes simple text', t => {
+    const tokenizer = new FallbackTokenizer();
+    const count = tokenizer.encode('Hello, world!');
+    // "Hello, world!" is 13 characters, so ~4 tokens (13/4 = 3.25, rounded up to 4)
+    t.is(count, 4);
+});
+test('FallbackTokenizer encodes empty string', t => {
+    const tokenizer = new FallbackTokenizer();
+    const count = tokenizer.encode('');
+    t.is(count, 0);
+});
+test('FallbackTokenizer encodes longer text', t => {
+    const tokenizer = new FallbackTokenizer();
+    const text = 'This is a longer piece of text that should have more tokens than a simple hello world.';
+    const count = tokenizer.encode(text);
+    // 88 characters / 4 = 22 tokens
+    t.is(count, 22);
+});
+test('FallbackTokenizer uses 4 chars per token ratio', t => {
+    const tokenizer = new FallbackTokenizer();
+    // Test exact multiples of 4
+    t.is(tokenizer.encode('1234'), 1);
+    t.is(tokenizer.encode('12345678'), 2);
+    t.is(tokenizer.encode('123456789012'), 3);
+});
+test('FallbackTokenizer rounds up partial tokens', t => {
+    const tokenizer = new FallbackTokenizer();
+    // 5 characters should round up to 2 tokens
+    t.is(tokenizer.encode('12345'), 2);
+    // 9 characters should round up to 3 tokens
+    t.is(tokenizer.encode('123456789'), 3);
+});
+test('FallbackTokenizer getName returns correct name', t => {
+    const tokenizer = new FallbackTokenizer();
+    t.is(tokenizer.getName(), 'fallback');
+});
+test('FallbackTokenizer countTokens for user message', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'user',
+        content: 'Hello, how are you?',
+    };
+    const count = tokenizer.countTokens(message);
+    // "Hello, how are you?" is 19 chars = 5 tokens
+    // "user" is 4 chars = 1 token
+    // Total = 6 tokens
+    t.is(count, 6);
+});
+test('FallbackTokenizer countTokens for assistant message', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'assistant',
+        content: 'I am doing well, thank you!',
+    };
+    const count = tokenizer.countTokens(message);
+    // "I am doing well, thank you!" is 27 chars = 7 tokens
+    // "assistant" is 9 chars = 3 tokens
+    // Total = 10 tokens
+    t.is(count, 10);
+});
+test('FallbackTokenizer countTokens for system message', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'system',
+        content: 'You are a helpful assistant.',
+    };
+    const count = tokenizer.countTokens(message);
+    // "You are a helpful assistant." is 28 chars = 7 tokens
+    // "system" is 6 chars = 2 tokens
+    // Total = 9 tokens
+    t.is(count, 9);
+});
+test('FallbackTokenizer countTokens handles empty content', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'user',
+        content: '',
+    };
+    const count = tokenizer.countTokens(message);
+    // Empty content = 0 tokens
+    // "user" is 4 chars = 1 token
+    // Total = 1 token
+    t.is(count, 1);
+});
+test('FallbackTokenizer countTokens handles missing content', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'user',
+    };
+    const count = tokenizer.countTokens(message);
+    // Missing content treated as empty string = 0 tokens
+    // "user" is 4 chars = 1 token
+    // Total = 1 token
+    t.is(count, 1);
+});
+test('FallbackTokenizer handles special characters', t => {
+    const tokenizer = new FallbackTokenizer();
+    const text = '你好世界 🌍 Привет мир';
+    const count = tokenizer.encode(text);
+    // Should handle all characters equally (character count / 4)
+    t.true(count > 0);
+    t.is(count, Math.ceil(text.length / 4));
+});
+test('FallbackTokenizer handles code snippets', t => {
+    const tokenizer = new FallbackTokenizer();
+    const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+    const count = tokenizer.encode(code);
+    t.true(count > 0);
+    t.is(count, Math.ceil(code.length / 4));
+});
+test('FallbackTokenizer handles long messages', t => {
+    const tokenizer = new FallbackTokenizer();
+    const longText = 'Hello '.repeat(1000);
+    const message = {
+        role: 'user',
+        content: longText,
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle long text without crashing
+    t.true(count > 1000);
+    const expectedContentTokens = Math.ceil(longText.length / 4);
+    const expectedRoleTokens = Math.ceil('user'.length / 4);
+    t.is(count, expectedContentTokens + expectedRoleTokens);
+});
+test('FallbackTokenizer countTokens with tool message', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        role: 'tool',
+        content: 'Tool result here',
+        tool_call_id: '123',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle tool messages
+    // "Tool result here" is 16 chars = 4 tokens
+    // "tool" is 4 chars = 1 token
+    // Total = 5 tokens
+    t.is(count, 5);
+});
+test('FallbackTokenizer handles single character', t => {
+    const tokenizer = new FallbackTokenizer();
+    const count = tokenizer.encode('a');
+    // 1 character should round up to 1 token
+    t.is(count, 1);
+});
+test('FallbackTokenizer handles whitespace', t => {
+    const tokenizer = new FallbackTokenizer();
+    const count = tokenizer.encode('    ');
+    // 4 spaces = 1 token
+    t.is(count, 1);
+});
+test('FallbackTokenizer handles newlines', t => {
+    const tokenizer = new FallbackTokenizer();
+    const count = tokenizer.encode('\n\n\n\n');
+    // 4 newlines = 1 token
+    t.is(count, 1);
+});
+test('FallbackTokenizer is consistent', t => {
+    const tokenizer = new FallbackTokenizer();
+    const text = 'The quick brown fox jumps over the lazy dog';
+    const count1 = tokenizer.encode(text);
+    const count2 = tokenizer.encode(text);
+    t.is(count1, count2);
+});
+test('FallbackTokenizer handles messages with missing role', t => {
+    const tokenizer = new FallbackTokenizer();
+    const message = {
+        content: 'Hello world',
+    };
+    const count = tokenizer.countTokens(message);
+    // Should handle gracefully, role treated as empty string
+    t.true(count > 0);
+});
+//# sourceMappingURL=fallback-tokenizer.spec.js.map

package/dist/tokenization/tokenizers/fallback-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fallback-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAE5C,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,gFAAgF;IAChF,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,gCAAgC;IAChC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,4BAA4B;IAC5B,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnC,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,+CAA+C;IAC/C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,uDAAuD;IACvD,oCAAoC;IACpC,oBAAoB;IACpB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,wDAAwD;IACxD,iCAAiC;IACjC,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,qDAAqD;IACrD,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,6DAA6D;IAC7D,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IACrB,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,GAAG,kBAAkB,CAAC,CAAC;AACzD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,4CAA4C;IAC5C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAEpC,yCAAyC;IACzC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAEvC,qBAAqB;IACrB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAE3C,uBAAuB;IACvB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE;IAC3C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,6CAA6C,CAAC;IAE3D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEtC,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,OAAO,EAAE,aAAa;KACX,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}