@gguf/coder 0.3.0 → 0.3.1

package/source/tokenization/tokenizers/anthropic-tokenizer.spec.ts

@@ -0,0 +1,200 @@
+/**
+ * Tests for anthropic-tokenizer.ts
+ */
+
+import type {Message} from '@/types/core.js';
+import test from 'ava';
+import {AnthropicTokenizer} from './anthropic-tokenizer.js';
+
+console.log(`\nantrhopic-tokenizer.spec.ts`);
+
+test('AnthropicTokenizer encodes simple text', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const count = tokenizer.encode('Hello, world!');
+
+	// Should return a positive token count
+	t.true(count > 0);
+	t.true(count < 10);
+});
+
+test('AnthropicTokenizer encodes empty string', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const count = tokenizer.encode('');
+
+	t.is(count, 0);
+});
+
+test('AnthropicTokenizer encodes longer text', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const text =
+		'This is a longer piece of text that should have more tokens than a simple hello world.';
+	const count = tokenizer.encode(text);
+
+	// Should have significantly more tokens
+	t.true(count > 10);
+	t.true(count < 50);
+});
+
+test('AnthropicTokenizer defaults to claude-3 when no model specified', t => {
+	const tokenizer = new AnthropicTokenizer();
+
+	t.is(tokenizer.getName(), 'anthropic-claude-3');
+});
+
+test('AnthropicTokenizer getName returns correct format', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
+
+	t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
+});
+
+test('AnthropicTokenizer countTokens for user message', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'user',
+		content: 'Hello, how are you?',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should include content tokens + role tokens + overhead
+	t.true(count > 5);
+	t.true(count < 20);
+});
+
+test('AnthropicTokenizer countTokens for assistant message', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'assistant',
+		content: 'I am doing well, thank you!',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('AnthropicTokenizer countTokens for system message', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'system',
+		content: 'You are a helpful assistant.',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('AnthropicTokenizer countTokens handles empty content', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'user',
+		content: '',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should still have overhead for role and message structure
+	t.true(count >= 3);
+});
+
+test('AnthropicTokenizer countTokens handles missing content', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'user',
+	} as Message;
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle gracefully
+	t.true(count >= 0);
+});
+
+test('AnthropicTokenizer countTokens includes message overhead', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const shortMessage: Message = {
+		role: 'user',
+		content: 'Hi',
+	};
+
+	const count = tokenizer.countTokens(shortMessage);
+	const contentOnly = tokenizer.encode('Hi');
+	const roleOnly = tokenizer.encode('user');
+
+	// Total should be more than just content + role due to overhead
+	t.true(count > contentOnly + roleOnly);
+});
+
+test('AnthropicTokenizer handles special characters', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const text = '你好世界 🌍 Привет мир';
+	const count = tokenizer.encode(text);
+
+	t.true(count > 0);
+});
+
+test('AnthropicTokenizer handles code snippets', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+	const count = tokenizer.encode(code);
+
+	t.true(count > 10);
+});
+
+test('AnthropicTokenizer works with claude-3-sonnet model', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
+});
+
+test('AnthropicTokenizer works with claude-3-haiku model', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-haiku');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'anthropic-claude-3-haiku');
+});
+
+test('AnthropicTokenizer handles long messages', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const longText = 'Hello '.repeat(1000);
+	const message: Message = {
+		role: 'user',
+		content: longText,
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle long text without crashing
+	t.true(count > 1000);
+});
+
+test('AnthropicTokenizer uses fallback on encoding error', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+
+	// The fallback should kick in for any edge cases
+	// Testing with normal text should still work
+	const count = tokenizer.encode('Normal text');
+
+	t.true(count > 0);
+});
+
+test('AnthropicTokenizer countTokens with tool message', t => {
+	const tokenizer = new AnthropicTokenizer('claude-3-opus');
+	const message: Message = {
+		role: 'tool',
+		content: 'Tool result here',
+		tool_call_id: '123',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle tool messages
+	t.true(count > 0);
+});

package/source/tokenization/tokenizers/anthropic-tokenizer.ts

@@ -0,0 +1,43 @@
+/**
+ * Anthropic tokenizer for Claude models
+ * Uses @anthropic-ai/tokenizer package
+ */
+
+import type {Message} from '@/types/core';
+import {countTokens as anthropicCountTokens} from '@anthropic-ai/tokenizer';
+import type {Tokenizer} from '../../types/tokenization';
+
+/**
+ * Anthropic tokenizer for Claude models
+ */
+export class AnthropicTokenizer implements Tokenizer {
+	private modelName: string;
+
+	constructor(modelId?: string) {
+		this.modelName = modelId || 'claude-3';
+	}
+
+	encode(text: string): number {
+		try {
+			return anthropicCountTokens(text);
+		} catch {
+			// Fallback to character-based estimation if tokenization fails
+			return Math.ceil(text.length / 4);
+		}
+	}
+
+	countTokens(message: Message): number {
+		const content = message.content || '';
+		const role = message.role || '';
+
+		// Anthropic format includes role in the message structure
+		// Approximate overhead for message formatting
+		const messageOverhead = 3;
+
+		return this.encode(content) + this.encode(role) + messageOverhead;
+	}
+
+	getName(): string {
+		return `anthropic-${this.modelName}`;
+	}
+}

package/source/tokenization/tokenizers/fallback-tokenizer.spec.ts

@@ -0,0 +1,236 @@
+/**
+ * Tests for fallback-tokenizer.ts
+ */
+
+import type {Message} from '@/types/core.js';
+import test from 'ava';
+import {FallbackTokenizer} from './fallback-tokenizer.js';
+
+console.log(`\nfallback-tokenizer.spec.ts`);
+
+test('FallbackTokenizer encodes simple text', t => {
+	const tokenizer = new FallbackTokenizer();
+	const count = tokenizer.encode('Hello, world!');
+
+	// "Hello, world!" is 13 characters, so ~4 tokens (13/4 = 3.25, rounded up to 4)
+	t.is(count, 4);
+});
+
+test('FallbackTokenizer encodes empty string', t => {
+	const tokenizer = new FallbackTokenizer();
+	const count = tokenizer.encode('');
+
+	t.is(count, 0);
+});
+
+test('FallbackTokenizer encodes longer text', t => {
+	const tokenizer = new FallbackTokenizer();
+	const text =
+		'This is a longer piece of text that should have more tokens than a simple hello world.';
+	const count = tokenizer.encode(text);
+
+	// 88 characters / 4 = 22 tokens
+	t.is(count, 22);
+});
+
+test('FallbackTokenizer uses 4 chars per token ratio', t => {
+	const tokenizer = new FallbackTokenizer();
+
+	// Test exact multiples of 4
+	t.is(tokenizer.encode('1234'), 1);
+	t.is(tokenizer.encode('12345678'), 2);
+	t.is(tokenizer.encode('123456789012'), 3);
+});
+
+test('FallbackTokenizer rounds up partial tokens', t => {
+	const tokenizer = new FallbackTokenizer();
+
+	// 5 characters should round up to 2 tokens
+	t.is(tokenizer.encode('12345'), 2);
+
+	// 9 characters should round up to 3 tokens
+	t.is(tokenizer.encode('123456789'), 3);
+});
+
+test('FallbackTokenizer getName returns correct name', t => {
+	const tokenizer = new FallbackTokenizer();
+
+	t.is(tokenizer.getName(), 'fallback');
+});
+
+test('FallbackTokenizer countTokens for user message', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'user',
+		content: 'Hello, how are you?',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// "Hello, how are you?" is 19 chars = 5 tokens
+	// "user" is 4 chars = 1 token
+	// Total = 6 tokens
+	t.is(count, 6);
+});
+
+test('FallbackTokenizer countTokens for assistant message', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'assistant',
+		content: 'I am doing well, thank you!',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// "I am doing well, thank you!" is 27 chars = 7 tokens
+	// "assistant" is 9 chars = 3 tokens
+	// Total = 10 tokens
+	t.is(count, 10);
+});
+
+test('FallbackTokenizer countTokens for system message', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'system',
+		content: 'You are a helpful assistant.',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// "You are a helpful assistant." is 28 chars = 7 tokens
+	// "system" is 6 chars = 2 tokens
+	// Total = 9 tokens
+	t.is(count, 9);
+});
+
+test('FallbackTokenizer countTokens handles empty content', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'user',
+		content: '',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Empty content = 0 tokens
+	// "user" is 4 chars = 1 token
+	// Total = 1 token
+	t.is(count, 1);
+});
+
+test('FallbackTokenizer countTokens handles missing content', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'user',
+	} as Message;
+
+	const count = tokenizer.countTokens(message);
+
+	// Missing content treated as empty string = 0 tokens
+	// "user" is 4 chars = 1 token
+	// Total = 1 token
+	t.is(count, 1);
+});
+
+test('FallbackTokenizer handles special characters', t => {
+	const tokenizer = new FallbackTokenizer();
+	const text = '你好世界 🌍 Привет мир';
+	const count = tokenizer.encode(text);
+
+	// Should handle all characters equally (character count / 4)
+	t.true(count > 0);
+	t.is(count, Math.ceil(text.length / 4));
+});
+
+test('FallbackTokenizer handles code snippets', t => {
+	const tokenizer = new FallbackTokenizer();
+	const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+	const count = tokenizer.encode(code);
+
+	t.true(count > 0);
+	t.is(count, Math.ceil(code.length / 4));
+});
+
+test('FallbackTokenizer handles long messages', t => {
+	const tokenizer = new FallbackTokenizer();
+	const longText = 'Hello '.repeat(1000);
+	const message: Message = {
+		role: 'user',
+		content: longText,
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle long text without crashing
+	t.true(count > 1000);
+	const expectedContentTokens = Math.ceil(longText.length / 4);
+	const expectedRoleTokens = Math.ceil('user'.length / 4);
+	t.is(count, expectedContentTokens + expectedRoleTokens);
+});
+
+test('FallbackTokenizer countTokens with tool message', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		role: 'tool',
+		content: 'Tool result here',
+		tool_call_id: '123',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle tool messages
+	// "Tool result here" is 16 chars = 4 tokens
+	// "tool" is 4 chars = 1 token
+	// Total = 5 tokens
+	t.is(count, 5);
+});
+
+test('FallbackTokenizer handles single character', t => {
+	const tokenizer = new FallbackTokenizer();
+	const count = tokenizer.encode('a');
+
+	// 1 character should round up to 1 token
+	t.is(count, 1);
+});
+
+test('FallbackTokenizer handles whitespace', t => {
+	const tokenizer = new FallbackTokenizer();
+	const count = tokenizer.encode('    ');
+
+	// 4 spaces = 1 token
+	t.is(count, 1);
+});
+
+test('FallbackTokenizer handles newlines', t => {
+	const tokenizer = new FallbackTokenizer();
+	const count = tokenizer.encode('\n\n\n\n');
+
+	// 4 newlines = 1 token
+	t.is(count, 1);
+});
+
+test('FallbackTokenizer is consistent', t => {
+	const tokenizer = new FallbackTokenizer();
+	const text = 'The quick brown fox jumps over the lazy dog';
+
+	const count1 = tokenizer.encode(text);
+	const count2 = tokenizer.encode(text);
+
+	t.is(count1, count2);
+});
+
+test('FallbackTokenizer handles messages with missing role', t => {
+	const tokenizer = new FallbackTokenizer();
+	const message: Message = {
+		content: 'Hello world',
+	} as Message;
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle gracefully, role treated as empty string
+	t.true(count > 0);
+});

package/source/tokenization/tokenizers/fallback-tokenizer.ts

@@ -0,0 +1,26 @@
+import type {Message} from '@/types/core';
+import type {Tokenizer} from '../../types/tokenization';
+
+/**
+ * Fallback tokenizer for unsupported models
+ * Uses a simple character-based estimation (4 chars per token)
+ */
+export class FallbackTokenizer implements Tokenizer {
+	private readonly CHARS_PER_TOKEN = 4;
+
+	encode(text: string): number {
+		return Math.ceil(text.length / this.CHARS_PER_TOKEN);
+	}
+
+	countTokens(message: Message): number {
+		const content = message.content || '';
+		const role = message.role || '';
+
+		// Count tokens for content + a small overhead for role and formatting
+		return this.encode(content) + Math.ceil(role.length / this.CHARS_PER_TOKEN);
+	}
+
+	getName(): string {
+		return 'fallback';
+	}
+}