@gguf/coder 0.3.1 → 0.3.2

package/source/tokenization/tokenizers/anthropic-tokenizer.spec.ts

@@ -1,200 +0,0 @@
-/**
- * Tests for anthropic-tokenizer.ts
- */
-
-import type {Message} from '@/types/core.js';
-import test from 'ava';
-import {AnthropicTokenizer} from './anthropic-tokenizer.js';
-
-console.log(`\nantrhopic-tokenizer.spec.ts`);
-
-test('AnthropicTokenizer encodes simple text', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const count = tokenizer.encode('Hello, world!');
-
-	// Should return a positive token count
-	t.true(count > 0);
-	t.true(count < 10);
-});
-
-test('AnthropicTokenizer encodes empty string', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const count = tokenizer.encode('');
-
-	t.is(count, 0);
-});
-
-test('AnthropicTokenizer encodes longer text', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const text =
-		'This is a longer piece of text that should have more tokens than a simple hello world.';
-	const count = tokenizer.encode(text);
-
-	// Should have significantly more tokens
-	t.true(count > 10);
-	t.true(count < 50);
-});
-
-test('AnthropicTokenizer defaults to claude-3 when no model specified', t => {
-	const tokenizer = new AnthropicTokenizer();
-
-	t.is(tokenizer.getName(), 'anthropic-claude-3');
-});
-
-test('AnthropicTokenizer getName returns correct format', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
-
-	t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
-});
-
-test('AnthropicTokenizer countTokens for user message', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'user',
-		content: 'Hello, how are you?',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should include content tokens + role tokens + overhead
-	t.true(count > 5);
-	t.true(count < 20);
-});
-
-test('AnthropicTokenizer countTokens for assistant message', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'assistant',
-		content: 'I am doing well, thank you!',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('AnthropicTokenizer countTokens for system message', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'system',
-		content: 'You are a helpful assistant.',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('AnthropicTokenizer countTokens handles empty content', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'user',
-		content: '',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should still have overhead for role and message structure
-	t.true(count >= 3);
-});
-
-test('AnthropicTokenizer countTokens handles missing content', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'user',
-	} as Message;
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle gracefully
-	t.true(count >= 0);
-});
-
-test('AnthropicTokenizer countTokens includes message overhead', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const shortMessage: Message = {
-		role: 'user',
-		content: 'Hi',
-	};
-
-	const count = tokenizer.countTokens(shortMessage);
-	const contentOnly = tokenizer.encode('Hi');
-	const roleOnly = tokenizer.encode('user');
-
-	// Total should be more than just content + role due to overhead
-	t.true(count > contentOnly + roleOnly);
-});
-
-test('AnthropicTokenizer handles special characters', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const text = '你好世界 🌍 Привет мир';
-	const count = tokenizer.encode(text);
-
-	t.true(count > 0);
-});
-
-test('AnthropicTokenizer handles code snippets', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-	const count = tokenizer.encode(code);
-
-	t.true(count > 10);
-});
-
-test('AnthropicTokenizer works with claude-3-sonnet model', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-sonnet');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'anthropic-claude-3-sonnet');
-});
-
-test('AnthropicTokenizer works with claude-3-haiku model', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-haiku');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'anthropic-claude-3-haiku');
-});
-
-test('AnthropicTokenizer handles long messages', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const longText = 'Hello '.repeat(1000);
-	const message: Message = {
-		role: 'user',
-		content: longText,
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle long text without crashing
-	t.true(count > 1000);
-});
-
-test('AnthropicTokenizer uses fallback on encoding error', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-
-	// The fallback should kick in for any edge cases
-	// Testing with normal text should still work
-	const count = tokenizer.encode('Normal text');
-
-	t.true(count > 0);
-});
-
-test('AnthropicTokenizer countTokens with tool message', t => {
-	const tokenizer = new AnthropicTokenizer('claude-3-opus');
-	const message: Message = {
-		role: 'tool',
-		content: 'Tool result here',
-		tool_call_id: '123',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle tool messages
-	t.true(count > 0);
-});

package/source/tokenization/tokenizers/anthropic-tokenizer.ts

@@ -1,43 +0,0 @@
-/**
- * Anthropic tokenizer for Claude models
- * Uses @anthropic-ai/tokenizer package
- */
-
-import type {Message} from '@/types/core';
-import {countTokens as anthropicCountTokens} from '@anthropic-ai/tokenizer';
-import type {Tokenizer} from '../../types/tokenization';
-
-/**
- * Anthropic tokenizer for Claude models
- */
-export class AnthropicTokenizer implements Tokenizer {
-	private modelName: string;
-
-	constructor(modelId?: string) {
-		this.modelName = modelId || 'claude-3';
-	}
-
-	encode(text: string): number {
-		try {
-			return anthropicCountTokens(text);
-		} catch {
-			// Fallback to character-based estimation if tokenization fails
-			return Math.ceil(text.length / 4);
-		}
-	}
-
-	countTokens(message: Message): number {
-		const content = message.content || '';
-		const role = message.role || '';
-
-		// Anthropic format includes role in the message structure
-		// Approximate overhead for message formatting
-		const messageOverhead = 3;
-
-		return this.encode(content) + this.encode(role) + messageOverhead;
-	}
-
-	getName(): string {
-		return `anthropic-${this.modelName}`;
-	}
-}

package/source/tokenization/tokenizers/fallback-tokenizer.spec.ts

@@ -1,236 +0,0 @@
-/**
- * Tests for fallback-tokenizer.ts
- */
-
-import type {Message} from '@/types/core.js';
-import test from 'ava';
-import {FallbackTokenizer} from './fallback-tokenizer.js';
-
-console.log(`\nfallback-tokenizer.spec.ts`);
-
-test('FallbackTokenizer encodes simple text', t => {
-	const tokenizer = new FallbackTokenizer();
-	const count = tokenizer.encode('Hello, world!');
-
-	// "Hello, world!" is 13 characters, so ~4 tokens (13/4 = 3.25, rounded up to 4)
-	t.is(count, 4);
-});
-
-test('FallbackTokenizer encodes empty string', t => {
-	const tokenizer = new FallbackTokenizer();
-	const count = tokenizer.encode('');
-
-	t.is(count, 0);
-});
-
-test('FallbackTokenizer encodes longer text', t => {
-	const tokenizer = new FallbackTokenizer();
-	const text =
-		'This is a longer piece of text that should have more tokens than a simple hello world.';
-	const count = tokenizer.encode(text);
-
-	// 88 characters / 4 = 22 tokens
-	t.is(count, 22);
-});
-
-test('FallbackTokenizer uses 4 chars per token ratio', t => {
-	const tokenizer = new FallbackTokenizer();
-
-	// Test exact multiples of 4
-	t.is(tokenizer.encode('1234'), 1);
-	t.is(tokenizer.encode('12345678'), 2);
-	t.is(tokenizer.encode('123456789012'), 3);
-});
-
-test('FallbackTokenizer rounds up partial tokens', t => {
-	const tokenizer = new FallbackTokenizer();
-
-	// 5 characters should round up to 2 tokens
-	t.is(tokenizer.encode('12345'), 2);
-
-	// 9 characters should round up to 3 tokens
-	t.is(tokenizer.encode('123456789'), 3);
-});
-
-test('FallbackTokenizer getName returns correct name', t => {
-	const tokenizer = new FallbackTokenizer();
-
-	t.is(tokenizer.getName(), 'fallback');
-});
-
-test('FallbackTokenizer countTokens for user message', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'user',
-		content: 'Hello, how are you?',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// "Hello, how are you?" is 19 chars = 5 tokens
-	// "user" is 4 chars = 1 token
-	// Total = 6 tokens
-	t.is(count, 6);
-});
-
-test('FallbackTokenizer countTokens for assistant message', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'assistant',
-		content: 'I am doing well, thank you!',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// "I am doing well, thank you!" is 27 chars = 7 tokens
-	// "assistant" is 9 chars = 3 tokens
-	// Total = 10 tokens
-	t.is(count, 10);
-});
-
-test('FallbackTokenizer countTokens for system message', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'system',
-		content: 'You are a helpful assistant.',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// "You are a helpful assistant." is 28 chars = 7 tokens
-	// "system" is 6 chars = 2 tokens
-	// Total = 9 tokens
-	t.is(count, 9);
-});
-
-test('FallbackTokenizer countTokens handles empty content', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'user',
-		content: '',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Empty content = 0 tokens
-	// "user" is 4 chars = 1 token
-	// Total = 1 token
-	t.is(count, 1);
-});
-
-test('FallbackTokenizer countTokens handles missing content', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'user',
-	} as Message;
-
-	const count = tokenizer.countTokens(message);
-
-	// Missing content treated as empty string = 0 tokens
-	// "user" is 4 chars = 1 token
-	// Total = 1 token
-	t.is(count, 1);
-});
-
-test('FallbackTokenizer handles special characters', t => {
-	const tokenizer = new FallbackTokenizer();
-	const text = '你好世界 🌍 Привет мир';
-	const count = tokenizer.encode(text);
-
-	// Should handle all characters equally (character count / 4)
-	t.true(count > 0);
-	t.is(count, Math.ceil(text.length / 4));
-});
-
-test('FallbackTokenizer handles code snippets', t => {
-	const tokenizer = new FallbackTokenizer();
-	const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-	const count = tokenizer.encode(code);
-
-	t.true(count > 0);
-	t.is(count, Math.ceil(code.length / 4));
-});
-
-test('FallbackTokenizer handles long messages', t => {
-	const tokenizer = new FallbackTokenizer();
-	const longText = 'Hello '.repeat(1000);
-	const message: Message = {
-		role: 'user',
-		content: longText,
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle long text without crashing
-	t.true(count > 1000);
-	const expectedContentTokens = Math.ceil(longText.length / 4);
-	const expectedRoleTokens = Math.ceil('user'.length / 4);
-	t.is(count, expectedContentTokens + expectedRoleTokens);
-});
-
-test('FallbackTokenizer countTokens with tool message', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		role: 'tool',
-		content: 'Tool result here',
-		tool_call_id: '123',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle tool messages
-	// "Tool result here" is 16 chars = 4 tokens
-	// "tool" is 4 chars = 1 token
-	// Total = 5 tokens
-	t.is(count, 5);
-});
-
-test('FallbackTokenizer handles single character', t => {
-	const tokenizer = new FallbackTokenizer();
-	const count = tokenizer.encode('a');
-
-	// 1 character should round up to 1 token
-	t.is(count, 1);
-});
-
-test('FallbackTokenizer handles whitespace', t => {
-	const tokenizer = new FallbackTokenizer();
-	const count = tokenizer.encode('    ');
-
-	// 4 spaces = 1 token
-	t.is(count, 1);
-});
-
-test('FallbackTokenizer handles newlines', t => {
-	const tokenizer = new FallbackTokenizer();
-	const count = tokenizer.encode('\n\n\n\n');
-
-	// 4 newlines = 1 token
-	t.is(count, 1);
-});
-
-test('FallbackTokenizer is consistent', t => {
-	const tokenizer = new FallbackTokenizer();
-	const text = 'The quick brown fox jumps over the lazy dog';
-
-	const count1 = tokenizer.encode(text);
-	const count2 = tokenizer.encode(text);
-
-	t.is(count1, count2);
-});
-
-test('FallbackTokenizer handles messages with missing role', t => {
-	const tokenizer = new FallbackTokenizer();
-	const message: Message = {
-		content: 'Hello world',
-	} as Message;
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle gracefully, role treated as empty string
-	t.true(count > 0);
-});

package/source/tokenization/tokenizers/fallback-tokenizer.ts

@@ -1,26 +0,0 @@
-import type {Message} from '@/types/core';
-import type {Tokenizer} from '../../types/tokenization';
-
-/**
- * Fallback tokenizer for unsupported models
- * Uses a simple character-based estimation (4 chars per token)
- */
-export class FallbackTokenizer implements Tokenizer {
-	private readonly CHARS_PER_TOKEN = 4;
-
-	encode(text: string): number {
-		return Math.ceil(text.length / this.CHARS_PER_TOKEN);
-	}
-
-	countTokens(message: Message): number {
-		const content = message.content || '';
-		const role = message.role || '';
-
-		// Count tokens for content + a small overhead for role and formatting
-		return this.encode(content) + Math.ceil(role.length / this.CHARS_PER_TOKEN);
-	}
-
-	getName(): string {
-		return 'fallback';
-	}
-}