@gguf/coder 0.2.9 → 0.3.1

package/source/tokenization/tokenizers/llama-tokenizer.spec.ts

@@ -0,0 +1,224 @@
+/**
+ * Tests for llama-tokenizer.ts
+ */
+
+import type {Message} from '@/types/core.js';
+import test from 'ava';
+import {LlamaTokenizer} from './llama-tokenizer.js';
+
+console.log(`\nllama-tokenizer.spec.ts`);
+
+test('LlamaTokenizer encodes simple text', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const count = tokenizer.encode('Hello, world!');
+
+	// Should return a positive token count
+	t.true(count > 0);
+	t.true(count < 10);
+});
+
+test('LlamaTokenizer encodes empty string', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const count = tokenizer.encode('');
+
+	t.is(count, 0);
+});
+
+test('LlamaTokenizer encodes longer text', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const text =
+		'This is a longer piece of text that should have more tokens than a simple hello world.';
+	const count = tokenizer.encode(text);
+
+	// Should have significantly more tokens
+	t.true(count > 10);
+	t.true(count < 50);
+});
+
+test('LlamaTokenizer defaults to llama when no model specified', t => {
+	const tokenizer = new LlamaTokenizer();
+
+	t.is(tokenizer.getName(), 'llama-llama');
+});
+
+test('LlamaTokenizer getName returns correct format', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-70b');
+
+	t.is(tokenizer.getName(), 'llama-llama-3-70b');
+});
+
+test('LlamaTokenizer countTokens for user message', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'user',
+		content: 'Hello, how are you?',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should include content tokens + role tokens + overhead
+	t.true(count > 5);
+	t.true(count < 25);
+});
+
+test('LlamaTokenizer countTokens for assistant message', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'assistant',
+		content: 'I am doing well, thank you!',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('LlamaTokenizer countTokens for system message', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'system',
+		content: 'You are a helpful assistant.',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('LlamaTokenizer countTokens handles empty content', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'user',
+		content: '',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should still have overhead for role and message structure
+	t.true(count >= 6);
+});
+
+test('LlamaTokenizer countTokens handles missing content', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'user',
+	} as Message;
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle gracefully
+	t.true(count >= 0);
+});
+
+test('LlamaTokenizer countTokens includes message overhead', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const shortMessage: Message = {
+		role: 'user',
+		content: 'Hi',
+	};
+
+	const count = tokenizer.countTokens(shortMessage);
+	const contentOnly = tokenizer.encode('Hi');
+	const roleOnly = tokenizer.encode('user');
+
+	// Total should be more than just content + role due to overhead
+	t.true(count > contentOnly + roleOnly);
+});
+
+test('LlamaTokenizer handles special characters', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const text = '你好世界 🌍 Привет мир';
+	const count = tokenizer.encode(text);
+
+	t.true(count > 0);
+});
+
+test('LlamaTokenizer handles code snippets', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+	const count = tokenizer.encode(code);
+
+	t.true(count > 10);
+});
+
+test('LlamaTokenizer works with mistral model', t => {
+	const tokenizer = new LlamaTokenizer('mistral-7b');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'llama-mistral-7b');
+});
+
+test('LlamaTokenizer works with qwen model', t => {
+	const tokenizer = new LlamaTokenizer('qwen-2.5');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'llama-qwen-2.5');
+});
+
+test('LlamaTokenizer works with codellama model', t => {
+	const tokenizer = new LlamaTokenizer('codellama-7b');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'llama-codellama-7b');
+});
+
+test('LlamaTokenizer handles long messages', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const longText = 'Hello '.repeat(1000);
+	const message: Message = {
+		role: 'user',
+		content: longText,
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle long text without crashing
+	t.true(count > 1000);
+});
+
+test('LlamaTokenizer uses fallback on encoding error', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+
+	// The fallback should kick in for any edge cases
+	// Testing with normal text should still work
+	const count = tokenizer.encode('Normal text');
+
+	t.true(count > 0);
+});
+
+test('LlamaTokenizer countTokens with tool message', t => {
+	const tokenizer = new LlamaTokenizer('llama-3-8b');
+	const message: Message = {
+		role: 'tool',
+		content: 'Tool result here',
+		tool_call_id: '123',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle tool messages
+	t.true(count > 0);
+});
+
+test('LlamaTokenizer handles deepseek model', t => {
+	const tokenizer = new LlamaTokenizer('deepseek-coder-33b');
+	const count = tokenizer.encode('const x = 42;');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'llama-deepseek-coder-33b');
+});
+
+test('LlamaTokenizer handles mixtral model', t => {
+	const tokenizer = new LlamaTokenizer('mixtral-8x7b');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'llama-mixtral-8x7b');
+});

package/source/tokenization/tokenizers/llama-tokenizer.ts

@@ -0,0 +1,41 @@
+/**
+ * Llama tokenizer for local models
+ * Uses llama-tokenizer-js package
+ */
+
+import type {Message} from '@/types/core';
+import llamaTokenizer from 'llama-tokenizer-js';
+import type {Tokenizer} from '../../types/tokenization';
+
+export class LlamaTokenizer implements Tokenizer {
+	private modelName: string;
+
+	constructor(modelId?: string) {
+		this.modelName = modelId || 'llama';
+	}
+
+	encode(text: string): number {
+		try {
+			const tokens = llamaTokenizer.encode(text);
+			return tokens.length;
+		} catch {
+			// Fallback to character-based estimation if tokenization fails
+			return Math.ceil(text.length / 4);
+		}
+	}
+
+	countTokens(message: Message): number {
+		const content = message.content || '';
+		const role = message.role || '';
+
+		// Llama format: <|start_header_id|>role<|end_header_id|>content<|eot_id|>
+		// Approximate overhead for message formatting
+		const messageOverhead = 6;
+
+		return this.encode(content) + this.encode(role) + messageOverhead;
+	}
+
+	getName(): string {
+		return `llama-${this.modelName}`;
+	}
+}

package/source/tokenization/tokenizers/openai-tokenizer.spec.ts

@@ -0,0 +1,184 @@
+/**
+ * Tests for openai-tokenizer.ts
+ */
+
+import type {Message} from '@/types/core.js';
+import test from 'ava';
+import {OpenAITokenizer} from './openai-tokenizer.js';
+
+console.log(`\nopenai-tokenizer.spec.ts`);
+
+test('OpenAITokenizer encodes simple text', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const count = tokenizer.encode('Hello, world!');
+
+	// "Hello, world!" should tokenize to around 4 tokens
+	t.true(count > 0);
+	t.true(count < 10);
+});
+
+test('OpenAITokenizer encodes empty string', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const count = tokenizer.encode('');
+
+	t.is(count, 0);
+});
+
+test('OpenAITokenizer encodes longer text', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const text =
+		'This is a longer piece of text that should have more tokens than a simple hello world.';
+	const count = tokenizer.encode(text);
+
+	// Should have significantly more tokens
+	t.true(count > 10);
+	t.true(count < 50);
+});
+
+test('OpenAITokenizer uses fallback encoding for unsupported model', t => {
+	const tokenizer = new OpenAITokenizer('unknown-model-xyz');
+	const count = tokenizer.encode('Hello, world!');
+
+	// Should still return a count using fallback
+	t.true(count > 0);
+});
+
+test('OpenAITokenizer defaults to gpt-4 when no model specified', t => {
+	const tokenizer = new OpenAITokenizer();
+
+	t.is(tokenizer.getName(), 'openai-gpt-4');
+});
+
+test('OpenAITokenizer getName returns correct format', t => {
+	const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+
+	t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+
+test('OpenAITokenizer countTokens for user message', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const message: Message = {
+		role: 'user',
+		content: 'Hello, how are you?',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should include content tokens + role tokens + overhead
+	t.true(count > 5);
+	t.true(count < 20);
+});
+
+test('OpenAITokenizer countTokens for assistant message', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const message: Message = {
+		role: 'assistant',
+		content: 'I am doing well, thank you!',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('OpenAITokenizer countTokens for system message', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const message: Message = {
+		role: 'system',
+		content: 'You are a helpful assistant.',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	t.true(count > 5);
+});
+
+test('OpenAITokenizer countTokens handles empty content', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const message: Message = {
+		role: 'user',
+		content: '',
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should still have overhead for role and message structure
+	t.true(count >= 4);
+});
+
+test('OpenAITokenizer countTokens handles missing content', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const message: Message = {
+		role: 'user',
+	} as Message;
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle gracefully
+	t.true(count >= 0);
+});
+
+test('OpenAITokenizer countTokens includes message overhead', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const shortMessage: Message = {
+		role: 'user',
+		content: 'Hi',
+	};
+
+	const count = tokenizer.countTokens(shortMessage);
+	const contentOnly = tokenizer.encode('Hi');
+	const roleOnly = tokenizer.encode('user');
+
+	// Total should be more than just content + role due to overhead
+	t.true(count > contentOnly + roleOnly);
+});
+
+test('OpenAITokenizer free method exists', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+
+	t.notThrows(() => {
+		tokenizer.free();
+	});
+});
+
+test('OpenAITokenizer handles special characters', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const text = '你好世界 🌍 Привет мир';
+	const count = tokenizer.encode(text);
+
+	t.true(count > 0);
+});
+
+test('OpenAITokenizer handles code snippets', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const code = `
+		function hello() {
+			console.log("Hello, world!");
+		}
+	`;
+	const count = tokenizer.encode(code);
+
+	t.true(count > 10);
+});
+
+test('OpenAITokenizer works with gpt-3.5-turbo model', t => {
+	const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
+	const count = tokenizer.encode('Hello, world!');
+
+	t.true(count > 0);
+	t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
+});
+
+test('OpenAITokenizer handles long messages', t => {
+	const tokenizer = new OpenAITokenizer('gpt-4');
+	const longText = 'Hello '.repeat(1000);
+	const message: Message = {
+		role: 'user',
+		content: longText,
+	};
+
+	const count = tokenizer.countTokens(message);
+
+	// Should handle long text without crashing
+	t.true(count > 1000);
+});

package/source/tokenization/tokenizers/openai-tokenizer.ts

@@ -0,0 +1,57 @@
+/**
+ * OpenAI tokenizer using tiktoken
+ * Supports GPT-3.5, GPT-4, and other OpenAI models
+ */
+
+import type {Message} from '@/types/core';
+import {type TiktokenModel, encoding_for_model, get_encoding} from 'tiktoken';
+import type {Tokenizer} from '../../types/tokenization';
+
+/**
+ * OpenAI tokenizer using tiktoken for accurate token counting
+ */
+export class OpenAITokenizer implements Tokenizer {
+	private encoding: ReturnType<typeof get_encoding>;
+	private modelName: string;
+
+	constructor(modelId?: string) {
+		this.modelName = modelId || 'gpt-4';
+
+		try {
+			this.encoding = encoding_for_model(modelId as TiktokenModel);
+		} catch {
+			this.encoding = get_encoding('cl100k_base');
+		}
+	}
+
+	encode(text: string): number {
+		try {
+			const tokens = this.encoding.encode(text);
+			return tokens.length;
+		} catch {
+			return Math.ceil(text.length / 4);
+		}
+	}
+
+	countTokens(message: Message): number {
+		const content = message.content || '';
+		const role = message.role || '';
+
+		// OpenAI format: each message has overhead for role markers
+		// <|im_start|>role\ncontent<|im_end|>
+		const messageOverhead = 4; // Approximate overhead per message
+
+		return this.encode(content) + this.encode(role) + messageOverhead;
+	}
+
+	getName(): string {
+		return `openai-${this.modelName}`;
+	}
+
+	/**
+	 * Clean up encoding resources
+	 */
+	free(): void {
+		this.encoding.free();
+	}
+}

package/source/tool-calling/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Tool calling utilities - main exports
+ */
+
+export {parseToolCalls} from '@/tool-calling/tool-parser';