@gguf/coder 0.3.1 → 0.3.3

package/source/tokenization/tokenizers/llama-tokenizer.spec.ts

@@ -1,224 +0,0 @@
-/**
- * Tests for llama-tokenizer.ts
- */
-
-import type {Message} from '@/types/core.js';
-import test from 'ava';
-import {LlamaTokenizer} from './llama-tokenizer.js';
-
-console.log(`\nllama-tokenizer.spec.ts`);
-
-test('LlamaTokenizer encodes simple text', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const count = tokenizer.encode('Hello, world!');
-
-	// Should return a positive token count
-	t.true(count > 0);
-	t.true(count < 10);
-});
-
-test('LlamaTokenizer encodes empty string', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const count = tokenizer.encode('');
-
-	t.is(count, 0);
-});
-
-test('LlamaTokenizer encodes longer text', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const text =
-		'This is a longer piece of text that should have more tokens than a simple hello world.';
-	const count = tokenizer.encode(text);
-
-	// Should have significantly more tokens
-	t.true(count > 10);
-	t.true(count < 50);
-});
-
-test('LlamaTokenizer defaults to llama when no model specified', t => {
-	const tokenizer = new LlamaTokenizer();
-
-	t.is(tokenizer.getName(), 'llama-llama');
-});
-
-test('LlamaTokenizer getName returns correct format', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-70b');
-
-	t.is(tokenizer.getName(), 'llama-llama-3-70b');
-});
-
-test('LlamaTokenizer countTokens for user message', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'user',
-		content: 'Hello, how are you?',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should include content tokens + role tokens + overhead
-	t.true(count > 5);
-	t.true(count < 25);
-});
-
-test('LlamaTokenizer countTokens for assistant message', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'assistant',
-		content: 'I am doing well, thank you!',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('LlamaTokenizer countTokens for system message', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'system',
-		content: 'You are a helpful assistant.',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('LlamaTokenizer countTokens handles empty content', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'user',
-		content: '',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should still have overhead for role and message structure
-	t.true(count >= 6);
-});
-
-test('LlamaTokenizer countTokens handles missing content', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'user',
-	} as Message;
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle gracefully
-	t.true(count >= 0);
-});
-
-test('LlamaTokenizer countTokens includes message overhead', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const shortMessage: Message = {
-		role: 'user',
-		content: 'Hi',
-	};
-
-	const count = tokenizer.countTokens(shortMessage);
-	const contentOnly = tokenizer.encode('Hi');
-	const roleOnly = tokenizer.encode('user');
-
-	// Total should be more than just content + role due to overhead
-	t.true(count > contentOnly + roleOnly);
-});
-
-test('LlamaTokenizer handles special characters', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const text = '你好世界 🌍 Привет мир';
-	const count = tokenizer.encode(text);
-
-	t.true(count > 0);
-});
-
-test('LlamaTokenizer handles code snippets', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-	const count = tokenizer.encode(code);
-
-	t.true(count > 10);
-});
-
-test('LlamaTokenizer works with mistral model', t => {
-	const tokenizer = new LlamaTokenizer('mistral-7b');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'llama-mistral-7b');
-});
-
-test('LlamaTokenizer works with qwen model', t => {
-	const tokenizer = new LlamaTokenizer('qwen-2.5');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'llama-qwen-2.5');
-});
-
-test('LlamaTokenizer works with codellama model', t => {
-	const tokenizer = new LlamaTokenizer('codellama-7b');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'llama-codellama-7b');
-});
-
-test('LlamaTokenizer handles long messages', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const longText = 'Hello '.repeat(1000);
-	const message: Message = {
-		role: 'user',
-		content: longText,
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle long text without crashing
-	t.true(count > 1000);
-});
-
-test('LlamaTokenizer uses fallback on encoding error', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-
-	// The fallback should kick in for any edge cases
-	// Testing with normal text should still work
-	const count = tokenizer.encode('Normal text');
-
-	t.true(count > 0);
-});
-
-test('LlamaTokenizer countTokens with tool message', t => {
-	const tokenizer = new LlamaTokenizer('llama-3-8b');
-	const message: Message = {
-		role: 'tool',
-		content: 'Tool result here',
-		tool_call_id: '123',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle tool messages
-	t.true(count > 0);
-});
-
-test('LlamaTokenizer handles deepseek model', t => {
-	const tokenizer = new LlamaTokenizer('deepseek-coder-33b');
-	const count = tokenizer.encode('const x = 42;');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'llama-deepseek-coder-33b');
-});
-
-test('LlamaTokenizer handles mixtral model', t => {
-	const tokenizer = new LlamaTokenizer('mixtral-8x7b');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'llama-mixtral-8x7b');
-});

package/source/tokenization/tokenizers/llama-tokenizer.ts

@@ -1,41 +0,0 @@
-/**
- * Llama tokenizer for local models
- * Uses llama-tokenizer-js package
- */
-
-import type {Message} from '@/types/core';
-import llamaTokenizer from 'llama-tokenizer-js';
-import type {Tokenizer} from '../../types/tokenization';
-
-export class LlamaTokenizer implements Tokenizer {
-	private modelName: string;
-
-	constructor(modelId?: string) {
-		this.modelName = modelId || 'llama';
-	}
-
-	encode(text: string): number {
-		try {
-			const tokens = llamaTokenizer.encode(text);
-			return tokens.length;
-		} catch {
-			// Fallback to character-based estimation if tokenization fails
-			return Math.ceil(text.length / 4);
-		}
-	}
-
-	countTokens(message: Message): number {
-		const content = message.content || '';
-		const role = message.role || '';
-
-		// Llama format: <|start_header_id|>role<|end_header_id|>content<|eot_id|>
-		// Approximate overhead for message formatting
-		const messageOverhead = 6;
-
-		return this.encode(content) + this.encode(role) + messageOverhead;
-	}
-
-	getName(): string {
-		return `llama-${this.modelName}`;
-	}
-}

package/source/tokenization/tokenizers/openai-tokenizer.spec.ts

@@ -1,184 +0,0 @@
-/**
- * Tests for openai-tokenizer.ts
- */
-
-import type {Message} from '@/types/core.js';
-import test from 'ava';
-import {OpenAITokenizer} from './openai-tokenizer.js';
-
-console.log(`\nopenai-tokenizer.spec.ts`);
-
-test('OpenAITokenizer encodes simple text', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const count = tokenizer.encode('Hello, world!');
-
-	// "Hello, world!" should tokenize to around 4 tokens
-	t.true(count > 0);
-	t.true(count < 10);
-});
-
-test('OpenAITokenizer encodes empty string', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const count = tokenizer.encode('');
-
-	t.is(count, 0);
-});
-
-test('OpenAITokenizer encodes longer text', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const text =
-		'This is a longer piece of text that should have more tokens than a simple hello world.';
-	const count = tokenizer.encode(text);
-
-	// Should have significantly more tokens
-	t.true(count > 10);
-	t.true(count < 50);
-});
-
-test('OpenAITokenizer uses fallback encoding for unsupported model', t => {
-	const tokenizer = new OpenAITokenizer('unknown-model-xyz');
-	const count = tokenizer.encode('Hello, world!');
-
-	// Should still return a count using fallback
-	t.true(count > 0);
-});
-
-test('OpenAITokenizer defaults to gpt-4 when no model specified', t => {
-	const tokenizer = new OpenAITokenizer();
-
-	t.is(tokenizer.getName(), 'openai-gpt-4');
-});
-
-test('OpenAITokenizer getName returns correct format', t => {
-	const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
-
-	t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
-});
-
-test('OpenAITokenizer countTokens for user message', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const message: Message = {
-		role: 'user',
-		content: 'Hello, how are you?',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should include content tokens + role tokens + overhead
-	t.true(count > 5);
-	t.true(count < 20);
-});
-
-test('OpenAITokenizer countTokens for assistant message', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const message: Message = {
-		role: 'assistant',
-		content: 'I am doing well, thank you!',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('OpenAITokenizer countTokens for system message', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const message: Message = {
-		role: 'system',
-		content: 'You are a helpful assistant.',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	t.true(count > 5);
-});
-
-test('OpenAITokenizer countTokens handles empty content', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const message: Message = {
-		role: 'user',
-		content: '',
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should still have overhead for role and message structure
-	t.true(count >= 4);
-});
-
-test('OpenAITokenizer countTokens handles missing content', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const message: Message = {
-		role: 'user',
-	} as Message;
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle gracefully
-	t.true(count >= 0);
-});
-
-test('OpenAITokenizer countTokens includes message overhead', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const shortMessage: Message = {
-		role: 'user',
-		content: 'Hi',
-	};
-
-	const count = tokenizer.countTokens(shortMessage);
-	const contentOnly = tokenizer.encode('Hi');
-	const roleOnly = tokenizer.encode('user');
-
-	// Total should be more than just content + role due to overhead
-	t.true(count > contentOnly + roleOnly);
-});
-
-test('OpenAITokenizer free method exists', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-
-	t.notThrows(() => {
-		tokenizer.free();
-	});
-});
-
-test('OpenAITokenizer handles special characters', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const text = '你好世界 🌍 Привет мир';
-	const count = tokenizer.encode(text);
-
-	t.true(count > 0);
-});
-
-test('OpenAITokenizer handles code snippets', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const code = `
-		function hello() {
-			console.log("Hello, world!");
-		}
-	`;
-	const count = tokenizer.encode(code);
-
-	t.true(count > 10);
-});
-
-test('OpenAITokenizer works with gpt-3.5-turbo model', t => {
-	const tokenizer = new OpenAITokenizer('gpt-3.5-turbo');
-	const count = tokenizer.encode('Hello, world!');
-
-	t.true(count > 0);
-	t.is(tokenizer.getName(), 'openai-gpt-3.5-turbo');
-});
-
-test('OpenAITokenizer handles long messages', t => {
-	const tokenizer = new OpenAITokenizer('gpt-4');
-	const longText = 'Hello '.repeat(1000);
-	const message: Message = {
-		role: 'user',
-		content: longText,
-	};
-
-	const count = tokenizer.countTokens(message);
-
-	// Should handle long text without crashing
-	t.true(count > 1000);
-});

package/source/tokenization/tokenizers/openai-tokenizer.ts

@@ -1,57 +0,0 @@
-/**
- * OpenAI tokenizer using tiktoken
- * Supports GPT-3.5, GPT-4, and other OpenAI models
- */
-
-import type {Message} from '@/types/core';
-import {type TiktokenModel, encoding_for_model, get_encoding} from 'tiktoken';
-import type {Tokenizer} from '../../types/tokenization';
-
-/**
- * OpenAI tokenizer using tiktoken for accurate token counting
- */
-export class OpenAITokenizer implements Tokenizer {
-	private encoding: ReturnType<typeof get_encoding>;
-	private modelName: string;
-
-	constructor(modelId?: string) {
-		this.modelName = modelId || 'gpt-4';
-
-		try {
-			this.encoding = encoding_for_model(modelId as TiktokenModel);
-		} catch {
-			this.encoding = get_encoding('cl100k_base');
-		}
-	}
-
-	encode(text: string): number {
-		try {
-			const tokens = this.encoding.encode(text);
-			return tokens.length;
-		} catch {
-			return Math.ceil(text.length / 4);
-		}
-	}
-
-	countTokens(message: Message): number {
-		const content = message.content || '';
-		const role = message.role || '';
-
-		// OpenAI format: each message has overhead for role markers
-		// <|im_start|>role\ncontent<|im_end|>
-		const messageOverhead = 4; // Approximate overhead per message
-
-		return this.encode(content) + this.encode(role) + messageOverhead;
-	}
-
-	getName(): string {
-		return `openai-${this.modelName}`;
-	}
-
-	/**
-	 * Clean up encoding resources
-	 */
-	free(): void {
-		this.encoding.free();
-	}
-}

package/source/tool-calling/index.ts

@@ -1,5 +0,0 @@
-/**
- * Tool calling utilities - main exports
- */
-
-export {parseToolCalls} from '@/tool-calling/tool-parser';