@gguf/coder 0.3.0 → 0.3.1

package/source/services/file-snapshot.ts

@@ -0,0 +1,220 @@
+import {execSync} from 'child_process';
+import {existsSync} from 'fs';
+import * as path from 'path';
+import {MAX_CHECKPOINT_FILES} from '@/constants';
+import {logWarning} from '@/utils/message-queue';
+import * as fs from 'fs/promises';
+
+/**
+ * Service for capturing and restoring file snapshots for checkpoints
+ */
+export class FileSnapshotService {
+	private readonly workspaceRoot: string;
+
+	constructor(workspaceRoot: string = process.cwd()) {
+		this.workspaceRoot = workspaceRoot;
+	}
+
+	/**
+	 * Capture the contents of specified files
+	 */
+	async captureFiles(filePaths: string[]): Promise<Map<string, string>> {
+		const snapshots = new Map<string, string>();
+
+		for (const filePath of filePaths) {
+			try {
+				const absolutePath = path.resolve(this.workspaceRoot, filePath); // nosemgrep
+				const content = await fs.readFile(absolutePath, 'utf-8');
+				const relativePath = path.relative(this.workspaceRoot, absolutePath);
+				snapshots.set(relativePath, content);
+			} catch (error) {
+				logWarning('Could not capture file', true, {
+					context: {
+						filePath,
+						error: error instanceof Error ? error.message : 'Unknown error',
+					},
+				});
+			}
+		}
+
+		return snapshots;
+	}
+
+	/**
+	 * Restore files from snapshots
+	 */
+	async restoreFiles(snapshots: Map<string, string>): Promise<void> {
+		const errors: string[] = [];
+
+		for (const [relativePath, content] of snapshots) {
+			try {
+				const absolutePath = path.resolve(this.workspaceRoot, relativePath);
+				const directory = path.dirname(absolutePath);
+
+				await fs.mkdir(directory, {recursive: true});
+				await fs.writeFile(absolutePath, content, 'utf-8');
+			} catch (error) {
+				errors.push(
+					`Failed to restore ${relativePath}: ${
+						error instanceof Error ? error.message : 'Unknown error'
+					}`,
+				);
+			}
+		}
+
+		if (errors.length > 0) {
+			throw new Error(`Failed to restore some files:\n${errors.join('\n')}`);
+		}
+	}
+
+	/**
+	 * Get list of modified files in the workspace
+	 * Uses git to detect modified files if available, otherwise returns empty array
+	 */
+	getModifiedFiles(): string[] {
+		try {
+			const modifiedOutput = execSync('git diff --name-only HEAD', {
+				cwd: this.workspaceRoot,
+				encoding: 'utf-8',
+				stdio: ['pipe', 'pipe', 'pipe'],
+			}).trim();
+
+			const untrackedOutput = execSync(
+				'git ls-files --others --exclude-standard',
+				{
+					cwd: this.workspaceRoot,
+					encoding: 'utf-8',
+					stdio: ['pipe', 'pipe', 'pipe'],
+				},
+			).trim();
+
+			const modifiedFiles = modifiedOutput
+				? modifiedOutput.split('\n').filter(Boolean)
+				: [];
+			const untrackedFiles = untrackedOutput
+				? untrackedOutput.split('\n').filter(Boolean)
+				: [];
+
+			const allFiles = [...new Set([...modifiedFiles, ...untrackedFiles])];
+
+			const filtered = allFiles.filter(file => {
+				const ignorePatterns = [
+					'node_modules/',
+					'dist/',
+					'build/',
+					'.git/',
+					'.coder/',
+					'coverage/',
+					'*.log',
+				];
+
+				return !ignorePatterns.some(pattern => {
+					if (pattern.endsWith('/')) {
+						return file.startsWith(pattern);
+					}
+					if (pattern.startsWith('*.')) {
+						return file.endsWith(pattern.slice(1));
+					}
+					return file.includes(pattern);
+				});
+			});
+
+			if (filtered.length > MAX_CHECKPOINT_FILES) {
+				logWarning(
+					'Too many modified files detected, limiting to maximum',
+					true,
+					{
+						context: {
+							fileCount: filtered.length,
+							maxFiles: MAX_CHECKPOINT_FILES,
+						},
+					},
+				);
+				return filtered.slice(0, MAX_CHECKPOINT_FILES);
+			}
+
+			return filtered;
+		} catch {
+			logWarning('Git not available for file tracking', true, {
+				context: {
+					workspaceRoot: this.workspaceRoot,
+				},
+			});
+			return [];
+		}
+	}
+
+	/**
+	 * Get the size of a file snapshot
+	 */
+	getSnapshotSize(snapshots: Map<string, string>): number {
+		let totalSize = 0;
+		for (const content of snapshots.values()) {
+			totalSize += Buffer.byteLength(content, 'utf-8');
+		}
+		return totalSize;
+	}
+
+	/**
+	 * Validate that all files in the snapshot can be written to their locations
+	 */
+	async validateRestorePath(
+		snapshots: Map<string, string>,
+	): Promise<{valid: boolean; errors: string[]}> {
+		const errors: string[] = [];
+
+		for (const relativePath of snapshots.keys()) {
+			const absolutePath = path.resolve(this.workspaceRoot, relativePath); // nosemgrep
+			const directory = path.dirname(absolutePath);
+
+			try {
+				let dirWritable = true;
+				try {
+					await fs.access(directory, fs.constants.W_OK);
+				} catch (_err) {
+					// Directory is not writable or does not exist, so try to create it
+					try {
+						await fs.mkdir(directory, {recursive: true});
+					} catch (mkdirError) {
+						dirWritable = false;
+						errors.push(
+							`Cannot create directory "${directory}": ${mkdirError instanceof Error ? mkdirError.message : 'Unknown error'}`,
+						);
+					}
+				}
+
+				// After attempting to create the directory, verify it's writable
+				if (dirWritable) {
+					try {
+						await fs.access(directory, fs.constants.W_OK);
+					} catch (accessError) {
+						dirWritable = false;
+						errors.push(
+							`Directory "${directory}" is not writable: ${accessError instanceof Error ? accessError.message : 'Unknown error'}`,
+						);
+					}
+				}
+
+				// If directory is not writable or was not successfully created, skip further checks for this file
+				if (!dirWritable) {
+					continue;
+				}
+
+				if (existsSync(absolutePath)) {
+					try {
+						await fs.access(absolutePath, fs.constants.W_OK);
+					} catch (fileError) {
+						errors.push(
+							`Cannot write to file "${absolutePath}": ${fileError instanceof Error ? fileError.message : 'Unknown error'}`,
+						);
+					}
+				}
+			} catch (error) {
+				errors.push(
+					`Cannot validate path for ${relativePath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
+				);
+			}
+		}
+		return {valid: errors.length === 0, errors};
+	}
+}

package/source/test-utils/render-with-theme.tsx

@@ -0,0 +1,48 @@
+import {ThemeContext} from '@/hooks/useTheme';
+import type {Colors, ThemePreset} from '@/types/ui';
+import {render} from 'ink-testing-library';
+import React from 'react';
+
+// Default test colors that match the structure used in the app
+const testColors: Colors = {
+	primary: 'blue',
+	secondary: 'gray',
+	white: 'white',
+	black: 'black',
+	info: 'cyan',
+	warning: 'yellow',
+	error: 'red',
+	success: 'green',
+	tool: 'magenta',
+	diffAdded: 'green',
+	diffRemoved: 'red',
+	diffAddedText: 'white',
+	diffRemovedText: 'white',
+};
+
+// Test theme context value
+const testThemeContext = {
+	currentTheme: 'tokyo-night' as ThemePreset,
+	colors: testColors,
+	setCurrentTheme: () => {},
+};
+
+/**
+ * Wrapper component that provides ThemeContext for tests
+ */
+function TestThemeProvider({children}: {children: React.ReactNode}) {
+	return (
+		<ThemeContext.Provider value={testThemeContext}>
+			{children}
+		</ThemeContext.Provider>
+	);
+}
+
+/**
+ * Render a component wrapped with ThemeContext for testing
+ */
+export function renderWithTheme(
+	element: React.ReactElement,
+): ReturnType<typeof render> {
+	return render(<TestThemeProvider>{element}</TestThemeProvider>);
+}

package/source/tokenization/index.ts

@@ -0,0 +1 @@
+export {createTokenizer} from './tokenizer-factory.js';

package/source/tokenization/tokenizer-factory.spec.ts

@@ -0,0 +1,170 @@
+/**
+ * Tests for tokenizer-factory.ts
+ */
+
+import test from 'ava';
+import {
+	createTokenizer,
+	createTokenizerForProvider,
+} from './tokenizer-factory.js';
+import {AnthropicTokenizer} from './tokenizers/anthropic-tokenizer.js';
+import {FallbackTokenizer} from './tokenizers/fallback-tokenizer.js';
+import {LlamaTokenizer} from './tokenizers/llama-tokenizer.js';
+import {OpenAITokenizer} from './tokenizers/openai-tokenizer.js';
+
+console.log(`\ntokenizer-factory.spec.ts`);
+
+// Test createTokenizer with OpenAI provider detection
+test('createTokenizer detects OpenAI from provider name', t => {
+	const tokenizer = createTokenizer('openai', 'custom-model');
+	t.true(tokenizer instanceof OpenAITokenizer);
+	t.is(tokenizer.getName(), 'openai-custom-model');
+});
+
+test('createTokenizer detects OpenAI from GPT model name', t => {
+	const tokenizer = createTokenizer('custom', 'gpt-4-turbo');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+test('createTokenizer detects OpenAI from model name with openai keyword', t => {
+	const tokenizer = createTokenizer('custom', 'openai-model');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+// Test createTokenizer with Anthropic provider detection
+test('createTokenizer detects Anthropic from provider name', t => {
+	const tokenizer = createTokenizer('anthropic', 'custom-model');
+	t.true(tokenizer instanceof AnthropicTokenizer);
+});
+
+test('createTokenizer detects Anthropic from provider name with claude keyword', t => {
+	const tokenizer = createTokenizer('claude-provider', 'model');
+	t.true(tokenizer instanceof AnthropicTokenizer);
+});
+
+test('createTokenizer detects Anthropic from claude model name', t => {
+	const tokenizer = createTokenizer('custom', 'claude-3-opus');
+	t.true(tokenizer instanceof AnthropicTokenizer);
+});
+
+// Test createTokenizer with Llama provider detection
+test('createTokenizer detects Llama from llama model name', t => {
+	const tokenizer = createTokenizer('custom', 'llama-3-8b');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from mistral model name', t => {
+	const tokenizer = createTokenizer('custom', 'mistral-7b');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from qwen model name', t => {
+	const tokenizer = createTokenizer('custom', 'qwen-2.5');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from gemma model name', t => {
+	const tokenizer = createTokenizer('custom', 'gemma-2b');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from phi model name', t => {
+	const tokenizer = createTokenizer('custom', 'phi-3');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from codellama model name', t => {
+	const tokenizer = createTokenizer('custom', 'codellama-7b');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from deepseek model name', t => {
+	const tokenizer = createTokenizer('custom', 'deepseek-coder');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from mixtral model name', t => {
+	const tokenizer = createTokenizer('custom', 'mixtral-8x7b');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from ollama provider', t => {
+	const tokenizer = createTokenizer('ollama', 'custom-model');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from llama.cpp provider', t => {
+	const tokenizer = createTokenizer('llama.cpp', 'custom-model');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizer detects Llama from local provider', t => {
+	const tokenizer = createTokenizer('local', 'custom-model');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+// Test createTokenizer with fallback
+test('createTokenizer returns FallbackTokenizer for unknown provider', t => {
+	const tokenizer = createTokenizer('unknown', 'unknown-model');
+	t.true(tokenizer instanceof FallbackTokenizer);
+});
+
+// Test cloud suffix stripping
+test('createTokenizer strips :cloud suffix from model name', t => {
+	const tokenizer = createTokenizer('ollama', 'llama-3:cloud');
+	t.true(tokenizer instanceof LlamaTokenizer);
+	t.is(tokenizer.getName(), 'llama-llama-3');
+});
+
+test('createTokenizer strips -cloud suffix from model name', t => {
+	const tokenizer = createTokenizer('ollama', 'llama-3-cloud');
+	t.true(tokenizer instanceof LlamaTokenizer);
+	t.is(tokenizer.getName(), 'llama-llama-3');
+});
+
+// Test case insensitivity
+test('createTokenizer is case insensitive for provider detection', t => {
+	const tokenizer = createTokenizer('OPENAI', 'model');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+test('createTokenizer is case insensitive for model detection', t => {
+	const tokenizer = createTokenizer('custom', 'GPT-4-TURBO');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+// Test createTokenizerForProvider
+test('createTokenizerForProvider creates OpenAI tokenizer', t => {
+	const tokenizer = createTokenizerForProvider('openai', 'gpt-4');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+test('createTokenizerForProvider creates Anthropic tokenizer', t => {
+	const tokenizer = createTokenizerForProvider('anthropic', 'claude-3');
+	t.true(tokenizer instanceof AnthropicTokenizer);
+});
+
+test('createTokenizerForProvider creates Llama tokenizer', t => {
+	const tokenizer = createTokenizerForProvider('llama', 'llama-3');
+	t.true(tokenizer instanceof LlamaTokenizer);
+});
+
+test('createTokenizerForProvider creates FallbackTokenizer', t => {
+	const tokenizer = createTokenizerForProvider('fallback');
+	t.true(tokenizer instanceof FallbackTokenizer);
+});
+
+test('createTokenizerForProvider handles auto mode with model ID', t => {
+	const tokenizer = createTokenizerForProvider('auto', 'gpt-4');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});
+
+test('createTokenizerForProvider handles auto mode without model ID', t => {
+	const tokenizer = createTokenizerForProvider('auto');
+	t.true(tokenizer instanceof FallbackTokenizer);
+});
+
+test('createTokenizerForProvider works without model ID', t => {
+	const tokenizer = createTokenizerForProvider('openai');
+	t.true(tokenizer instanceof OpenAITokenizer);
+});

package/source/tokenization/tokenizer-factory.ts

@@ -0,0 +1,125 @@
+/**
+ * Tokenizer factory
+ * Creates appropriate tokenizer based on provider and model
+ */
+
+import type {Tokenizer, TokenizerProvider} from '../types/tokenization.js';
+import {AnthropicTokenizer} from './tokenizers/anthropic-tokenizer.js';
+import {FallbackTokenizer} from './tokenizers/fallback-tokenizer.js';
+import {LlamaTokenizer} from './tokenizers/llama-tokenizer.js';
+import {OpenAITokenizer} from './tokenizers/openai-tokenizer.js';
+
+/**
+ * Detect provider from model ID or provider name
+ */
+function detectProvider(
+	providerName: string,
+	modelId: string,
+): TokenizerProvider {
+	const lowerProvider = providerName.toLowerCase();
+	const lowerModel = modelId.toLowerCase();
+
+	// Check provider name
+	if (lowerProvider.includes('openai')) {
+		return 'openai';
+	}
+
+	if (lowerProvider.includes('anthropic') || lowerProvider.includes('claude')) {
+		return 'anthropic';
+	}
+
+	// Check model ID for common patterns
+	if (lowerModel.includes('gpt') || lowerModel.includes('openai')) {
+		return 'openai';
+	}
+
+	if (lowerModel.includes('claude')) {
+		return 'anthropic';
+	}
+
+	if (
+		lowerModel.includes('llama') ||
+		lowerModel.includes('mistral') ||
+		lowerModel.includes('qwen') ||
+		lowerModel.includes('gemma') ||
+		lowerModel.includes('phi') ||
+		lowerModel.includes('codellama') ||
+		lowerModel.includes('deepseek') ||
+		lowerModel.includes('mixtral')
+	) {
+		return 'llama';
+	}
+
+	// Default to llama for local models (most common for local inference)
+	if (
+		lowerProvider.includes('ollama') ||
+		lowerProvider.includes('llama.cpp') ||
+		lowerProvider.includes('local')
+	) {
+		return 'llama';
+	}
+
+	return 'fallback';
+}
+
+/**
+ * Create a tokenizer based on provider and model
+ */
+export function createTokenizer(
+	providerName: string,
+	modelId: string,
+): Tokenizer {
+	// Strip :cloud suffix if present (Ollama cloud models)
+	const normalizedModelId =
+		modelId.endsWith(':cloud') || modelId.endsWith('-cloud')
+			? modelId.slice(0, -6)
+			: modelId;
+
+	const provider = detectProvider(providerName, normalizedModelId);
+
+	switch (provider) {
+		case 'openai':
+			return new OpenAITokenizer(normalizedModelId);
+
+		case 'anthropic':
+			return new AnthropicTokenizer(normalizedModelId);
+
+		case 'llama':
+			return new LlamaTokenizer(normalizedModelId);
+
+		case 'fallback':
+		default:
+			return new FallbackTokenizer();
+	}
+}
+
+/**
+ * Create a tokenizer with explicit provider
+ */
+export function createTokenizerForProvider(
+	provider: TokenizerProvider,
+	modelId?: string,
+): Tokenizer {
+	switch (provider) {
+		case 'openai':
+			return new OpenAITokenizer(modelId);
+
+		case 'anthropic':
+			return new AnthropicTokenizer(modelId);
+
+		case 'llama':
+			return new LlamaTokenizer(modelId);
+
+		case 'auto':
+			// Auto-detect based on model ID if provided
+			if (modelId) {
+				return createTokenizer('', modelId);
+			}
+
+			return new FallbackTokenizer();
+
+		case 'fallback':
+		default:
+			return new FallbackTokenizer();
+	}
+}