@gguf/coder 0.3.1 → 0.3.2

package/source/services/file-snapshot.ts

@@ -1,220 +0,0 @@
-import {execSync} from 'child_process';
-import {existsSync} from 'fs';
-import * as path from 'path';
-import {MAX_CHECKPOINT_FILES} from '@/constants';
-import {logWarning} from '@/utils/message-queue';
-import * as fs from 'fs/promises';
-
-/**
- * Service for capturing and restoring file snapshots for checkpoints
- */
-export class FileSnapshotService {
-	private readonly workspaceRoot: string;
-
-	constructor(workspaceRoot: string = process.cwd()) {
-		this.workspaceRoot = workspaceRoot;
-	}
-
-	/**
-	 * Capture the contents of specified files
-	 */
-	async captureFiles(filePaths: string[]): Promise<Map<string, string>> {
-		const snapshots = new Map<string, string>();
-
-		for (const filePath of filePaths) {
-			try {
-				const absolutePath = path.resolve(this.workspaceRoot, filePath); // nosemgrep
-				const content = await fs.readFile(absolutePath, 'utf-8');
-				const relativePath = path.relative(this.workspaceRoot, absolutePath);
-				snapshots.set(relativePath, content);
-			} catch (error) {
-				logWarning('Could not capture file', true, {
-					context: {
-						filePath,
-						error: error instanceof Error ? error.message : 'Unknown error',
-					},
-				});
-			}
-		}
-
-		return snapshots;
-	}
-
-	/**
-	 * Restore files from snapshots
-	 */
-	async restoreFiles(snapshots: Map<string, string>): Promise<void> {
-		const errors: string[] = [];
-
-		for (const [relativePath, content] of snapshots) {
-			try {
-				const absolutePath = path.resolve(this.workspaceRoot, relativePath);
-				const directory = path.dirname(absolutePath);
-
-				await fs.mkdir(directory, {recursive: true});
-				await fs.writeFile(absolutePath, content, 'utf-8');
-			} catch (error) {
-				errors.push(
-					`Failed to restore ${relativePath}: ${
-						error instanceof Error ? error.message : 'Unknown error'
-					}`,
-				);
-			}
-		}
-
-		if (errors.length > 0) {
-			throw new Error(`Failed to restore some files:\n${errors.join('\n')}`);
-		}
-	}
-
-	/**
-	 * Get list of modified files in the workspace
-	 * Uses git to detect modified files if available, otherwise returns empty array
-	 */
-	getModifiedFiles(): string[] {
-		try {
-			const modifiedOutput = execSync('git diff --name-only HEAD', {
-				cwd: this.workspaceRoot,
-				encoding: 'utf-8',
-				stdio: ['pipe', 'pipe', 'pipe'],
-			}).trim();
-
-			const untrackedOutput = execSync(
-				'git ls-files --others --exclude-standard',
-				{
-					cwd: this.workspaceRoot,
-					encoding: 'utf-8',
-					stdio: ['pipe', 'pipe', 'pipe'],
-				},
-			).trim();
-
-			const modifiedFiles = modifiedOutput
-				? modifiedOutput.split('\n').filter(Boolean)
-				: [];
-			const untrackedFiles = untrackedOutput
-				? untrackedOutput.split('\n').filter(Boolean)
-				: [];
-
-			const allFiles = [...new Set([...modifiedFiles, ...untrackedFiles])];
-
-			const filtered = allFiles.filter(file => {
-				const ignorePatterns = [
-					'node_modules/',
-					'dist/',
-					'build/',
-					'.git/',
-					'.coder/',
-					'coverage/',
-					'*.log',
-				];
-
-				return !ignorePatterns.some(pattern => {
-					if (pattern.endsWith('/')) {
-						return file.startsWith(pattern);
-					}
-					if (pattern.startsWith('*.')) {
-						return file.endsWith(pattern.slice(1));
-					}
-					return file.includes(pattern);
-				});
-			});
-
-			if (filtered.length > MAX_CHECKPOINT_FILES) {
-				logWarning(
-					'Too many modified files detected, limiting to maximum',
-					true,
-					{
-						context: {
-							fileCount: filtered.length,
-							maxFiles: MAX_CHECKPOINT_FILES,
-						},
-					},
-				);
-				return filtered.slice(0, MAX_CHECKPOINT_FILES);
-			}
-
-			return filtered;
-		} catch {
-			logWarning('Git not available for file tracking', true, {
-				context: {
-					workspaceRoot: this.workspaceRoot,
-				},
-			});
-			return [];
-		}
-	}
-
-	/**
-	 * Get the size of a file snapshot
-	 */
-	getSnapshotSize(snapshots: Map<string, string>): number {
-		let totalSize = 0;
-		for (const content of snapshots.values()) {
-			totalSize += Buffer.byteLength(content, 'utf-8');
-		}
-		return totalSize;
-	}
-
-	/**
-	 * Validate that all files in the snapshot can be written to their locations
-	 */
-	async validateRestorePath(
-		snapshots: Map<string, string>,
-	): Promise<{valid: boolean; errors: string[]}> {
-		const errors: string[] = [];
-
-		for (const relativePath of snapshots.keys()) {
-			const absolutePath = path.resolve(this.workspaceRoot, relativePath); // nosemgrep
-			const directory = path.dirname(absolutePath);
-
-			try {
-				let dirWritable = true;
-				try {
-					await fs.access(directory, fs.constants.W_OK);
-				} catch (_err) {
-					// Directory is not writable or does not exist, so try to create it
-					try {
-						await fs.mkdir(directory, {recursive: true});
-					} catch (mkdirError) {
-						dirWritable = false;
-						errors.push(
-							`Cannot create directory "${directory}": ${mkdirError instanceof Error ? mkdirError.message : 'Unknown error'}`,
-						);
-					}
-				}
-
-				// After attempting to create the directory, verify it's writable
-				if (dirWritable) {
-					try {
-						await fs.access(directory, fs.constants.W_OK);
-					} catch (accessError) {
-						dirWritable = false;
-						errors.push(
-							`Directory "${directory}" is not writable: ${accessError instanceof Error ? accessError.message : 'Unknown error'}`,
-						);
-					}
-				}
-
-				// If directory is not writable or was not successfully created, skip further checks for this file
-				if (!dirWritable) {
-					continue;
-				}
-
-				if (existsSync(absolutePath)) {
-					try {
-						await fs.access(absolutePath, fs.constants.W_OK);
-					} catch (fileError) {
-						errors.push(
-							`Cannot write to file "${absolutePath}": ${fileError instanceof Error ? fileError.message : 'Unknown error'}`,
-						);
-					}
-				}
-			} catch (error) {
-				errors.push(
-					`Cannot validate path for ${relativePath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
-				);
-			}
-		}
-		return {valid: errors.length === 0, errors};
-	}
-}

package/source/test-utils/render-with-theme.tsx

@@ -1,48 +0,0 @@
-import {ThemeContext} from '@/hooks/useTheme';
-import type {Colors, ThemePreset} from '@/types/ui';
-import {render} from 'ink-testing-library';
-import React from 'react';
-
-// Default test colors that match the structure used in the app
-const testColors: Colors = {
-	primary: 'blue',
-	secondary: 'gray',
-	white: 'white',
-	black: 'black',
-	info: 'cyan',
-	warning: 'yellow',
-	error: 'red',
-	success: 'green',
-	tool: 'magenta',
-	diffAdded: 'green',
-	diffRemoved: 'red',
-	diffAddedText: 'white',
-	diffRemovedText: 'white',
-};
-
-// Test theme context value
-const testThemeContext = {
-	currentTheme: 'tokyo-night' as ThemePreset,
-	colors: testColors,
-	setCurrentTheme: () => {},
-};
-
-/**
- * Wrapper component that provides ThemeContext for tests
- */
-function TestThemeProvider({children}: {children: React.ReactNode}) {
-	return (
-		<ThemeContext.Provider value={testThemeContext}>
-			{children}
-		</ThemeContext.Provider>
-	);
-}
-
-/**
- * Render a component wrapped with ThemeContext for testing
- */
-export function renderWithTheme(
-	element: React.ReactElement,
-): ReturnType<typeof render> {
-	return render(<TestThemeProvider>{element}</TestThemeProvider>);
-}

package/source/tokenization/index.ts

@@ -1 +0,0 @@
-export {createTokenizer} from './tokenizer-factory.js';

package/source/tokenization/tokenizer-factory.spec.ts

@@ -1,170 +0,0 @@
-/**
- * Tests for tokenizer-factory.ts
- */
-
-import test from 'ava';
-import {
-	createTokenizer,
-	createTokenizerForProvider,
-} from './tokenizer-factory.js';
-import {AnthropicTokenizer} from './tokenizers/anthropic-tokenizer.js';
-import {FallbackTokenizer} from './tokenizers/fallback-tokenizer.js';
-import {LlamaTokenizer} from './tokenizers/llama-tokenizer.js';
-import {OpenAITokenizer} from './tokenizers/openai-tokenizer.js';
-
-console.log(`\ntokenizer-factory.spec.ts`);
-
-// Test createTokenizer with OpenAI provider detection
-test('createTokenizer detects OpenAI from provider name', t => {
-	const tokenizer = createTokenizer('openai', 'custom-model');
-	t.true(tokenizer instanceof OpenAITokenizer);
-	t.is(tokenizer.getName(), 'openai-custom-model');
-});
-
-test('createTokenizer detects OpenAI from GPT model name', t => {
-	const tokenizer = createTokenizer('custom', 'gpt-4-turbo');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-test('createTokenizer detects OpenAI from model name with openai keyword', t => {
-	const tokenizer = createTokenizer('custom', 'openai-model');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-// Test createTokenizer with Anthropic provider detection
-test('createTokenizer detects Anthropic from provider name', t => {
-	const tokenizer = createTokenizer('anthropic', 'custom-model');
-	t.true(tokenizer instanceof AnthropicTokenizer);
-});
-
-test('createTokenizer detects Anthropic from provider name with claude keyword', t => {
-	const tokenizer = createTokenizer('claude-provider', 'model');
-	t.true(tokenizer instanceof AnthropicTokenizer);
-});
-
-test('createTokenizer detects Anthropic from claude model name', t => {
-	const tokenizer = createTokenizer('custom', 'claude-3-opus');
-	t.true(tokenizer instanceof AnthropicTokenizer);
-});
-
-// Test createTokenizer with Llama provider detection
-test('createTokenizer detects Llama from llama model name', t => {
-	const tokenizer = createTokenizer('custom', 'llama-3-8b');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from mistral model name', t => {
-	const tokenizer = createTokenizer('custom', 'mistral-7b');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from qwen model name', t => {
-	const tokenizer = createTokenizer('custom', 'qwen-2.5');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from gemma model name', t => {
-	const tokenizer = createTokenizer('custom', 'gemma-2b');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from phi model name', t => {
-	const tokenizer = createTokenizer('custom', 'phi-3');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from codellama model name', t => {
-	const tokenizer = createTokenizer('custom', 'codellama-7b');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from deepseek model name', t => {
-	const tokenizer = createTokenizer('custom', 'deepseek-coder');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from mixtral model name', t => {
-	const tokenizer = createTokenizer('custom', 'mixtral-8x7b');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from ollama provider', t => {
-	const tokenizer = createTokenizer('ollama', 'custom-model');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from llama.cpp provider', t => {
-	const tokenizer = createTokenizer('llama.cpp', 'custom-model');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizer detects Llama from local provider', t => {
-	const tokenizer = createTokenizer('local', 'custom-model');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-// Test createTokenizer with fallback
-test('createTokenizer returns FallbackTokenizer for unknown provider', t => {
-	const tokenizer = createTokenizer('unknown', 'unknown-model');
-	t.true(tokenizer instanceof FallbackTokenizer);
-});
-
-// Test cloud suffix stripping
-test('createTokenizer strips :cloud suffix from model name', t => {
-	const tokenizer = createTokenizer('ollama', 'llama-3:cloud');
-	t.true(tokenizer instanceof LlamaTokenizer);
-	t.is(tokenizer.getName(), 'llama-llama-3');
-});
-
-test('createTokenizer strips -cloud suffix from model name', t => {
-	const tokenizer = createTokenizer('ollama', 'llama-3-cloud');
-	t.true(tokenizer instanceof LlamaTokenizer);
-	t.is(tokenizer.getName(), 'llama-llama-3');
-});
-
-// Test case insensitivity
-test('createTokenizer is case insensitive for provider detection', t => {
-	const tokenizer = createTokenizer('OPENAI', 'model');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-test('createTokenizer is case insensitive for model detection', t => {
-	const tokenizer = createTokenizer('custom', 'GPT-4-TURBO');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-// Test createTokenizerForProvider
-test('createTokenizerForProvider creates OpenAI tokenizer', t => {
-	const tokenizer = createTokenizerForProvider('openai', 'gpt-4');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-test('createTokenizerForProvider creates Anthropic tokenizer', t => {
-	const tokenizer = createTokenizerForProvider('anthropic', 'claude-3');
-	t.true(tokenizer instanceof AnthropicTokenizer);
-});
-
-test('createTokenizerForProvider creates Llama tokenizer', t => {
-	const tokenizer = createTokenizerForProvider('llama', 'llama-3');
-	t.true(tokenizer instanceof LlamaTokenizer);
-});
-
-test('createTokenizerForProvider creates FallbackTokenizer', t => {
-	const tokenizer = createTokenizerForProvider('fallback');
-	t.true(tokenizer instanceof FallbackTokenizer);
-});
-
-test('createTokenizerForProvider handles auto mode with model ID', t => {
-	const tokenizer = createTokenizerForProvider('auto', 'gpt-4');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});
-
-test('createTokenizerForProvider handles auto mode without model ID', t => {
-	const tokenizer = createTokenizerForProvider('auto');
-	t.true(tokenizer instanceof FallbackTokenizer);
-});
-
-test('createTokenizerForProvider works without model ID', t => {
-	const tokenizer = createTokenizerForProvider('openai');
-	t.true(tokenizer instanceof OpenAITokenizer);
-});

package/source/tokenization/tokenizer-factory.ts

@@ -1,125 +0,0 @@
-/**
- * Tokenizer factory
- * Creates appropriate tokenizer based on provider and model
- */
-
-import type {Tokenizer, TokenizerProvider} from '../types/tokenization.js';
-import {AnthropicTokenizer} from './tokenizers/anthropic-tokenizer.js';
-import {FallbackTokenizer} from './tokenizers/fallback-tokenizer.js';
-import {LlamaTokenizer} from './tokenizers/llama-tokenizer.js';
-import {OpenAITokenizer} from './tokenizers/openai-tokenizer.js';
-
-/**
- * Detect provider from model ID or provider name
- */
-function detectProvider(
-	providerName: string,
-	modelId: string,
-): TokenizerProvider {
-	const lowerProvider = providerName.toLowerCase();
-	const lowerModel = modelId.toLowerCase();
-
-	// Check provider name
-	if (lowerProvider.includes('openai')) {
-		return 'openai';
-	}
-
-	if (lowerProvider.includes('anthropic') || lowerProvider.includes('claude')) {
-		return 'anthropic';
-	}
-
-	// Check model ID for common patterns
-	if (lowerModel.includes('gpt') || lowerModel.includes('openai')) {
-		return 'openai';
-	}
-
-	if (lowerModel.includes('claude')) {
-		return 'anthropic';
-	}
-
-	if (
-		lowerModel.includes('llama') ||
-		lowerModel.includes('mistral') ||
-		lowerModel.includes('qwen') ||
-		lowerModel.includes('gemma') ||
-		lowerModel.includes('phi') ||
-		lowerModel.includes('codellama') ||
-		lowerModel.includes('deepseek') ||
-		lowerModel.includes('mixtral')
-	) {
-		return 'llama';
-	}
-
-	// Default to llama for local models (most common for local inference)
-	if (
-		lowerProvider.includes('ollama') ||
-		lowerProvider.includes('llama.cpp') ||
-		lowerProvider.includes('local')
-	) {
-		return 'llama';
-	}
-
-	return 'fallback';
-}
-
-/**
- * Create a tokenizer based on provider and model
- */
-export function createTokenizer(
-	providerName: string,
-	modelId: string,
-): Tokenizer {
-	// Strip :cloud suffix if present (Ollama cloud models)
-	const normalizedModelId =
-		modelId.endsWith(':cloud') || modelId.endsWith('-cloud')
-			? modelId.slice(0, -6)
-			: modelId;
-
-	const provider = detectProvider(providerName, normalizedModelId);
-
-	switch (provider) {
-		case 'openai':
-			return new OpenAITokenizer(normalizedModelId);
-
-		case 'anthropic':
-			return new AnthropicTokenizer(normalizedModelId);
-
-		case 'llama':
-			return new LlamaTokenizer(normalizedModelId);
-
-		case 'fallback':
-		default:
-			return new FallbackTokenizer();
-	}
-}
-
-/**
- * Create a tokenizer with explicit provider
- */
-export function createTokenizerForProvider(
-	provider: TokenizerProvider,
-	modelId?: string,
-): Tokenizer {
-	switch (provider) {
-		case 'openai':
-			return new OpenAITokenizer(modelId);
-
-		case 'anthropic':
-			return new AnthropicTokenizer(modelId);
-
-		case 'llama':
-			return new LlamaTokenizer(modelId);
-
-		case 'auto':
-			// Auto-detect based on model ID if provided
-			if (modelId) {
-				return createTokenizer('', modelId);
-			}
-
-			return new FallbackTokenizer();
-
-		case 'fallback':
-		default:
-			return new FallbackTokenizer();
-	}
-}