byterover-cli 0.2.1 → 0.3.0

package/dist/infra/cipher/llm/tokenizers/default-tokenizer.d.ts

@@ -0,0 +1,31 @@
+import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.js';
+/**
+ * Default fallback tokenizer for unknown or unsupported models.
+ *
+ * This tokenizer provides a generic character-based approximation
+ * that can be used when no provider-specific tokenizer is available.
+ * It uses a simple heuristic that works reasonably well for English text.
+ */
+export declare class DefaultTokenizer implements ITokenizer {
+    /**
+     * Approximates token count using a character-based heuristic.
+     *
+     * Assumes roughly 4 characters per token, which is a common approximation
+     * for English text with modern tokenizers (like GPT, Claude, Gemini).
+     *
+     * This heuristic may be less accurate for:
+     * - Non-English languages (especially character-based languages)
+     * - Code (which often has different tokenization patterns)
+     * - Text with many special characters or formatting
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    /**
+     * Gets the provider name for this tokenizer.
+     *
+     * @returns 'default' as the provider identifier
+     */
+    getProviderName(): string;
+}

package/dist/infra/cipher/llm/tokenizers/default-tokenizer.js

@@ -0,0 +1,38 @@
+/**
+ * Default fallback tokenizer for unknown or unsupported models.
+ *
+ * This tokenizer provides a generic character-based approximation
+ * that can be used when no provider-specific tokenizer is available.
+ * It uses a simple heuristic that works reasonably well for English text.
+ */
+export class DefaultTokenizer {
+    /**
+     * Approximates token count using a character-based heuristic.
+     *
+     * Assumes roughly 4 characters per token, which is a common approximation
+     * for English text with modern tokenizers (like GPT, Claude, Gemini).
+     *
+     * This heuristic may be less accurate for:
+     * - Non-English languages (especially character-based languages)
+     * - Code (which often has different tokenization patterns)
+     * - Text with many special characters or formatting
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text) {
+        if (!text) {
+            return 0;
+        }
+        // Simple heuristic: ~4 characters per token
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Gets the provider name for this tokenizer.
+     *
+     * @returns 'default' as the provider identifier
+     */
+    getProviderName() {
+        return 'default';
+    }
+}

package/dist/infra/cipher/llm/tokenizers/gemini-tokenizer.d.ts

@@ -0,0 +1,37 @@
+import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.js';
+/**
+ * Tokenizer for Google Gemini models - CURRENTLY USING APPROXIMATION.
+ *
+ * This implementation uses a character-based approximation rather than
+ * accurate token counting. This is a temporary solution due to the
+ * asynchronous nature of the official Gemini countTokens API.
+ *
+ * TODO: Consider these improvements:
+ * 1. Use the official @google/genai countTokens method (requires async handling)
+ * 2. Implement a WASM-based tokenizer for accurate synchronous counting
+ * 3. Cache token counts for frequently used text
+ * 4. Model-specific adjustments based on actual Gemini tokenization patterns
+ */
+export declare class GeminiTokenizer implements ITokenizer {
+    private readonly modelName;
+    /**
+     * Creates a new Gemini tokenizer instance.
+     *
+     * @param model - The Gemini model name (e.g., 'gemini-2.5-flash', 'gemini-pro')
+     *                Currently not used for approximation, but stored for future improvements
+     */
+    constructor(model: string);
+    /**
+     * Approximates the token count for Google Gemini models.
+     *
+     * Uses a rough character-based approximation: ~4 characters per token.
+     * This is based on common estimates for English text with modern tokenizers.
+     *
+     * IMPORTANT: This is NOT accurate for Gemini models and should be replaced
+     * with a proper implementation when possible.
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+}

package/dist/infra/cipher/llm/tokenizers/gemini-tokenizer.js

@@ -0,0 +1,45 @@
+/**
+ * Tokenizer for Google Gemini models - CURRENTLY USING APPROXIMATION.
+ *
+ * This implementation uses a character-based approximation rather than
+ * accurate token counting. This is a temporary solution due to the
+ * asynchronous nature of the official Gemini countTokens API.
+ *
+ * TODO: Consider these improvements:
+ * 1. Use the official @google/genai countTokens method (requires async handling)
+ * 2. Implement a WASM-based tokenizer for accurate synchronous counting
+ * 3. Cache token counts for frequently used text
+ * 4. Model-specific adjustments based on actual Gemini tokenization patterns
+ */
+export class GeminiTokenizer {
+    modelName;
+    /**
+     * Creates a new Gemini tokenizer instance.
+     *
+     * @param model - The Gemini model name (e.g., 'gemini-2.5-flash', 'gemini-pro')
+     *                Currently not used for approximation, but stored for future improvements
+     */
+    constructor(model) {
+        this.modelName = model;
+    }
+    /**
+     * Approximates the token count for Google Gemini models.
+     *
+     * Uses a rough character-based approximation: ~4 characters per token.
+     * This is based on common estimates for English text with modern tokenizers.
+     *
+     * IMPORTANT: This is NOT accurate for Gemini models and should be replaced
+     * with a proper implementation when possible.
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text) {
+        if (!text) {
+            return 0;
+        }
+        // Rough approximation: ~4 characters per token
+        // This is a simplified heuristic and varies by language and content type
+        return Math.ceil(text.length / 4);
+    }
+}

package/dist/infra/cipher/llm/tokenizers/openrouter-tokenizer.d.ts

@@ -0,0 +1,29 @@
+import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.js';
+/**
+ * Tokenizer for OpenRouter API.
+ *
+ * OpenRouter supports multiple model providers (OpenAI, Anthropic, etc.),
+ * each with different tokenization schemes. This tokenizer provides a
+ * generic approximation that works reasonably well across different models.
+ *
+ * For more accurate token counting, model-specific tokenizers could be
+ * implemented based on the selected model's provider.
+ */
+export declare class OpenRouterTokenizer implements ITokenizer {
+    /**
+     * Approximates token count using a character-based heuristic.
+     *
+     * Uses ~4 characters per token, which is a common approximation
+     * for modern tokenizers (GPT, Claude, Gemini, etc.).
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text: string): number;
+    /**
+     * Gets the provider name for this tokenizer.
+     *
+     * @returns 'openrouter' as the provider identifier
+     */
+    getProviderName(): string;
+}

package/dist/infra/cipher/llm/tokenizers/openrouter-tokenizer.js

@@ -0,0 +1,37 @@
+/**
+ * Tokenizer for OpenRouter API.
+ *
+ * OpenRouter supports multiple model providers (OpenAI, Anthropic, etc.),
+ * each with different tokenization schemes. This tokenizer provides a
+ * generic approximation that works reasonably well across different models.
+ *
+ * For more accurate token counting, model-specific tokenizers could be
+ * implemented based on the selected model's provider.
+ */
+export class OpenRouterTokenizer {
+    /**
+     * Approximates token count using a character-based heuristic.
+     *
+     * Uses ~4 characters per token, which is a common approximation
+     * for modern tokenizers (GPT, Claude, Gemini, etc.).
+     *
+     * @param text - Text content to count tokens for
+     * @returns Approximate number of tokens
+     */
+    countTokens(text) {
+        if (!text) {
+            return 0;
+        }
+        // Simple heuristic: ~4 characters per token
+        // This works reasonably well for most OpenRouter models
+        return Math.ceil(text.length / 4);
+    }
+    /**
+     * Gets the provider name for this tokenizer.
+     *
+     * @returns 'openrouter' as the provider identifier
+     */
+    getProviderName() {
+        return 'openrouter';
+    }
+}

package/dist/infra/cipher/llm/tool-output-processor.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Tool Output Processing Layer
+ *
+ * Handles truncation and file saving for large tool outputs.
+ * Prevents context overflow by truncating outputs while preserving critical information.
+ */
+/**
+ * Configuration for output truncation behavior
+ */
+export interface TruncationConfig {
+    /**
+     * Whether to enable truncation
+     * @default true
+     */
+    enabled?: boolean;
+    /**
+     * Number of lines to keep from the end of the output
+     * @default 250
+     */
+    headLines?: number;
+    /**
+     * Number of lines to keep from the start of the output
+     * @default 250
+     */
+    tailLines?: number;
+    /**
+     * Character threshold for triggering truncation
+     * @default 50000
+     */
+    threshold?: number;
+}
+/**
+ * Result of output processing
+ */
+export interface ProcessedOutput {
+    /**
+     * Processed content (truncated if necessary)
+     */
+    content: string;
+    /**
+     * Metadata about the processing
+     */
+    metadata?: {
+        /**
+         * Original content length before truncation
+         */
+        originalLength?: number;
+        /**
+         * Path to file where full output was saved
+         */
+        savedToFile?: string;
+        /**
+         * Whether output was truncated
+         */
+        truncated?: boolean;
+    };
+}
+/**
+ * Default truncation configuration
+ */
+export declare const DEFAULT_TRUNCATION_CONFIG: Required<TruncationConfig>;
+/**
+ * Tool output processor with truncation and file saving capabilities
+ */
+export declare class ToolOutputProcessor {
+    private readonly config;
+    /**
+     * Create a new tool output processor
+     *
+     * @param config - Truncation configuration
+     */
+    constructor(config?: TruncationConfig);
+    /**
+     * Process tool output with truncation and file saving
+     *
+     * If output exceeds threshold:
+     * 1. Saves full output to temp file
+     * 2. Returns truncated content (head + tail with omission notice)
+     * 3. Includes metadata about truncation
+     *
+     * @param toolName - Name of the tool that produced the output
+     * @param output - Raw tool output (any type, will be stringified)
+     * @returns Processed output with metadata
+     */
+    processOutput(toolName: string, output: unknown): Promise<ProcessedOutput>;
+    /**
+     * Save content to a temporary file
+     *
+     * Creates a unique temp file for storing full tool output.
+     *
+     * @param toolName - Name of the tool
+     * @param content - Content to save
+     * @returns Path to saved file
+     */
+    private saveToTempFile;
+    /**
+     * Safely stringify any value
+     *
+     * Handles circular references and special types.
+     *
+     * @param value - Value to stringify
+     * @returns String representation
+     */
+    private stringify;
+    /**
+     * Truncate content keeping head and tail lines
+     *
+     * Format:
+     * [First N lines]
+     * ... [omission notice with line count and file reference] ...
+     * [Last N lines]
+     *
+     * @param content - Content to truncate
+     * @returns Truncated content
+     */
+    private truncateContent;
+}

package/dist/infra/cipher/llm/tool-output-processor.js

@@ -0,0 +1,153 @@
+/**
+ * Tool Output Processing Layer
+ *
+ * Handles truncation and file saving for large tool outputs.
+ * Prevents context overflow by truncating outputs while preserving critical information.
+ */
+import { existsSync, promises as fsPromises, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+/**
+ * Default truncation configuration
+ */
+export const DEFAULT_TRUNCATION_CONFIG = {
+    enabled: true,
+    headLines: 250,
+    tailLines: 250,
+    threshold: 50_000,
+};
+/**
+ * Tool output processor with truncation and file saving capabilities
+ */
+export class ToolOutputProcessor {
+    config;
+    /**
+     * Create a new tool output processor
+     *
+     * @param config - Truncation configuration
+     */
+    constructor(config) {
+        this.config = {
+            ...DEFAULT_TRUNCATION_CONFIG,
+            ...config,
+        };
+    }
+    /**
+     * Process tool output with truncation and file saving
+     *
+     * If output exceeds threshold:
+     * 1. Saves full output to temp file
+     * 2. Returns truncated content (head + tail with omission notice)
+     * 3. Includes metadata about truncation
+     *
+     * @param toolName - Name of the tool that produced the output
+     * @param output - Raw tool output (any type, will be stringified)
+     * @returns Processed output with metadata
+     */
+    async processOutput(toolName, output) {
+        // Convert output to string
+        const contentString = this.stringify(output);
+        const originalLength = contentString.length;
+        // Check if truncation is needed
+        if (!this.config.enabled || originalLength <= this.config.threshold) {
+            return {
+                content: contentString,
+            };
+        }
+        // Save full output to temp file
+        const savedFilePath = await this.saveToTempFile(toolName, contentString);
+        // Truncate content
+        const truncatedContent = this.truncateContent(contentString);
+        return {
+            content: truncatedContent,
+            metadata: {
+                originalLength,
+                savedToFile: savedFilePath,
+                truncated: true,
+            },
+        };
+    }
+    /**
+     * Save content to a temporary file
+     *
+     * Creates a unique temp file for storing full tool output.
+     *
+     * @param toolName - Name of the tool
+     * @param content - Content to save
+     * @returns Path to saved file
+     */
+    async saveToTempFile(toolName, content) {
+        // Create temp directory if it doesn't exist
+        const tempDir = join(tmpdir(), 'byterover-tool-outputs');
+        if (!existsSync(tempDir)) {
+            mkdirSync(tempDir, { recursive: true });
+        }
+        // Generate unique filename
+        const timestamp = Date.now();
+        const sanitizedToolName = toolName.replaceAll(/[^a-z0-9-_]/gi, '_');
+        const filename = `${sanitizedToolName}_${timestamp}.txt`;
+        const filePath = join(tempDir, filename);
+        // Write content to file
+        await fsPromises.writeFile(filePath, content, 'utf8');
+        return filePath;
+    }
+    /**
+     * Safely stringify any value
+     *
+     * Handles circular references and special types.
+     *
+     * @param value - Value to stringify
+     * @returns String representation
+     */
+    stringify(value) {
+        if (typeof value === 'string') {
+            return value;
+        }
+        if (value === null || value === undefined) {
+            return String(value);
+        }
+        try {
+            // Try JSON.stringify with circular reference handling
+            return JSON.stringify(value, (_, val) => {
+                // Handle circular references by converting to string
+                if (typeof val === 'object' && val !== null) {
+                    return val;
+                }
+                return val;
+            }, 2);
+        }
+        catch {
+            // Fallback to String() if JSON.stringify fails
+            return String(value);
+        }
+    }
+    /**
+     * Truncate content keeping head and tail lines
+     *
+     * Format:
+     * [First N lines]
+     * ... [omission notice with line count and file reference] ...
+     * [Last N lines]
+     *
+     * @param content - Content to truncate
+     * @returns Truncated content
+     */
+    truncateContent(content) {
+        const lines = content.split('\n');
+        const totalLines = lines.length;
+        // Calculate how many lines to keep
+        const headLines = Math.min(this.config.headLines, totalLines);
+        const tailLines = Math.min(this.config.tailLines, totalLines);
+        // If content is small enough to fit, return as-is
+        if (headLines + tailLines >= totalLines) {
+            return content;
+        }
+        // Extract head and tail
+        const head = lines.slice(0, headLines).join('\n');
+        const tail = lines.slice(-tailLines).join('\n');
+        // Calculate omitted lines
+        const omittedLines = totalLines - headLines - tailLines;
+        const omissionNotice = `\n\n... [${omittedLines} lines omitted - full output saved to file] ...\n\n`;
+        return head + omissionNotice + tail;
+    }
+}

package/dist/infra/cipher/logger/console-logger.d.ts

@@ -0,0 +1,42 @@
+import type { ILogger } from '../../../core/interfaces/cipher/i-logger.js';
+/**
+ * Console-based logger implementation.
+ *
+ * This is a concrete logger that writes to the console with color formatting.
+ * Used at the command/presentation layer to display logs to the user.
+ *
+ * Design principles:
+ * - Only used in command layer (not in domain/infrastructure)
+ * - Formats messages with chalk for better UX
+ * - Includes timestamps for debugging
+ * - Can be swapped with other implementations (file, remote, etc.)
+ *
+ * @example
+ * ```typescript
+ * const logger = new ConsoleLogger({ verbose: true });
+ * logger.info('Server started', { port: 3000 });
+ * // Output: [2025-01-26 10:30:45] INFO: Server started {"port":3000}
+ * ```
+ */
+export declare class ConsoleLogger implements ILogger {
+    private readonly verbose;
+    /**
+     * Create a new console logger.
+     *
+     * @param options - Logger options
+     * @param options.verbose - Enable verbose (debug) output
+     */
+    constructor(options?: {
+        verbose?: boolean;
+    });
+    debug(message: string, context?: Record<string, unknown>): void;
+    error(message: string, context?: Record<string, unknown>): void;
+    info(message: string, context?: Record<string, unknown>): void;
+    warn(message: string, context?: Record<string, unknown>): void;
+    /**
+     * Format current timestamp for log messages.
+     *
+     * @returns Formatted timestamp string (YYYY-MM-DD HH:MM:SS)
+     */
+    private formatTimestamp;
+}

package/dist/infra/cipher/logger/console-logger.js

@@ -0,0 +1,63 @@
+import chalk from 'chalk';
+/**
+ * Console-based logger implementation.
+ *
+ * This is a concrete logger that writes to the console with color formatting.
+ * Used at the command/presentation layer to display logs to the user.
+ *
+ * Design principles:
+ * - Only used in command layer (not in domain/infrastructure)
+ * - Formats messages with chalk for better UX
+ * - Includes timestamps for debugging
+ * - Can be swapped with other implementations (file, remote, etc.)
+ *
+ * @example
+ * ```typescript
+ * const logger = new ConsoleLogger({ verbose: true });
+ * logger.info('Server started', { port: 3000 });
+ * // Output: [2025-01-26 10:30:45] INFO: Server started {"port":3000}
+ * ```
+ */
+export class ConsoleLogger {
+    verbose;
+    /**
+     * Create a new console logger.
+     *
+     * @param options - Logger options
+     * @param options.verbose - Enable verbose (debug) output
+     */
+    constructor(options = {}) {
+        this.verbose = options.verbose ?? false;
+    }
+    debug(message, context) {
+        if (!this.verbose)
+            return;
+        const timestamp = this.formatTimestamp();
+        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+        console.log(chalk.gray(`[${timestamp}] DEBUG: ${message}${contextStr}`));
+    }
+    error(message, context) {
+        const timestamp = this.formatTimestamp();
+        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+        console.error(chalk.red(`[${timestamp}] ERROR: ${message}${contextStr}`));
+    }
+    info(message, context) {
+        const timestamp = this.formatTimestamp();
+        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+        console.log(chalk.cyan(`[${timestamp}] INFO: ${message}${contextStr}`));
+    }
+    warn(message, context) {
+        const timestamp = this.formatTimestamp();
+        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+        console.warn(chalk.yellow(`[${timestamp}] WARN: ${message}${contextStr}`));
+    }
+    /**
+     * Format current timestamp for log messages.
+     *
+     * @returns Formatted timestamp string (YYYY-MM-DD HH:MM:SS)
+     */
+    formatTimestamp() {
+        const now = new Date();
+        return now.toISOString().replace('T', ' ').slice(0, 19);
+    }
+}

package/dist/infra/cipher/logger/event-based-logger.d.ts

@@ -0,0 +1,54 @@
+import type { ILogger } from '../../../core/interfaces/cipher/i-logger.js';
+import type { AgentEventBus } from '../events/event-emitter.js';
+/**
+ * Event-based logger that emits cipher:log events instead of writing directly.
+ *
+ * This follows Clean Architecture by decoupling logging from infrastructure.
+ * The logger emits events that can be handled by any listener (console, file, remote, etc.).
+ *
+ * Design principles:
+ * - Domain/infrastructure layers use this logger
+ * - Events are emitted to AgentEventBus
+ * - Command layer listens to events and decides how to handle them
+ * - Supports optional source and sessionId for context tracking
+ *
+ * @example
+ * ```typescript
+ * const logger = new EventBasedLogger(agentBus, 'MyService');
+ * logger.info('User logged in', { userId: '123' });
+ * // Emits: cipher:log { level: 'info', message: '...', source: 'MyService', ... }
+ * ```
+ */
+export declare class EventBasedLogger implements ILogger {
+    private readonly eventBus;
+    private readonly source?;
+    private readonly sessionId?;
+    /**
+     * Create a new event-based logger.
+     *
+     * @param eventBus - Agent event bus to emit log events to
+     * @param source - Optional source identifier (e.g., class name)
+     * @param sessionId - Optional session ID for session-scoped logs
+     */
+    constructor(eventBus: AgentEventBus, source?: string | undefined, sessionId?: string | undefined);
+    debug(message: string, context?: Record<string, unknown>): void;
+    error(message: string, context?: Record<string, unknown>): void;
+    info(message: string, context?: Record<string, unknown>): void;
+    warn(message: string, context?: Record<string, unknown>): void;
+    /**
+     * Create a child logger with a different session ID.
+     * Useful for creating session-scoped loggers.
+     *
+     * @param sessionId - Session ID for the child logger
+     * @returns New logger instance with the specified session ID
+     */
+    withSessionId(sessionId: string): EventBasedLogger;
+    /**
+     * Create a child logger with a different source.
+     * Useful for creating component-scoped loggers.
+     *
+     * @param source - Source identifier for the child logger
+     * @returns New logger instance with the specified source
+     */
+    withSource(source: string): EventBasedLogger;
+}