byterover-cli 0.4.1 → 1.0.1

package/dist/infra/cipher/llm/stream-processor.js

@@ -0,0 +1,276 @@
+/**
+ * Stream Processor
+ *
+ * Handles granular LLM stream processing following the OpenCode pattern.
+ * Provides real-time event emission for streaming text, tool calls, and step tracking.
+ *
+ * Key features:
+ * - Delta-based text updates for responsive UI
+ * - Tool call lifecycle tracking (pending → running → completed/error)
+ * - Step-level cost and token tracking
+ * - Part creation and updates with unique IDs
+ */
+/**
+ * Stream Processor class.
+ *
+ * Processes streaming events from LLM providers and emits granular events
+ * for real-time UI updates. Follows the OpenCode pattern of part-based
+ * message construction with delta updates.
+ *
+ * @example
+ * ```typescript
+ * const processor = new StreamProcessor()
+ *
+ * const state = await processor.process(streamEvents, {
+ *   eventBus: sessionEventBus,
+ *   generateId: () => crypto.randomUUID(),
+ *   sessionId: 'session-123',
+ * })
+ *
+ * console.log('Accumulated text:', state.textContent)
+ * console.log('Parts created:', state.parts.length)
+ * ```
+ */
+export class StreamProcessor {
+    /**
+     * Process a stream of events and emit granular updates.
+     *
+     * @param stream - Async iterable of stream events
+     * @param context - Processing context with event bus and utilities
+     * @returns Final processor state with accumulated parts
+     */
+    async process(stream, context) {
+        const state = {
+            currentStepIndex: 0,
+            parts: [],
+            textContent: '',
+            toolParts: new Map(),
+        };
+        for await (const event of stream) {
+            await this.handleEvent(event, state, context);
+        }
+        return state;
+    }
+    /**
+     * Finalize text part if there's accumulated content.
+     */
+    finalizeTextPart(state, context) {
+        if (state.textContent.length > 0) {
+            const textPart = {
+                text: state.textContent,
+                type: 'text',
+            };
+            state.parts.push(textPart);
+            // Emit final chunk
+            context.eventBus.emit('llmservice:chunk', {
+                content: '',
+                isComplete: true,
+                type: 'text',
+            });
+        }
+    }
+    /**
+     * Handle a single stream event.
+     */
+    async handleEvent(event, state, context) {
+        switch (event.type) {
+            case 'finish': {
+                // Finalize any pending text part
+                this.finalizeTextPart(state, context);
+                break;
+            }
+            case 'reasoning-delta': {
+                // Emit reasoning chunk for UI streaming
+                context.eventBus.emit('llmservice:chunk', {
+                    content: event.delta,
+                    type: 'reasoning',
+                });
+                break;
+            }
+            case 'step-finish': {
+                this.handleStepFinish({
+                    cost: event.cost,
+                    finishReason: event.finishReason,
+                    stepIndex: event.stepIndex,
+                    tokens: event.tokens,
+                }, state, context);
+                break;
+            }
+            case 'step-start': {
+                this.handleStepStart(event.stepIndex, state, context);
+                break;
+            }
+            case 'text-delta': {
+                this.handleTextDelta(event.delta, state, context);
+                break;
+            }
+            case 'tool-call-complete': {
+                this.handleToolCallComplete(event.callId, event.output, state);
+                break;
+            }
+            case 'tool-call-error': {
+                this.handleToolCallError(event.callId, event.error, state);
+                break;
+            }
+            case 'tool-call-input': {
+                this.handleToolCallInput(event.callId, event.input, state);
+                break;
+            }
+            case 'tool-call-running': {
+                this.handleToolCallRunning(event.callId, state);
+                break;
+            }
+            case 'tool-call-start': {
+                this.handleToolCallStart(event.callId, event.toolName, state, context);
+                break;
+            }
+        }
+    }
+    /**
+     * Handle step finish event.
+     */
+    handleStepFinish(options, state, context) {
+        const stepFinishPart = {
+            cost: options.cost,
+            finishReason: options.finishReason,
+            id: context.generateId(),
+            stepIndex: options.stepIndex,
+            timestamp: Date.now(),
+            tokens: options.tokens,
+            type: 'step_finish',
+        };
+        state.parts.push(stepFinishPart);
+        // Emit step finished event
+        context.eventBus.emit('step:finished', {
+            cost: options.cost,
+            finishReason: options.finishReason,
+            stepIndex: options.stepIndex,
+            tokens: options.tokens,
+        });
+    }
+    /**
+     * Handle step start event.
+     */
+    handleStepStart(stepIndex, state, context) {
+        state.currentStepIndex = stepIndex;
+        const stepStartPart = {
+            id: context.generateId(),
+            stepIndex,
+            timestamp: Date.now(),
+            type: 'step_start',
+        };
+        state.parts.push(stepStartPart);
+        // Emit step started event
+        context.eventBus.emit('step:started', {
+            stepIndex,
+        });
+    }
+    /**
+     * Handle text delta - accumulate and emit.
+     */
+    handleTextDelta(delta, state, context) {
+        state.textContent += delta;
+        // Emit chunk with delta for real-time UI update
+        context.eventBus.emit('llmservice:chunk', {
+            content: delta,
+            type: 'text',
+        });
+    }
+    /**
+     * Handle tool call completion.
+     */
+    handleToolCallComplete(callId, output, state) {
+        const toolPart = state.toolParts.get(callId);
+        if (toolPart && toolPart.state.status === 'running') {
+            const startTime = toolPart.state.startedAt;
+            toolPart.state = {
+                input: toolPart.state.input,
+                output,
+                status: 'completed',
+                time: {
+                    end: Date.now(),
+                    start: startTime,
+                },
+            };
+        }
+    }
+    /**
+     * Handle tool call error.
+     */
+    handleToolCallError(callId, error, state) {
+        const toolPart = state.toolParts.get(callId);
+        if (toolPart) {
+            const startTime = toolPart.state.status === 'running' ? toolPart.state.startedAt : Date.now();
+            toolPart.state = {
+                error,
+                input: toolPart.state.input,
+                status: 'error',
+                time: {
+                    end: Date.now(),
+                    start: startTime,
+                },
+            };
+        }
+    }
+    /**
+     * Handle tool call input received.
+     */
+    handleToolCallInput(callId, input, state) {
+        const toolPart = state.toolParts.get(callId);
+        if (toolPart && toolPart.state.status === 'pending') {
+            toolPart.state = {
+                input,
+                status: 'pending',
+            };
+        }
+    }
+    /**
+     * Handle tool call transition to running.
+     */
+    handleToolCallRunning(callId, state) {
+        const toolPart = state.toolParts.get(callId);
+        if (toolPart && toolPart.state.status === 'pending') {
+            toolPart.state = {
+                input: toolPart.state.input,
+                startedAt: Date.now(),
+                status: 'running',
+            };
+        }
+    }
+    /**
+     * Handle tool call start - create pending tool part.
+     */
+    handleToolCallStart(callId, toolName, state, context) {
+        // Finalize any pending text before tool call
+        this.finalizeTextPart(state, context);
+        state.textContent = '';
+        const toolPart = {
+            callId,
+            state: {
+                input: {},
+                status: 'pending',
+            },
+            toolName,
+            type: 'tool',
+        };
+        state.toolParts.set(callId, toolPart);
+        state.parts.push(toolPart);
+        // Emit tool call event
+        context.eventBus.emit('llmservice:toolCall', {
+            args: {},
+            callId,
+            toolName,
+        });
+    }
+}
+/**
+ * Helper to create a unique ID generator.
+ */
+export function createIdGenerator() {
+    let counter = 0;
+    return () => `part-${Date.now()}-${++counter}`;
+}
+/**
+ * Singleton stream processor instance.
+ */
+export const streamProcessor = new StreamProcessor();

package/dist/infra/cipher/llm/tokenizers/claude-tokenizer.d.ts

@@ -1,44 +1,37 @@
 import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.js';
 /**
- * Tokenizer for Anthropic Claude models - CURRENTLY USING APPROXIMATION.
+ * Tokenizer for Anthropic Claude models.
  *
- * This implementation uses a character-based approximation rather than
- * accurate token counting. This is a temporary solution due to the
- * complexity of integrating Anthropic's token counting API.
+ * Uses the LLM registry for model-specific character-per-token ratios,
+ * providing better estimation accuracy across different Claude models.
  *
- * Claude models use a tokenizer similar to GPT's, with approximately
- * 3.5-4 characters per token for English text. We use 3.5 as a middle ground.
+ * This implementation uses a character-based approximation rather than
+ * accurate token counting. The ratio is now model-aware via the registry.
  *
  * TODO: Consider these improvements:
- * 1. Use Anthropic's official token counting API (requires async handling or separate service)
+ * 1. Use Anthropic's official token counting API (requires async handling)
  * 2. Implement a WASM-based tokenizer for accurate synchronous counting
  * 3. Cache token counts for frequently used text
- * 4. Model-specific adjustments based on actual Claude tokenization patterns
  *
  * Reference: https://docs.anthropic.com/en/docs/about-claude/models
  */
 export declare class ClaudeTokenizer implements ITokenizer {
-    private readonly modelName;
+    private readonly charsPerToken;
     /**
      * Creates a new Claude tokenizer instance.
      *
-     * @param model - The Claude model name (e.g., 'claude-3-5-sonnet-20241022', 'claude-3-opus-20240229')
-     *                Currently not used for approximation, but stored for future improvements
+     * @param model - The Claude model name (e.g., 'claude-sonnet-4-20250514', 'claude-3-5-sonnet-20241022')
+     *                Used to look up model-specific token ratio from registry
      */
     constructor(model: string);
     /**
      * Approximates the token count for Anthropic Claude models.
      *
-     * Uses a character-based approximation: ~3.5 characters per token.
-     * This is based on Anthropic's documentation and common patterns for
-     * English text with Claude's tokenizer.
+     * Uses model-specific character-per-token ratio from the LLM registry.
+     * Default for Claude models is ~3.5 characters per token.
      *
-     * IMPORTANT: This is NOT accurate for Claude models and should be replaced
-     * with a proper implementation when possible. The actual token count can vary
-     * significantly based on:
-     * - Language (non-English text may have different ratios)
-     * - Content type (code vs prose)
-     * - Special characters and formatting
+     * IMPORTANT: This is still an approximation. The actual token count can vary
+     * based on language, content type (code vs prose), and special characters.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens

package/dist/infra/cipher/llm/tokenizers/claude-tokenizer.js

@@ -1,45 +1,40 @@
+import { DEFAULT_CHARS_PER_TOKEN, getCharsPerToken } from '../../../../core/domain/cipher/llm/index.js';
 /**
- * Tokenizer for Anthropic Claude models - CURRENTLY USING APPROXIMATION.
+ * Tokenizer for Anthropic Claude models.
  *
- * This implementation uses a character-based approximation rather than
- * accurate token counting. This is a temporary solution due to the
- * complexity of integrating Anthropic's token counting API.
+ * Uses the LLM registry for model-specific character-per-token ratios,
+ * providing better estimation accuracy across different Claude models.
  *
- * Claude models use a tokenizer similar to GPT's, with approximately
- * 3.5-4 characters per token for English text. We use 3.5 as a middle ground.
+ * This implementation uses a character-based approximation rather than
+ * accurate token counting. The ratio is now model-aware via the registry.
  *
  * TODO: Consider these improvements:
- * 1. Use Anthropic's official token counting API (requires async handling or separate service)
+ * 1. Use Anthropic's official token counting API (requires async handling)
  * 2. Implement a WASM-based tokenizer for accurate synchronous counting
  * 3. Cache token counts for frequently used text
- * 4. Model-specific adjustments based on actual Claude tokenization patterns
  *
  * Reference: https://docs.anthropic.com/en/docs/about-claude/models
  */
 export class ClaudeTokenizer {
-    modelName;
+    charsPerToken;
     /**
      * Creates a new Claude tokenizer instance.
      *
-     * @param model - The Claude model name (e.g., 'claude-3-5-sonnet-20241022', 'claude-3-opus-20240229')
-     *                Currently not used for approximation, but stored for future improvements
+     * @param model - The Claude model name (e.g., 'claude-sonnet-4-20250514', 'claude-3-5-sonnet-20241022')
+     *                Used to look up model-specific token ratio from registry
      */
     constructor(model) {
-        this.modelName = model;
+        // Look up model-specific ratio from registry, fallback to default
+        this.charsPerToken = getCharsPerToken('claude', model) ?? DEFAULT_CHARS_PER_TOKEN;
     }
     /**
      * Approximates the token count for Anthropic Claude models.
      *
-     * Uses a character-based approximation: ~3.5 characters per token.
-     * This is based on Anthropic's documentation and common patterns for
-     * English text with Claude's tokenizer.
+     * Uses model-specific character-per-token ratio from the LLM registry.
+     * Default for Claude models is ~3.5 characters per token.
      *
-     * IMPORTANT: This is NOT accurate for Claude models and should be replaced
-     * with a proper implementation when possible. The actual token count can vary
-     * significantly based on:
-     * - Language (non-English text may have different ratios)
-     * - Content type (code vs prose)
-     * - Special characters and formatting
+     * IMPORTANT: This is still an approximation. The actual token count can vary
+     * based on language, content type (code vs prose), and special characters.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens
@@ -48,8 +43,6 @@ export class ClaudeTokenizer {
         if (!text) {
             return 0;
         }
-        // Approximation: ~3.5 characters per token
-        // Claude's tokenizer is similar to GPT's, typically more efficient than 4 chars/token
-        return Math.ceil(text.length / 3.5);
+        return Math.ceil(text.length / this.charsPerToken);
     }
 }

package/dist/infra/cipher/llm/tokenizers/gemini-tokenizer.d.ts

@@ -1,34 +1,35 @@
 import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.js';
 /**
- * Tokenizer for Google Gemini models - CURRENTLY USING APPROXIMATION.
+ * Tokenizer for Google Gemini models.
+ *
+ * Uses the LLM registry for model-specific character-per-token ratios,
+ * providing better estimation accuracy across different Gemini models.
  *
  * This implementation uses a character-based approximation rather than
- * accurate token counting. This is a temporary solution due to the
- * asynchronous nature of the official Gemini countTokens API.
+ * accurate token counting. The ratio is now model-aware via the registry.
  *
  * TODO: Consider these improvements:
  * 1. Use the official @google/genai countTokens method (requires async handling)
  * 2. Implement a WASM-based tokenizer for accurate synchronous counting
  * 3. Cache token counts for frequently used text
- * 4. Model-specific adjustments based on actual Gemini tokenization patterns
  */
 export declare class GeminiTokenizer implements ITokenizer {
-    private readonly modelName;
+    private readonly charsPerToken;
     /**
      * Creates a new Gemini tokenizer instance.
      *
-     * @param model - The Gemini model name (e.g., 'gemini-2.5-flash', 'gemini-pro')
-     *                Currently not used for approximation, but stored for future improvements
+     * @param model - The Gemini model name (e.g., 'gemini-2.0-flash', 'gemini-1.5-pro')
+     *                Used to look up model-specific token ratio from registry
      */
     constructor(model: string);
     /**
      * Approximates the token count for Google Gemini models.
      *
-     * Uses a rough character-based approximation: ~4 characters per token.
-     * This is based on common estimates for English text with modern tokenizers.
+     * Uses model-specific character-per-token ratio from the LLM registry.
+     * Default for Gemini models is ~4 characters per token.
      *
-     * IMPORTANT: This is NOT accurate for Gemini models and should be replaced
-     * with a proper implementation when possible.
+     * IMPORTANT: This is still an approximation. The actual token count can vary
+     * based on language, content type (code vs prose), and special characters.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens

package/dist/infra/cipher/llm/tokenizers/gemini-tokenizer.js

@@ -1,35 +1,38 @@
+import { DEFAULT_CHARS_PER_TOKEN, getCharsPerToken } from '../../../../core/domain/cipher/llm/index.js';
 /**
- * Tokenizer for Google Gemini models - CURRENTLY USING APPROXIMATION.
+ * Tokenizer for Google Gemini models.
+ *
+ * Uses the LLM registry for model-specific character-per-token ratios,
+ * providing better estimation accuracy across different Gemini models.
  *
  * This implementation uses a character-based approximation rather than
- * accurate token counting. This is a temporary solution due to the
- * asynchronous nature of the official Gemini countTokens API.
+ * accurate token counting. The ratio is now model-aware via the registry.
  *
  * TODO: Consider these improvements:
  * 1. Use the official @google/genai countTokens method (requires async handling)
  * 2. Implement a WASM-based tokenizer for accurate synchronous counting
  * 3. Cache token counts for frequently used text
- * 4. Model-specific adjustments based on actual Gemini tokenization patterns
  */
 export class GeminiTokenizer {
-    modelName;
+    charsPerToken;
     /**
      * Creates a new Gemini tokenizer instance.
      *
-     * @param model - The Gemini model name (e.g., 'gemini-2.5-flash', 'gemini-pro')
-     *                Currently not used for approximation, but stored for future improvements
+     * @param model - The Gemini model name (e.g., 'gemini-2.0-flash', 'gemini-1.5-pro')
+     *                Used to look up model-specific token ratio from registry
      */
     constructor(model) {
-        this.modelName = model;
+        // Look up model-specific ratio from registry, fallback to default
+        this.charsPerToken = getCharsPerToken('gemini', model) ?? DEFAULT_CHARS_PER_TOKEN;
     }
     /**
      * Approximates the token count for Google Gemini models.
      *
-     * Uses a rough character-based approximation: ~4 characters per token.
-     * This is based on common estimates for English text with modern tokenizers.
+     * Uses model-specific character-per-token ratio from the LLM registry.
+     * Default for Gemini models is ~4 characters per token.
      *
-     * IMPORTANT: This is NOT accurate for Gemini models and should be replaced
-     * with a proper implementation when possible.
+     * IMPORTANT: This is still an approximation. The actual token count can vary
+     * based on language, content type (code vs prose), and special characters.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens
@@ -38,8 +41,6 @@ export class GeminiTokenizer {
         if (!text) {
             return 0;
         }
-        // Rough approximation: ~4 characters per token
-        // This is a simplified heuristic and varies by language and content type
-        return Math.ceil(text.length / 4);
+        return Math.ceil(text.length / this.charsPerToken);
     }
 }

package/dist/infra/cipher/llm/tokenizers/openrouter-tokenizer.d.ts

@@ -3,18 +3,26 @@ import type { ITokenizer } from '../../../../core/interfaces/cipher/i-tokenizer.
  * Tokenizer for OpenRouter API.
  *
  * OpenRouter supports multiple model providers (OpenAI, Anthropic, etc.),
- * each with different tokenization schemes. This tokenizer provides a
- * generic approximation that works reasonably well across different models.
+ * each with different tokenization schemes. This tokenizer uses the LLM
+ * registry to look up model-specific character-per-token ratios when available.
  *
- * For more accurate token counting, model-specific tokenizers could be
- * implemented based on the selected model's provider.
+ * For models not in the registry, falls back to a generic approximation
+ * that works reasonably well across different model providers.
  */
 export declare class OpenRouterTokenizer implements ITokenizer {
+    private readonly charsPerToken;
     /**
-     * Approximates token count using a character-based heuristic.
+     * Creates a new OpenRouter tokenizer instance.
      *
-     * Uses ~4 characters per token, which is a common approximation
-     * for modern tokenizers (GPT, Claude, Gemini, etc.).
+     * @param model - The OpenRouter model name (e.g., 'anthropic/claude-sonnet-4', 'openai/gpt-4o')
+     *                Used to look up model-specific token ratio from registry
+     */
+    constructor(model?: string);
+    /**
+     * Approximates token count using model-specific character-per-token ratio.
+     *
+     * Uses the LLM registry when available, otherwise defaults to ~4 characters
+     * per token which is a common approximation for modern tokenizers.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens

package/dist/infra/cipher/llm/tokenizers/openrouter-tokenizer.js

@@ -1,19 +1,33 @@
+import { DEFAULT_CHARS_PER_TOKEN, getCharsPerToken } from '../../../../core/domain/cipher/llm/index.js';
 /**
  * Tokenizer for OpenRouter API.
  *
  * OpenRouter supports multiple model providers (OpenAI, Anthropic, etc.),
- * each with different tokenization schemes. This tokenizer provides a
- * generic approximation that works reasonably well across different models.
+ * each with different tokenization schemes. This tokenizer uses the LLM
+ * registry to look up model-specific character-per-token ratios when available.
  *
- * For more accurate token counting, model-specific tokenizers could be
- * implemented based on the selected model's provider.
+ * For models not in the registry, falls back to a generic approximation
+ * that works reasonably well across different model providers.
  */
 export class OpenRouterTokenizer {
+    charsPerToken;
     /**
-     * Approximates token count using a character-based heuristic.
+     * Creates a new OpenRouter tokenizer instance.
      *
-     * Uses ~4 characters per token, which is a common approximation
-     * for modern tokenizers (GPT, Claude, Gemini, etc.).
+     * @param model - The OpenRouter model name (e.g., 'anthropic/claude-sonnet-4', 'openai/gpt-4o')
+     *                Used to look up model-specific token ratio from registry
+     */
+    constructor(model) {
+        // Look up model-specific ratio from registry, fallback to default
+        this.charsPerToken = model
+            ? (getCharsPerToken('openrouter', model) ?? DEFAULT_CHARS_PER_TOKEN)
+            : DEFAULT_CHARS_PER_TOKEN;
+    }
+    /**
+     * Approximates token count using model-specific character-per-token ratio.
+     *
+     * Uses the LLM registry when available, otherwise defaults to ~4 characters
+     * per token which is a common approximation for modern tokenizers.
      *
      * @param text - Text content to count tokens for
      * @returns Approximate number of tokens
@@ -22,9 +36,7 @@ export class OpenRouterTokenizer {
         if (!text) {
             return 0;
         }
-        // Simple heuristic: ~4 characters per token
-        // This works reasonably well for most OpenRouter models
-        return Math.ceil(text.length / 4);
+        return Math.ceil(text.length / this.charsPerToken);
     }
     /**
      * Gets the provider name for this tokenizer.