byterover-cli 0.4.0 → 1.0.0

package/dist/infra/cipher/llm/sanitization/tool-sanitizer.js

@@ -0,0 +1,398 @@
+/**
+ * Tool output sanitization pipeline.
+ *
+ * This module provides a comprehensive sanitization pipeline for tool outputs:
+ * 1. normalize() - Convert arbitrary tool output to MessagePart[]
+ * 2. persistMedia() - Store large media as blob references
+ * 3. applySanitization() - Apply final cleanup (truncation, etc.)
+ *
+ */
+import { DEFAULT_MAX_INLINE_MEDIA_BYTES, DEFAULT_TEXT_TRUNCATION, inferResourceKind, shouldAlwaysBlob, } from '../../../../core/interfaces/cipher/sanitization-types.js';
+import { base64LengthToBytes, formatByteSize, isLikelyBase64String, parseDataUri, } from './base64-utils.js';
+/**
+ * Tool output sanitizer.
+ *
+ * Provides a pipeline for sanitizing tool outputs:
+ * - Normalizes arbitrary output formats to MessagePart[]
+ * - Stores large media in blob storage with lazy references
+ * - Applies text truncation and cleanup
+ */
+export class ToolSanitizer {
+    blobStorage;
+    maxInlineMediaBytes;
+    maxTextLength;
+    constructor(options) {
+        this.blobStorage = options?.blobStorage;
+        this.maxInlineMediaBytes = options?.maxInlineMediaBytes ?? DEFAULT_MAX_INLINE_MEDIA_BYTES;
+        this.maxTextLength = options?.maxTextLength ?? DEFAULT_TEXT_TRUNCATION.maxLength;
+    }
+    /**
+     * Step 3: Apply final sanitization (truncation, etc.).
+     */
+    applySanitization(parts, maxTextLength) {
+        const maxLen = maxTextLength ?? this.maxTextLength;
+        return parts.map((part) => {
+            if (part.type === 'text' && part.text.length > maxLen) {
+                return {
+                    text: this.truncateText(part.text, maxLen),
+                    type: 'text',
+                };
+            }
+            return part;
+        });
+    }
+    /**
+     * Step 1: Normalize arbitrary tool output to MessagePart array.
+     */
+    normalize(result) {
+        const detectedMimeTypes = [];
+        let hasBinaryContent = false;
+        const parts = this.normalizeValue(result, detectedMimeTypes, (isBinary) => {
+            if (isBinary)
+                hasBinaryContent = true;
+        });
+        return {
+            detectedMimeTypes: [...new Set(detectedMimeTypes)],
+            hasBinaryContent,
+            parts,
+        };
+    }
+    /**
+     * Step 2: Persist large media to blob storage.
+     */
+    async persistMedia(parts, toolName, toolCallId, maxInlineBytes) {
+        const threshold = maxInlineBytes ?? this.maxInlineMediaBytes;
+        const resources = [];
+        const persistedParts = [];
+        let blobsCreated = 0;
+        let bytesStored = 0;
+        const persistPromises = parts.map(async (part) => {
+            if (part.type === 'image' && typeof part.image === 'string') {
+                return this.persistImageIfNeeded(part, toolName, toolCallId, threshold);
+            }
+            if (part.type === 'file' && typeof part.data === 'string') {
+                return this.persistFileIfNeeded(part, toolName, toolCallId, threshold);
+            }
+            return { part };
+        });
+        const persistedResults = await Promise.all(persistPromises);
+        for (const persisted of persistedResults) {
+            persistedParts.push(persisted.part);
+            if (persisted.resource) {
+                resources.push(persisted.resource);
+                blobsCreated++;
+                bytesStored += persisted.resource.size ?? 0;
+            }
+        }
+        return { blobsCreated, bytesStored, parts: persistedParts, resources };
+    }
+    /**
+     * Sanitize a tool result through the full pipeline.
+     */
+    async sanitize(result, options) {
+        // Step 1: Normalize to MessagePart[]
+        const normalized = this.normalize(result);
+        // Step 2: Persist large media to blob storage
+        const persisted = await this.persistMedia(normalized.parts, options.toolName, options.toolCallId, options.maxInlineMediaBytes);
+        // Step 3: Apply final sanitization
+        const sanitizedParts = this.applySanitization(persisted.parts, options.maxTextLength);
+        return {
+            content: sanitizedParts,
+            ...(persisted.resources.length > 0 && { resources: persisted.resources }),
+            meta: {
+                success: options.success,
+                toolCallId: options.toolCallId,
+                toolName: options.toolName,
+                ...(options.durationMs !== undefined && { durationMs: options.durationMs }),
+            },
+        };
+    }
+    extractData(data) {
+        if (typeof data === 'string')
+            return data;
+        if (Buffer.isBuffer(data))
+            return data.toString('base64');
+        if (data instanceof Uint8Array)
+            return Buffer.from(data).toString('base64');
+        if (data instanceof ArrayBuffer)
+            return Buffer.from(new Uint8Array(data)).toString('base64');
+        return String(data);
+    }
+    generateBlobId(toolName, toolCallId) {
+        const sanitizedTool = toolName.replaceAll(/[^a-z0-9]/gi, '_').slice(0, 20);
+        const sanitizedCall = toolCallId.replaceAll(/[^a-z0-9]/gi, '_').slice(0, 10);
+        const timestamp = Date.now().toString(36);
+        const random = Math.random().toString(36).slice(2, 6);
+        return `${sanitizedTool}_${sanitizedCall}_${timestamp}_${random}`;
+    }
+    normalizeArray(array, detectedMimeTypes, onBinary) {
+        const parts = [];
+        for (const item of array) {
+            if (!item || typeof item !== 'object')
+                continue;
+            const obj = item;
+            const itemParts = this.normalizeArrayItem(obj, detectedMimeTypes, onBinary);
+            if (itemParts.length > 0) {
+                parts.push(...itemParts);
+            }
+        }
+        return parts.length > 0 ? parts : [{ text: JSON.stringify(array), type: 'text' }];
+    }
+    normalizeArrayItem(obj, detectedMimeTypes, onBinary) {
+        // MCP text content
+        if (obj.type === 'text' && typeof obj.text === 'string') {
+            return [{ text: obj.text, type: 'text' }];
+        }
+        // MCP image content
+        if (obj.type === 'image' && obj.data && obj.mimeType) {
+            const mimeType = obj.mimeType;
+            detectedMimeTypes.push(mimeType);
+            onBinary(true);
+            return [{
+                    image: obj.data,
+                    mimeType,
+                    type: 'image',
+                }];
+        }
+        // MCP resource content
+        if (obj.type === 'resource' && obj.resource) {
+            return this.normalizeResourceContent(obj.resource, detectedMimeTypes, onBinary);
+        }
+        // Nested object - recursively normalize
+        return this.normalizeValue(obj, detectedMimeTypes, onBinary);
+    }
+    normalizeObject(obj, detectedMimeTypes, onBinary) {
+        // Check for nested content array
+        if ('content' in obj && Array.isArray(obj.content)) {
+            return this.normalizeArray(obj.content, detectedMimeTypes, onBinary);
+        }
+        // { image, mimeType? }
+        if ('image' in obj) {
+            const mimeType = obj.mimeType || 'image/jpeg';
+            detectedMimeTypes.push(mimeType);
+            onBinary(true);
+            return [{
+                    image: this.extractData(obj.image),
+                    mimeType,
+                    type: 'image',
+                }];
+        }
+        // { data, mimeType }
+        if ('data' in obj && obj.mimeType) {
+            const mimeType = obj.mimeType;
+            detectedMimeTypes.push(mimeType);
+            onBinary(true);
+            const filePart = {
+                data: this.extractData(obj.data),
+                mimeType,
+                type: 'file',
+            };
+            if (obj.filename && typeof obj.filename === 'string') {
+                filePart.filename = obj.filename;
+            }
+            return [filePart];
+        }
+        // Generic object - sanitize and stringify
+        const cleaned = this.sanitizeDeepObject(obj);
+        return [{ text: JSON.stringify(cleaned, null, 2), type: 'text' }];
+    }
+    normalizeResourceContent(resource, detectedMimeTypes, onBinary) {
+        if (!resource.text && !resource.blob) {
+            return [];
+        }
+        const mimeType = resource.mimeType || 'application/octet-stream';
+        const data = (resource.blob || resource.text);
+        detectedMimeTypes.push(mimeType);
+        if (mimeType.startsWith('image/')) {
+            onBinary(true);
+            return [{ image: data, mimeType, type: 'image' }];
+        }
+        if (resource.blob || isLikelyBase64String(data)) {
+            onBinary(true);
+            const filePart = {
+                data,
+                mimeType,
+                type: 'file',
+            };
+            if (resource.title && typeof resource.title === 'string') {
+                filePart.filename = resource.title;
+            }
+            return [filePart];
+        }
+        return [{ text: data, type: 'text' }];
+    }
+    normalizeString(value, detectedMimeTypes, onBinary) {
+        // Check for data URI
+        const dataUri = parseDataUri(value);
+        if (dataUri) {
+            detectedMimeTypes.push(dataUri.mediaType);
+            onBinary(true);
+            if (dataUri.mediaType.startsWith('image/')) {
+                return [{ image: dataUri.base64, mimeType: dataUri.mediaType, type: 'image' }];
+            }
+            return [{ data: dataUri.base64, mimeType: dataUri.mediaType, type: 'file' }];
+        }
+        // Check for likely base64 blob
+        if (isLikelyBase64String(value)) {
+            onBinary(true);
+            // Can't determine MIME type, treat as binary
+            return [{ data: value, mimeType: 'application/octet-stream', type: 'file' }];
+        }
+        // Regular text
+        return [{ text: value, type: 'text' }];
+    }
+    normalizeValue(value, detectedMimeTypes, onBinary) {
+        // Null/undefined
+        if (value === null || value === undefined) {
+            return [{ text: '', type: 'text' }];
+        }
+        // String input
+        if (typeof value === 'string') {
+            return this.normalizeString(value, detectedMimeTypes, onBinary);
+        }
+        // Array input (MCP-style content)
+        if (Array.isArray(value)) {
+            return this.normalizeArray(value, detectedMimeTypes, onBinary);
+        }
+        // Object input
+        if (typeof value === 'object') {
+            return this.normalizeObject(value, detectedMimeTypes, onBinary);
+        }
+        // Primitive fallback
+        return [{ text: String(value), type: 'text' }];
+    }
+    async persistFileIfNeeded(part, toolName, toolCallId, threshold) {
+        const { data, filename, mimeType } = part;
+        const dataString = data;
+        const approxBytes = base64LengthToBytes(dataString.length);
+        const shouldPersist = shouldAlwaysBlob(mimeType) || approxBytes >= threshold;
+        if (!shouldPersist || !this.blobStorage) {
+            return { part };
+        }
+        const blobId = await this.storeAsBlob({
+            data: dataString,
+            filename,
+            mimeType,
+            toolCallId,
+            toolName,
+        });
+        if (!blobId) {
+            return { part };
+        }
+        return {
+            part: {
+                data: `@blob:${blobId}`,
+                ...(filename && { filename }),
+                mimeType,
+                type: 'file',
+            },
+            resource: {
+                ...(filename && { filename }),
+                kind: inferResourceKind(mimeType),
+                mimeType,
+                size: approxBytes,
+                uri: `blob:${blobId}`,
+            },
+        };
+    }
+    async persistImageIfNeeded(part, toolName, toolCallId, threshold) {
+        const data = part.image;
+        const mimeType = part.mimeType || 'image/jpeg';
+        const approxBytes = base64LengthToBytes(data.length);
+        const shouldPersist = shouldAlwaysBlob(mimeType) || approxBytes >= threshold;
+        if (!shouldPersist || !this.blobStorage) {
+            return { part };
+        }
+        const blobId = await this.storeAsBlob({
+            data,
+            mimeType,
+            toolCallId,
+            toolName,
+        });
+        if (!blobId) {
+            return { part };
+        }
+        return {
+            part: {
+                image: `@blob:${blobId}`,
+                mimeType,
+                type: 'image',
+            },
+            resource: {
+                kind: 'image',
+                mimeType,
+                size: approxBytes,
+                uri: `blob:${blobId}`,
+            },
+        };
+    }
+    /**
+     * Recursively sanitize object, replacing large base64 with placeholders.
+     */
+    sanitizeDeepObject(obj) {
+        if (obj === null || obj === undefined)
+            return obj;
+        if (typeof obj === 'string') {
+            if (isLikelyBase64String(obj)) {
+                const approxBytes = base64LengthToBytes(obj.length);
+                return `[binary data omitted ~${formatByteSize(approxBytes)}]`;
+            }
+            return obj;
+        }
+        if (Array.isArray(obj)) {
+            return obj.map((x) => this.sanitizeDeepObject(x));
+        }
+        if (typeof obj === 'object') {
+            const out = {};
+            for (const [k, v] of Object.entries(obj)) {
+                out[k] = this.sanitizeDeepObject(v);
+            }
+            return out;
+        }
+        return obj;
+    }
+    async storeAsBlob(options) {
+        if (!this.blobStorage)
+            return null;
+        try {
+            const blobId = this.generateBlobId(options.toolName, options.toolCallId);
+            const buffer = Buffer.from(options.data, 'base64');
+            await this.blobStorage.store(blobId, buffer, {
+                contentType: options.mimeType,
+                originalName: options.filename,
+                tags: { source: 'tool', toolCallId: options.toolCallId, toolName: options.toolName },
+            });
+            return blobId;
+        }
+        catch {
+            return null;
+        }
+    }
+    truncateText(text, maxLength) {
+        if (text.length <= maxLength)
+            return text;
+        const { headLength, placeholder, tailLength } = DEFAULT_TEXT_TRUNCATION;
+        const omittedCount = text.length - headLength - tailLength;
+        const head = text.slice(0, headLength);
+        const tail = text.slice(-tailLength);
+        const replacedPlaceholder = placeholder.replace('{count}', String(omittedCount));
+        return `${head}${replacedPlaceholder}${tail}`;
+    }
+}
+/**
+ * Create a ToolSanitizer instance.
+ */
+export function createToolSanitizer(options) {
+    return new ToolSanitizer(options);
+}
+/**
+ * Convenience function to sanitize a tool result with default options.
+ */
+export async function sanitizeToolResult(result, options) {
+    const sanitizer = new ToolSanitizer({
+        blobStorage: options.blobStorage,
+        maxInlineMediaBytes: options.maxInlineMediaBytes,
+        maxTextLength: options.maxTextLength,
+    });
+    return sanitizer.sanitize(result, options);
+}

package/dist/infra/cipher/llm/stream-processor.d.ts

@@ -0,0 +1,158 @@
+/**
+ * Stream Processor
+ *
+ * Handles granular LLM stream processing following the OpenCode pattern.
+ * Provides real-time event emission for streaming text, tool calls, and step tracking.
+ *
+ * Key features:
+ * - Delta-based text updates for responsive UI
+ * - Tool call lifecycle tracking (pending → running → completed/error)
+ * - Step-level cost and token tracking
+ * - Part creation and updates with unique IDs
+ */
+import type { StepTokenUsage } from '../../../core/domain/cipher/agent-events/types.js';
+import type { CompactionPart, PatchPart, RetryPart, SnapshotPart, StepFinishPart, StepStartPart, TextPart, ToolPart } from '../../../core/interfaces/cipher/message-types.js';
+import type { SessionEventBus } from '../events/event-emitter.js';
+/**
+ * Stream event types that the processor can handle.
+ */
+export type StreamEvent = {
+    callId: string;
+    error: string;
+    type: 'tool-call-error';
+} | {
+    callId: string;
+    input: Record<string, unknown>;
+    type: 'tool-call-input';
+} | {
+    callId: string;
+    output: string;
+    type: 'tool-call-complete';
+} | {
+    callId: string;
+    toolName: string;
+    type: 'tool-call-start';
+} | {
+    callId: string;
+    type: 'tool-call-running';
+} | {
+    cost: number;
+    finishReason: 'max_tokens' | 'stop' | 'tool_calls';
+    stepIndex: number;
+    tokens: StepTokenUsage;
+    type: 'step-finish';
+} | {
+    delta: string;
+    type: 'reasoning-delta';
+} | {
+    delta: string;
+    type: 'text-delta';
+} | {
+    stepIndex: number;
+    type: 'step-start';
+} | {
+    type: 'finish';
+};
+/**
+ * Context provided to the stream processor.
+ */
+export interface ProcessorContext {
+    /** Event bus for emitting events */
+    eventBus: SessionEventBus;
+    /** Function to generate unique IDs */
+    generateId: () => string;
+    /** Session ID for event context */
+    sessionId: string;
+}
+/**
+ * Accumulated state during stream processing.
+ */
+export interface ProcessorState {
+    /** Current step index */
+    currentStepIndex: number;
+    /** Parts created during processing */
+    parts: Array<CompactionPart | PatchPart | RetryPart | SnapshotPart | StepFinishPart | StepStartPart | TextPart | ToolPart>;
+    /** Accumulated text content */
+    textContent: string;
+    /** Tool parts indexed by call ID */
+    toolParts: Map<string, ToolPart>;
+}
+/**
+ * Stream Processor class.
+ *
+ * Processes streaming events from LLM providers and emits granular events
+ * for real-time UI updates. Follows the OpenCode pattern of part-based
+ * message construction with delta updates.
+ *
+ * @example
+ * ```typescript
+ * const processor = new StreamProcessor()
+ *
+ * const state = await processor.process(streamEvents, {
+ *   eventBus: sessionEventBus,
+ *   generateId: () => crypto.randomUUID(),
+ *   sessionId: 'session-123',
+ * })
+ *
+ * console.log('Accumulated text:', state.textContent)
+ * console.log('Parts created:', state.parts.length)
+ * ```
+ */
+export declare class StreamProcessor {
+    /**
+     * Process a stream of events and emit granular updates.
+     *
+     * @param stream - Async iterable of stream events
+     * @param context - Processing context with event bus and utilities
+     * @returns Final processor state with accumulated parts
+     */
+    process(stream: AsyncIterable<StreamEvent>, context: ProcessorContext): Promise<ProcessorState>;
+    /**
+     * Finalize text part if there's accumulated content.
+     */
+    private finalizeTextPart;
+    /**
+     * Handle a single stream event.
+     */
+    private handleEvent;
+    /**
+     * Handle step finish event.
+     */
+    private handleStepFinish;
+    /**
+     * Handle step start event.
+     */
+    private handleStepStart;
+    /**
+     * Handle text delta - accumulate and emit.
+     */
+    private handleTextDelta;
+    /**
+     * Handle tool call completion.
+     */
+    private handleToolCallComplete;
+    /**
+     * Handle tool call error.
+     */
+    private handleToolCallError;
+    /**
+     * Handle tool call input received.
+     */
+    private handleToolCallInput;
+    /**
+     * Handle tool call transition to running.
+     */
+    private handleToolCallRunning;
+    /**
+     * Handle tool call start - create pending tool part.
+     */
+    private handleToolCallStart;
+}
+/**
+ * Helper to create a unique ID generator.
+ */
+export declare function createIdGenerator(): () => string;
+/**
+ * Singleton stream processor instance.
+ */
+export declare const streamProcessor: StreamProcessor;