byterover-cli 0.4.0 → 1.0.0

package/dist/infra/cipher/storage/message-storage-service.js

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 import { COMPACTED_TOOL_OUTPUT_PLACEHOLDER } from '../../../core/domain/cipher/storage/message-storage-types.js';
+import { ToolPartFactory } from './tool-part-factory.js';
 /**
  * Service for granular message and part storage.
  *
@@ -19,6 +20,66 @@ export class MessageStorageService {
     constructor(keyStorage) {
         this.keyStorage = keyStorage;
     }
+    /**
+     * Add attachments to a completed tool part.
+     *
+     * @param messageId - ID of the message containing the part
+     * @param partId - ID of the tool part
+     * @param attachments - Attachments to add
+     */
+    async addToolPartAttachments(messageId, partId, attachments) {
+        await this.keyStorage.update(this.partKey(messageId, partId), (prev) => {
+            if (!prev)
+                throw new Error(`Part ${partId} not found`);
+            if (prev.type !== 'tool' || !prev.toolState) {
+                throw new Error(`Part ${partId} is not a tool part`);
+            }
+            const existingAttachments = prev.toolState.attachments ?? [];
+            const newAttachments = attachments.map((att) => ({
+                data: att.data,
+                filename: att.filename,
+                mime: att.mime,
+                type: att.type,
+            }));
+            return {
+                ...prev,
+                toolState: {
+                    ...prev.toolState,
+                    attachments: [...existingAttachments, ...newAttachments],
+                },
+            };
+        });
+    }
+    /**
+     * Create a tool part in pending state and add it to a message.
+     *
+     * @param options - Options for creating the tool part
+     * @returns The created StoredPart
+     */
+    async createToolPart(options) {
+        const { callId, input, messageId, sessionId, toolName } = options;
+        // Verify message exists
+        const message = await this.keyStorage.get(this.messageKey(sessionId, messageId));
+        if (!message) {
+            throw new Error(`Message ${messageId} not found in session ${sessionId}`);
+        }
+        // Create the tool part in pending state
+        const toolPart = ToolPartFactory.createPending(messageId, callId, toolName, input);
+        // Save the part
+        await this.keyStorage.set(this.partKey(messageId, toolPart.id), toolPart);
+        // Update message's partIds
+        const now = Date.now();
+        await this.keyStorage.update(this.messageKey(sessionId, messageId), (prev) => {
+            if (!prev)
+                throw new Error(`Message ${messageId} not found`);
+            return {
+                ...prev,
+                partIds: [...prev.partIds, toolPart.id],
+                updatedAt: now,
+            };
+        });
+        return toolPart;
+    }
     /**
      * Delete a session and all its messages and parts.
      */
@@ -51,6 +112,24 @@ export class MessageStorageService {
     async getSession(sessionId) {
         return this.keyStorage.get(this.sessionKey(sessionId));
     }
+    /**
+     * Get a tool part by call ID from a message.
+     *
+     * @param messageId - ID of the message
+     * @param callId - Call ID of the tool part
+     * @returns The tool part if found
+     */
+    async getToolPartByCallId(messageId, callId) {
+        const partKeys = await this.keyStorage.list(this.partPrefix(messageId));
+        for (const partKey of partKeys) {
+            // eslint-disable-next-line no-await-in-loop
+            const part = await this.keyStorage.get(partKey);
+            if (part?.type === 'tool' && part.toolState?.callId === callId) {
+                return part;
+            }
+        }
+        return undefined;
+    }
     /**
      * Check if a session exists in granular format.
      */
@@ -145,15 +224,20 @@ export class MessageStorageService {
     /**
      * Prune old tool outputs by marking them as compacted.
      * Keeps the most recent tool outputs up to the specified token limit.
+     *
+     * Turn-based protection: Protects tool outputs in the most recent N user turns.
+     * Minimum threshold: Only executes if the minimum token threshold can be saved.
      */
     async pruneToolOutputs(options) {
-        const { keepTokens = 40_000, sessionId } = options;
+        const { keepTokens = 40_000, minimumTokens = 20_000, protectedTurns = 2, sessionId } = options;
         const session = await this.getSession(sessionId);
         if (!session || !session.newestMessageId) {
             return { compactedCount: 0, tokensSaved: 0 };
         }
         // Collect all tool output parts, newest first
+        // Track user turns for turn-based protection
         const toolOutputParts = [];
+        let userTurnCount = 0;
         // Traverse messages from newest to oldest
         let currentMessageId = session.newestMessageId;
         while (currentMessageId) {
@@ -161,6 +245,15 @@ export class MessageStorageService {
             const storedMsg = await this.keyStorage.get(this.messageKey(sessionId, currentMessageId));
             if (!storedMsg)
                 break;
+            // Stop at existing compaction boundary
+            if (storedMsg.compactionBoundary)
+                break;
+            // Track user turns for turn-based protection
+            if (storedMsg.role === 'user') {
+                userTurnCount++;
+            }
+            // Determine if this message's tool outputs are protected
+            const isProtected = userTurnCount <= protectedTurns;
             // Collect tool output parts from this message
             for (const partId of storedMsg.partIds) {
                 // eslint-disable-next-line no-await-in-loop
@@ -169,33 +262,46 @@ export class MessageStorageService {
                     toolOutputParts.push({
                         key: this.partKey(storedMsg.id, partId),
                         part,
+                        protected: isProtected,
                     });
                 }
             }
             currentMessageId = storedMsg.prevMessageId;
         }
-        // Estimate tokens (rough: 1 token ≈ 4 chars)
+        // First pass: Calculate potential tokens saved (only from unprotected parts)
         let keptTokens = 0;
-        let compactedCount = 0;
-        let tokensSaved = 0;
-        const now = Date.now();
-        for (const { key, part } of toolOutputParts) {
+        let potentialTokensSaved = 0;
+        const partsToCompact = [];
+        for (const { key, part, protected: isProtected } of toolOutputParts) {
             const estimatedTokens = Math.ceil(part.content.length / 4);
-            if (keptTokens < keepTokens) {
+            if (isProtected || keptTokens < keepTokens) {
+                // Keep this part (either protected or within token limit)
                 keptTokens += estimatedTokens;
             }
             else {
-                // Mark this part as compacted
-                // eslint-disable-next-line no-await-in-loop
-                await this.keyStorage.set(key, {
-                    ...part,
-                    compactedAt: now,
-                    content: '', // Clear the content to save space
-                });
-                compactedCount++;
-                tokensSaved += estimatedTokens;
+                // This part is a candidate for compaction
+                partsToCompact.push({ key, part, tokens: estimatedTokens });
+                potentialTokensSaved += estimatedTokens;
             }
         }
+        // Check minimum threshold: only proceed if we can save enough tokens
+        if (potentialTokensSaved < minimumTokens) {
+            return { compactedCount: 0, tokensSaved: 0 };
+        }
+        // Second pass: Actually compact the parts
+        let compactedCount = 0;
+        let tokensSaved = 0;
+        const now = Date.now();
+        for (const { key, part, tokens } of partsToCompact) {
+            // eslint-disable-next-line no-await-in-loop
+            await this.keyStorage.set(key, {
+                ...part,
+                compactedAt: now,
+                content: '', // Clear the content to save space
+            });
+            compactedCount++;
+            tokensSaved += tokens;
+        }
         return { compactedCount, tokensSaved };
     }
     /**
@@ -324,46 +430,16 @@ export class MessageStorageService {
         if (parts.length > 0) {
             const contentParts = [];
             for (const part of parts) {
-                if (part.compactedAt) {
-                    // Show placeholder for compacted parts
-                    contentParts.push({
-                        text: part.type === 'tool_output' ? COMPACTED_TOOL_OUTPUT_PLACEHOLDER : '[Content cleared]',
-                        type: 'text',
-                    });
+                const converted = this.convertStoredPartToMessagePart(part);
+                if (converted.messagePart) {
+                    contentParts.push(converted.messagePart);
                 }
-                else {
-                    switch (part.type) {
-                        case 'file': {
-                            contentParts.push({
-                                data: part.content,
-                                filename: part.filename,
-                                mimeType: part.mimeType || 'application/octet-stream',
-                                type: 'file',
-                            });
-                            break;
-                        }
-                        case 'image': {
-                            contentParts.push({
-                                image: part.content,
-                                mimeType: part.mimeType,
-                                type: 'image',
-                            });
-                            break;
-                        }
-                        case 'text': {
-                            contentParts.push({ text: part.content, type: 'text' });
-                            break;
-                        }
-                        case 'tool_output': {
-                            // Tool output becomes text content for tool messages
-                            content = part.content;
-                            break;
-                        }
-                    }
+                if (converted.toolOutputContent !== undefined) {
+                    content = converted.toolOutputContent;
                 }
             }
-            // Only use content parts array if we have non-tool parts
-            if (contentParts.length > 0 && message.role === 'user') {
+            // Only use content parts array if we have non-tool parts or tool parts
+            if (contentParts.length > 0 && (message.role === 'user' || message.role === 'assistant')) {
                 content = contentParts;
             }
         }
@@ -385,6 +461,95 @@ export class MessageStorageService {
     toInternalMessages(messages) {
         return messages.map((msg) => this.toInternalMessage(msg));
     }
+    /**
+     * Update a tool part's state.
+     * Used for transitioning through pending → running → completed/error.
+     *
+     * @param messageId - ID of the message containing the part
+     * @param partId - ID of the tool part to update
+     * @param update - Partial tool state update
+     */
+    async updateToolPartState(messageId, partId, update) {
+        await this.keyStorage.update(this.partKey(messageId, partId), (prev) => {
+            if (!prev)
+                throw new Error(`Part ${partId} not found`);
+            if (prev.type !== 'tool' || !prev.toolState) {
+                throw new Error(`Part ${partId} is not a tool part`);
+            }
+            return {
+                ...prev,
+                toolState: {
+                    ...prev.toolState,
+                    ...update,
+                },
+            };
+        });
+    }
+    /**
+     * Convert a StoredPart to a MessagePart for reconstruction.
+     * Returns the converted part and optionally tool output content.
+     */
+    convertStoredPartToMessagePart(part) {
+        // Handle compacted parts
+        if (part.compactedAt) {
+            return {
+                messagePart: {
+                    text: part.type === 'tool_output' ? COMPACTED_TOOL_OUTPUT_PLACEHOLDER : '[Content cleared]',
+                    type: 'text',
+                },
+            };
+        }
+        // Convert based on part type
+        switch (part.type) {
+            case 'file': {
+                return {
+                    messagePart: {
+                        data: part.content,
+                        filename: part.filename,
+                        mimeType: part.mimeType || 'application/octet-stream',
+                        type: 'file',
+                    },
+                };
+            }
+            case 'image': {
+                return {
+                    messagePart: {
+                        image: part.content,
+                        mimeType: part.mimeType,
+                        type: 'image',
+                    },
+                };
+            }
+            case 'reasoning': {
+                return {
+                    messagePart: {
+                        summary: part.reasoningSummary,
+                        text: part.content,
+                        type: 'reasoning',
+                    },
+                };
+            }
+            case 'text': {
+                return {
+                    messagePart: { text: part.content, type: 'text' },
+                };
+            }
+            case 'tool': {
+                if (!part.toolState)
+                    return {};
+                return {
+                    messagePart: this.storedPartToToolPart(part),
+                };
+            }
+            case 'tool_output': {
+                // Tool output becomes text content for tool messages
+                return { toolOutputContent: part.content };
+            }
+            default: {
+                return {};
+            }
+        }
+    }
     /**
      * Extract parts from an InternalMessage.
      */
@@ -443,6 +608,17 @@ export class MessageStorageService {
                         });
                         break;
                     }
+                    case 'reasoning': {
+                        parts.push({
+                            content: part.text,
+                            createdAt: now,
+                            id: partId,
+                            messageId,
+                            reasoningSummary: part.summary,
+                            type: 'reasoning',
+                        });
+                        break;
+                    }
                     case 'text': {
                         parts.push({
                             content: part.text,
@@ -453,9 +629,14 @@ export class MessageStorageService {
                         });
                         break;
                     }
+                    case 'tool': {
+                        // Store tool part with full state
+                        parts.push(this.toolPartToStoredPart(part, partId, messageId, now));
+                        break;
+                    }
                 }
             }
-            // For user messages, content is reconstructed from parts
+            // For user/assistant messages, content is reconstructed from parts
             // but we can store a text preview
             const textParts = message.content.filter((p) => p.type === 'text');
             if (textParts.length > 0) {
@@ -520,6 +701,71 @@ export class MessageStorageService {
     sessionKey(sessionId) {
         return ['session', sessionId];
     }
+    /**
+     * Convert a stored part with tool state to a ToolPart.
+     */
+    storedPartToToolPart(part) {
+        if (!part.toolState || !part.toolName) {
+            throw new Error('Cannot convert non-tool part to ToolPart');
+        }
+        return {
+            callId: part.toolState.callId,
+            state: ToolPartFactory.toToolPartState(part.toolState),
+            toolName: part.toolName,
+            type: 'tool',
+        };
+    }
+    /**
+     * Convert a ToolPart to a StoredPart.
+     */
+    toolPartToStoredPart(part, partId, messageId, now) {
+        const { callId, state, toolName } = part;
+        // Build stored tool state from ToolPart state
+        const toolState = {
+            callId,
+            input: state.input,
+            status: state.status,
+        };
+        // Add state-specific fields
+        switch (state.status) {
+            case 'completed': {
+                toolState.attachments = state.attachments?.map((att) => ({
+                    data: att.data,
+                    filename: att.filename,
+                    mime: att.mime,
+                    type: att.type,
+                }));
+                toolState.completedAt = state.time.end;
+                toolState.output = state.output;
+                toolState.startedAt = state.time.start;
+                toolState.title = state.title;
+                break;
+            }
+            case 'error': {
+                toolState.completedAt = state.time.end;
+                toolState.error = state.error;
+                toolState.startedAt = state.time.start;
+                break;
+            }
+            case 'pending': {
+                // No additional fields for pending state
+                break;
+            }
+            case 'running': {
+                toolState.startedAt = state.startedAt;
+                break;
+            }
+        }
+        return {
+            content: '',
+            createdAt: now,
+            id: partId,
+            messageId,
+            toolName,
+            toolState,
+            type: 'tool',
+        };
+    }
 }
 /**
  * Factory function to create MessageStorageService.

package/dist/infra/cipher/storage/tool-part-factory.d.ts

@@ -0,0 +1,116 @@
+/**
+ * Factory for creating and managing tool parts with state machine.
+ *
+ * This module provides utilities for:
+ * - Creating tool parts in pending state
+ * - Transitioning tool parts through their lifecycle (pending → running → completed/error)
+ * - Marking tool parts as compacted for pruning
+ *
+ * Based on OpenCode's tool state machine pattern.
+ */
+import type { StoredPart, StoredToolState } from '../../../core/domain/cipher/storage/message-storage-types.js';
+import type { AttachmentPart } from '../../../core/interfaces/cipher/message-types.js';
+/**
+ * Options for creating a completed tool state.
+ */
+export interface ToolCompletionOptions {
+    /** Attachments produced by the tool (images, files) */
+    attachments?: AttachmentPart[];
+    /** Additional metadata about the execution */
+    metadata?: Record<string, unknown>;
+    /** Human-readable title for display */
+    title?: string;
+}
+/**
+ * Factory for creating and managing tool parts with state machine.
+ *
+ * Tool parts track the full lifecycle of a tool call:
+ * 1. Pending: Tool call received from LLM, not yet started
+ * 2. Running: Execution has started
+ * 3. Completed: Execution finished successfully with output
+ * 4. Error: Execution failed with error message
+ *
+ * This enables:
+ * - Real-time UI feedback during tool execution
+ * - Better conversation history representation
+ * - Efficient compaction (mark individual outputs as compacted)
+ */
+export declare const ToolPartFactory: {
+    /**
+     * Create a new tool part in pending state.
+     *
+     * @param messageId - ID of the message this part belongs to
+     * @param callId - Unique identifier for this tool call
+     * @param toolName - Name of the tool being called
+     * @param input - Parsed input arguments
+     * @returns StoredPart with pending tool state
+     */
+    createPending(messageId: string, callId: string, toolName: string, input: Record<string, unknown>): StoredPart;
+    /**
+     * Mark a tool part as compacted.
+     * Clears the output while preserving the state structure.
+     *
+     * @param part - The tool part to mark as compacted
+     * @returns Updated StoredPart with compactedAt timestamp
+     */
+    markCompacted(part: StoredPart): StoredPart;
+    /**
+     * Convert a StoredPart's tool state to the format used in message-types.ts.
+     * Useful for reconstructing InternalMessage from stored parts.
+     *
+     * @param toolState - The stored tool state
+     * @returns Tool state in the format expected by ToolPart
+     */
+    toToolPartState(toolState: StoredToolState): {
+        attachments?: AttachmentPart[];
+        compactedAt?: number;
+        input: Record<string, unknown>;
+        metadata?: Record<string, unknown>;
+        output: string;
+        status: "completed";
+        time: {
+            end: number;
+            start: number;
+        };
+        title?: string;
+    } | {
+        error: string;
+        input: Record<string, unknown>;
+        status: "error";
+        time: {
+            end: number;
+            start: number;
+        };
+    } | {
+        input: Record<string, unknown>;
+        startedAt: number;
+        status: "running";
+    } | {
+        input: Record<string, unknown>;
+        status: "pending";
+    };
+    /**
+     * Transition a tool part to completed state.
+     *
+     * @param part - The tool part to update
+     * @param output - Tool output content
+     * @param options - Additional options (title, metadata, attachments)
+     * @returns Updated StoredPart with completed state
+     */
+    transitionToCompleted(part: StoredPart, output: string, options?: ToolCompletionOptions): StoredPart;
+    /**
+     * Transition a tool part to error state.
+     *
+     * @param part - The tool part to update
+     * @param error - Error message
+     * @returns Updated StoredPart with error state
+     */
+    transitionToError(part: StoredPart, error: string): StoredPart;
+    /**
+     * Transition a tool part to running state.
+     *
+     * @param part - The tool part to update
+     * @returns Updated StoredPart with running state
+     */
+    transitionToRunning(part: StoredPart): StoredPart;
+};