@sschepis/robodev 1.0.0

package/src/core/history-manager.mjs

@@ -0,0 +1,330 @@
+// History management and token estimation
+// Handles context limits and conversation history optimization
+
+import { consoleStyler } from '../ui/console-styler.mjs';
+import { config } from '../config.mjs';
+
+// Average characters per token (approximate for English text)
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Manages conversation history, token estimation, and context limits.
+ */
+export class HistoryManager {
+    /**
+     * @param {number|null} maxTokens - Maximum allow tokens
+     * @param {number|null} contextWindowSize - Total context window size
+     */
+    constructor(maxTokens = null, contextWindowSize = null) {
+        this.maxTokens = maxTokens || config.ai.maxTokens || 4096;
+        this.contextWindowSize = contextWindowSize || config.ai.contextWindowSize || 128000;
+        
+        /** @type {Array<Object>} */
+        this.history = [];
+        this.systemMessage = null;
+    }
+
+    /**
+     * Initialize history with a system message
+     * @param {string} systemPrompt 
+     */
+    initialize(systemPrompt) {
+        this.systemMessage = { 
+            role: 'system', 
+            content: systemPrompt 
+        };
+        this.history = [this.systemMessage];
+    }
+
+    /**
+     * Add a message to history
+     * @param {string} role - 'user', 'assistant', 'system', or 'tool'
+     * @param {string} content - Message content
+     * @param {Array<Object>|null} toolCalls - Optional tool calls
+     * @param {string|null} toolCallId - Optional tool call ID
+     * @param {string|null} name - Optional name for tool messages
+     */
+    addMessage(role, content, toolCalls = null, toolCallId = null, name = null) {
+        const message = { role, content };
+        
+        if (toolCalls) {
+            message.tool_calls = toolCalls;
+        }
+        
+        if (toolCallId) {
+            message.tool_call_id = toolCallId;
+        }
+
+        if (name) {
+            message.name = name;
+        }
+        
+        this.history.push(message);
+        
+        // Check context limits and optimize if needed
+        this.enforceContextLimits();
+    }
+
+    /**
+     * Add a complete message object directly
+     * @param {Object} message 
+     */
+    pushMessage(message) {
+        this.history.push(message);
+        this.enforceContextLimits();
+    }
+
+    /**
+     * Get the full history
+     * @returns {Array<Object>}
+     */
+    getHistory() {
+        return this.history;
+    }
+
+    /**
+     * Update the system prompt
+     * @param {string} newContent 
+     */
+    updateSystemPrompt(newContent) {
+        if (this.history.length > 0 && this.history[0].role === 'system') {
+            this.history[0].content = newContent;
+            this.systemMessage.content = newContent;
+        } else {
+            this.systemMessage = { role: 'system', content: newContent };
+            this.history.unshift(this.systemMessage);
+        }
+    }
+
+    /**
+     * Delete specified number of exchanges from history (backwards)
+     * @param {number} count 
+     * @returns {number} Number of exchanges deleted
+     */
+    deleteHistoryExchanges(count) {
+        if (count <= 0) return 0;
+        
+        let deletedExchanges = 0;
+        
+        // Work backwards through history, preserving system message at index 0
+        for (let i = 0; i < count; i++) {
+            // Find the most recent complete exchange (user + assistant + any tools)
+            let foundUserMessage = false;
+            let messagesToDelete = [];
+            
+            // Scan backwards from end of history
+            for (let j = this.history.length - 1; j > 0; j--) { // Start from index 1 to preserve system message
+                const message = this.history[j];
+                
+                if (message.role === 'user' && !foundUserMessage) {
+                    // Found the start of the exchange to delete
+                    foundUserMessage = true;
+                    messagesToDelete.unshift(j); // Add to beginning since we're going backwards
+                } else if (foundUserMessage) {
+                    // Part of the exchange (assistant, tool responses)
+                    messagesToDelete.unshift(j);
+                    
+                    // Check if this completes an exchange (hit previous user message or system)
+                    if (message.role === 'user' || message.role === 'system') {
+                        break;
+                    }
+                } else if (message.role === 'assistant' || message.role === 'tool') {
+                    // Dangling assistant/tool message, include it in deletion
+                    messagesToDelete.unshift(j);
+                }
+            }
+            
+            // Delete the identified messages
+            if (messagesToDelete.length > 0) {
+                // Sort in descending order to delete from end first (preserves indices)
+                messagesToDelete.sort((a, b) => b - a);
+                for (const index of messagesToDelete) {
+                    this.history.splice(index, 1);
+                }
+                deletedExchanges++;
+            } else {
+                // No more exchanges to delete
+                break;
+            }
+        }
+        
+        return deletedExchanges;
+    }
+
+    /**
+     * Reset history to just the system prompt
+     */
+    reset() {
+        if (this.systemMessage) {
+            this.history = [this.systemMessage];
+        } else {
+            this.history = [];
+        }
+    }
+
+    /**
+     * Set history to a specific set of messages (e.g. for retries)
+     * @param {Array<Object>} messages 
+     */
+    setHistory(messages) {
+        this.history = [...messages];
+        // Ensure system message is tracked
+        if (this.history.length > 0 && this.history[0].role === 'system') {
+            this.systemMessage = this.history[0];
+        }
+    }
+
+    /**
+     * Estimate token count for a string
+     * @param {string} text 
+     * @returns {number}
+     */
+    estimateTokens(text) {
+        if (!text) return 0;
+        return Math.ceil(text.length / CHARS_PER_TOKEN);
+    }
+
+    /**
+     * Estimate total tokens in history
+     * @returns {number}
+     */
+    getTotalTokens() {
+        let total = 0;
+        for (const msg of this.history) {
+            // Content tokens
+            total += this.estimateTokens(msg.content || '');
+            
+            // Role overhead (approximate)
+            total += 4; 
+            
+            // Tool calls tokens
+            if (msg.tool_calls) {
+                for (const call of msg.tool_calls) {
+                    total += this.estimateTokens(call.function.name);
+                    total += this.estimateTokens(call.function.arguments);
+                }
+            }
+        }
+        return total;
+    }
+
+    /**
+     * Enforce context window limits by summarizing or truncating
+     */
+    enforceContextLimits() {
+        const currentTokens = this.getTotalTokens();
+        
+        // If we're approaching the limit (90% capacity)
+        if (currentTokens > this.contextWindowSize * 0.9) {
+            consoleStyler.log('system', `⚠️ Context limit approaching (${currentTokens}/${this.contextWindowSize} tokens). Optimizing history...`);
+            
+            // Calculate how many tokens we need to free up (aim for 70% capacity)
+            const targetTokens = Math.floor(this.contextWindowSize * 0.7);
+            
+            // Simple strategy: Remove oldest exchanges (after system prompt)
+            // A smarter strategy would be to summarize, but that requires an LLM call
+            
+            let attempts = 0;
+            const maxAttempts = 20; // Prevent infinite loops
+            
+            while (this.getTotalTokens() > targetTokens && this.history.length > 2 && attempts < maxAttempts) {
+                // Remove the oldest exchange (index 1 is usually the first user message)
+                // We use deleteHistoryExchanges logic but target specific indices
+                
+                // Find first user message after system prompt
+                let firstUserIndex = -1;
+                for (let i = 1; i < this.history.length; i++) {
+                    if (this.history[i].role === 'user') {
+                        firstUserIndex = i;
+                        break;
+                    }
+                }
+                
+                if (firstUserIndex === -1) break; // No user messages found
+                
+                // Find the next user message to define the exchange boundary
+                let nextUserIndex = -1;
+                for (let i = firstUserIndex + 1; i < this.history.length; i++) {
+                    if (this.history[i].role === 'user') {
+                        nextUserIndex = i;
+                        break;
+                    }
+                }
+                
+                // If no next user message, we're at the last exchange - don't delete recent context unless critical
+                if (nextUserIndex === -1 && currentTokens < this.contextWindowSize) {
+                    break; 
+                }
+                
+                // Delete everything from firstUserIndex up to (but not including) nextUserIndex
+                // If nextUserIndex is -1, delete until end (only if we are critically over limit)
+                const deleteCount = (nextUserIndex !== -1 ? nextUserIndex : this.history.length) - firstUserIndex;
+                
+                this.history.splice(firstUserIndex, deleteCount);
+                attempts++;
+            }
+            
+            consoleStyler.log('system', `✓ History optimized. New token count: ~${this.getTotalTokens()}`);
+        }
+    }
+
+    /**
+     * Get context statistics
+     * @returns {Object}
+     */
+    getStats() {
+        return {
+            messageCount: this.history.length,
+            estimatedTokens: this.getTotalTokens(),
+            contextWindowSize: this.contextWindowSize,
+            utilizationPercent: Math.round((this.getTotalTokens() / this.contextWindowSize) * 100)
+        };
+    }
+
+    /**
+     * Save history to a file
+     * @param {string} filePath 
+     * @returns {Promise<boolean>}
+     */
+    async save(filePath) {
+        try {
+            const fs = await import('fs');
+            const data = {
+                timestamp: new Date().toISOString(),
+                history: this.history,
+                systemMessage: this.systemMessage
+            };
+            fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf8');
+            return true;
+        } catch (error) {
+            consoleStyler.log('error', `Failed to save history: ${error.message}`);
+            return false;
+        }
+    }
+
+    /**
+     * Load history from a file
+     * @param {string} filePath 
+     * @returns {Promise<boolean>}
+     */
+    async load(filePath) {
+        try {
+            const fs = await import('fs');
+            if (!fs.existsSync(filePath)) {
+                return false;
+            }
+            
+            const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+            
+            if (data.history && Array.isArray(data.history)) {
+                this.history = data.history;
+                this.systemMessage = data.systemMessage || this.history.find(msg => msg.role === 'system');
+                return true;
+            }
+            return false;
+        } catch (error) {
+            consoleStyler.log('error', `Failed to load history: ${error.message}`);
+            return false;
+        }
+    }
+}

package/src/core/system-prompt.mjs

@@ -0,0 +1,182 @@
+// System prompt generation
+// Creates the system prompt with workspace context and guidelines
+
+export function createSystemPrompt(workingDir, workspace = null, manifestContent = null) {
+    let prompt = `You are a JavaScript/Node.js command executor. Your output consists of direct commands and concise results, not explanations.
+
+**Working Directory:** ${workingDir}
+The user is executing commands from this directory. When working with files or paths, consider this as the current working directory unless otherwise specified.`;
+
+    // Add Living Manifest if available
+    if (manifestContent) {
+        prompt += `
+
+**LIVING MANIFEST (SYSTEM_MAP.md):**
+The following is the authoritative state of the system. You MUST adhere to Global Invariants and respect Feature Locks.
+
+${manifestContent}
+
+**STRUCTURED DEVELOPMENT RULES:**
+1. Check "Global Invariants" before writing any code.
+2. Check "Feature Registry" to see if a feature is Locked.
+   - If "Interface" lock is active, you CANNOT change API signatures without a refactor request.
+   - If "None" or "Discovery", you are free to design.
+3. Update the manifest using the provided tools as you progress through phases (Discovery -> Interface -> Implementation).
+`;
+    }
+
+    // Add workspace context if active
+    if (workspace) {
+        prompt += `
+
+**ACTIVE WORKSPACE:**
+• Task Goal: ${workspace.task_goal}
+• Current Step: ${workspace.current_step}
+• Status: ${workspace.status}
+• Progress Data: ${JSON.stringify(workspace.progress_data)}
+• Next Steps: ${workspace.next_steps.join(', ')}
+
+IMPORTANT: You are continuing work on the above task. Use the workspace context to maintain continuity. Update the workspace as you make progress using the manage_workspace tool.`;
+    }
+
+    prompt += `
+
+**Core Principles:**
+* **Truthfulness:** Be strictly truthful. Never fabricate outcomes, always report failures accurately, and admit when you cannot complete a task.
+* **Language:** Default to modern ES6+ JavaScript and \`async/await\`. Interpret requests to "create" or "build" as "write JavaScript code."
+* **Workspace Management:** For complex multi-step tasks, use the \`manage_workspace\` tool to maintain context across retries and quality evaluations.
+* **Work Reporting:** ALWAYS include a \`workPerformed\` field in your responses when you perform any action or use tools. This should be a brief, clear statement like "I executed JavaScript code to fetch data from the API" or "I created a file with the requested content". This helps users understand what work was completed.
+
+Before answering, work through the request step-by-step:
+
+1. UNDERSTAND: What is the core question being asked?
+2. ANALYZE: What are the key factors/components involved?
+3. REASON: What logical connections can I make?
+4. SYNTHESIZE: How do these elements combine?
+5. CONCLUDE: What is the most accurate/helpful response?
+
+Then provide your answer.
+
+**Execution Protocol:**
+1.  **Plan:** Analyze the request and formulate a step-by-step technical plan. For complex tasks, create a workspace to track progress.
+2.  **Execute:** Carry out the plan using your available tools. Update workspace as you progress.
+3.  **Recover:** On error, use your \`analyze_and_recover\` tool to find an alternative solution before giving up.
+4.  **Report:** State the final, factual result. Update workspace status when task is complete.
+
+**Technical Constraints:**
+* For Node.js v18 compatibility, prefer built-in modules (\`fetch\`) over packages with known issues (\`axios\`, \`undici\`).
+* If a primary tool like \`cheerio\` fails, use a fallback like regex or built-in DOM parsing.
+
+**Node.js v18 Compatibility Guidelines:**
+* ALWAYS use built-in fetch instead of axios for HTTP requests
+* For web scraping: Use regex patterns or built-in string methods instead of cheerio
+* Avoid these packages: axios, undici, node-fetch, cheerio (they have File API issues in Node v18)
+* When scraping HTML, use patterns like: /<h[1-6][^>]*>(.*?)<\/h[1-6]>/gi for headlines
+* For complex HTML parsing, use built-in DOMParser alternatives or regex
+
+Execute commands. Report results. Recover from errors. Move to next step.`;
+
+    return prompt;
+}
+
+// Create enhanced system prompt with work reporting instruction
+export function createEnhancedSystemPrompt(workingDir, workspace = null) {
+    const basePrompt = createSystemPrompt(workingDir, workspace);
+    
+    return basePrompt + `
+
+**IMPORTANT RESPONSE FORMAT:**
+Always structure your responses to include actionable information and clear work reporting. When you use tools or execute code, explicitly state what was accomplished in a \`workPerformed\` field or section.`;
+}
+
+// Create system prompt for quality evaluation
+export function createQualityEvaluationPrompt() {
+    return `You are an AI response quality evaluator. Your job is to objectively assess whether AI responses appropriately address user queries.
+
+**Evaluation Criteria:**
+- **Completeness:** Does the response fully address all parts of the user's request?
+- **Accuracy:** Is the information provided correct and factual?
+- **Usefulness:** Does the response provide practical value to the user?
+- **Tool Usage:** If tools were used, were they appropriate and effective?
+- **Clarity:** Is the response clear and well-structured?
+
+**Scoring Scale:**
+- 9-10: Excellent response that exceeds expectations
+- 7-8: Good response that meets expectations well
+- 5-6: Adequate response with minor issues
+- 3-4: Poor response with significant problems
+- 1-2: Completely inadequate response
+
+**Important:** Consider BOTH the text response AND any tools that were executed. A brief text response paired with successful tool execution that accomplishes the user's goal should be rated highly.`;
+}
+
+// Create system prompt for tool generation
+export function createToolGenerationPrompt() {
+    return `You are a JavaScript function generator. Your job is to convert code snippets into reusable, parameterized functions.
+
+**Requirements:**
+1. Extract hardcoded values as function parameters
+2. Add comprehensive error handling
+3. Include detailed JSDoc comments
+4. Return meaningful data structures
+5. Handle edge cases and validation
+6. Use modern ES6+ syntax with async/await
+7. Make functions self-contained
+
+**Output:** Return ONLY the function code, no explanations or markdown formatting.`;
+}
+
+// Create system prompt for schema generation
+export function createSchemaGenerationPrompt() {
+    return `You are a JSON schema generator for OpenAI function calling format.
+
+**Requirements:**
+1. Analyze function parameters and their types
+2. Provide clear descriptions for each parameter
+3. Identify required vs optional parameters
+4. Use proper JSON schema types and formats
+5. Follow OpenAI function calling specification
+
+**Output:** Return ONLY the JSON schema object, no explanations or markdown formatting.`;
+}
+
+// Get appropriate system prompt based on context
+export function getSystemPrompt(context = {}) {
+    const {
+        type = 'default',
+        workingDir = process.cwd(),
+        workspace = null,
+        manifestContent = null,
+        enhanced = false
+    } = context;
+
+    switch (type) {
+        case 'quality':
+            return createQualityEvaluationPrompt();
+        
+        case 'tool-generation':
+            return createToolGenerationPrompt();
+        
+        case 'schema-generation':
+            return createSchemaGenerationPrompt();
+        
+        case 'enhanced':
+            return createEnhancedSystemPrompt(workingDir, workspace);
+        
+        default:
+            return enhanced
+                ? createEnhancedSystemPrompt(workingDir, workspace)
+                : createSystemPrompt(workingDir, workspace, manifestContent);
+    }
+}
+
+// Add work performed instruction to existing messages
+export function enhanceMessagesWithWorkReporting(messages) {
+    if (messages.length > 1) {
+        const lastUserMessage = messages[messages.length - 1];
+        if (lastUserMessage.role === 'user') {
+            lastUserMessage.content += `\n\nIMPORTANT: Please include a 'workPerformed' field in your response with a brief summary of any work completed (e.g., "I executed JavaScript code to analyze the data" or "I created a file with the requested content").`;
+        }
+    }
+    return messages;
+}