codeep 1.1.36 → 1.2.0

package/dist/utils/agent.js

@@ -1,6 +1,9 @@
 /**
  * Agent loop - autonomous task execution
  */
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { recordTokenUsage, extractOpenAIUsage, extractAnthropicUsage } from './tokenTracker.js';
 // Debug logging helper - only logs when CODEEP_DEBUG=1
 const debug = (...args) => {
     if (process.env.CODEEP_DEBUG === '1') {
@@ -47,6 +50,31 @@ const DEFAULT_OPTIONS = {
     maxDuration: 20 * 60 * 1000, // 20 minutes
     usePlanning: false, // Disable task planning - causes more problems than it solves
 };
+/**
+ * Load project rules from .codeep/rules.md or CODEEP.md
+ * Returns the rules content formatted for system prompt, or empty string if no rules found
+ */
+export function loadProjectRules(projectRoot) {
+    const candidates = [
+        join(projectRoot, '.codeep', 'rules.md'),
+        join(projectRoot, 'CODEEP.md'),
+    ];
+    for (const filePath of candidates) {
+        if (existsSync(filePath)) {
+            try {
+                const content = readFileSync(filePath, 'utf-8').trim();
+                if (content) {
+                    debug('Loaded project rules from', filePath);
+                    return `\n\n## Project Rules\nThe following rules are defined by the project owner. You MUST follow these rules:\n\n${content}`;
+                }
+            }
+            catch (err) {
+                debug('Failed to read project rules from', filePath, err);
+            }
+        }
+    }
+    return '';
+}
 /**
  * Generate system prompt for agent mode (used with native tool calling)
  */
@@ -202,6 +230,7 @@ async function agentChat(messages, systemPrompt, onChunk, abortSignal, dynamicTi
     try {
         let endpoint;
         let body;
+        const useStreaming = Boolean(onChunk);
         if (protocol === 'openai') {
             endpoint = `${baseUrl}/chat/completions`;
             body = {
@@ -212,8 +241,9 @@ async function agentChat(messages, systemPrompt, onChunk, abortSignal, dynamicTi
                 ],
                 tools: getOpenAITools(),
                 tool_choice: 'auto',
+                stream: useStreaming,
                 temperature: config.get('temperature'),
-                max_tokens: Math.max(config.get('maxTokens'), 16384), // Ensure enough tokens for large file generation
+                max_tokens: Math.max(config.get('maxTokens'), 16384),
             };
         }
         else {
@@ -223,8 +253,9 @@ async function agentChat(messages, systemPrompt, onChunk, abortSignal, dynamicTi
                 system: systemPrompt,
                 messages: messages,
                 tools: getAnthropicTools(),
+                stream: useStreaming,
                 temperature: config.get('temperature'),
-                max_tokens: Math.max(config.get('maxTokens'), 16384), // Ensure enough tokens for large file generation
+                max_tokens: Math.max(config.get('maxTokens'), 16384),
             };
         }
         const response = await fetch(endpoint, {
@@ -241,7 +272,22 @@ async function agentChat(messages, systemPrompt, onChunk, abortSignal, dynamicTi
             }
             throw new Error(`API error: ${response.status} - ${errorText}`);
         }
+        // Streaming path — parse tool calls from SSE deltas
+        if (useStreaming && response.body) {
+            if (protocol === 'openai') {
+                return await handleOpenAIAgentStream(response.body, onChunk, model, providerId);
+            }
+            else {
+                return await handleAnthropicAgentStream(response.body, onChunk, model, providerId);
+            }
+        }
+        // Non-streaming path (fallback if no body)
         const data = await response.json();
+        // Track token usage
+        const usageExtractor = protocol === 'openai' ? extractOpenAIUsage : extractAnthropicUsage;
+        const usage = usageExtractor(data);
+        if (usage)
+            recordTokenUsage(usage, model, providerId);
         debug('Raw API response:', JSON.stringify(data, null, 2).substring(0, 1500));
         if (protocol === 'openai') {
             const message = data.choices?.[0]?.message;
@@ -388,6 +434,11 @@ async function agentChatFallback(messages, systemPrompt, onChunk, abortSignal, d
         }
         else {
             const data = await response.json();
+            // Track token usage
+            const fallbackUsageExtractor = protocol === 'openai' ? extractOpenAIUsage : extractAnthropicUsage;
+            const fallbackUsage = fallbackUsageExtractor(data);
+            if (fallbackUsage)
+                recordTokenUsage(fallbackUsage, model, providerId);
             if (protocol === 'openai') {
                 content = data.choices?.[0]?.message?.content || '';
             }
@@ -416,7 +467,186 @@ async function agentChatFallback(messages, systemPrompt, onChunk, abortSignal, d
     }
 }
 /**
- * Handle streaming response
+ * Handle OpenAI streaming response with tool call accumulation
+ */
+async function handleOpenAIAgentStream(body, onChunk, model, providerId) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let content = '';
+    // Accumulate tool calls from deltas
+    const toolCallMap = new Map();
+    let usageData = null;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+        for (const line of lines) {
+            if (!line.startsWith('data: '))
+                continue;
+            const data = line.slice(6);
+            if (data === '[DONE]')
+                continue;
+            try {
+                const parsed = JSON.parse(data);
+                // Track usage from final chunk
+                if (parsed.usage) {
+                    usageData = parsed;
+                }
+                const delta = parsed.choices?.[0]?.delta;
+                if (!delta)
+                    continue;
+                // Accumulate text content
+                if (delta.content) {
+                    content += delta.content;
+                    onChunk(delta.content);
+                }
+                // Accumulate tool calls
+                if (delta.tool_calls) {
+                    for (const tc of delta.tool_calls) {
+                        const idx = tc.index ?? 0;
+                        if (!toolCallMap.has(idx)) {
+                            toolCallMap.set(idx, {
+                                id: tc.id || '',
+                                name: tc.function?.name || '',
+                                arguments: '',
+                            });
+                        }
+                        const entry = toolCallMap.get(idx);
+                        if (tc.id)
+                            entry.id = tc.id;
+                        if (tc.function?.name)
+                            entry.name = tc.function.name;
+                        if (tc.function?.arguments)
+                            entry.arguments += tc.function.arguments;
+                    }
+                }
+            }
+            catch {
+                // Ignore parse errors
+            }
+        }
+    }
+    // Track token usage if available
+    if (usageData) {
+        const usage = extractOpenAIUsage(usageData);
+        if (usage)
+            recordTokenUsage(usage, model, providerId);
+    }
+    // Convert accumulated tool calls
+    const rawToolCalls = Array.from(toolCallMap.values()).map(tc => ({
+        id: tc.id,
+        type: 'function',
+        function: { name: tc.name, arguments: tc.arguments },
+    }));
+    const toolCalls = parseOpenAIToolCalls(rawToolCalls);
+    debug('Stream parsed tool calls:', toolCalls.length, toolCalls.map(t => t.tool));
+    // If no native tool calls, try text-based parsing
+    if (toolCalls.length === 0 && content) {
+        const textToolCalls = parseToolCalls(content);
+        if (textToolCalls.length > 0) {
+            return { content, toolCalls: textToolCalls, usedNativeTools: false };
+        }
+    }
+    return { content, toolCalls, usedNativeTools: true };
+}
+/**
+ * Handle Anthropic streaming response with tool call accumulation
+ */
+async function handleAnthropicAgentStream(body, onChunk, model, providerId) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let content = '';
+    // Accumulate content blocks for tool use
+    const contentBlocks = [];
+    let currentBlockIndex = -1;
+    let currentBlockType = '';
+    let currentToolName = '';
+    let currentToolId = '';
+    let currentToolInput = '';
+    let usageData = null;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split('\n');
+        buffer = lines.pop() || '';
+        for (const line of lines) {
+            if (!line.startsWith('data: '))
+                continue;
+            const data = line.slice(6);
+            try {
+                const parsed = JSON.parse(data);
+                // Track usage
+                if (parsed.usage) {
+                    usageData = parsed;
+                }
+                if (parsed.type === 'message_delta' && parsed.usage) {
+                    usageData = parsed;
+                }
+                if (parsed.type === 'content_block_start') {
+                    currentBlockIndex = parsed.index;
+                    const block = parsed.content_block;
+                    if (block.type === 'text') {
+                        currentBlockType = 'text';
+                    }
+                    else if (block.type === 'tool_use') {
+                        currentBlockType = 'tool_use';
+                        currentToolName = block.name || '';
+                        currentToolId = block.id || '';
+                        currentToolInput = '';
+                    }
+                }
+                else if (parsed.type === 'content_block_delta') {
+                    if (currentBlockType === 'text' && parsed.delta?.text) {
+                        content += parsed.delta.text;
+                        onChunk(parsed.delta.text);
+                    }
+                    else if (currentBlockType === 'tool_use' && parsed.delta?.partial_json) {
+                        currentToolInput += parsed.delta.partial_json;
+                    }
+                }
+                else if (parsed.type === 'content_block_stop') {
+                    if (currentBlockType === 'tool_use') {
+                        contentBlocks.push({
+                            type: 'tool_use',
+                            id: currentToolId,
+                            name: currentToolName,
+                            input: tryParseJSON(currentToolInput),
+                        });
+                    }
+                    currentBlockType = '';
+                }
+            }
+            catch {
+                // Ignore parse errors
+            }
+        }
+    }
+    // Track token usage
+    if (usageData) {
+        const usage = extractAnthropicUsage(usageData);
+        if (usage)
+            recordTokenUsage(usage, model, providerId);
+    }
+    const toolCalls = parseAnthropicToolCalls(contentBlocks);
+    return { content, toolCalls, usedNativeTools: true };
+}
+function tryParseJSON(str) {
+    try {
+        return JSON.parse(str);
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * Handle streaming response (text-based fallback)
  */
 async function handleStream(body, protocol, onChunk) {
     const reader = body.getReader();
@@ -515,6 +745,11 @@ export async function runAgent(prompt, projectContext, options = {}) {
     let systemPrompt = useNativeTools
         ? getAgentSystemPrompt(projectContext)
         : getFallbackSystemPrompt(projectContext);
+    // Inject project rules (from .codeep/rules.md or CODEEP.md)
+    const projectRules = loadProjectRules(projectContext.root);
+    if (projectRules) {
+        systemPrompt += projectRules;
+    }
     if (smartContextStr) {
         systemPrompt += '\n\n' + smartContextStr;
     }

package/dist/utils/agent.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/agent.test.js

@@ -0,0 +1,250 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+// Mock 'fs' before importing the module under test - use importOriginal to
+// preserve all fs exports that transitive dependencies need (e.g. mkdirSync).
+vi.mock('fs', async (importOriginal) => {
+    const actual = await importOriginal();
+    return {
+        ...actual,
+        existsSync: vi.fn(),
+        readFileSync: vi.fn(),
+    };
+});
+import { existsSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { loadProjectRules, formatAgentResult } from './agent.js';
+// Cast mocked functions for convenience
+const mockExistsSync = existsSync;
+const mockReadFileSync = readFileSync;
+describe('loadProjectRules', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    it('should return formatted rules when .codeep/rules.md exists', () => {
+        const projectRoot = '/my/project';
+        const rulesContent = 'Always use TypeScript.\nNo console.log in production.';
+        mockExistsSync.mockImplementation((filePath) => {
+            return filePath === join(projectRoot, '.codeep', 'rules.md');
+        });
+        mockReadFileSync.mockReturnValue(rulesContent);
+        const result = loadProjectRules(projectRoot);
+        expect(result).toContain('## Project Rules');
+        expect(result).toContain('Always use TypeScript.');
+        expect(result).toContain('No console.log in production.');
+        expect(mockExistsSync).toHaveBeenCalledWith(join(projectRoot, '.codeep', 'rules.md'));
+        expect(mockReadFileSync).toHaveBeenCalledWith(join(projectRoot, '.codeep', 'rules.md'), 'utf-8');
+    });
+    it('should fall back to CODEEP.md when .codeep/rules.md does not exist', () => {
+        const projectRoot = '/my/project';
+        const rulesContent = 'Follow the style guide.';
+        mockExistsSync.mockImplementation((filePath) => {
+            return filePath === join(projectRoot, 'CODEEP.md');
+        });
+        mockReadFileSync.mockReturnValue(rulesContent);
+        const result = loadProjectRules(projectRoot);
+        expect(result).toContain('## Project Rules');
+        expect(result).toContain('Follow the style guide.');
+        // Should have checked .codeep/rules.md first
+        expect(mockExistsSync).toHaveBeenCalledWith(join(projectRoot, '.codeep', 'rules.md'));
+        // Then checked CODEEP.md
+        expect(mockExistsSync).toHaveBeenCalledWith(join(projectRoot, 'CODEEP.md'));
+        expect(mockReadFileSync).toHaveBeenCalledWith(join(projectRoot, 'CODEEP.md'), 'utf-8');
+    });
+    it('should return empty string when neither rules file exists', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockReturnValue(false);
+        const result = loadProjectRules(projectRoot);
+        expect(result).toBe('');
+        expect(mockExistsSync).toHaveBeenCalledTimes(2);
+        expect(mockReadFileSync).not.toHaveBeenCalled();
+    });
+    it('should return empty string when rules file exists but is empty', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockImplementation((filePath) => {
+            return filePath === join(projectRoot, '.codeep', 'rules.md');
+        });
+        mockReadFileSync.mockReturnValue('   \n  \n  ');
+        const result = loadProjectRules(projectRoot);
+        // Empty after trim, so should skip and check next candidate
+        expect(mockExistsSync).toHaveBeenCalledWith(join(projectRoot, '.codeep', 'rules.md'));
+        // Since the first file was empty (whitespace-only), it checks the second
+        expect(mockExistsSync).toHaveBeenCalledWith(join(projectRoot, 'CODEEP.md'));
+    });
+    it('should return empty string when both files exist but are empty', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockReturnValue(true);
+        mockReadFileSync.mockReturnValue('   ');
+        const result = loadProjectRules(projectRoot);
+        expect(result).toBe('');
+    });
+    it('should return empty string when readFileSync throws an error', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockReturnValue(true);
+        mockReadFileSync.mockImplementation(() => {
+            throw new Error('EACCES: permission denied');
+        });
+        const result = loadProjectRules(projectRoot);
+        // Both candidates exist but both throw on read, so should return ''
+        expect(result).toBe('');
+        // Should have attempted to read both files
+        expect(mockReadFileSync).toHaveBeenCalledTimes(2);
+    });
+    it('should fall back to CODEEP.md when .codeep/rules.md read throws', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockReturnValue(true);
+        mockReadFileSync.mockImplementation((filePath) => {
+            if (filePath === join(projectRoot, '.codeep', 'rules.md')) {
+                throw new Error('EACCES: permission denied');
+            }
+            return 'Fallback rules content';
+        });
+        const result = loadProjectRules(projectRoot);
+        expect(result).toContain('## Project Rules');
+        expect(result).toContain('Fallback rules content');
+        expect(mockReadFileSync).toHaveBeenCalledTimes(2);
+    });
+    it('should prefer .codeep/rules.md over CODEEP.md when both exist', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockReturnValue(true);
+        mockReadFileSync.mockImplementation((filePath) => {
+            if (filePath === join(projectRoot, '.codeep', 'rules.md')) {
+                return 'Primary rules';
+            }
+            return 'Secondary rules';
+        });
+        const result = loadProjectRules(projectRoot);
+        expect(result).toContain('Primary rules');
+        expect(result).not.toContain('Secondary rules');
+        // Should only read the first file since it had content
+        expect(mockReadFileSync).toHaveBeenCalledTimes(1);
+    });
+    it('should include the MUST follow preamble in the returned string', () => {
+        const projectRoot = '/my/project';
+        mockExistsSync.mockImplementation((filePath) => {
+            return filePath === join(projectRoot, '.codeep', 'rules.md');
+        });
+        mockReadFileSync.mockReturnValue('Some rules');
+        const result = loadProjectRules(projectRoot);
+        expect(result).toContain('You MUST follow these rules');
+    });
+});
+describe('formatAgentResult', () => {
+    it('should format a successful result with iterations', () => {
+        const result = {
+            success: true,
+            iterations: 3,
+            actions: [],
+            finalResponse: 'All done',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Agent completed in 3 iteration(s)');
+    });
+    it('should format a failed result with error message', () => {
+        const result = {
+            success: false,
+            iterations: 5,
+            actions: [],
+            finalResponse: '',
+            error: 'Exceeded maximum duration',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Agent failed: Exceeded maximum duration');
+    });
+    it('should format an aborted result', () => {
+        const result = {
+            success: false,
+            iterations: 2,
+            actions: [],
+            finalResponse: 'Agent was stopped by user',
+            aborted: true,
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Agent was stopped by user');
+    });
+    it('should list actions when present', () => {
+        const result = {
+            success: true,
+            iterations: 2,
+            actions: [
+                { type: 'read', target: 'src/index.ts', result: 'success', timestamp: Date.now() },
+                { type: 'write', target: 'src/new-file.ts', result: 'success', timestamp: Date.now() },
+                { type: 'command', target: 'npm install', result: 'error', timestamp: Date.now() },
+            ],
+            finalResponse: 'Done',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Actions performed:');
+        expect(formatted).toContain('read: src/index.ts');
+        expect(formatted).toContain('write: src/new-file.ts');
+        expect(formatted).toContain('command: npm install');
+    });
+    it('should show check mark for successful actions and cross for errors', () => {
+        const result = {
+            success: true,
+            iterations: 1,
+            actions: [
+                { type: 'write', target: 'file.ts', result: 'success', timestamp: Date.now() },
+                { type: 'edit', target: 'other.ts', result: 'error', timestamp: Date.now() },
+            ],
+            finalResponse: 'Done',
+        };
+        const formatted = formatAgentResult(result);
+        const lines = formatted.split('\n');
+        const successLine = lines.find(l => l.includes('write: file.ts'));
+        const errorLine = lines.find(l => l.includes('edit: other.ts'));
+        expect(successLine).toMatch(/✓/);
+        expect(errorLine).toMatch(/✗/);
+    });
+    it('should not show actions section when there are no actions', () => {
+        const result = {
+            success: true,
+            iterations: 1,
+            actions: [],
+            finalResponse: 'Nothing to do',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).not.toContain('Actions performed:');
+    });
+    it('should format a failed result without aborted flag', () => {
+        const result = {
+            success: false,
+            iterations: 10,
+            actions: [
+                { type: 'read', target: 'config.json', result: 'success', timestamp: Date.now() },
+            ],
+            finalResponse: '',
+            error: 'Exceeded maximum of 10 iterations',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Agent failed: Exceeded maximum of 10 iterations');
+        expect(formatted).toContain('Actions performed:');
+        expect(formatted).toContain('read: config.json');
+    });
+    it('should format result with single iteration correctly', () => {
+        const result = {
+            success: true,
+            iterations: 1,
+            actions: [],
+            finalResponse: 'Quick task',
+        };
+        const formatted = formatAgentResult(result);
+        expect(formatted).toContain('Agent completed in 1 iteration(s)');
+    });
+    it('should handle all action types', () => {
+        const actionTypes = ['read', 'write', 'edit', 'delete', 'command', 'search', 'list', 'mkdir', 'fetch'];
+        const result = {
+            success: true,
+            iterations: 5,
+            actions: actionTypes.map(type => ({
+                type,
+                target: `target-for-${type}`,
+                result: 'success',
+                timestamp: Date.now(),
+            })),
+            finalResponse: 'Done',
+        };
+        const formatted = formatAgentResult(result);
+        for (const type of actionTypes) {
+            expect(formatted).toContain(`${type}: target-for-${type}`);
+        }
+    });
+});