gekto 0.0.8 → 0.0.9

package/dist/agents/HeadlessAgent.js

@@ -35,9 +35,15 @@ export class HeadlessAgent {
     isRunning() {
         return this.currentProc !== null && !this.currentProc.killed;
     }
-    async send(message, callbacks) {
+    async send(message, callbacks, imagePaths) {
+        // Append image file paths to the message so Claude can read them
+        let finalMessage = message;
+        if (imagePaths && imagePaths.length > 0) {
+            const imageRefs = imagePaths.map(p => `  - ${p}`).join('\n');
+            finalMessage += `\n\n[The user attached ${imagePaths.length} image(s). Use the Read tool to view them:\n${imageRefs}]`;
+        }
         const args = [
-            '-p', message,
+            '-p', finalMessage,
             '--output-format', 'stream-json',
             '--verbose',
             '--model', 'claude-opus-4-5-20251101',
@@ -71,6 +77,8 @@ export class HeadlessAgent {
             let buffer = '';
             let lastResult = null;
             let currentTool = null;
+            const toolUseIdToName = new Map();
+            const streamState = { receivedDeltas: false };
             proc.stdout.on('data', (data) => {
                 buffer += data.toString();
                 const lines = buffer.split('\n');
@@ -80,7 +88,7 @@ export class HeadlessAgent {
                         continue;
                     try {
                         const event = JSON.parse(line);
-                        this.processStreamEvent(event, callbacks, (tool) => {
+                        this.processStreamEvent(event, callbacks, toolUseIdToName, streamState, (tool) => {
                             currentTool = tool;
                         }, () => {
                             currentTool = null;
@@ -125,7 +133,7 @@ export class HeadlessAgent {
             proc.on('error', reject);
         });
     }
-    processStreamEvent(event, callbacks, setCurrentTool, clearCurrentTool) {
+    processStreamEvent(event, callbacks, toolUseIdToName, streamState, setCurrentTool, clearCurrentTool) {
         if (event.type === 'assistant' && event.message) {
             const message = event.message;
             if (message.content) {
@@ -133,6 +141,10 @@ export class HeadlessAgent {
                     if (block.type === 'tool_use' && block.name) {
                         setCurrentTool?.(block.name);
                         callbacks?.onToolStart?.(block.name, block.input);
+                        // Track tool name by ID for tool_result matching
+                        if (block.id) {
+                            toolUseIdToName?.set(block.id, block.name);
+                        }
                         // Track file changes for Write/Edit tools
                         if ((block.name === 'Write' || block.name === 'Edit') && block.id && block.input?.file_path) {
                             const filePath = String(block.input.file_path);
@@ -144,19 +156,55 @@ export class HeadlessAgent {
                             });
                         }
                     }
-                    // Extract text content from assistant messages
-                    if (block.type === 'text' && block.text) {
+                    // Fallback: emit full text block only if no streaming deltas were received
+                    if (block.type === 'text' && block.text && !streamState?.receivedDeltas) {
                         callbacks?.onText?.(block.text);
                     }
+                    // Thinking block from assistant event (fallback when no thinking_delta)
+                    if (block.type === 'thinking' && block.thinking && !streamState?.receivedDeltas) {
+                        callbacks?.onThinking?.(block.thinking);
+                    }
                 }
             }
         }
+        // Streaming deltas — character-by-character text and thinking
+        if (event.type === 'content_block_delta') {
+            const delta = event.delta;
+            if (delta?.type === 'text_delta' && delta.text) {
+                if (streamState)
+                    streamState.receivedDeltas = true;
+                callbacks?.onText?.(delta.text);
+            }
+            else if (delta?.type === 'thinking_delta' && delta.thinking) {
+                if (streamState)
+                    streamState.receivedDeltas = true;
+                callbacks?.onThinking?.(delta.thinking);
+            }
+        }
         if (event.type === 'user' && event.message) {
             const message = event.message;
             if (message.content) {
                 for (const block of message.content) {
                     if (block.type === 'tool_result') {
                         clearCurrentTool?.();
+                        // Extract tool result content
+                        if (block.tool_use_id && block.content != null) {
+                            const toolName = toolUseIdToName?.get(block.tool_use_id) || 'unknown';
+                            let resultText = '';
+                            if (typeof block.content === 'string') {
+                                resultText = block.content;
+                            }
+                            else if (Array.isArray(block.content)) {
+                                // content is array of {type: 'text', text: '...'} blocks
+                                resultText = block.content
+                                    .filter(c => c.type === 'text' && c.text)
+                                    .map(c => c.text)
+                                    .join('\n');
+                            }
+                            if (resultText) {
+                                callbacks?.onToolResult?.(toolName, resultText, block.tool_use_id);
+                            }
+                        }
                         // Complete file change tracking
                         if (block.tool_use_id && this.pendingFileChanges.has(block.tool_use_id)) {
                             const pending = this.pendingFileChanges.get(block.tool_use_id);

package/dist/agents/agentPool.js

@@ -1,7 +1,10 @@
 import path from 'path';
 import { writeFileSync, unlinkSync, existsSync, mkdirSync } from 'fs';
 import { dirname } from 'path';
+import { randomUUID } from 'crypto';
+import { tmpdir } from 'os';
 import { HeadlessAgent } from './HeadlessAgent.js';
+import { getState, mutate, broadcastFileChange, broadcastAgent } from '../state.js';
 // Per-lizard sessions
 const sessions = new Map();
 // Summarize tool input for display
@@ -100,14 +103,51 @@ export function getQueueLength(lizardId) {
 function safeSend(session, data) {
     const ws = session.currentWs;
     if (ws && ws.readyState === ws.OPEN) {
+        const type = data.type;
+        const lizardId = data.lizardId;
+        // Log outgoing messages (skip noisy streaming deltas)
+        if (type !== 'text' && type !== 'thinking') {
+            console.log(`[WS→] ${type}${lizardId ? ` [${lizardId}]` : ''}${data.tool ? ` tool=${data.tool}` : ''}${data.state ? ` state=${data.state}` : ''}`);
+        }
         ws.send(JSON.stringify(data));
     }
 }
-export async function sendMessage(lizardId, message, ws) {
+// Save base64 data URL images to temp files and return file paths
+export function saveImagesToTempFiles(images) {
+    const paths = [];
+    const dir = path.join(tmpdir(), 'gekto-images');
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    for (const dataUrl of images) {
+        const match = dataUrl.match(/^data:image\/(\w+);base64,(.+)$/);
+        if (!match)
+            continue;
+        const ext = match[1] === 'jpeg' ? 'jpg' : match[1];
+        const buffer = Buffer.from(match[2], 'base64');
+        const filePath = path.join(dir, `gekto-img-${randomUUID()}.${ext}`);
+        writeFileSync(filePath, buffer);
+        paths.push(filePath);
+    }
+    return paths;
+}
+export async function sendMessage(lizardId, message, ws, images) {
     const session = getOrCreateSession(lizardId, ws);
+    // Save images to temp files and build the final message
+    let finalMessage = message;
+    let imagePaths;
+    if (images && images.length > 0) {
+        imagePaths = saveImagesToTempFiles(images);
+    }
+    // Accumulators for streaming deltas — reset on each tool start
+    let accumulatedText = '';
+    let accumulatedThinking = '';
     // Create streaming callbacks that use session's current WebSocket
     const callbacks = {
         onToolStart: (tool, input) => {
+            // Reset accumulators when a new tool starts (text block ended)
+            accumulatedText = '';
+            accumulatedThinking = '';
             safeSend(session, {
                 type: 'tool',
                 lizardId,
@@ -126,24 +166,75 @@ export async function sendMessage(lizardId, message, ws) {
             });
         },
         onText: (text) => {
+            accumulatedText += text;
             safeSend(session, {
                 type: 'text',
                 lizardId,
-                text,
+                text: accumulatedText,
+            });
+        },
+        onThinking: (text) => {
+            accumulatedThinking += text;
+            safeSend(session, {
+                type: 'thinking',
+                lizardId,
+                text: accumulatedThinking,
+            });
+        },
+        onToolResult: (tool, content, toolUseId) => {
+            safeSend(session, {
+                type: 'tool_result',
+                lizardId,
+                tool,
+                content: content.length > 2000 ? content.substring(0, 2000) + '…' : content,
+                toolUseId,
             });
         },
         onFileChange: (change) => {
+            // Encode path for use as key
+            const encodedPath = change.filePath.replace(/\//g, '--');
+            // Enrich change with metadata
+            const enrichedChange = {
+                ...change,
+                agentId: lizardId,
+                taskId: getState().agents[lizardId]?.taskId,
+                timestamp: new Date().toISOString(),
+            };
+            // Write to top-level fileChanges collection
+            mutate(`fileChanges.${encodedPath}`, enrichedChange);
+            broadcastFileChange(encodedPath);
+            // Also update agent's fileChangePaths for reference
+            const agent = getState().agents[lizardId];
+            if (agent) {
+                const paths = agent.fileChangePaths ?? [];
+                if (!paths.includes(change.filePath)) {
+                    mutate(`agents.${lizardId}.fileChangePaths`, [...paths, change.filePath]);
+                }
+                // Keep backward-compat fileChanges on agent
+                const existing = agent.fileChanges ?? [];
+                const existingIndex = existing.findIndex(fc => fc.filePath === change.filePath);
+                let updated;
+                if (existingIndex >= 0) {
+                    updated = [...existing];
+                    updated[existingIndex] = { ...updated[existingIndex], after: change.after, tool: change.tool };
+                }
+                else {
+                    updated = [...existing, enrichedChange];
+                }
+                mutate(`agents.${lizardId}.fileChanges`, updated);
+                broadcastAgent(lizardId);
+            }
             safeSend(session, {
                 type: 'file_change',
                 lizardId,
-                change,
+                change: enrichedChange,
             });
         },
     };
     // If already processing, queue the message
     if (session.isProcessing) {
         return new Promise((resolve, reject) => {
-            session.queue.push({ message, ws, callbacks, resolve, reject });
+            session.queue.push({ message: finalMessage, ws, callbacks, resolve, reject, imagePaths });
             const position = session.queue.length;
             ws.send(JSON.stringify({
                 type: 'queued',
@@ -153,14 +244,14 @@ export async function sendMessage(lizardId, message, ws) {
         });
     }
     // Process immediately
-    return processMessage(lizardId, session, message, ws, callbacks);
+    return processMessage(lizardId, session, finalMessage, ws, callbacks, imagePaths);
 }
 async function processMessage(lizardId, session, message, _ws, // Kept for queue compatibility, but we use session.currentWs
-callbacks) {
+callbacks, imagePaths) {
     session.isProcessing = true;
     safeSend(session, { type: 'state', lizardId, state: 'working' });
     try {
-        const response = await session.agent.send(message, callbacks);
+        const response = await session.agent.send(message, callbacks, imagePaths);
         safeSend(session, {
             type: 'response',
             lizardId,
@@ -185,7 +276,7 @@ callbacks) {
         // Process next queued message if any
         if (session.queue.length > 0) {
             const next = session.queue.shift();
-            processMessage(lizardId, session, next.message, next.ws, next.callbacks)
+            processMessage(lizardId, session, next.message, next.ws, next.callbacks, next.imagePaths)
                 .then(next.resolve)
                 .catch(next.reject);
         }