@zds-ai/cli 0.1.3 → 0.1.4

package/README.md

@@ -339,6 +339,38 @@ Add to `~/.grok/user-settings.json`:
 
 **Model Priority**: `--model` flag > `GROK_MODEL` environment variable > user default model > system default (grok-code-fast-1)
 
+### Image Support
+
+zai-cli supports sending images to vision-capable AI models.  Use the `@` prefix to reference image files in your messages:
+
+```sh
+# Absolute path
+zai-cli --prompt "What's in this image? @/Users/joseph/photos/image.jpg"
+
+# Relative path
+zai-cli --prompt "Analyze @./screenshot.png"
+
+# Tilde expansion
+zai-cli --prompt "Describe @~/Pictures/photo.jpg"
+
+# Paths with spaces (quoted)
+zai-cli --prompt 'Compare these images: @"~/My Pictures/photo1.jpg" @"~/My Pictures/photo2.jpg"'
+
+# Paths with spaces (escaped)
+zai-cli --prompt "What's here? @/Users/joseph/My\ Documents/image.png"
+```
+
+**Supported Image Formats**: .jpg, .jpeg, .png, .gif, .webp, .bmp
+
+**Vision-Capable Models**: Image support works with vision models like:
+- `grok-4-1-fast-reasoning`
+- `grok-vision-beta`
+- Other vision-enabled models (via custom base URLs)
+
+**Automatic Fallback**: If you send an image to a model that doesn't support vision, zai-cli will automatically detect the error and retry with text-only content.
+
+**Interactive Mode**: The `@` syntax works in both interactive and headless (`--prompt`) modes.
+
 ### Command Line Options
 
 ```sh

package/dist/agent/grok-agent.d.ts

@@ -1,9 +1,10 @@
 import { GrokMessage, GrokToolCall } from "../grok/client.js";
+import type { ChatCompletionContentPart } from "openai/resources/chat/completions.js";
 import { ToolResult } from "../types/index.js";
 import { EventEmitter } from "events";
 export interface ChatEntry {
     type: "user" | "assistant" | "tool_result" | "tool_call" | "system";
-    content?: string;
+    content?: string | ChatCompletionContentPart[];
     timestamp: Date;
     tool_calls?: GrokToolCall[];
     toolCall?: GrokToolCall;
@@ -15,6 +16,10 @@ export interface ChatEntry {
     };
     isStreaming?: boolean;
     preserveFormatting?: boolean;
+    metadata?: {
+        rephrased_note?: string;
+        [key: string]: any;
+    };
 }
 export interface StreamingChunk {
     type: "content" | "tool_calls" | "tool_result" | "done" | "token_count" | "user_message";
@@ -62,6 +67,7 @@ export declare class GrokAgent extends EventEmitter {
     private activeTaskColor;
     private apiKeyEnvVar;
     private pendingContextEditSession;
+    private rephraseState;
     constructor(apiKey: string, baseURL?: string, model?: string, maxToolRounds?: number, debugLogFile?: string, startupHookOutput?: string, temperature?: number, maxTokens?: number);
     private startupHookOutput?;
     private systemPrompt;
@@ -125,6 +131,14 @@ export declare class GrokAgent extends EventEmitter {
         contextFilePath: string;
     } | null;
     clearPendingContextEditSession(): void;
+    setRephraseState(originalAssistantMessageIndex: number, rephraseRequestIndex: number, newResponseIndex: number, messageType: "user" | "system"): void;
+    getRephraseState(): {
+        originalAssistantMessageIndex: number;
+        rephraseRequestIndex: number;
+        newResponseIndex: number;
+        messageType: "user" | "system";
+    } | null;
+    clearRephraseState(): void;
     setPersona(persona: string, color?: string): Promise<{
         success: boolean;
         error?: string;

package/dist/agent/grok-agent.js

@@ -3,6 +3,8 @@ import { getAllGrokTools, getMCPManager, initializeMCPServers, } from "../grok/t
 import { loadMCPConfig } from "../mcp/config.js";
 import { ChatHistoryManager } from "../utils/chat-history-manager.js";
 import { logApiError } from "../utils/error-logger.js";
+import { parseImagesFromMessage, hasImageReferences } from "../utils/image-encoder.js";
+import { getTextContent } from "../utils/content-utils.js";
 import fs from "fs";
 import { TextEditorTool, MorphEditorTool, ZshTool, ConfirmationTool, SearchTool, EnvTool, IntrospectTool, ClearCacheTool, CharacterTool, TaskTool, InternetTool, ImageTool, FileConversionTool, RestartTool } from "../tools/index.js";
 import { EventEmitter } from "events";
@@ -106,6 +108,7 @@ export class GrokAgent extends EventEmitter {
     activeTaskColor = "white";
     apiKeyEnvVar = "GROK_API_KEY";
     pendingContextEditSession = null;
+    rephraseState = null;
     constructor(apiKey, baseURL, model, maxToolRounds, debugLogFile, startupHookOutput, temperature, maxTokens) {
         super();
         const manager = getSettingsManager();
@@ -229,21 +232,24 @@ Current working directory: ${process.cwd()}`;
             switch (entry.type) {
                 case "system":
                     // All system messages from chatHistory go into conversation (persona, mood, etc.)
+                    // System messages must always be strings
                     historyMessages.push({
                         role: "system",
-                        content: entry.content,
+                        content: getTextContent(entry.content),
                     });
                     break;
                 case "user":
+                    // User messages can have images (content arrays)
                     historyMessages.push({
                         role: "user",
-                        content: entry.content,
+                        content: entry.content || "",
                     });
                     break;
                 case "assistant":
+                    // Assistant messages are always text (no images in responses)
                     const assistantMessage = {
                         role: "assistant",
-                        content: entry.content || "", // Ensure content is never null/undefined
+                        content: getTextContent(entry.content) || "", // Ensure content is never null/undefined
                     };
                     if (entry.tool_calls && entry.tool_calls.length > 0) {
                         // For assistant messages with tool calls, collect the tool results that correspond to them
@@ -344,6 +350,39 @@ Current working directory: ${process.cwd()}`;
         return false;
     }
     async processUserMessage(message) {
+        // Detect rephrase commands
+        let isRephraseCommand = false;
+        let isSystemRephrase = false;
+        let messageToSend = message;
+        let messageType = "user";
+        if (message.startsWith("/system rephrase")) {
+            isRephraseCommand = true;
+            isSystemRephrase = true;
+            messageToSend = message.substring(8).trim(); // Strip "/system " (8 chars including space)
+            messageType = "system";
+        }
+        else if (message.startsWith("/rephrase")) {
+            isRephraseCommand = true;
+            messageToSend = message; // Keep full text including "/rephrase"
+            messageType = "user";
+        }
+        // If this is a rephrase command, find the last assistant message
+        if (isRephraseCommand) {
+            // Find index of last assistant message in chatHistory
+            let lastAssistantIndex = -1;
+            for (let i = this.chatHistory.length - 1; i >= 0; i--) {
+                if (this.chatHistory[i].type === "assistant") {
+                    lastAssistantIndex = i;
+                    break;
+                }
+            }
+            if (lastAssistantIndex === -1) {
+                throw new Error("No previous assistant message to rephrase");
+            }
+            // Store rephrase state (will be updated with newResponseIndex after response)
+            // For now, just mark that we're in rephrase mode
+            this.setRephraseState(lastAssistantIndex, this.chatHistory.length, -1, messageType);
+        }
         // Before adding the new user message, check if there are incomplete tool calls
         // from a previous interrupted turn. This prevents malformed message sequences
         // that cause Ollama 500 errors.
@@ -374,14 +413,40 @@ Current working directory: ${process.cwd()}`;
                 }
             }
         }
-        // Add user message to conversation
+        // Add user/system message to conversation
+        // Check for image references and parse if present
+        // Note: System messages can only have string content, so images are only supported for user messages
+        const supportsVision = this.grokClient.getSupportsVision();
+        let messageContent = messageToSend;
+        if (messageType === "user" && hasImageReferences(messageToSend) && supportsVision) {
+            // Parse images from message (only for user messages)
+            const parsed = parseImagesFromMessage(messageToSend);
+            if (parsed.images.length > 0) {
+                // Construct content array with text and images
+                messageContent = [
+                    { type: "text", text: parsed.text },
+                    ...parsed.images
+                ];
+            }
+            else {
+                // No valid images found (errors will be in parsed.text)
+                messageContent = parsed.text;
+            }
+        }
         const userEntry = {
-            type: "user",
-            content: message,
+            type: messageType,
+            content: messageContent,
             timestamp: new Date(),
         };
         this.chatHistory.push(userEntry);
-        this.messages.push({ role: "user", content: message });
+        // Push to messages array with proper typing based on role
+        if (messageType === "user") {
+            this.messages.push({ role: "user", content: messageContent });
+        }
+        else {
+            // System messages must have string content only
+            this.messages.push({ role: "system", content: typeof messageContent === "string" ? messageContent : messageToSend });
+        }
         await this.emitContextChange();
         const newEntries = [userEntry];
         const maxToolRounds = this.maxToolRounds; // Prevent infinite loops
@@ -529,10 +594,11 @@ Current working directory: ${process.cwd()}`;
                         // Collect system messages that appeared after this assistant message
                         for (let i = assistantIndex + 1; i < this.chatHistory.length; i++) {
                             const entry = this.chatHistory[i];
-                            if (entry.type === 'system' && entry.content && entry.content.trim()) {
+                            const content = getTextContent(entry.content);
+                            if (entry.type === 'system' && content && content.trim()) {
                                 this.messages.push({
                                     role: 'system',
-                                    content: entry.content
+                                    content: content
                                 });
                             }
                             // Stop if we hit another assistant or user message (next turn)
@@ -573,6 +639,11 @@ Current working directory: ${process.cwd()}`;
                             content: trimmedContent,
                         });
                         newEntries.push(responseEntry);
+                        // Update rephrase state with the new response index
+                        if (this.rephraseState && this.rephraseState.newResponseIndex === -1) {
+                            const newResponseIndex = this.chatHistory.length - 1;
+                            this.setRephraseState(this.rephraseState.originalAssistantMessageIndex, this.rephraseState.rephraseRequestIndex, newResponseIndex, this.rephraseState.messageType);
+                        }
                     }
                     // TODO: HACK - This is a temporary fix to prevent duplicate responses.
                     // We need a proper way for the bot to signal task completion, such as:
@@ -728,6 +799,39 @@ Current working directory: ${process.cwd()}`;
     async *processUserMessageStream(message) {
         // Create new abort controller for this request
         this.abortController = new AbortController();
+        // Detect rephrase commands
+        let isRephraseCommand = false;
+        let isSystemRephrase = false;
+        let messageToSend = message;
+        let messageType = "user";
+        if (message.startsWith("/system rephrase")) {
+            isRephraseCommand = true;
+            isSystemRephrase = true;
+            messageToSend = message.substring(8).trim(); // Strip "/system " (8 chars including space)
+            messageType = "system";
+        }
+        else if (message.startsWith("/rephrase")) {
+            isRephraseCommand = true;
+            messageToSend = message; // Keep full text including "/rephrase"
+            messageType = "user";
+        }
+        // If this is a rephrase command, find the last assistant message
+        if (isRephraseCommand) {
+            // Find index of last assistant message in chatHistory
+            let lastAssistantIndex = -1;
+            for (let i = this.chatHistory.length - 1; i >= 0; i--) {
+                if (this.chatHistory[i].type === "assistant") {
+                    lastAssistantIndex = i;
+                    break;
+                }
+            }
+            if (lastAssistantIndex === -1) {
+                throw new Error("No previous assistant message to rephrase");
+            }
+            // Store rephrase state (will be updated with newResponseIndex after response)
+            // For now, just mark that we're in rephrase mode
+            this.setRephraseState(lastAssistantIndex, this.chatHistory.length, -1, messageType);
+        }
         // Before adding the new user message, check if there are incomplete tool calls
         // from a previous interrupted turn. This prevents malformed message sequences
         // that cause Ollama 500 errors.
@@ -758,14 +862,40 @@ Current working directory: ${process.cwd()}`;
                 }
             }
         }
-        // Add user message to both API conversation and chat history
+        // Add user/system message to both API conversation and chat history
+        // Check for image references and parse if present
+        // Note: System messages can only have string content, so images are only supported for user messages
+        const supportsVision = this.grokClient.getSupportsVision();
+        let messageContent = messageToSend;
+        if (messageType === "user" && hasImageReferences(messageToSend) && supportsVision) {
+            // Parse images from message (only for user messages)
+            const parsed = parseImagesFromMessage(messageToSend);
+            if (parsed.images.length > 0) {
+                // Construct content array with text and images
+                messageContent = [
+                    { type: "text", text: parsed.text },
+                    ...parsed.images
+                ];
+            }
+            else {
+                // No valid images found (errors will be in parsed.text)
+                messageContent = parsed.text;
+            }
+        }
         const userEntry = {
-            type: "user",
-            content: message,
+            type: messageType,
+            content: messageContent,
             timestamp: new Date(),
         };
         this.chatHistory.push(userEntry);
-        this.messages.push({ role: "user", content: message });
+        // Push to messages array with proper typing based on role
+        if (messageType === "user") {
+            this.messages.push({ role: "user", content: messageContent });
+        }
+        else {
+            // System messages must have string content only
+            this.messages.push({ role: "system", content: typeof messageContent === "string" ? messageContent : messageToSend });
+        }
         await this.emitContextChange();
         // Yield user message so UI can display it immediately
         yield {
@@ -934,6 +1064,11 @@ Current working directory: ${process.cwd()}`;
                 };
                 this.chatHistory.push(assistantEntry);
                 await this.emitContextChange();
+                // Update rephrase state if this is a final response (no tool calls)
+                if (this.rephraseState && this.rephraseState.newResponseIndex === -1 && (!accumulatedMessage.tool_calls || accumulatedMessage.tool_calls.length === 0)) {
+                    const newResponseIndex = this.chatHistory.length - 1;
+                    this.setRephraseState(this.rephraseState.originalAssistantMessageIndex, this.rephraseState.rephraseRequestIndex, newResponseIndex, this.rephraseState.messageType);
+                }
                 // Handle tool calls if present
                 if (accumulatedMessage.tool_calls?.length > 0) {
                     toolRounds++;
@@ -1048,10 +1183,11 @@ Current working directory: ${process.cwd()}`;
                         // Collect system messages that appeared after this assistant message
                         for (let i = assistantIndex + 1; i < this.chatHistory.length; i++) {
                             const entry = this.chatHistory[i];
-                            if (entry.type === 'system' && entry.content && entry.content.trim()) {
+                            const content = getTextContent(entry.content);
+                            if (entry.type === 'system' && content && content.trim()) {
                                 this.messages.push({
                                     role: 'system',
-                                    content: entry.content
+                                    content: content
                                 });
                             }
                             // Stop if we hit another assistant or user message (next turn)
@@ -1109,7 +1245,7 @@ Current working directory: ${process.cwd()}`;
             this.chatHistory.push(errorEntry);
             yield {
                 type: "content",
-                content: errorEntry.content,
+                content: getTextContent(errorEntry.content),
             };
             // Mark first message as processed even on error
             this.firstMessageProcessed = true;
@@ -1245,18 +1381,9 @@ Current working directory: ${process.cwd()}`;
             // Validate tool arguments against schema
             const validationError = await this.validateToolArguments(toolCall.function.name, args);
             if (validationError) {
-                // Add system message explaining the validation error
-                const systemMsg = `Tool call validation failed: ${validationError}. Please try again with correct parameters.`;
-                console.warn(`[VALIDATION ERROR] ${systemMsg}`);
-                this.messages.push({
-                    role: 'system',
-                    content: systemMsg
-                });
-                this.chatHistory.push({
-                    type: 'system',
-                    content: systemMsg,
-                    timestamp: new Date()
-                });
+                // Validation failed - return error
+                const errorMsg = `Tool call validation failed: ${validationError}. Please try again with correct parameters.`;
+                console.warn(`[VALIDATION ERROR] ${errorMsg}`);
                 return {
                     success: false,
                     error: validationError
@@ -1372,6 +1499,8 @@ Current working directory: ${process.cwd()}`;
                     return await this.imageTool.captionImage(args.filename, args.prompt);
                 case "pngInfo":
                     return await this.imageTool.pngInfo(args.filename);
+                case "listImageModels":
+                    return await this.imageTool.listImageModels();
                 case "readXlsx":
                     return await this.fileConversionTool.readXlsx(args.filename, args.sheetName, args.outputFormat, args.output);
                 case "listXlsxSheets":
@@ -1493,17 +1622,17 @@ Current working directory: ${process.cwd()}`;
             const timestamp = entry.timestamp.toLocaleTimeString();
             if (entry.type === 'user') {
                 lines.push(`(${msgNum}) ${userName} (user) - ${timestamp}`);
-                lines.push(entry.content || "");
+                lines.push(getTextContent(entry.content) || "");
                 lines.push("");
             }
             else if (entry.type === 'assistant') {
                 lines.push(`(${msgNum}) ${agentName} (assistant) - ${timestamp}`);
-                lines.push(entry.content || "");
+                lines.push(getTextContent(entry.content) || "");
                 lines.push("");
             }
             else if (entry.type === 'system') {
                 lines.push(`(${msgNum}) System (system) - ${timestamp}`);
-                lines.push(entry.content || "");
+                lines.push(getTextContent(entry.content) || "");
                 lines.push("");
             }
             else if (entry.type === 'tool_call') {
@@ -1518,7 +1647,7 @@ Current working directory: ${process.cwd()}`;
                 const toolCall = entry.toolCall;
                 const toolName = toolCall?.function?.name || "unknown";
                 lines.push(`(${msgNum}) System (tool_result: ${toolName}) - ${timestamp}`);
-                lines.push(entry.content || "");
+                lines.push(getTextContent(entry.content) || "");
                 lines.push("");
             }
         });
@@ -1554,6 +1683,15 @@ Current working directory: ${process.cwd()}`;
     clearPendingContextEditSession() {
         this.pendingContextEditSession = null;
     }
+    setRephraseState(originalAssistantMessageIndex, rephraseRequestIndex, newResponseIndex, messageType) {
+        this.rephraseState = { originalAssistantMessageIndex, rephraseRequestIndex, newResponseIndex, messageType };
+    }
+    getRephraseState() {
+        return this.rephraseState;
+    }
+    clearRephraseState() {
+        this.rephraseState = null;
+    }
     async setPersona(persona, color) {
         // Execute hook if configured
         const settings = getSettingsManager();
@@ -1561,12 +1699,6 @@ Current working directory: ${process.cwd()}`;
         const hookMandatory = settings.isPersonaHookMandatory();
         if (!hookPath && hookMandatory) {
             const reason = "Persona hook is mandatory but not configured";
-            // Note: Don't add to this.messages during tool execution - only chatHistory
-            this.chatHistory.push({
-                type: 'system',
-                content: `Failed to change persona to "${persona}": ${reason}`,
-                timestamp: new Date()
-            });
             return {
                 success: false,
                 error: reason
@@ -1584,24 +1716,13 @@ Current working directory: ${process.cwd()}`;
                 // Even in rejection, we process commands (might have MODEL change)
                 await this.processHookResult(hookResult);
                 // Note: We ignore the return value here since we're already rejecting the persona
-                // Note: Don't add to this.messages during tool execution - only chatHistory
-                this.chatHistory.push({
-                    type: 'system',
-                    content: `Failed to change persona to "${persona}": ${reason}`,
-                    timestamp: new Date()
-                });
                 return {
                     success: false,
                     error: reason
                 };
             }
             if (hookResult.timedOut) {
-                // Note: Don't add to this.messages during tool execution - only chatHistory
-                this.chatHistory.push({
-                    type: 'system',
-                    content: `Persona hook timed out (auto-approved)`,
-                    timestamp: new Date()
-                });
+                // Hook timed out but was auto-approved
             }
             // Process hook commands (ENV, MODEL, SYSTEM)
             const result = await this.processHookResult(hookResult, 'ZDS_AI_AGENT_PERSONA');
@@ -1622,25 +1743,7 @@ Current working directory: ${process.cwd()}`;
         this.persona = persona;
         this.personaColor = color || "white";
         process.env.ZDS_AI_AGENT_PERSONA = persona;
-        // Add system message for recordkeeping
-        let systemContent;
-        if (oldPersona) {
-            const oldColorStr = oldColor && oldColor !== "white" ? ` (${oldColor})` : "";
-            const newColorStr = this.personaColor && this.personaColor !== "white" ? ` (${this.personaColor})` : "";
-            systemContent = `Assistant changed the persona from "${oldPersona}"${oldColorStr} to "${this.persona}"${newColorStr}`;
-        }
-        else {
-            const colorStr = this.personaColor && this.personaColor !== "white" ? ` (${this.personaColor})` : "";
-            systemContent = `Assistant set the persona to "${this.persona}"${colorStr}`;
-        }
-        // Note: Don't add to this.messages during tool execution - only chatHistory
-        // System messages added during tool execution create invalid message sequences
-        // because they get inserted between tool_calls and tool_results
-        this.chatHistory.push({
-            type: 'system',
-            content: systemContent,
-            timestamp: new Date()
-        });
+        // Persona hook generates success message - no need for redundant CLI message
         this.emit('personaChange', {
             persona: this.persona,
             color: this.personaColor
@@ -1654,12 +1757,6 @@ Current working directory: ${process.cwd()}`;
         const hookMandatory = settings.isMoodHookMandatory();
         if (!hookPath && hookMandatory) {
             const reason = "Mood hook is mandatory but not configured";
-            // Note: Don't add to this.messages during tool execution - only chatHistory
-            this.chatHistory.push({
-                type: 'system',
-                content: `Failed to change mood to "${mood}": ${reason}`,
-                timestamp: new Date()
-            });
             return {
                 success: false,
                 error: reason
@@ -1675,24 +1772,13 @@ Current working directory: ${process.cwd()}`;
                 const reason = hookResult.reason || "Hook rejected mood change";
                 // Process rejection commands (MODEL, SYSTEM)
                 await this.processHookResult(hookResult);
-                // Note: Don't add to this.messages during tool execution - only chatHistory
-                this.chatHistory.push({
-                    type: 'system',
-                    content: `Failed to change mood to "${mood}": ${reason}`,
-                    timestamp: new Date()
-                });
                 return {
                     success: false,
                     error: reason
                 };
             }
             if (hookResult.timedOut) {
-                // Note: Don't add to this.messages during tool execution - only chatHistory
-                this.chatHistory.push({
-                    type: 'system',
-                    content: `Mood hook timed out (auto-approved)`,
-                    timestamp: new Date()
-                });
+                // Hook timed out but was auto-approved
             }
             // Process hook commands (ENV, MODEL, SYSTEM)
             const result = await this.processHookResult(hookResult, 'ZDS_AI_AGENT_MOOD');
@@ -1760,20 +1846,13 @@ Current working directory: ${process.cwd()}`;
             await this.processHookResult(hookResult);
             if (!hookResult.approved) {
                 const reason = hookResult.reason || "Hook rejected task start";
-                this.messages.push({
-                    role: 'system',
-                    content: `Failed to start task "${activeTask}": ${reason}`
-                });
                 return {
                     success: false,
                     error: reason
                 };
             }
             if (hookResult.timedOut) {
-                this.messages.push({
-                    role: 'system',
-                    content: `Task start hook timed out (auto-approved)`
-                });
+                // Hook timed out but was auto-approved
             }
         }
         // Set the task
@@ -1815,20 +1894,13 @@ Current working directory: ${process.cwd()}`;
             await this.processHookResult(hookResult);
             if (!hookResult.approved) {
                 const reason = hookResult.reason || "Hook rejected task status transition";
-                this.messages.push({
-                    role: 'system',
-                    content: `Failed to transition task "${this.activeTask}" from ${this.activeTaskAction} to ${action}: ${reason}`
-                });
                 return {
                     success: false,
                     error: reason
                 };
             }
             if (hookResult.timedOut) {
-                this.messages.push({
-                    role: 'system',
-                    content: `Task transition hook timed out (auto-approved)`
-                });
+                // Hook timed out but was auto-approved
             }
         }
         // Store old action for system message
@@ -1874,20 +1946,13 @@ Current working directory: ${process.cwd()}`;
             await this.processHookResult(hookResult);
             if (!hookResult.approved) {
                 const hookReason = hookResult.reason || "Hook rejected task stop";
-                this.messages.push({
-                    role: 'system',
-                    content: `Failed to stop task "${this.activeTask}": ${hookReason}`
-                });
                 return {
                     success: false,
                     error: hookReason
                 };
             }
             if (hookResult.timedOut) {
-                this.messages.push({
-                    role: 'system',
-                    content: `Task stop hook timed out (auto-approved)`
-                });
+                // Hook timed out but was auto-approved
             }
         }
         // Calculate remaining time to meet 3-second minimum