orquesta-cli 0.2.42 → 0.2.43

package/dist/core/llm/llm-client.js

@@ -138,11 +138,17 @@ export class LLMClient {
     }
     preprocessMessages(messages, modelId) {
         return messages.map((msg) => {
+            let processedMsg = { ...msg };
+            const multimodal = processedMsg.multimodal;
+            if (multimodal && Array.isArray(multimodal)) {
+                processedMsg.content = multimodal;
+                delete processedMsg.multimodal;
+                return processedMsg;
+            }
             if (msg.role !== 'assistant') {
-                return msg;
+                return processedMsg;
             }
-            const msgAny = msg;
-            let processedMsg = { ...msg };
+            const msgAny = processedMsg;
             if (msgAny.reasoning_content && (!msg.content || msg.content.trim() === '')) {
                 processedMsg.content = msgAny.reasoning_content;
                 delete processedMsg.reasoning_content;

package/dist/types/index.d.ts

@@ -21,13 +21,25 @@ export interface ModelInfo {
     lastHealthCheck?: Date;
     healthStatus?: 'healthy' | 'degraded' | 'unhealthy';
 }
+export type ContentPart = {
+    type: 'text';
+    text: string;
+} | {
+    type: 'image_url';
+    image_url: {
+        url: string;
+        detail?: 'auto' | 'low' | 'high';
+    };
+};
 export interface Message {
     role: 'system' | 'user' | 'assistant' | 'tool' | 'error';
     content: string;
+    multimodal?: ContentPart[];
     name?: string;
     tool_calls?: ToolCall[];
     tool_call_id?: string;
 }
+export declare function messageText(content: string | ContentPart[] | undefined | null): string;
 export interface ToolCall {
     id: string;
     type: 'function';

package/dist/types/index.js

@@ -1,2 +1,8 @@
-export {};
+export function messageText(content) {
+    if (!content)
+        return '';
+    if (typeof content === 'string')
+        return content;
+    return content.filter(p => p.type === 'text').map(p => p.text).join('\n');
+}
 //# sourceMappingURL=index.js.map

package/dist/ui/components/PlanExecuteApp.js

@@ -923,14 +923,19 @@ export const PlanExecuteApp = ({ llmClient: initialLlmClient, modelInfo }) => {
             content: userMessage,
         });
         const processedResult = await processFileReferences(userMessage);
-        const processedMessage = processedResult.content;
+        const processedMessage = typeof processedResult.content === 'string' ? processedResult.content : userMessage;
+        const multimodal = Array.isArray(processedResult.content) ? processedResult.content : undefined;
         if (processedResult.includedFiles.length > 0) {
             logger.debug('Files included in message', {
                 files: processedResult.includedFiles,
                 failedFiles: processedResult.failedFiles,
+                hasImages: !!multimodal,
             });
         }
-        let updatedMessages = [...messages, { role: 'user', content: processedMessage }];
+        const userMsg = { role: 'user', content: processedMessage };
+        if (multimodal)
+            userMsg.multimodal = multimodal;
+        let updatedMessages = [...messages, userMsg];
         setMessages(updatedMessages);
         setIsProcessing(true);
         setActivityStartTime(Date.now());
@@ -1052,7 +1057,8 @@ export const PlanExecuteApp = ({ llmClient: initialLlmClient, modelInfo }) => {
             });
             const processAndExecute = async () => {
                 const processedResult = await processFileReferences(queuedMessage);
-                const processedMessage = processedResult.content;
+                const processedMessage = typeof processedResult.content === 'string' ? processedResult.content : queuedMessage;
+                const queuedMultimodal = Array.isArray(processedResult.content) ? processedResult.content : undefined;
                 setIsProcessing(true);
                 setActivityStartTime(Date.now());
                 try {
@@ -1062,7 +1068,10 @@ export const PlanExecuteApp = ({ llmClient: initialLlmClient, modelInfo }) => {
                     }
                     else {
                         logger.flow('No pending TODOs - starting fresh execution');
-                        const updatedMessages = [...messages, { role: 'user', content: processedMessage }];
+                        const queuedMsg = { role: 'user', content: processedMessage };
+                        if (queuedMultimodal)
+                            queuedMsg.multimodal = queuedMultimodal;
+                        const updatedMessages = [...messages, queuedMsg];
                         setMessages(updatedMessages);
                         await planExecutionState.executeAutoMode(processedMessage, llmClient, updatedMessages, setMessages);
                     }

package/dist/ui/hooks/atFileProcessor.d.ts

@@ -6,7 +6,14 @@ export interface AtTriggerInfo {
 export declare function detectAtTrigger(input: string): AtTriggerInfo;
 export declare function insertFilePaths(input: string, atPosition: number, filterLength: number, filePaths: string[]): string;
 export interface ProcessedMessage {
-    content: string;
+    content: string | Array<{
+        type: string;
+        text?: string;
+        image_url?: {
+            url: string;
+            detail?: string;
+        };
+    }>;
     includedFiles: string[];
     failedFiles: string[];
 }

package/dist/ui/hooks/atFileProcessor.js

@@ -48,6 +48,12 @@ export async function processFileReferences(input) {
     const includedFiles = [];
     const failedFiles = [];
     const fileContents = [];
+    const imageBlocks = [];
+    const IMAGE_EXTS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp']);
+    const MIME_MAP = {
+        '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg',
+        '.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
+    };
     for (const filePath of fileRefs) {
         try {
             const resolvedPath = path.resolve(process.cwd(), filePath);
@@ -60,15 +66,41 @@ export async function processFileReferences(input) {
                 const entries = await fs.readdir(resolvedPath);
                 fileContents.push(`\n--- Directory: ${filePath} ---\n${entries.join('\n')}\n---`);
                 includedFiles.push(filePath);
+                continue;
             }
-            else {
-                const content = await fs.readFile(resolvedPath, 'utf-8');
-                const ext = path.extname(filePath).slice(1) || 'txt';
-                fileContents.push(`\n--- File: ${filePath} ---\n\`\`\`${ext}\n${content}\n\`\`\`\n---`);
+            const ext = path.extname(filePath).toLowerCase();
+            if (IMAGE_EXTS.has(ext)) {
+                const buffer = await fs.readFile(resolvedPath);
+                const base64 = buffer.toString('base64');
+                const mime = MIME_MAP[ext] || 'image/png';
+                imageBlocks.push({
+                    type: 'image_url',
+                    image_url: { url: `data:${mime};base64,${base64}`, detail: 'auto' },
+                });
                 includedFiles.push(filePath);
+                continue;
+            }
+            if (ext === '.pdf') {
+                try {
+                    const buffer = await fs.readFile(resolvedPath);
+                    const pdfParse = (await import('pdf-parse')).default;
+                    const data = await pdfParse(buffer);
+                    const text = data.text.slice(0, 50000);
+                    fileContents.push(`\n--- PDF: ${filePath} (${data.numpages} pages) ---\n${text}\n---`);
+                    includedFiles.push(filePath);
+                }
+                catch {
+                    fileContents.push(`\n--- PDF: ${filePath} (text extraction unavailable — install pdf-parse) ---`);
+                    failedFiles.push(filePath);
+                }
+                continue;
             }
+            const content = await fs.readFile(resolvedPath, 'utf-8');
+            const langExt = ext.slice(1) || 'txt';
+            fileContents.push(`\n--- File: ${filePath} ---\n\`\`\`${langExt}\n${content}\n\`\`\`\n---`);
+            includedFiles.push(filePath);
         }
-        catch (error) {
+        catch {
             failedFiles.push(filePath);
         }
     }
@@ -76,6 +108,19 @@ export async function processFileReferences(input) {
     for (const filePath of new Set(fileRefs)) {
         cleanedInput = cleanedInput.replaceAll(`@${filePath}`, `[${filePath}]`);
     }
+    if (imageBlocks.length > 0) {
+        const textContent = fileContents.length > 0
+            ? `${cleanedInput}\n\n<attached_files>${fileContents.join('\n')}</attached_files>`
+            : cleanedInput;
+        return {
+            content: [
+                { type: 'text', text: textContent },
+                ...imageBlocks,
+            ],
+            includedFiles,
+            failedFiles,
+        };
+    }
     const finalContent = fileContents.length > 0
         ? `${cleanedInput}\n\n<attached_files>${fileContents.join('\n')}</attached_files>`
         : cleanedInput;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "orquesta-cli",
-  "version": "0.2.42",
+  "version": "0.2.43",
   "description": "Orquesta CLI - AI-powered coding assistant with team collaboration",
   "type": "module",
   "main": "dist/index.js",
@@ -99,6 +99,7 @@
     "ink-text-input": "^5.0.1",
     "inquirer": "^8.2.6",
     "ora": "^5.4.1",
+    "pdf-parse": "^1.1.1",
     "semver": "^7.7.3",
     "socket.io-client": "^4.8.3",
     "ws": "^8.18.3"
@@ -106,6 +107,7 @@
   "devDependencies": {
     "@types/inquirer": "^9.0.7",
     "@types/node": "^20.10.5",
+    "@types/pdf-parse": "^1.1.5",
     "@types/react": "^19.2.2",
     "@types/semver": "^7.7.1",
     "@types/ws": "^8.18.1",