onbuzz 3.3.0

package/src/tools/visionTool.js

@@ -0,0 +1,507 @@
+/**
+ * Vision Tool — Image analysis for agents
+ *
+ * Purpose:
+ * - Analyze images from disk with natural language questions
+ * - Session-based: first call uploads image, subsequent calls reuse it
+ * - Uses the best available vision model (prefers Anthropic Sonnet/Opus)
+ * - Returns structured answers only (no filler text)
+ *
+ * Actions:
+ * - analyze: Send an image + question, get a structured answer + session ID
+ * - ask: Ask a follow-up question about an image in an existing session
+ * - list-sessions: List active vision sessions
+ * - close-session: Close a vision session to free memory
+ */
+
+import { BaseTool } from './baseTool.js';
+import { promises as fs } from 'fs';
+import path from 'path';
+
+// Dynamic vision model selection priority:
+// 1. Model with "opus" in name + vision capability
+// 2. Model with "sonnet" in name + vision capability
+// 3. Agent's current model if it has vision capability
+// 4. First model in the list with vision capability
+const VISION_MODEL_PRIORITY = ['opus', 'sonnet'];
+
+const MAX_IMAGE_SIZE_BYTES = 20 * 1024 * 1024; // 20MB
+const SESSION_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
+const CLEANUP_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+
+class VisionTool extends BaseTool {
+  constructor(config = {}, logger = null) {
+    super(config, logger);
+
+    this.aiService = null;
+    this.modelsService = null;
+    this.requiresProject = false;
+    this.isAsync = false;
+    this.timeout = 120000; // 2 minutes (vision can be slow)
+
+    // Session storage: sessionId → { imageBase64, mimeType, history, createdAt, lastActivity }
+    this.sessions = new Map();
+    this.cleanupTimer = null;
+  }
+
+  // --- Dependency Injection ---
+
+  setAIService(aiService) { this.aiService = aiService; }
+  setModelsService(modelsService) { this.modelsService = modelsService; }
+  setAgentPool(agentPool) { this.agentPool = agentPool; }
+
+  getDescription() {
+    return `Vision Tool: Analyze images with AI vision models.
+
+Send an image file + a question and receive a structured answer. Supports follow-up questions on the same image without re-uploading.
+
+ACTIONS:
+
+1. ANALYZE — Send image + question (creates a session):
+\`\`\`json
+{ "toolId": "vision", "action": "analyze", "imagePath": "/path/to/image.png", "question": "What objects are in this image?" }
+\`\`\`
+Returns: { answer, sessionId, model }
+
+2. ASK — Follow-up question on an existing session:
+\`\`\`json
+{ "toolId": "vision", "action": "ask", "sessionId": "vs-...", "question": "What color is the largest object?" }
+\`\`\`
+Returns: { answer, sessionId, model }
+
+3. LIST-SESSIONS — Show active vision sessions:
+\`\`\`json
+{ "toolId": "vision", "action": "list-sessions" }
+\`\`\`
+
+4. CLOSE-SESSION — Free a session:
+\`\`\`json
+{ "toolId": "vision", "action": "close-session", "sessionId": "vs-..." }
+\`\`\`
+
+SUPPORTED FORMATS: JPEG, PNG, GIF, WebP, BMP (max 20MB)
+MODEL: Automatically selects the best available vision model (prefers Anthropic Sonnet/Opus).
+ANSWERS: Structured, direct answers only — no filler text.`;
+  }
+
+  parseParameters(content) {
+    return content;
+  }
+
+  getRequiredParameters() {
+    return ['action'];
+  }
+
+  getSupportedActions() {
+    return ['analyze', 'ask', 'list-sessions', 'close-session'];
+  }
+
+  validateParameterTypes(params) {
+    const errors = [];
+    if (params.action && typeof params.action !== 'string') errors.push('action must be a string');
+    if (params.imagePath !== undefined && typeof params.imagePath !== 'string') errors.push('imagePath must be a string');
+    if (params.question !== undefined && typeof params.question !== 'string') errors.push('question must be a string');
+    if (params.sessionId !== undefined && typeof params.sessionId !== 'string') errors.push('sessionId must be a string');
+    return errors;
+  }
+
+  customValidateParameters(params) {
+    const errors = [];
+    const validActions = this.getSupportedActions();
+    if (!validActions.includes(params.action)) {
+      errors.push(`Invalid action: "${params.action}". Valid: ${validActions.join(', ')}`);
+      return errors;
+    }
+    if (params.action === 'analyze') {
+      if (!params.imagePath) errors.push('"imagePath" is required for analyze');
+      if (!params.question) errors.push('"question" is required for analyze');
+    }
+    if (params.action === 'ask') {
+      if (!params.sessionId) errors.push('"sessionId" is required for ask');
+      if (!params.question) errors.push('"question" is required for ask');
+    }
+    if (params.action === 'close-session') {
+      if (!params.sessionId) errors.push('"sessionId" is required for close-session');
+    }
+    return errors;
+  }
+
+  async execute(params, context = {}) {
+    this._ensureCleanup();
+
+    try {
+      switch (params.action) {
+        case 'analyze':
+          return await this._analyze(params.imagePath, params.question, context);
+        case 'ask':
+          return await this._ask(params.sessionId, params.question, context);
+        case 'list-sessions':
+          return this._listSessions();
+        case 'close-session':
+          return this._closeSession(params.sessionId);
+        default:
+          return { success: false, error: `Unknown action: ${params.action}` };
+      }
+    } catch (error) {
+      return { success: false, error: error.message };
+    }
+  }
+
+  // --- Core Actions ---
+
+  _resolveImagePath(imagePath, context) {
+    // Resolve relative to agent's working directory (synced with terminal/filesystem tools)
+    if (path.isAbsolute(imagePath)) return imagePath;
+    const workingDir = context.directoryAccess?.workingDirectory || context.projectDir || process.cwd();
+    return path.resolve(workingDir, imagePath);
+  }
+
+  async _analyze(imagePath, question, context) {
+    // Read and validate image — resolve relative to agent's working directory
+    const resolvedPath = this._resolveImagePath(imagePath, context);
+    let imageBuffer;
+    try {
+      imageBuffer = await fs.readFile(resolvedPath);
+    } catch {
+      throw new Error(`Image not found: ${imagePath}`);
+    }
+
+    if (imageBuffer.length > MAX_IMAGE_SIZE_BYTES) {
+      throw new Error(`Image too large (${(imageBuffer.length / 1024 / 1024).toFixed(1)}MB). Max: ${MAX_IMAGE_SIZE_BYTES / 1024 / 1024}MB`);
+    }
+
+    const ext = path.extname(resolvedPath).toLowerCase();
+    const mimeMap = { '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png', '.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp' };
+    const mimeType = mimeMap[ext];
+    if (!mimeType) {
+      throw new Error(`Unsupported image format: ${ext}. Supported: ${Object.keys(mimeMap).join(', ')}`);
+    }
+
+    const imageBase64 = imageBuffer.toString('base64');
+
+    // Create session
+    const sessionId = `vs-${Date.now()}-${Math.random().toString(36).substr(2, 6)}`;
+    this.sessions.set(sessionId, {
+      imageBase64,
+      mimeType,
+      imagePath: resolvedPath,
+      history: [],
+      createdAt: Date.now(),
+      lastActivity: Date.now()
+    });
+
+    // Send to vision model
+    const answer = await this._sendVisionRequest(imageBase64, mimeType, question, [], context);
+
+    // Store in history
+    this.sessions.get(sessionId).history.push(
+      { role: 'user', content: question },
+      { role: 'assistant', content: answer }
+    );
+
+    return {
+      success: true,
+      result: {
+        answer,
+        sessionId,
+        model: this._getSelectedModel(),
+        imagePath: resolvedPath
+      }
+    };
+  }
+
+  async _ask(sessionId, question, context) {
+    const session = this.sessions.get(sessionId);
+    if (!session) {
+      throw new Error(`Vision session not found: ${sessionId}. Use "analyze" to start a new session.`);
+    }
+
+    session.lastActivity = Date.now();
+
+    // Send with conversation history
+    const answer = await this._sendVisionRequest(
+      session.imageBase64,
+      session.mimeType,
+      question,
+      session.history,
+      context
+    );
+
+    // Append to history
+    session.history.push(
+      { role: 'user', content: question },
+      { role: 'assistant', content: answer }
+    );
+
+    return {
+      success: true,
+      result: {
+        answer,
+        sessionId,
+        model: this._getSelectedModel(),
+        questionsInSession: session.history.filter(h => h.role === 'user').length
+      }
+    };
+  }
+
+  _listSessions() {
+    const sessions = [];
+    for (const [id, session] of this.sessions) {
+      sessions.push({
+        sessionId: id,
+        imagePath: session.imagePath,
+        questionsAsked: session.history.filter(h => h.role === 'user').length,
+        createdAt: new Date(session.createdAt).toISOString(),
+        lastActivity: new Date(session.lastActivity).toISOString()
+      });
+    }
+    return { success: true, result: { sessions, count: sessions.length } };
+  }
+
+  _closeSession(sessionId) {
+    if (!this.sessions.has(sessionId)) {
+      return { success: false, error: `Session not found: ${sessionId}` };
+    }
+    this.sessions.delete(sessionId);
+    return { success: true, result: { closed: sessionId } };
+  }
+
+  // --- Vision API ---
+
+  async _sendVisionRequest(imageBase64, mimeType, question, history, context) {
+    if (!this.aiService) {
+      throw new Error('AI service not available. Vision tool requires aiService dependency.');
+    }
+
+    const model = this._selectVisionModel(context);
+    if (!model) {
+      throw new Error('No vision-capable model available. Ensure at least one model with vision support is configured.');
+    }
+
+    this.logger?.info('[VisionTool] Selected vision model', { model, visionModelsAvailable: this._getVisionModels().map(m => m.name) });
+
+    const systemPrompt = 'You are a precise image analysis assistant. Answer the question about the image directly and concisely. Provide only the answer — no preamble, no filler, no commentary. If the image contains text, transcribe it accurately. If asked about objects, list them clearly. Structure your response for programmatic consumption.';
+
+    // Build messages with image
+    const messages = [];
+
+    // System prompt
+    messages.push({ role: 'system', content: systemPrompt });
+
+    // First message includes the image
+    const imageContent = [
+      { type: 'image_url', image_url: { url: `data:${mimeType};base64,${imageBase64}` } },
+      { type: 'text', text: history.length === 0 ? question : history[0].content }
+    ];
+    messages.push({ role: 'user', content: imageContent });
+
+    // Add conversation history (skip first user message, already included with image)
+    for (let i = 1; i < history.length; i++) {
+      const h = history[i];
+      messages.push({ role: h.role, content: h.content });
+    }
+
+    // Add current question (if there's history, this is a follow-up)
+    if (history.length > 0) {
+      messages.push({ role: 'user', content: question });
+    }
+
+    // Primary: send through chat completions with multimodal content
+    // This works with all vision models via the /chat/completions endpoint
+    try {
+      const response = await this._callChatCompletionsWithImage(imageBase64, mimeType, question, history, model, systemPrompt, context);
+      return response;
+    } catch (chatError) {
+      this.logger?.warn('[VisionTool] Chat completions with image failed, trying /llm/vision endpoint', { error: chatError.message });
+    }
+
+    // Fallback: dedicated /llm/vision endpoint (supports fewer models)
+    try {
+      const response = await this._callVisionEndpoint(imageBase64, question, model, context);
+      return response;
+    } catch (visionError) {
+      throw new Error(`Vision analysis failed: ${visionError.message}`);
+    }
+  }
+
+  async _callChatCompletionsWithImage(imageBase64, mimeType, question, history, model, systemPrompt, context) {
+    const backendUrl = this.aiService?.backendUrl || 'https://loxia-api-g7hrb8bxdae8a2h7.z02.azurefd.net';
+    const sessionId = context.sessionId;
+
+    let apiKey = null;
+    if (this.aiService?.apiKeyManager) {
+      const keys = this.aiService.apiKeyManager.getKeysForRequest(sessionId, { platformProvided: true });
+      apiKey = keys.loxiaApiKey;
+    }
+
+    // Build messages array with image in the first user message
+    const messages = [];
+
+    // First user message includes the image
+    // Format image for the selected model's provider
+    const isAnthropic = model.toLowerCase().includes('claude') || model.toLowerCase().includes('anthropic');
+    const imageBlock = isAnthropic
+      ? { type: 'image', source: { type: 'base64', media_type: mimeType, data: imageBase64 } }
+      : { type: 'image_url', image_url: { url: `data:${mimeType};base64,${imageBase64}` } };
+
+    const firstQuestion = history.length > 0 ? history[0].content : question;
+    messages.push({
+      role: 'user',
+      content: [
+        imageBlock,
+        { type: 'text', text: firstQuestion }
+      ]
+    });
+
+    // Add conversation history (skip first user message, already included)
+    for (let i = 1; i < history.length; i++) {
+      messages.push({ role: history[i].role, content: history[i].content });
+    }
+
+    // Add current question if this is a follow-up
+    if (history.length > 0) {
+      messages.push({ role: 'user', content: question });
+    }
+
+    // Build payload matching the /llm/chat endpoint format
+    const firstUserContent = messages[0]?.content;
+    const messageText = Array.isArray(firstUserContent)
+      ? firstUserContent.find(c => c.type === 'text')?.text || question
+      : question;
+
+    const payload = {
+      model,
+      message: messageText,
+      systemPrompt,
+      messages, // Multimodal messages array
+      options: {
+        max_tokens: 2048,
+        temperature: 0.1
+      },
+      metadata: {
+        requestId: `vision-${Date.now()}`,
+        agentId: context.agentId
+      }
+    };
+
+    const response = await fetch(`${backendUrl}/llm/chat`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(apiKey && { 'Authorization': `Bearer ${apiKey}` })
+      },
+      body: JSON.stringify(payload),
+      signal: AbortSignal.timeout(120000)
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json().catch(() => ({}));
+      throw new Error(errorData.error?.message || errorData.error || `Chat completions returned ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data.content || data.choices?.[0]?.message?.content || '';
+  }
+
+  async _callVisionEndpoint(imageBase64, prompt, model, context) {
+    // Call the backend /llm/vision endpoint directly
+    const backendUrl = this.aiService?.backendUrl || 'https://loxia-api-g7hrb8bxdae8a2h7.z02.azurefd.net';
+    const sessionId = context.sessionId;
+
+    // Get API key
+    let apiKey = null;
+    if (this.aiService?.apiKeyManager) {
+      const keys = this.aiService.apiKeyManager.getKeysForRequest(sessionId, { platformProvided: true });
+      apiKey = keys.loxiaApiKey;
+    }
+
+    const response = await fetch(`${backendUrl}/llm/vision`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        ...(apiKey && { 'Authorization': `Bearer ${apiKey}` })
+      },
+      body: JSON.stringify({ image: imageBase64, prompt, model, metadata: {} }),
+      signal: AbortSignal.timeout(120000)
+    });
+
+    if (!response.ok) {
+      const errorData = await response.json().catch(() => ({}));
+      throw new Error(errorData.error || `Vision API returned ${response.status}`);
+    }
+
+    const data = await response.json();
+    return data.content || '';
+  }
+
+  _selectVisionModel(context = {}) {
+    const visionModels = this._getVisionModels();
+
+    // 1. Search by priority keywords (opus, sonnet)
+    for (const keyword of VISION_MODEL_PRIORITY) {
+      const match = visionModels.find(m => m.name.toLowerCase().includes(keyword));
+      if (match) {
+        this._selectedModel = match.name;
+        return this._selectedModel;
+      }
+    }
+
+    // 2. Agent's current model if it has vision capability
+    const agentModel = context.currentModel;
+    if (agentModel && visionModels.some(m => m.name === agentModel)) {
+      this._selectedModel = agentModel;
+      return this._selectedModel;
+    }
+
+    // 3. First available vision model
+    if (visionModels.length > 0) {
+      this._selectedModel = visionModels[0].name;
+      return this._selectedModel;
+    }
+
+    this.logger?.warn('[VisionTool] No vision-capable model found in model list');
+    return null;
+  }
+
+  _getVisionModels() {
+    if (!this.modelsService) return [];
+    const allModels = this.modelsService.getModels?.() || [];
+    return allModels.filter(m => m.supportsVision === true);
+  }
+
+  _getSelectedModel() {
+    return this._selectedModel || null;
+  }
+
+  // --- Cleanup ---
+
+  _ensureCleanup() {
+    if (this.cleanupTimer) return;
+    this.cleanupTimer = setInterval(() => {
+      const now = Date.now();
+      for (const [id, session] of this.sessions) {
+        if (now - session.lastActivity > SESSION_TIMEOUT_MS) {
+          this.sessions.delete(id);
+          this.logger?.debug('[VisionTool] Session expired', { sessionId: id });
+        }
+      }
+    }, CLEANUP_INTERVAL_MS);
+    // Don't prevent process exit
+    if (this.cleanupTimer.unref) this.cleanupTimer.unref();
+  }
+
+  getParameterSchema() {
+    return {
+      type: 'object',
+      required: ['action'],
+      properties: {
+        action: { type: 'string', enum: ['analyze', 'ask', 'list-sessions', 'close-session'] },
+        imagePath: { type: 'string', description: 'Path to image file (for analyze)' },
+        question: { type: 'string', description: 'Question about the image' },
+        sessionId: { type: 'string', description: 'Session ID (for ask/close-session)' }
+      }
+    };
+  }
+}
+
+export default VisionTool;