crewswarm 0.8.1-beta

package/lib/integrations/multimodal.mjs

@@ -0,0 +1,313 @@
+#!/usr/bin/env node
+/**
+ * multimodal.mjs — Shared image/audio processing for crewswarm
+ * 
+ * Supports:
+ *   - Image analysis via Groq Vision (Llama 4 Scout) or Gemini 2.0 Flash
+ *   - Audio transcription via Groq Whisper or Gemini
+ *   - Used by Telegram, WhatsApp, dashboard, and CrewChat
+ */
+
+import { readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { loadSwarmConfig } from "../runtime/config.mjs";
+
+// ── Config ────────────────────────────────────────────────────────────────────
+
+const cfg = loadSwarmConfig();
+const providers = cfg.providers || {};
+
+// Provider selection priority: Groq (fast/cheap) → Gemini → OpenAI → fail
+const GROQ_KEY = providers.groq?.apiKey || "";
+const GEMINI_KEY = providers.google?.apiKey || providers.gemini?.apiKey || "";
+const OPENAI_KEY = providers.openai?.apiKey || "";
+
+const GROQ_VISION_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct";
+const GROQ_WHISPER_MODEL = "whisper-large-v3-turbo"; // 216x realtime, $0.04/hr
+const GEMINI_MODEL = "gemini-2.0-flash-exp";
+
+// ── Image Analysis ────────────────────────────────────────────────────────────
+
+/**
+ * Analyze an image via URL or base64
+ * @param {string} imageUrlOrBase64 - Public URL or base64 data URI
+ * @param {string} prompt - Question/instruction for the model
+ * @param {object} options - { provider: 'groq'|'gemini'|'auto' }
+ * @returns {Promise<string>} - Model's response
+ */
+export async function analyzeImage(imageUrlOrBase64, prompt = "Describe this image in detail.", options = {}) {
+  const provider = options.provider || "auto";
+  
+  // Auto-select: prefer Groq (cheaper), fallback to Gemini
+  if (provider === "auto") {
+    if (GROQ_KEY) return analyzeImageGroq(imageUrlOrBase64, prompt);
+    if (GEMINI_KEY) return analyzeImageGemini(imageUrlOrBase64, prompt);
+    throw new Error("No vision provider configured (need Groq or Gemini API key)");
+  }
+  
+  if (provider === "groq") return analyzeImageGroq(imageUrlOrBase64, prompt);
+  if (provider === "gemini") return analyzeImageGemini(imageUrlOrBase64, prompt);
+  throw new Error(`Unknown vision provider: ${provider}`);
+}
+
+async function analyzeImageGroq(imageUrlOrBase64, prompt) {
+  if (!GROQ_KEY) throw new Error("Groq API key missing (providers.groq.apiKey)");
+  
+  // Convert to message format Groq expects
+  const isUrl = imageUrlOrBase64.startsWith("http://") || imageUrlOrBase64.startsWith("https://");
+  const imageContent = isUrl 
+    ? { type: "image_url", image_url: { url: imageUrlOrBase64 } }
+    : { type: "image_url", image_url: { url: imageUrlOrBase64 } }; // base64 also uses image_url
+  
+  const res = await fetch("https://api.groq.com/openai/v1/chat/completions", {
+    method: "POST",
+    headers: {
+      "Authorization": `Bearer ${GROQ_KEY}`,
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify({
+      model: GROQ_VISION_MODEL,
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: prompt },
+            imageContent
+          ]
+        }
+      ],
+      temperature: 0.7,
+      max_tokens: 1024
+    })
+  });
+  
+  if (!res.ok) {
+    const error = await res.text();
+    throw new Error(`Groq vision API error: ${res.status} ${error}`);
+  }
+  
+  const data = await res.json();
+  return data.choices?.[0]?.message?.content || "No response from vision model";
+}
+
+async function analyzeImageGemini(imageUrlOrBase64, prompt) {
+  if (!GEMINI_KEY) throw new Error("Gemini API key missing (providers.google.apiKey)");
+  
+  // Gemini expects base64 inline parts
+  let imagePart;
+  if (imageUrlOrBase64.startsWith("data:image/")) {
+    // Extract mime and base64 from data URI
+    const [mime, base64] = imageUrlOrBase64.replace(/^data:/, "").split(";base64,");
+    imagePart = {
+      inline_data: {
+        mime_type: mime,
+        data: base64
+      }
+    };
+  } else if (imageUrlOrBase64.startsWith("http")) {
+    // Download and convert to base64
+    const imgRes = await fetch(imageUrlOrBase64);
+    const buffer = Buffer.from(await imgRes.arrayBuffer());
+    const base64 = buffer.toString("base64");
+    const mime = imgRes.headers.get("content-type") || "image/jpeg";
+    imagePart = {
+      inline_data: {
+        mime_type: mime,
+        data: base64
+      }
+    };
+  } else {
+    // Already base64
+    imagePart = {
+      inline_data: {
+        mime_type: "image/jpeg",
+        data: imageUrlOrBase64
+      }
+    };
+  }
+  
+  const res = await fetch(
+    `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${GEMINI_KEY}`,
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        contents: [
+          {
+            parts: [
+              { text: prompt },
+              imagePart
+            ]
+          }
+        ]
+      })
+    }
+  );
+  
+  if (!res.ok) {
+    const error = await res.text();
+    throw new Error(`Gemini API error: ${res.status} ${error}`);
+  }
+  
+  const data = await res.json();
+  return data.candidates?.[0]?.content?.parts?.[0]?.text || "No response from Gemini";
+}
+
+// ── Audio Transcription ───────────────────────────────────────────────────────
+
+/**
+ * Transcribe audio file to text
+ * @param {Buffer} audioBuffer - Raw audio file data
+ * @param {object} options - { provider, language, mimeType: 'audio/m4a'|'audio/webm'|... }
+ * @returns {Promise<string>} - Transcribed text
+ */
+export async function transcribeAudio(audioBuffer, options = {}) {
+  const provider = options.provider || "auto";
+  
+  // Auto-select: Groq → Gemini → OpenAI (Whisper handles m4a well)
+  if (provider === "auto") {
+    if (GROQ_KEY) return transcribeAudioGroq(audioBuffer, options);
+    if (GEMINI_KEY) return transcribeAudioGemini(audioBuffer, options);
+    if (OPENAI_KEY) return transcribeAudioOpenAI(audioBuffer, options);
+    throw new Error("No audio provider configured (need Groq, Gemini, or OpenAI API key)");
+  }
+  if (provider === "groq") return transcribeAudioGroq(audioBuffer, options);
+  if (provider === "gemini") return transcribeAudioGemini(audioBuffer, options);
+  if (provider === "openai") return transcribeAudioOpenAI(audioBuffer, options);
+  throw new Error(`Unknown audio provider: ${provider}`);
+}
+
+async function transcribeAudioGroq(audioBuffer, options) {
+  if (!GROQ_KEY) throw new Error("Groq API key missing (providers.groq.apiKey)");
+  const mime = options.mimeType || "audio/m4a";
+  const filename = mime.includes("m4a") || mime === "audio/mp4" ? "audio.m4a" : mime.includes("webm") ? "audio.webm" : "audio.m4a";
+  const formData = new FormData();
+  formData.append("file", new Blob([audioBuffer], { type: mime }), filename);
+  formData.append("model", GROQ_WHISPER_MODEL);
+  formData.append("response_format", "json"); // Explicit per Groq docs
+  if (options.language && options.language !== "auto") {
+    formData.append("language", options.language);
+  }
+  
+  const res = await fetch("https://api.groq.com/openai/v1/audio/transcriptions", {
+    method: "POST",
+    headers: {
+      "Authorization": `Bearer ${GROQ_KEY}`
+    },
+    body: formData
+  });
+  
+  if (!res.ok) {
+    const error = await res.text();
+    throw new Error(`Groq Whisper API error: ${res.status} ${error}`);
+  }
+  
+  const data = await res.json();
+  const text = (data.text || "").trim();
+  if (!text) throw new Error("Groq returned empty transcription (check audio format and length)");
+  return text;
+}
+
+async function transcribeAudioGemini(audioBuffer, options) {
+  if (!GEMINI_KEY) throw new Error("Gemini API key missing (providers.google.apiKey)");
+  // Gemini expects audio/mp4 for m4a, not audio/m4a
+  const raw = options.mimeType || "audio/m4a";
+  const mimeType = raw === "audio/m4a" || raw.includes("m4a") ? "audio/mp4" : raw;
+  const base64 = audioBuffer.toString("base64");
+  const res = await fetch(
+    `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${GEMINI_KEY}`,
+    {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        contents: [
+          {
+            parts: [
+              { text: "Transcribe this audio to text. Only return the transcription, no other commentary." },
+              {
+                inline_data: {
+                  mime_type: mimeType,
+                  data: base64
+                }
+              }
+            ]
+          }
+        ]
+      })
+    }
+  );
+  
+  if (!res.ok) {
+    const error = await res.text();
+    throw new Error(`Gemini audio API error: ${res.status} ${error}`);
+  }
+  
+  const data = await res.json();
+  const text = (data.candidates?.[0]?.content?.parts?.[0]?.text || "").trim();
+  if (!text) throw new Error("Gemini returned empty transcription (check audio format and length)");
+  return text;
+}
+
+async function transcribeAudioOpenAI(audioBuffer, options) {
+  if (!OPENAI_KEY) throw new Error("OpenAI API key missing (providers.openai.apiKey)");
+  const mime = options.mimeType || "audio/m4a";
+  const filename = mime.includes("m4a") || mime === "audio/mp4" ? "audio.m4a" : mime.includes("webm") ? "audio.webm" : "audio.m4a";
+  const formData = new FormData();
+  formData.append("file", new Blob([audioBuffer], { type: mime }), filename);
+  formData.append("model", "whisper-1");
+  const res = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+    method: "POST",
+    headers: { "Authorization": `Bearer ${OPENAI_KEY}` },
+    body: formData
+  });
+  if (!res.ok) {
+    const err = await res.text();
+    throw new Error(`OpenAI Whisper API error: ${res.status} ${err}`);
+  }
+  const data = await res.json();
+  const text = (data.text || "").trim();
+  if (!text) throw new Error("OpenAI returned empty transcription (check audio format and length)");
+  return text;
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+/**
+ * Download file from URL to Buffer
+ */
+export async function downloadToBuffer(url) {
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Download failed: ${res.status}`);
+  return Buffer.from(await res.arrayBuffer());
+}
+
+/**
+ * Convert file path to base64 data URI
+ */
+export function fileToBase64DataUri(filePath, mimeType = "image/jpeg") {
+  const buffer = readFileSync(filePath);
+  const base64 = buffer.toString("base64");
+  return `data:${mimeType};base64,${base64}`;
+}
+
+/**
+ * Check if provider is configured
+ */
+export function hasVisionProvider() {
+  return !!(GROQ_KEY || GEMINI_KEY);
+}
+
+export function hasAudioProvider() {
+  return !!(GROQ_KEY || GEMINI_KEY || OPENAI_KEY);
+}
+
+/**
+ * Get active provider names
+ */
+export function getActiveProviders() {
+  const providers = [];
+  if (GROQ_KEY) providers.push("groq");
+  if (GEMINI_KEY) providers.push("gemini");
+  return providers;
+}

package/lib/integrations/telegram-streaming.mjs

@@ -0,0 +1,153 @@
+/**
+ * Telegram Native Streaming Handler
+ * Uses sendMessageDraft (Bot API 9.5) for progressive text display
+ */
+
+/**
+ * Stream LLM response progressively to Telegram using sendMessageDraft
+ * @param {number} chatId - Telegram chat ID
+ * @param {number|null} threadId - Topic ID (if in forum group)
+ * @param {string} targetAgent - Agent identifier (for logging)
+ * @param {Object} provider - Provider config { baseUrl, apiKey }
+ * @param {string} modelId - Model identifier
+ * @param {Array} messages - Message array for LLM
+ * @param {Function} tgRequest - Telegram API request function
+ * @param {Function} log - Logger function
+ * @returns {Promise<string>} Final response text
+ */
+export async function streamToTelegram({
+  chatId,
+  threadId = null,
+  targetAgent,
+  provider,
+  modelId,
+  messages,
+  tgRequest,
+  log
+}) {
+  const draftId = Date.now(); // Unique draft ID for this stream
+  let fullText = "";
+  let lastUpdate = 0;
+  const UPDATE_INTERVAL_MS = 100; // Update every 100ms (smooth but not spammy)
+  
+  try {
+    // Call LLM with streaming enabled
+    const response = await fetch(provider.baseUrl + "/chat/completions", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${provider.apiKey}`
+      },
+      body: JSON.stringify({
+        model: modelId,
+        messages,
+        temperature: 0.7,
+        stream: true // Enable streaming
+      })
+    });
+    
+    if (!response.ok) {
+      throw new Error(`LLM API returned ${response.status}`);
+    }
+    
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() || "";
+      
+      for (const line of lines) {
+        if (!line.trim() || !line.startsWith("data: ")) continue;
+        
+        const data = line.slice(6);
+        if (data === "[DONE]") continue;
+        
+        try {
+          const parsed = JSON.parse(data);
+          const content = parsed.choices?.[0]?.delta?.content;
+          
+          if (content) {
+            fullText += content;
+            
+            // Throttle updates - only send every 100ms
+            const now = Date.now();
+            if (now - lastUpdate >= UPDATE_INTERVAL_MS) {
+              await tgRequest("sendMessageDraft", {
+                chat_id: chatId,
+                ...(threadId && { message_thread_id: threadId }),
+                draft_id: draftId,
+                text: fullText,
+                parse_mode: "Markdown"
+              }).catch(err => {
+                // Fallback: if Markdown fails, try without parse_mode
+                return tgRequest("sendMessageDraft", {
+                  chat_id: chatId,
+                  ...(threadId && { message_thread_id: threadId }),
+                  draft_id: draftId,
+                  text: fullText
+                });
+              });
+              
+              lastUpdate = now;
+            }
+          }
+        } catch (parseErr) {
+          log("warn", "Failed to parse SSE chunk", { error: parseErr.message });
+        }
+      }
+    }
+    
+    // Send final update (in case last chunk didn't trigger threshold)
+    if (fullText && Date.now() - lastUpdate >= 50) {
+      await tgRequest("sendMessageDraft", {
+        chat_id: chatId,
+        ...(threadId && { message_thread_id: threadId }),
+        draft_id: draftId,
+        text: fullText,
+        parse_mode: "Markdown"
+      }).catch(err => {
+        return tgRequest("sendMessageDraft", {
+          chat_id: chatId,
+          ...(threadId && { message_thread_id: threadId }),
+          draft_id: draftId,
+          text: fullText
+        });
+      });
+    }
+    
+    log("info", "Streaming complete", { 
+      targetAgent, 
+      chatId, 
+      threadId, 
+      length: fullText.length 
+    });
+    
+    return fullText;
+    
+  } catch (err) {
+    log("error", "Streaming failed", { 
+      targetAgent, 
+      error: err.message 
+    });
+    throw err;
+  }
+}
+
+/**
+ * Check if native streaming is supported for this chat
+ * sendMessageDraft only works in private chats (DMs and private topics)
+ * @param {number} chatId - Telegram chat ID
+ * @returns {boolean}
+ */
+export function supportsNativeStreaming(chatId) {
+  // Private chats have positive IDs
+  // Groups/supergroups have negative IDs
+  // BUT: Private chats with topics ARE supported
+  return chatId > 0;
+}