openfox 0.2.3 → 1.0.0

package/dist/{chunk-5ZUU23XF.js → chunk-U6BUT37A.js}

@@ -1,6 +1,10 @@
+import {
+  describeImageFromDataUrl,
+  ensureVisionFallbackConfigLoaded
+} from "./chunk-M2C5GEQR.js";
 import {
   logger
-} from "./chunk-U62ZECSQ.js";
+} from "./chunk-PNBH3RAX.js";
 
 // src/server/llm/backend.ts
 var BACKEND_CAPABILITIES = {
@@ -288,7 +292,8 @@ var DEFAULT_PROFILE = {
   topP: 0.9,
   supportsReasoning: false,
   reasoningAsContent: false,
-  defaultMaxTokens: 4096
+  defaultMaxTokens: 4096,
+  supportsVision: true
 };
 var MOCK_PROFILE = {
   name: "mock",
@@ -296,7 +301,8 @@ var MOCK_PROFILE = {
   topP: 0.9,
   supportsReasoning: false,
   reasoningAsContent: false,
-  defaultMaxTokens: 1024
+  defaultMaxTokens: 1024,
+  supportsVision: false
 };
 var MODEL_PROFILES = [
   {
@@ -307,7 +313,8 @@ var MODEL_PROFILES = [
       topP: 0.9,
       supportsReasoning: false,
       reasoningAsContent: false,
-      defaultMaxTokens: 4096
+      defaultMaxTokens: 16384,
+      supportsVision: false
     }
   },
   {
@@ -321,7 +328,8 @@ var MODEL_PROFILES = [
       // "This model supports only non-thinking mode and does not generate <think></think> blocks"
       supportsReasoning: false,
       reasoningAsContent: false,
-      defaultMaxTokens: 8192
+      defaultMaxTokens: 16384,
+      supportsVision: false
     }
   },
   {
@@ -332,7 +340,20 @@ var MODEL_PROFILES = [
       topP: 0.9,
       supportsReasoning: true,
       reasoningAsContent: false,
-      defaultMaxTokens: 4096
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "qwen3-vl",
+    profile: {
+      name: "Qwen3-VL",
+      temperature: 0.7,
+      topP: 0.9,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: true
     }
   },
   {
@@ -343,7 +364,72 @@ var MODEL_PROFILES = [
       topP: 0.95,
       supportsReasoning: true,
       reasoningAsContent: false,
-      defaultMaxTokens: 4096
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "minimax-m2.5",
+    profile: {
+      name: "MiniMax-M2.5",
+      temperature: 1,
+      topP: 0.95,
+      topK: 40,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "minimax-m2.7",
+    profile: {
+      name: "MiniMax-M2.7",
+      temperature: 1,
+      topP: 0.95,
+      topK: 40,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "minimax",
+    profile: {
+      name: "MiniMax",
+      temperature: 1,
+      topP: 0.95,
+      topK: 40,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "minimax-m3",
+    profile: {
+      name: "MiniMax-M3",
+      temperature: 1,
+      topP: 0.95,
+      topK: 40,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: true
+    }
+  },
+  {
+    pattern: "llava",
+    profile: {
+      name: "LLaVA",
+      temperature: 0.7,
+      topP: 0.9,
+      supportsReasoning: false,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: true
     }
   },
   {
@@ -354,7 +440,8 @@ var MODEL_PROFILES = [
       topP: 0.9,
       supportsReasoning: false,
       reasoningAsContent: false,
-      defaultMaxTokens: 4096
+      defaultMaxTokens: 16384,
+      supportsVision: false
     }
   },
   {
@@ -365,7 +452,20 @@ var MODEL_PROFILES = [
       topP: 0.9,
       supportsReasoning: true,
       reasoningAsContent: false,
-      defaultMaxTokens: 4096
+      defaultMaxTokens: 16384,
+      supportsVision: false
+    }
+  },
+  {
+    pattern: "gemma-4",
+    profile: {
+      name: "Gemma 4",
+      temperature: 0.7,
+      topP: 0.9,
+      supportsReasoning: true,
+      reasoningAsContent: false,
+      defaultMaxTokens: 16384,
+      supportsVision: true
     }
   }
 ];
@@ -383,9 +483,50 @@ function getModelProfile(modelName) {
 }
 
 // src/server/llm/client-pure.ts
-function convertMessages(messages) {
+function convertAttachmentSync(attachment, modelSupportsVision) {
+  if (modelSupportsVision) {
+    return {
+      type: "image_url",
+      image_url: { url: attachment.data }
+    };
+  }
+  return {
+    type: "text",
+    text: `[Image: ${attachment.filename || "image"}] (vision not supported, cannot describe)`
+  };
+}
+async function convertAttachmentWithFallback(attachment, options) {
+  logger.debug("[VisionFallback] convertAttachmentWithFallback called", { filename: attachment.filename, id: attachment.id, hasCallbacks: !!options.onVisionFallbackStart });
+  if (options.modelSupportsVision) {
+    logger.debug("[VisionFallback] Model supports vision - passing image directly");
+    return {
+      type: "image_url",
+      image_url: { url: attachment.data }
+    };
+  }
+  if (!options.visionFallbackEnabled) {
+    logger.debug("[VisionFallback] Fallback disabled - returning placeholder");
+    return {
+      type: "text",
+      text: `[Image: ${attachment.filename || "image"}] (vision not supported)`
+    };
+  }
+  const attachmentId = attachment.id ?? crypto.randomUUID();
+  const filename = attachment.filename;
+  logger.debug("[VisionFallback] Starting delegation for:", { attachmentId, filename });
+  options.onVisionFallbackStart?.(attachmentId, filename);
+  const context = filename ? `File: ${filename}` : void 0;
+  const description = await describeImageFromDataUrl(attachment.data, context ? { context } : {});
+  logger.debug("[VisionFallback] Delegation complete:", { attachmentId, descriptionLength: description.length });
+  options.onVisionFallbackDone?.(attachmentId, description);
+  return {
+    type: "text",
+    text: `[Image: ${attachment.filename || "image"}] ${description}`
+  };
+}
+function convertMessages(messages, options) {
   const filtered = messages.filter((msg) => {
-    return !(msg.role === "assistant" && !msg.content?.trim() && !msg.toolCalls?.length);
+    return !(msg.role === "assistant" && !msg.content?.trim() && (!msg.toolCalls || msg.toolCalls.length === 0));
   });
   return filtered.map((msg) => {
     if (msg.role === "tool") {
@@ -395,10 +536,7 @@ function convertMessages(messages) {
           content.push({ type: "text", text: msg.content });
         }
         for (const attachment of msg.attachments) {
-          content.push({
-            type: "image_url",
-            image_url: { url: attachment.data }
-          });
+          content.push(convertAttachmentSync(attachment, options.modelSupportsVision));
         }
         return {
           role: "tool",
@@ -432,13 +570,7 @@ function convertMessages(messages) {
         content.push({ type: "text", text: msg.content });
       }
       for (const attachment of msg.attachments) {
-        content.push({
-          type: "image_url",
-          image_url: {
-            url: attachment.data
-            // base64 data URL
-          }
-        });
+        content.push(convertAttachmentSync(attachment, options.modelSupportsVision));
       }
       return {
         role: "user",
@@ -451,6 +583,80 @@ function convertMessages(messages) {
     };
   });
 }
+async function convertMessagesWithFallback(messages, options) {
+  logger.debug("[VisionFallback] convertMessagesWithFallback called", { messageCount: messages.length });
+  const filtered = messages.filter((msg) => {
+    return !(msg.role === "assistant" && !msg.content?.trim() && (!msg.toolCalls || msg.toolCalls.length === 0));
+  });
+  const converted = [];
+  for (const msg of filtered) {
+    if (msg.role === "tool") {
+      if (msg.attachments && msg.attachments.length > 0) {
+        const content = [];
+        if (msg.content?.trim()) {
+          content.push({ type: "text", text: msg.content });
+        }
+        for (const attachment of msg.attachments) {
+          const convertedContent = await convertAttachmentWithFallback(
+            { data: attachment.data, filename: attachment.filename, id: attachment.id },
+            options
+          );
+          content.push(convertedContent);
+        }
+        converted.push({
+          role: "tool",
+          content,
+          tool_call_id: msg.toolCallId
+        });
+      } else {
+        converted.push({
+          role: "tool",
+          content: msg.content,
+          tool_call_id: msg.toolCallId
+        });
+      }
+      continue;
+    }
+    if (msg.role === "assistant" && msg.toolCalls?.length) {
+      converted.push({
+        role: "assistant",
+        content: msg.content || null,
+        tool_calls: msg.toolCalls.map((toolCall) => ({
+          id: toolCall.id,
+          type: "function",
+          function: {
+            name: toolCall.name,
+            arguments: JSON.stringify(toolCall.arguments)
+          }
+        }))
+      });
+      continue;
+    }
+    if (msg.role === "user" && msg.attachments && msg.attachments.length > 0) {
+      const content = [];
+      if (msg.content?.trim()) {
+        content.push({ type: "text", text: msg.content });
+      }
+      for (const attachment of msg.attachments) {
+        const convertedContent = await convertAttachmentWithFallback(
+          { data: attachment.data, filename: attachment.filename, id: attachment.id },
+          options
+        );
+        content.push(convertedContent);
+      }
+      converted.push({
+        role: "user",
+        content
+      });
+      continue;
+    }
+    converted.push({
+      role: msg.role,
+      content: msg.content
+    });
+  }
+  return converted;
+}
 function convertTools(tools) {
   return tools.map((tool) => ({
     type: "function",
@@ -461,11 +667,33 @@ function convertTools(tools) {
     }
   }));
 }
-function buildNonStreamingCreateParams(input) {
-  const { model, request, profile, capabilities, disableThinking } = input;
+function needsVisionFallback(messages, modelSupportsVision, visionFallbackEnabled) {
+  const hasAttachments = messages.some(
+    (msg) => msg.attachments && msg.attachments.length > 0 || msg.role === "tool" && msg.attachments && msg.attachments.length > 0
+  );
+  const result = hasAttachments && !modelSupportsVision && visionFallbackEnabled;
+  logger.debug("[VisionFallback] needsVisionFallback check", {
+    hasAttachments,
+    modelSupportsVision,
+    visionFallbackEnabled,
+    result
+  });
+  return result;
+}
+async function buildNonStreamingCreateParams(input) {
+  const { model, request, profile, capabilities, disableThinking, visionFallbackEnabled = false, onVisionFallbackStart, onVisionFallbackDone } = input;
+  const messages = request.messages;
+  const modelSupportsVision = profile.supportsVision ?? false;
+  const options = {
+    modelSupportsVision,
+    visionFallbackEnabled,
+    onVisionFallbackStart,
+    onVisionFallbackDone
+  };
+  const convertedMessages = needsVisionFallback(messages, modelSupportsVision, visionFallbackEnabled) ? await convertMessagesWithFallback(messages, options) : convertMessages(messages, { modelSupportsVision, visionFallbackEnabled: false });
   const params = {
     model,
-    messages: convertMessages(request.messages),
+    messages: convertedMessages,
     ...request.tools ? { tools: convertTools(request.tools) } : {},
     ...request.toolChoice ? { tool_choice: request.toolChoice } : {},
     temperature: request.temperature ?? profile.temperature,
@@ -483,11 +711,15 @@ function buildNonStreamingCreateParams(input) {
   }
   return params;
 }
-function buildStreamingCreateParams(input) {
-  const { model, request, profile, capabilities, disableThinking } = input;
+async function buildStreamingCreateParams(input) {
+  const { model, request, profile, capabilities, disableThinking, visionFallbackEnabled = false, onVisionFallbackStart, onVisionFallbackDone } = input;
+  const messages = request.messages;
+  const modelSupportsVision = profile.supportsVision ?? false;
+  const options = { modelSupportsVision, visionFallbackEnabled, onVisionFallbackStart, onVisionFallbackDone };
+  const convertedMessages = needsVisionFallback(messages, modelSupportsVision, visionFallbackEnabled) ? await convertMessagesWithFallback(messages, options) : convertMessages(messages, { modelSupportsVision, visionFallbackEnabled: false });
   const params = {
     model,
-    messages: convertMessages(request.messages),
+    messages: convertedMessages,
     ...request.tools ? { tools: convertTools(request.tools) } : {},
     ...request.toolChoice ? { tool_choice: request.toolChoice } : {},
     temperature: request.temperature ?? profile.temperature,
@@ -585,13 +817,23 @@ function createLLMClient(config, initialBackend = "unknown") {
       });
       try {
         const shouldDisableThinking = disableThinking || request.disableThinking === true;
-        const createParams = buildNonStreamingCreateParams({
+        await ensureVisionFallbackConfigLoaded();
+        const { isVisionFallbackEnabled } = await import("./vision-fallback-63VPBSEN.js");
+        const paramsOptions = {
           model,
           request,
           profile,
           capabilities,
-          disableThinking: shouldDisableThinking
-        });
+          disableThinking: shouldDisableThinking,
+          visionFallbackEnabled: isVisionFallbackEnabled()
+        };
+        if (request.onVisionFallbackStart) {
+          paramsOptions.onVisionFallbackStart = request.onVisionFallbackStart;
+        }
+        if (request.onVisionFallbackDone) {
+          paramsOptions.onVisionFallbackDone = request.onVisionFallbackDone;
+        }
+        const createParams = await buildNonStreamingCreateParams(paramsOptions);
         const response = await openai.chat.completions.create(createParams, {
           signal: request.signal
         });
@@ -653,8 +895,19 @@ function createLLMClient(config, initialBackend = "unknown") {
         idleTimeout
       });
       try {
+        await ensureVisionFallbackConfigLoaded();
+        const { isVisionFallbackEnabled } = await import("./vision-fallback-63VPBSEN.js");
         const shouldDisableThinking = disableThinking || request.disableThinking === true;
-        const createParams = buildStreamingCreateParams({ model, request, profile, capabilities, disableThinking: shouldDisableThinking });
+        const createParams = await buildStreamingCreateParams({
+          model,
+          request,
+          profile,
+          capabilities,
+          disableThinking: shouldDisableThinking,
+          visionFallbackEnabled: isVisionFallbackEnabled(),
+          onVisionFallbackStart: request.onVisionFallbackStart,
+          onVisionFallbackDone: request.onVisionFallbackDone
+        });
         const stream = await openai.chat.completions.create(createParams, {
           signal: request.signal
         });
@@ -858,4 +1111,4 @@ export {
   getLlmStatus,
   clearModelCache
 };
-//# sourceMappingURL=chunk-5ZUU23XF.js.map
+//# sourceMappingURL=chunk-U6BUT37A.js.map