nitrostack 1.0.73 → 1.0.75

package/src/studio/components/ops/nodes/index.ts

@@ -6,3 +6,4 @@ export { ToolResultNode } from './ToolResultNode';
 export { WidgetRenderNode } from './WidgetRenderNode';
 export { LLMResponseNode } from './LLMResponseNode';
 
+

package/src/studio/lib/llm-service.ts

@@ -466,10 +466,7 @@ User: "list all resources"
       } else {
         // Regular user or assistant message
         const parts: GeminiFunctionPart[] = [];
-
-        if (msg.content) {
-          parts.push({ text: msg.content });
-        }
+        let hasImageData = false;
 
         if (msg.role === 'user' && msg.file) {
           // Extract base64 and mime type
@@ -486,6 +483,7 @@ User: "list all resources"
               mimeType.startsWith('audio/');
 
             if (isSupported) {
+              hasImageData = true;
               parts.push({
                 inlineData: {
                   mimeType: mimeType,
@@ -516,10 +514,22 @@ User: "list all resources"
           }
         }
 
-        contents.push({
-          role: msg.role === 'assistant' ? 'model' : 'user',
-          parts,
-        });
+        // Add text content - IMPORTANT: Gemini REQUIRES a text prompt when processing images
+        if (msg.content) {
+          parts.push({ text: msg.content });
+        } else if (hasImageData) {
+          // If we have image data but no text, add a default prompt
+          // Gemini requires at least some text instruction with images
+          parts.push({ text: `Please analyze this image (${msg.file?.name || 'uploaded file'}) and process it with the appropriate tool if needed.` });
+        }
+
+        // Only add content if we have parts
+        if (parts.length > 0) {
+          contents.push({
+            role: msg.role === 'assistant' ? 'model' : 'user',
+            parts,
+          });
+        }
         i++;
       }
     }
@@ -582,8 +592,36 @@ User: "list all resources"
       // console.log('Sending tools to Gemini:', JSON.stringify(requestBody.tools, null, 2));
     }
 
+    // Add generation config to ensure we get a response
+    requestBody.generationConfig = {
+      temperature: 0.7,
+      topP: 0.95,
+      topK: 40,
+      maxOutputTokens: 8192,
+    };
+
+    // Add safety settings to be more permissive (for image processing)
+    requestBody.safetySettings = [
+      { category: 'HARM_CATEGORY_HARASSMENT', threshold: 'BLOCK_ONLY_HIGH' },
+      { category: 'HARM_CATEGORY_HATE_SPEECH', threshold: 'BLOCK_ONLY_HIGH' },
+      { category: 'HARM_CATEGORY_SEXUALLY_EXPLICIT', threshold: 'BLOCK_ONLY_HIGH' },
+      { category: 'HARM_CATEGORY_DANGEROUS_CONTENT', threshold: 'BLOCK_ONLY_HIGH' },
+    ];
+
+    // Log request for debugging (truncate large data)
+    const debugRequest = JSON.parse(JSON.stringify(requestBody));
+    if (debugRequest.contents) {
+      debugRequest.contents = debugRequest.contents.map((c: { parts?: Array<{ inlineData?: { data?: string } }> }) => ({
+        ...c,
+        parts: c.parts?.map((p: { inlineData?: { data?: string } }) => 
+          p.inlineData ? { ...p, inlineData: { ...p.inlineData, data: '[BASE64_TRUNCATED]' } } : p
+        )
+      }));
+    }
+    console.log('Gemini request body (truncated):', JSON.stringify(debugRequest, null, 2));
+
     // Use Gemini 2.0 Flash Experimental (latest model with function calling)
-    // The v1beta API uses 'gemini-2.0-flash-exp' for the newest features
+    // Fallback to gemini-1.5-flash if issues persist
     const model = 'gemini-2.0-flash-exp';
     const response = await fetch(
       `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`,
@@ -605,7 +643,36 @@ User: "list all resources"
 
     const candidate = data.candidates?.[0];
     if (!candidate) {
-      throw new Error('No response from Gemini');
+      // Check for prompt feedback (content filtering)
+      if (data.promptFeedback) {
+        const feedback = data.promptFeedback;
+        if (feedback.blockReason) {
+          throw new Error(`Gemini blocked the request: ${feedback.blockReason}. ${feedback.blockReasonMessage || ''}`);
+        }
+        if (feedback.safetyRatings) {
+          const blockedRatings = feedback.safetyRatings.filter(
+            (r: { probability: string }) => r.probability === 'HIGH' || r.probability === 'MEDIUM'
+          );
+          if (blockedRatings.length > 0) {
+            throw new Error(`Gemini safety filter triggered: ${JSON.stringify(blockedRatings)}`);
+          }
+        }
+      }
+      
+      // Check if there's a candidates array but it's empty with finish reason
+      if (data.candidates && data.candidates.length === 0) {
+        throw new Error('Gemini returned empty candidates - content may have been filtered');
+      }
+      
+      // Generic error with full response for debugging
+      console.error('Full Gemini response without candidates:', JSON.stringify(data, null, 2));
+      throw new Error(`No response from Gemini. Response metadata: ${JSON.stringify(data.usageMetadata || {})}`);
+    }
+    
+    // Check if candidate was blocked
+    if (candidate.finishReason === 'SAFETY' || candidate.finishReason === 'BLOCKED') {
+      const safetyRatings = candidate.safetyRatings || [];
+      throw new Error(`Gemini blocked response due to safety: ${JSON.stringify(safetyRatings)}`);
     }
 
     const parts = candidate.content?.parts || [];