graphlit-client 1.0.20250611020 → 1.0.20250612002

package/dist/client.js

@@ -1658,12 +1658,21 @@ class Graphlit {
         // Use the formatted message from formatConversation which already includes
         // all context, RAG results, and conversation history
         if (formattedMessage) {
-            messages.push({
+            const messageToAdd = {
                 __typename: "ConversationMessage",
                 role: formattedMessage.role || Types.ConversationRoleTypes.User,
                 message: formattedMessage.message,
                 timestamp: formattedMessage.timestamp || new Date().toISOString(),
-            });
+            };
+            // Add image data if provided
+            if (mimeType && data) {
+                messageToAdd.mimeType = mimeType;
+                messageToAdd.data = data;
+                if (process.env.DEBUG_GRAPHLIT_STREAMING) {
+                    console.log(`\n🖼️ [Streaming] Adding image data to message: ${mimeType}, ${data.length} chars`);
+                }
+            }
+            messages.push(messageToAdd);
         }
         else {
             throw new Error("No formatted message returned from formatConversation");
@@ -2028,14 +2037,15 @@ class Graphlit {
      * Stream with OpenAI client
      */
     async streamWithOpenAI(specification, messages, tools, uiAdapter, onComplete) {
-        if (!OpenAI) {
+        // Check if we have either the OpenAI module or a provided client
+        if (!OpenAI && !this.openaiClient) {
             throw new Error("OpenAI client not available");
         }
         // Use provided client or create a new one
         const openaiClient = this.openaiClient ||
-            new OpenAI({
+            (OpenAI ? new OpenAI({
                 apiKey: process.env.OPENAI_API_KEY || "",
-            });
+            }) : (() => { throw new Error("OpenAI module not available"); })());
         if (process.env.DEBUG_GRAPHLIT_STREAMING) {
             console.log("\n🚀 [Graphlit SDK] Routing to OpenAI streaming provider");
             console.log(`  📋 Specification: ${specification.name} (${specification.id})`);
@@ -2048,14 +2058,15 @@ class Graphlit {
      * Stream with Anthropic client
      */
     async streamWithAnthropic(specification, messages, systemPrompt, tools, uiAdapter, onComplete) {
-        if (!Anthropic) {
+        // Check if we have either the Anthropic module or a provided client
+        if (!Anthropic && !this.anthropicClient) {
             throw new Error("Anthropic client not available");
         }
         // Use provided client or create a new one
         const anthropicClient = this.anthropicClient ||
-            new Anthropic({
+            (Anthropic ? new Anthropic({
                 apiKey: process.env.ANTHROPIC_API_KEY || "",
-            });
+            }) : (() => { throw new Error("Anthropic module not available"); })());
         if (process.env.DEBUG_GRAPHLIT_STREAMING) {
             console.log("\n🚀 [Graphlit SDK] Routing to Anthropic streaming provider");
             console.log(`  📋 Specification: ${specification.name} (${specification.id})`);
@@ -2069,12 +2080,13 @@ class Graphlit {
      * Stream with Google client
      */
     async streamWithGoogle(specification, messages, systemPrompt, tools, uiAdapter, onComplete) {
-        if (!GoogleGenerativeAI) {
+        // Check if we have either the Google module or a provided client
+        if (!GoogleGenerativeAI && !this.googleClient) {
             throw new Error("Google GenerativeAI client not available");
         }
         // Use provided client or create a new one
         const googleClient = this.googleClient ||
-            new GoogleGenerativeAI(process.env.GOOGLE_API_KEY || "");
+            (GoogleGenerativeAI ? new GoogleGenerativeAI(process.env.GOOGLE_API_KEY || "") : (() => { throw new Error("Google GenerativeAI module not available"); })());
         if (process.env.DEBUG_GRAPHLIT_STREAMING) {
             console.log("\n🚀 [Graphlit SDK] Routing to Google streaming provider");
             console.log(`  📋 Specification: ${specification.name} (${specification.id})`);

package/dist/streaming/llm-formatters.d.ts

@@ -4,7 +4,13 @@ import { ConversationMessage } from "../generated/graphql-types.js";
  */
 export interface OpenAIMessage {
     role: "system" | "user" | "assistant" | "tool";
-    content?: string;
+    content?: string | Array<{
+        type: "text" | "image_url";
+        text?: string;
+        image_url?: {
+            url: string;
+        };
+    }>;
     tool_calls?: Array<{
         id: string;
         type: "function";
@@ -21,8 +27,13 @@ export interface OpenAIMessage {
 export interface AnthropicMessage {
     role: "user" | "assistant";
     content: string | Array<{
-        type: "text" | "tool_use" | "tool_result";
+        type: "text" | "image" | "tool_use" | "tool_result";
         text?: string;
+        source?: {
+            type: "base64";
+            media_type: string;
+            data: string;
+        };
         id?: string;
         name?: string;
         input?: unknown;
@@ -37,6 +48,10 @@ export interface GoogleMessage {
     role: "user" | "model";
     parts: Array<{
         text?: string;
+        inlineData?: {
+            mimeType: string;
+            data: string;
+        };
         functionCall?: {
             name: string;
             args: unknown;

package/dist/streaming/llm-formatters.js

@@ -53,10 +53,36 @@ export function formatMessagesForOpenAI(messages) {
                 });
                 break;
             default: // User messages
-                formattedMessages.push({
-                    role: "user",
-                    content: trimmedMessage,
-                });
+                // Check if this message has image data
+                if (message.mimeType && message.data) {
+                    // Multi-modal message with image
+                    const contentParts = [];
+                    // Add text content if present
+                    if (trimmedMessage) {
+                        contentParts.push({
+                            type: "text",
+                            text: trimmedMessage,
+                        });
+                    }
+                    // Add image content
+                    contentParts.push({
+                        type: "image_url",
+                        image_url: {
+                            url: `data:${message.mimeType};base64,${message.data}`,
+                        },
+                    });
+                    formattedMessages.push({
+                        role: "user",
+                        content: contentParts,
+                    });
+                }
+                else {
+                    // Text-only message
+                    formattedMessages.push({
+                        role: "user",
+                        content: trimmedMessage,
+                    });
+                }
                 break;
         }
     }
@@ -122,10 +148,38 @@ export function formatMessagesForAnthropic(messages) {
                 });
                 break;
             default: // User messages
-                formattedMessages.push({
-                    role: "user",
-                    content: trimmedMessage,
-                });
+                // Check if this message has image data
+                if (message.mimeType && message.data) {
+                    // Multi-modal message with image
+                    const contentParts = [];
+                    // Add text content if present
+                    if (trimmedMessage) {
+                        contentParts.push({
+                            type: "text",
+                            text: trimmedMessage,
+                        });
+                    }
+                    // Add image content
+                    contentParts.push({
+                        type: "image",
+                        source: {
+                            type: "base64",
+                            media_type: message.mimeType,
+                            data: message.data,
+                        },
+                    });
+                    formattedMessages.push({
+                        role: "user",
+                        content: contentParts,
+                    });
+                }
+                else {
+                    // Text-only message
+                    formattedMessages.push({
+                        role: "user",
+                        content: trimmedMessage,
+                    });
+                }
                 break;
         }
     }
@@ -138,9 +192,14 @@ export function formatMessagesForAnthropic(messages) {
 export function formatMessagesForGoogle(messages) {
     const formattedMessages = [];
     for (const message of messages) {
-        if (!message.role || !message.message?.trim())
+        if (!message.role)
             continue;
-        const trimmedMessage = message.message.trim();
+        // Allow messages with image data even if they have no text content
+        const hasContent = message.message?.trim();
+        const hasImageData = message.mimeType && message.data;
+        if (!hasContent && !hasImageData)
+            continue;
+        const trimmedMessage = message.message?.trim() || "";
         switch (message.role) {
             case ConversationRoleTypes.System:
                 // Google handles system prompts differently, usually as part of the first user message
@@ -176,10 +235,33 @@ export function formatMessagesForGoogle(messages) {
                 });
                 break;
             default: // User messages
-                formattedMessages.push({
-                    role: "user",
-                    parts: [{ text: trimmedMessage }],
-                });
+                // Check if this message has image data
+                if (message.mimeType && message.data) {
+                    // Multi-modal message with image
+                    const parts = [];
+                    // Add text content if present
+                    if (trimmedMessage) {
+                        parts.push({ text: trimmedMessage });
+                    }
+                    // Add image content
+                    parts.push({
+                        inlineData: {
+                            mimeType: message.mimeType,
+                            data: message.data,
+                        },
+                    });
+                    formattedMessages.push({
+                        role: "user",
+                        parts,
+                    });
+                }
+                else {
+                    // Text-only message
+                    formattedMessages.push({
+                        role: "user",
+                        parts: [{ text: trimmedMessage }],
+                    });
+                }
                 break;
         }
     }

package/dist/streaming/providers.js

@@ -201,10 +201,7 @@ onEvent, onComplete) {
         onComplete(fullMessage, toolCalls);
     }
     catch (error) {
-        onEvent({
-            type: "error",
-            error: error instanceof Error ? error.message : "OpenAI streaming failed",
-        });
+        // Don't emit error event here - let the client handle it to avoid duplicates
         throw error;
     }
 }
@@ -427,10 +424,7 @@ onEvent, onComplete) {
         onComplete(fullMessage, validToolCalls);
     }
     catch (error) {
-        onEvent({
-            type: "error",
-            error: error instanceof Error ? error.message : "Anthropic streaming failed",
-        });
+        // Don't emit error event here - let the client handle it to avoid duplicates
         throw error;
     }
 }
@@ -678,10 +672,7 @@ onEvent, onComplete) {
         onComplete(fullMessage, toolCalls);
     }
     catch (error) {
-        onEvent({
-            type: "error",
-            error: error instanceof Error ? error.message : "Google streaming failed",
-        });
+        // Don't emit error event here - let the client handle it to avoid duplicates
         throw error;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "graphlit-client",
-  "version": "1.0.20250611020",
+  "version": "1.0.20250612002",
   "description": "Graphlit API Client for TypeScript",
   "type": "module",
   "main": "./dist/client.js",