@llumiverse/drivers 0.23.0 → 0.24.0-dev.20260203.164053Z

package/src/vertexai/models/gemini.ts

@@ -5,12 +5,20 @@ import {
 } from "@google/genai";
 import {
     AIModel, Completion, CompletionChunkObject, CompletionResult, ExecutionOptions,
-    ExecutionTokenUsage, getMaxTokensLimitVertexAi, JSONObject, JSONSchema, ModelType, PromptOptions, PromptRole,
-    PromptSegment, readStreamAsBase64, StatelessExecutionOptions, ToolDefinition, ToolUse,
+    ExecutionTokenUsage,
+    getConversationMeta,
+    getMaxTokensLimitVertexAi,
+    incrementConversationTurn,
+    JSONObject, JSONSchema, ModelType, PromptOptions, PromptRole,
+    PromptSegment, readStreamAsBase64, StatelessExecutionOptions,
+    stripBase64ImagesFromConversation,
+    ToolDefinition, ToolUse,
+    truncateLargeTextInConversation,
+    unwrapConversationArray,
     VertexAIGeminiOptions
 } from "@llumiverse/core";
 import { asyncMap } from "@llumiverse/core/async";
-import { VertexAIDriver, GenerateContentPrompt } from "../index.js";
+import { GenerateContentPrompt, VertexAIDriver } from "../index.js";
 import { ModelDefinition } from "../models.js";
 
 function supportsStructuredOutput(options: PromptOptions): boolean {
@@ -467,11 +475,17 @@ function collectToolUseParts(content: Content): ToolUse[] | undefined {
     const parts = content.parts ?? [];
     for (const part of parts) {
         if (part.functionCall) {
-            out.push({
+            const toolUse: ToolUse = {
                 id: part.functionCall.name ?? '',
                 tool_name: part.functionCall.name ?? '',
                 tool_input: part.functionCall.args as JSONObject,
-            });
+            };
+            // Capture thought_signature for Gemini thinking models (2.5+/3.0+)
+            // This must be passed back with the function response
+            if (part.thoughtSignature) {
+                toolUse.thought_signature = part.thoughtSignature;
+            }
+            out.push(toolUse);
         }
     }
     return out.length > 0 ? out : undefined;
@@ -545,7 +559,7 @@ function geminiThinkingConfig(option: StatelessExecutionOptions): ThinkingConfig
     const model_options = option.model_options as VertexAIGeminiOptions | undefined;
     const include_thoughts = model_options?.include_thoughts ?? false;
     if (model_options?.thinking_budget_tokens) {
-        return {includeThoughts: include_thoughts, thinkingBudget: model_options.thinking_budget_tokens};
+        return { includeThoughts: include_thoughts, thinkingBudget: model_options.thinking_budget_tokens };
     }
 
     // Set minimum thinking level by default.
@@ -623,16 +637,18 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
                 if (!msg.tool_use_id) {
                     throw new Error("Tool response missing tool_use_id");
                 }
+                // Build functionResponse part with optional thought_signature for Gemini thinking models
+                const functionResponsePart: Part = {
+                    functionResponse: {
+                        name: msg.tool_use_id,
+                        response: formatFunctionResponse(msg.content || ''),
+                    },
+                    // Include thought_signature if provided (required for Gemini 2.5+/3.0+ thinking models)
+                    thoughtSignature: msg.thought_signature,
+                };
                 contents.push({
                     role: 'user',
-                    parts: [
-                        {
-                            functionResponse: {
-                                name: msg.tool_use_id,
-                                response: formatFunctionResponse(msg.content || ''),
-                            }
-                        }
-                    ]
+                    parts: [functionResponsePart]
                 });
             } else {    // PromptRole.user, PromptRole.assistant, PromptRole.safety
                 const parts: Part[] = [];
@@ -646,14 +662,27 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
                 // File content handling
                 if (msg.files) {
                     for (const f of msg.files) {
-                        const stream = await f.getStream();
-                        const data = await readStreamAsBase64(stream);
-                        parts.push({
-                            inlineData: {
-                                data,
-                                mimeType: f.mime_type
-                            }
-                        });
+                        let fileUrl = await f.getURL();                     
+                        const isGsUrl = fileUrl.startsWith('gs://') || fileUrl.startsWith('https://storage.googleapis.com/');
+
+                        if (isGsUrl) {
+                            parts.push({
+                                fileData: {
+                                    fileUri: fileUrl,
+                                    mimeType: f.mime_type
+                                }
+                            });
+                        } else {
+                            // Inline data handling
+                            const stream = await f.getStream();
+                            const data = await readStreamAsBase64(stream); 
+                            parts.push({
+                                inlineData: {
+                                    data,
+                                    mimeType: f.mime_type
+                                }
+                            });
+                        }
                     }
                 }
 
@@ -742,7 +771,7 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 
-        let conversation = updateConversation(options.conversation as Content[], prompt.contents);
+        let conversation = updateConversation(options.conversation, prompt.contents);
         prompt.contents = conversation;
 
         // TODO: Remove hack, use global endpoint manually if needed.
@@ -792,12 +821,27 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
             finish_reason = "tool_use";
         }
 
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation) as Content[];
+
+        // Strip large base64 image data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+
         return {
             result: result && result.length > 0 ? result : [{ type: "text" as const, value: '' }],
             token_usage: token_usage,
             finish_reason: finish_reason,
             original_response: options.include_original_response ? response : undefined,
-            conversation,
+            conversation: processedConversation,
             tool_use
         } satisfies Completion;
     }
@@ -811,6 +855,10 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 
+        // Include conversation history in prompt contents (same as non-streaming)
+        const conversation = updateConversation(options.conversation, prompt.contents);
+        prompt.contents = conversation;
+
         if (options.model.includes("gemini-2.5-flash-image")) {
             region = "global"; // Gemini Flash Image only available in global region, this is for nano-banana model
         }
@@ -897,16 +945,19 @@ function getToolFunction(tool: ToolDefinition): FunctionDeclaration {
     };
 }
 
-
 /**
  * Update the conversation messages
  * @param prompt
  * @param response
  * @returns
  */
-function updateConversation(conversation: Content[], prompt: Content[]): Content[] {
-    return (conversation || [] satisfies Content[]).concat(prompt);
+function updateConversation(conversation: unknown, prompt: Content[]): Content[] {
+    // Unwrap array if wrapped, otherwise treat as array
+    const unwrapped = unwrapConversationArray<Content>(conversation);
+    const convArray = unwrapped ?? (conversation as Content[] || []);
+    return convArray.concat(prompt);
 }
+
 /**
  *
  * Gemini supports JSON output in the response. so we test if the response is a valid JSON object. otherwise we treat the response as a string.

package/src/xai/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, Completion, DriverOptions, ExecutionOptions, PromptOptions, PromptSegment } from "@llumiverse/core";
+import { AIModel, DriverOptions, PromptOptions, PromptSegment, Providers } from "@llumiverse/core";
 import { formatOpenAILikeMultimodalPrompt, OpenAIPromptFormatterOptions } from "../openai/openai_format.js";
 import { FetchClient } from "@vertesia/api-fetch-client";
 import OpenAI from "openai";
@@ -15,7 +15,7 @@ export interface xAiDriverOptions extends DriverOptions {
 export class xAIDriver extends BaseOpenAIDriver {
 
     service: OpenAI;
-    provider: "xai";
+    readonly provider = Providers.xai;
     xai_service: FetchClient;
     DEFAULT_ENDPOINT = "https://api.x.ai/v1";
 
@@ -31,7 +31,6 @@ export class xAIDriver extends BaseOpenAIDriver {
             baseURL: opts.endpoint ?? this.DEFAULT_ENDPOINT,
         });
         this.xai_service = new FetchClient(opts.endpoint ?? this.DEFAULT_ENDPOINT).withAuthCallback(async () => `Bearer ${opts.apiKey}`);
-        this.provider = "xai";
         //this.formatPrompt = this._formatPrompt; //TODO: fix xai prompt formatting
     }
 
@@ -49,17 +48,9 @@ export class xAIDriver extends BaseOpenAIDriver {
 
     }
 
-    extractDataFromResponse(_options: ExecutionOptions, result: OpenAI.Chat.Completions.ChatCompletion): Completion {
-        return {
-            result: result.choices[0].message.content ? [{ type: "text", value: result.choices[0].message.content }] : [],
-            finish_reason: result.choices[0].finish_reason,
-            token_usage: {
-                prompt: result.usage?.prompt_tokens,
-                result: result.usage?.completion_tokens,
-                total: result.usage?.total_tokens,
-            }
-        }
-    }
+    // Note: We intentionally do NOT override extractDataFromResponse here.
+    // The base class implementation properly handles tool_calls extraction.
+    // xAI's API is OpenAI-compatible and returns tool_calls in the same format.
 
     async listModels(): Promise<AIModel[]> {
         const [lm, em] = await Promise.all([
@@ -76,10 +67,12 @@ export class xAIDriver extends BaseOpenAIDriver {
             return {
                 id: model.id,
                 provider: this.provider,
-                name: model.object,
-                description: model.object,
+                name: model.id,
+                description: `${model.id} by ${model.owned_by}`,
                 is_multimodal: model.input_modalities.length > 1,
-                tags: [...model.input_modalities.map(m => `ì:${m}`), ...model.output_modalities.map(m => `ì:${m}`)],
+                input_modalities: model.input_modalities,
+                output_modalities: model.output_modalities,
+                tags: [...model.input_modalities.map(m => `i:${m}`), ...model.output_modalities.map(m => `o:${m}`)],
             } satisfies AIModel;
         });
 

package/lib/cjs/adobe/firefly.js

@@ -1,120 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FireflyDriver = void 0;
-const core_1 = require("@llumiverse/core");
-class FireflyDriver extends core_1.AbstractDriver {
-    static PROVIDER = "firefly";
-    provider = FireflyDriver.PROVIDER;
-    endpoint;
-    constructor(options) {
-        super(options);
-        if (!options.apiKey) {
-            throw new Error("No API key provided for Firefly driver");
-        }
-        this.endpoint = options.endpoint || "https://firefly-api.adobe.io/v3";
-    }
-    async requestTextCompletion(_prompt, _options) {
-        throw new Error("Text completion not supported by Firefly");
-    }
-    async requestTextCompletionStream(_prompt, _options) {
-        throw new Error("Text completion streaming not supported by Firefly");
-    }
-    async requestImageGeneration(segments, options) {
-        this.logger.debug(`[${this.provider}] Generating image with model ${options.model}`);
-        const prompt = segments.map(s => s.content).join("\n\n");
-        try {
-            const payload = {
-                prompt: prompt,
-            };
-            const response = await fetch(`${this.endpoint}/images/generate`, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                    'x-api-key': this.options.apiKey,
-                },
-                body: JSON.stringify(payload)
-            });
-            if (!response.ok) {
-                const error = await response.json();
-                throw new Error(`Firefly API error: ${error.message || response.statusText}`);
-            }
-            const result = await response.json();
-            if (result.promptHasDeniedWords || result.promptHasBlockedArtists) {
-                return {
-                    result: [],
-                    error: {
-                        message: "Prompt contains denied words or blocked artists",
-                        code: "content_policy_violation"
-                    }
-                };
-            }
-            return {
-                result: result.outputs.map(output => ({
-                    type: "image",
-                    value: output.image.url
-                }))
-            };
-        }
-        catch (error) {
-            this.logger.error({ error }, "[Firefly] Image generation failed");
-            return {
-                result: [],
-                error: {
-                    message: error.message,
-                    code: error.code || 'GENERATION_FAILED'
-                }
-            };
-        }
-    }
-    mapSize(size) {
-        // Default to 1024x1024 if no size specified
-        if (!size)
-            return { width: 1024, height: 1024 };
-        const [width, height] = size.split('x').map(Number);
-        return { width, height };
-    }
-    async listModels(_params) {
-        return [
-            {
-                id: "firefly-v3-text-to-image",
-                name: "Firefly v3 Text to Image",
-                provider: this.provider,
-                description: "Adobe Firefly v3 text to image generation model",
-                tags: ["image-generation"]
-            },
-            {
-                id: "firefly-v3-image-to-image",
-                name: "Firefly v3 Image to Image",
-                provider: this.provider,
-                description: "Adobe Firefly v3 image to image generation model",
-                tags: ["image-generation"]
-            },
-            {
-                id: "firefly-v3-inpainting",
-                name: "Firefly v3 Inpainting",
-                provider: this.provider,
-                description: "Adobe Firefly v3 inpainting model",
-                tags: ["image-generation"]
-            }
-        ];
-    }
-    async validateConnection() {
-        try {
-            const response = await fetch(`${this.endpoint}/auth/validate`, {
-                headers: {
-                    'x-api-key': this.options.apiKey
-                }
-            });
-            return response.ok;
-        }
-        catch (error) {
-            this.logger.error({ error }, "[Firefly] Connection validation failed");
-            return false;
-        }
-    }
-    async generateEmbeddings(_options) {
-        throw new Error("Embeddings not supported by Firefly");
-    }
-}
-exports.FireflyDriver = FireflyDriver;
-//# sourceMappingURL=firefly.js.map

package/lib/cjs/adobe/firefly.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"firefly.js","sourceRoot":"","sources":["../../../src/adobe/firefly.ts"],"names":[],"mappings":";;;AAAA,2CAAuM;AAoEvM,MAAa,aAAc,SAAQ,qBAAoC;IACnE,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC;IAEjB,QAAQ,CAAS;IAElC,YAAY,OAA6B;QACrC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEf,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iCAAiC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,OAAe,EAAE,QAA0B;QACnE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,OAAe,EAAE,QAA0B;QACzE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,sBAAsB,CAAC,QAAyB,EAAE,OAAyB;QAC7E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,iCAAiC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAGzD,IAAI,CAAC;YACD,MAAM,OAAO,GAA2B;gBACpC,MAAM,EAAE,MAAgB;aAC3B,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,kBAAkB,EAAE;gBAC7D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACL,cAAc,EAAE,kBAAkB;oBAClC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;aAChC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,MAAM,MAAM,GAA4B,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE9D,IAAI,MAAM,CAAC,oBAAoB,IAAI,MAAM,CAAC,uBAAuB,EAAE,CAAC;gBAChE,OAAO;oBACH,MAAM,EAAE,EAAE;oBACV,KAAK,EAAE;wBACH,OAAO,EAAE,iDAAiD;wBAC1D,IAAI,EAAE,0BAA0B;qBACnC;iBACJ,CAAC;YACN,CAAC;YAED,OAAO;gBACH,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBAClC,IAAI,EAAE,OAAgB;oBACtB,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG;iBAC1B,CAAC,CAAC;aACN,CAAC;QAEN,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,mCAAmC,CAAC,CAAC;YAClE,OAAO;gBACH,MAAM,EAAE,EAAE;gBACV,KAAK,EAAE;oBACH,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,mBAAmB;iBAC1C;aACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,OAAO,CAAC,IAAa;QACjB,4CAA4C;QAC5C,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAEhD,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAA4B;QACzC,OAAO;YACH;gBACI,EAAE,EAAE,0BAA0B;gBAC9B,IAAI,EAAE,0BAA0B;gBAChC,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,iDAAiD;gBAC9D,IAAI,EAAE,CAAC,kBAAkB,CAAC;aAC7B;YACD;gBACI,EAAE,EAAE,2BAA2B;gBAC/B,IAAI,EAAE,2BAA2B;gBACjC,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,kDAAkD;gBAC/D,IAAI,EAAE,CAAC,kBAAkB,CAAC;aAC7B;YACD;gBACI,EAAE,EAAE,uBAAuB;gBAC3B,IAAI,EAAE,uBAAuB;gBAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,WAAW,EAAE,mCAAmC;gBAChD,IAAI,EAAE,CAAC,kBAAkB,CAAC;aAC7B;SACJ,CAAC;IACN,CAAC;IAED,KAAK,CAAC,kBAAkB;QACpB,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,gBAAgB,EAAE;gBAC3D,OAAO,EAAE;oBACL,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;iBACnC;aACJ,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,EAAE,wCAAwC,CAAC,CAAC;YACvE,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,QAA2B;QAChD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3D,CAAC;;AAjIL,sCAkIC"}