@llumiverse/drivers 1.0.0-dev.20260224.234313Z → 1.0.0

package/lib/esm/openai/index.js

@@ -1,4 +1,5 @@
-import { AbstractDriver, ModelType, TrainingJobStatus, getConversationMeta, getModelCapabilities, incrementConversationTurn, modelModalitiesToArray, stripBase64ImagesFromConversation, supportsToolUse, truncateLargeTextInConversation, unwrapConversationArray, } from "@llumiverse/core";
+import { AbstractDriver, LlumiverseError, ModelType, TrainingJobStatus, getConversationMeta, getModelCapabilities, incrementConversationTurn, modelModalitiesToArray, stripBase64ImagesFromConversation, stripHeartbeatsFromConversation, supportsToolUse, truncateLargeTextInConversation, unwrapConversationArray, } from "@llumiverse/core";
+import { APIConnectionError, APIConnectionTimeoutError, APIError, AuthenticationError, BadRequestError, ConflictError, ContentFilterFinishReasonError, InternalServerError, LengthFinishReasonError, NotFoundError, OpenAIError, PermissionDeniedError, RateLimitError, UnprocessableEntityError, } from 'openai/error';
 import { formatOpenAILikeMultimodalPrompt } from "./openai_format.js";
 // Helper function to convert string to CompletionResult[]
 function textToCompletionResult(text) {
@@ -20,13 +21,14 @@ export class BaseOpenAIDriver extends AbstractDriver {
     extractDataFromResponse(_options, result) {
         const tokenInfo = mapUsage(result.usage);
         const tools = collectTools(result.output);
-        const data = extractTextFromResponse(result);
-        if (!data && !tools) {
+        // Collect all parts in order (text and images)
+        const allResults = extractCompletionResults(result.output);
+        if (allResults.length === 0 && !tools) {
             this.logger.error({ result }, "[OpenAI] Response is not valid");
             throw new Error("Response is not valid: no data");
         }
         return {
-            result: textToCompletionResult(data || ''),
+            result: allResults,
             token_usage: tokenInfo,
             finish_reason: responseFinishReason(result, tools),
             tool_use: tools,
@@ -37,9 +39,15 @@ export class BaseOpenAIDriver extends AbstractDriver {
             this.logger.warn({ options: options.model_options }, "Invalid model options");
         }
         // Include conversation history (same as non-streaming)
-        const conversation = updateConversation(options.conversation, prompt);
+        // Fix orphaned function_call items (can occur when agent is stopped mid-tool-execution)
+        let conversation = fixOrphanedToolUse(updateConversation(options.conversation, prompt));
         const toolDefs = getToolDefinitions(options.tools);
         const useTools = toolDefs ? supportsToolUse(options.model, this.provider, true) : false;
+        // When no tools are provided but conversation contains function_call/function_call_output
+        // items (e.g. checkpoint summary calls), convert them to text to avoid API errors
+        if (!useTools) {
+            conversation = convertOpenAIFunctionItemsToText(conversation);
+        }
         convertRoles(prompt, options.model);
         const model_options = options.model_options;
         insert_image_detail(prompt, model_options?.image_detail ?? "auto");
@@ -86,7 +94,13 @@ export class BaseOpenAIDriver extends AbstractDriver {
         insert_image_detail(prompt, model_options?.image_detail ?? "auto");
         const toolDefs = getToolDefinitions(options.tools);
         const useTools = toolDefs ? supportsToolUse(options.model, this.provider) : false;
-        let conversation = updateConversation(options.conversation, prompt);
+        // Fix orphaned function_call items (can occur when agent is stopped mid-tool-execution)
+        let conversation = fixOrphanedToolUse(updateConversation(options.conversation, prompt));
+        // When no tools are provided but conversation contains function_call/function_call_output
+        // items (e.g. checkpoint summary calls), convert them to text to avoid API errors
+        if (!useTools) {
+            conversation = convertOpenAIFunctionItemsToText(conversation);
+        }
         let parsedSchema = undefined;
         let strictMode = false;
         if (options.result_schema && supportsSchema(options.model)) {
@@ -136,10 +150,21 @@ export class BaseOpenAIDriver extends AbstractDriver {
         let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
         // Truncate large text content if configured
         processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        // Strip old heartbeat status messages
+        processedConversation = stripHeartbeatsFromConversation(processedConversation, {
+            keepForTurns: options.stripHeartbeatsAfterTurns ?? 1,
+            currentTurn,
+        });
         completion.conversation = processedConversation;
         return completion;
     }
     canStream(_options) {
+        // Image generation models don't support streaming
+        if (_options.model.includes("dall-e")
+            || _options.model.includes("gpt-image")
+            || _options.model.includes("chatgpt-image")) {
+            return Promise.resolve(false);
+        }
         if (_options.model.includes("o1")
             && !(_options.model.includes("mini") || _options.model.includes("preview"))) {
             //o1 full does not support streaming
@@ -187,6 +212,10 @@ export class BaseOpenAIDriver extends AbstractDriver {
         };
         let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
         processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        processedConversation = stripHeartbeatsFromConversation(processedConversation, {
+            keepForTurns: options.stripHeartbeatsAfterTurns ?? 1,
+            currentTurn,
+        });
         return processedConversation;
     }
     createTrainingPrompt(options) {
@@ -240,7 +269,7 @@ export class BaseOpenAIDriver extends AbstractDriver {
         //Some of these use the completions API instead of the chat completions API.
         //Others are for non-text input modalities. Therefore common to both.
         const wordBlacklist = ["embed", "whisper", "transcribe", "audio", "moderation", "tts",
-            "realtime", "dall-e", "babbage", "davinci", "codex", "o1-pro", "computer-use", "sora"];
+            "realtime", "babbage", "davinci", "codex", "o1-pro", "computer-use", "sora"];
         //OpenAI has very little information, filtering based on name.
         result = result.filter((m) => {
             return !wordBlacklist.some((word) => m.id.includes(word));
@@ -252,14 +281,17 @@ export class BaseOpenAIDriver extends AbstractDriver {
             if (owner == "system") {
                 owner = "openai";
             }
+            // Determine model type based on capabilities
+            let modelType = ModelType.Text;
+            if (m.id.includes("dall-e") || m.id.includes("gpt-image")) {
+                modelType = ModelType.Image;
+            }
             return {
                 id: m.id,
                 name: m.id,
                 provider: this.provider,
                 owner: owner,
-                type: m.object === "model" ? ModelType.Text : ModelType.Unknown,
-                can_stream: true,
-                is_multimodal: m.id.includes("gpt-4"),
+                type: modelType,
                 input_modalities: modelModalitiesToArray(modelCapability.input),
                 output_modalities: modelModalitiesToArray(modelCapability.output),
                 tool_support: modelCapability.tool_support,
@@ -284,6 +316,289 @@ export class BaseOpenAIDriver extends AbstractDriver {
         }
         return { values: embeddings, model };
     }
+    imageModels = ["dall-e", "gpt-image", "chatgpt-image"];
+    /**
+     * Determine if a model is specifically an image generation model (not conversational image model)
+     */
+    isImageModel(model) {
+        // DALL-E models are standalone image generation
+        // gpt-image models can generate images in conversations, not standalone
+        return this.imageModels.some(imageModel => model.includes(imageModel));
+    }
+    /**
+     * Request image generation from standalone Images API
+     * Supports: DALL-E 2, DALL-E 3, GPT-image models (for edit/variation)
+     */
+    async requestImageGeneration(prompt, options) {
+        this.logger.debug(`[${this.provider}] Generating image with model ${options.model}`);
+        const model_options = options.model_options;
+        // Extract prompt text from ResponseInputItem[]
+        let promptText = "";
+        for (const item of prompt) {
+            if ('content' in item && typeof item.content === 'string') {
+                promptText += item.content + "\\n";
+            }
+            else if ('content' in item && Array.isArray(item.content)) {
+                // Extract text from content array
+                for (const part of item.content) {
+                    if ('type' in part && part.type === 'input_text' && 'text' in part) {
+                        promptText += part.text + "\\n";
+                    }
+                }
+            }
+        }
+        promptText = promptText.trim();
+        try {
+            const generateParams = {
+                model: options.model,
+                prompt: promptText,
+                size: model_options?.size || "1024x1024",
+            };
+            // Add DALL-E specific options
+            if (options.model.includes("dall-e") || model_options?._option_id === "openai-dalle") {
+                const dalleOptions = model_options;
+                generateParams.n = dalleOptions?.n || 1;
+                generateParams.response_format = dalleOptions?.response_format || "b64_json";
+                if (options.model.includes("dall-e-3")) {
+                    generateParams.quality = dalleOptions?.image_quality || "standard";
+                    if (dalleOptions?.style) {
+                        generateParams.style = dalleOptions.style;
+                    }
+                }
+            }
+            else {
+                // Default for other models
+                generateParams.n = 1;
+            }
+            const response = await this.service.images.generate(generateParams);
+            // Convert response to CompletionResults
+            const results = [];
+            if (response.data) {
+                for (const image of response.data) {
+                    let imageValue;
+                    if (image.b64_json) {
+                        // Base64 format
+                        imageValue = `data:image/png;base64,${image.b64_json}`;
+                    }
+                    else if (image.url) {
+                        // URL format
+                        imageValue = image.url;
+                    }
+                    else {
+                        continue;
+                    }
+                    results.push({
+                        type: "image",
+                        value: imageValue
+                    });
+                }
+            }
+            return {
+                result: results
+            };
+        }
+        catch (error) {
+            this.logger.error({ error }, `[${this.provider}] Image generation failed`);
+            return {
+                result: [],
+                error: {
+                    message: error.message,
+                    code: error.code || 'GENERATION_FAILED'
+                }
+            };
+        }
+    }
+    /**
+     * Format OpenAI API errors into LlumiverseError with proper status codes and retryability.
+     *
+     * OpenAI API errors have a specific structure:
+     * - APIError.status: HTTP status code (400, 401, 403, 404, 409, 422, 429, 500+)
+     * - APIError.error: Error object with type, message, param, code
+     * - APIError.requestID: Request ID for support
+     * - APIError.code: Error code (e.g., 'invalid_api_key', 'rate_limit_exceeded')
+     * - APIError.param: Parameter that caused the error (optional)
+     * - APIError.type: Error type (optional)
+     *
+     * Common error types:
+     * - BadRequestError (400): Invalid request parameters
+     * - AuthenticationError (401): Invalid API key
+     * - PermissionDeniedError (403): Insufficient permissions
+     * - NotFoundError (404): Resource not found
+     * - ConflictError (409): Resource conflict
+     * - UnprocessableEntityError (422): Validation error
+     * - RateLimitError (429): Rate limit exceeded
+     * - InternalServerError (500+): Server-side errors
+     * - APIConnectionError: Connection issues (no status code)
+     * - APIConnectionTimeoutError: Request timeout (no status code)
+     * - LengthFinishReasonError: Response truncated due to length
+     * - ContentFilterFinishReasonError: Content filtered
+     *
+     * This implementation works for:
+     * - OpenAI API
+     * - Azure OpenAI
+     * - xAI (uses OpenAI-compatible API)
+     * - Azure Foundry (OpenAI-compatible)
+     * - Other OpenAI-compatible APIs
+     *
+     * @see https://platform.openai.com/docs/guides/error-codes
+     */
+    formatLlumiverseError(error, context) {
+        // Check if it's an OpenAI API error
+        const isOpenAIError = this.isOpenAIApiError(error);
+        if (!isOpenAIError) {
+            // Not an OpenAI API error, use default handling
+            throw error;
+        }
+        const apiError = error;
+        const httpStatusCode = apiError.status;
+        // Extract error message
+        const message = apiError.message || String(error);
+        // Extract additional error details (only available on APIError)
+        const errorCode = apiError.code;
+        const errorParam = apiError.param;
+        const errorType = apiError.type;
+        // Build user-facing message with status code
+        let userMessage = message;
+        // Include status code in message (for end-user visibility)
+        if (httpStatusCode) {
+            userMessage = `[${httpStatusCode}] ${userMessage}`;
+        }
+        // Add error code if available and not already in message
+        if (errorCode && !userMessage.includes(errorCode)) {
+            userMessage += ` (code: ${errorCode})`;
+        }
+        // Add parameter info if available and helpful
+        if (errorParam && !userMessage.toLowerCase().includes(errorParam.toLowerCase())) {
+            userMessage += ` [param: ${errorParam}]`;
+        }
+        // Add request ID if available (useful for OpenAI support)
+        if (apiError.requestID) {
+            userMessage += ` (Request ID: ${apiError.requestID})`;
+        }
+        // Determine retryability based on OpenAI error types
+        const retryable = this.isOpenAIErrorRetryable(error, httpStatusCode, errorCode, errorType);
+        // Use the error constructor name as the error name
+        const errorName = error.constructor?.name || 'OpenAIError';
+        return new LlumiverseError(`[${context.provider}] ${userMessage}`, retryable, context, error, httpStatusCode, errorName);
+    }
+    /**
+     * Type guard to check if error is an OpenAI API error or OpenAI-specific error.
+     */
+    isOpenAIApiError(error) {
+        return (error !== null &&
+            typeof error === 'object' &&
+            (error instanceof APIError || error instanceof OpenAIError));
+    }
+    /**
+     * Determine if an OpenAI API error is retryable.
+     *
+     * Retryable errors:
+     * - RateLimitError (429): Rate limit exceeded, retry with backoff
+     * - InternalServerError (500+): Server-side errors
+     * - APIConnectionTimeoutError: Request timeout
+     * - Error codes: 'timeout', 'server_error', 'service_unavailable'
+     * - Status codes: 408, 429, 502, 503, 504, 529, 5xx
+     *
+     * Non-retryable errors:
+     * - BadRequestError (400): Invalid request parameters
+     * - AuthenticationError (401): Invalid API key
+     * - PermissionDeniedError (403): Insufficient permissions
+     * - NotFoundError (404): Resource not found
+     * - ConflictError (409): Resource conflict
+     * - UnprocessableEntityError (422): Validation error
+     * - LengthFinishReasonError: Length limit reached
+     * - ContentFilterFinishReasonError: Content filtered
+     * - Error codes: 'invalid_api_key', 'invalid_request_error', 'model_not_found'
+     * - Other 4xx client errors
+     *
+     * @param error - The error object
+     * @param httpStatusCode - The HTTP status code if available
+     * @param errorCode - The error code if available
+     * @param errorType - The error type if available
+     * @returns True if retryable, false if not retryable, undefined if unknown
+     */
+    isOpenAIErrorRetryable(error, httpStatusCode, errorCode, errorType) {
+        // Check specific OpenAI error types by class
+        if (error instanceof RateLimitError)
+            return true;
+        if (error instanceof InternalServerError)
+            return true;
+        if (error instanceof APIConnectionTimeoutError)
+            return true;
+        // Non-retryable by error type
+        if (error instanceof BadRequestError)
+            return false;
+        if (error instanceof AuthenticationError)
+            return false;
+        if (error instanceof PermissionDeniedError)
+            return false;
+        if (error instanceof NotFoundError)
+            return false;
+        if (error instanceof ConflictError)
+            return false;
+        if (error instanceof UnprocessableEntityError)
+            return false;
+        if (error instanceof LengthFinishReasonError)
+            return false;
+        if (error instanceof ContentFilterFinishReasonError)
+            return false;
+        // Check error codes (OpenAI specific)
+        if (errorCode) {
+            // Retryable error codes
+            if (errorCode === 'timeout')
+                return true;
+            if (errorCode === 'server_error')
+                return true;
+            if (errorCode === 'service_unavailable')
+                return true;
+            if (errorCode === 'rate_limit_exceeded')
+                return true;
+            // Non-retryable error codes
+            if (errorCode === 'invalid_api_key')
+                return false;
+            if (errorCode === 'invalid_request_error')
+                return false;
+            if (errorCode === 'model_not_found')
+                return false;
+            if (errorCode === 'insufficient_quota')
+                return false;
+            if (errorCode === 'invalid_model')
+                return false;
+            if (errorCode.includes('invalid_'))
+                return false;
+        }
+        // Check error type
+        if (errorType === 'invalid_request_error')
+            return false;
+        if (errorType === 'authentication_error')
+            return false;
+        // Use HTTP status code
+        if (httpStatusCode !== undefined) {
+            if (httpStatusCode === 429)
+                return true; // Rate limit
+            if (httpStatusCode === 408)
+                return true; // Request timeout
+            if (httpStatusCode === 502)
+                return true; // Bad gateway
+            if (httpStatusCode === 503)
+                return true; // Service unavailable
+            if (httpStatusCode === 504)
+                return true; // Gateway timeout
+            if (httpStatusCode === 529)
+                return true; // Overloaded
+            if (httpStatusCode >= 500 && httpStatusCode < 600)
+                return true; // Server errors
+            if (httpStatusCode >= 400 && httpStatusCode < 500)
+                return false; // Client errors
+        }
+        // Connection errors without status codes
+        if (error instanceof APIConnectionError && !(error instanceof APIConnectionTimeoutError)) {
+            // Generic connection errors might be retryable (network issues)
+            return true;
+        }
+        // Unknown error type - let consumer decide retry strategy
+        return undefined;
+    }
 }
 function jobInfo(job) {
     //validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`.
@@ -484,6 +799,39 @@ function supportsSchema(model) {
     }
     return supportsToolUse(model, "openai");
 }
+/**
+ * Converts function_call and function_call_output items to text messages in OpenAI conversation.
+ * Preserves tool call information while removing structured items that require
+ * tools to be defined in the API request.
+ */
+export function convertOpenAIFunctionItemsToText(items) {
+    const hasFunctionItems = items.some(item => {
+        const type = item.type;
+        return type === 'function_call' || type === 'function_call_output';
+    });
+    if (!hasFunctionItems)
+        return items;
+    return items.map(item => {
+        const typed = item;
+        if (typed.type === 'function_call') {
+            const argsStr = typed.arguments || '';
+            const truncated = argsStr.length > 500 ? argsStr.substring(0, 500) + '...' : argsStr;
+            return {
+                role: 'assistant',
+                content: `[Tool call: ${typed.name}(${truncated})]`,
+            };
+        }
+        if (typed.type === 'function_call_output') {
+            const output = typed.output || 'No output';
+            const truncated = output.length > 500 ? output.substring(0, 500) + '...' : output;
+            return {
+                role: 'user',
+                content: `[Tool result: ${truncated}]`,
+            };
+        }
+        return item;
+    });
+}
 function getToolDefinitions(tools) {
     return tools ? tools.map(getToolDefinition) : undefined;
 }
@@ -544,6 +892,42 @@ export function collectTools(output) {
     }
     return tools.length > 0 ? tools : undefined;
 }
+/**
+ * Collect all parts (text and images) from response output in order.
+ * This preserves the original ordering of text and image parts.
+ */
+function extractCompletionResults(output) {
+    if (!output) {
+        return [];
+    }
+    const results = [];
+    for (const item of output) {
+        if (item.type === 'message') {
+            // Extract text from message content
+            for (const part of item.content) {
+                if (part.type === 'output_text' && part.text) {
+                    results.push({
+                        type: "text",
+                        value: part.text
+                    });
+                }
+            }
+        }
+        else if (item.type === 'image_generation_call' && 'result' in item && item.result) {
+            // GPT-image models return base64 encoded images in result field
+            const base64Data = item.result;
+            // Format as data URL for consistency with other image outputs
+            const imageUrl = base64Data.startsWith('data:')
+                ? base64Data
+                : `data:image/png;base64,${base64Data}`;
+            results.push({
+                type: "image",
+                value: imageUrl
+            });
+        }
+    }
+    return results;
+}
 //For strict mode false
 function limitedSchemaFormat(schema) {
     const formattedSchema = { ...schema };
@@ -618,23 +1002,6 @@ function openAISchemaFormat(schema, nesting = 0) {
     }
     return formattedSchema;
 }
-function extractTextFromResponse(response) {
-    if (response.output_text) {
-        return response.output_text;
-    }
-    const collected = [];
-    for (const item of response.output ?? []) {
-        if (item.type === 'message') {
-            const text = item.content
-                .map(part => part.type === 'output_text' ? part.text : '')
-                .join('');
-            if (text) {
-                collected.push(text);
-            }
-        }
-    }
-    return collected.join("\n");
-}
 function responseFinishReason(response, tools) {
     if (tools && tools.length > 0) {
         return "tool_use";
@@ -650,6 +1017,68 @@ function responseFinishReason(response, tools) {
     }
     return 'stop';
 }
+/**
+ * Fix orphaned function_call items in the OpenAI Responses API conversation.
+ *
+ * When an agent is stopped mid-tool-execution, the conversation may contain
+ * function_call items without matching function_call_output items. The OpenAI
+ * Responses API requires every function_call to have a matching function_call_output.
+ *
+ * This function detects such cases and injects synthetic function_call_output items
+ * indicating the tools were interrupted, allowing the conversation to continue.
+ */
+export function fixOrphanedToolUse(items) {
+    if (items.length < 2)
+        return items;
+    // First pass: collect all function_call_output call_ids
+    const outputCallIds = new Set();
+    for (const item of items) {
+        if ('type' in item && item.type === 'function_call_output') {
+            outputCallIds.add(item.call_id);
+        }
+    }
+    // Second pass: build result, injecting synthetic outputs for orphaned function_calls
+    const result = [];
+    const pendingCalls = new Map(); // call_id -> tool name
+    for (const item of items) {
+        if ('type' in item && item.type === 'function_call') {
+            const fc = item;
+            // Only track if there's no matching output anywhere in the conversation
+            if (!outputCallIds.has(fc.call_id)) {
+                pendingCalls.set(fc.call_id, fc.name ?? 'unknown');
+            }
+            result.push(item);
+        }
+        else if ('type' in item && item.type === 'function_call_output') {
+            result.push(item);
+        }
+        else {
+            // Before any non-function item, flush pending orphaned calls
+            if (pendingCalls.size > 0) {
+                for (const [callId, toolName] of pendingCalls) {
+                    result.push({
+                        type: 'function_call_output',
+                        call_id: callId,
+                        output: `[Tool interrupted: The user stopped the operation before "${toolName}" could execute.]`,
+                    });
+                }
+                pendingCalls.clear();
+            }
+            result.push(item);
+        }
+    }
+    // Handle trailing orphans at the end of the conversation
+    if (pendingCalls.size > 0) {
+        for (const [callId, toolName] of pendingCalls) {
+            result.push({
+                type: 'function_call_output',
+                call_id: callId,
+                output: `[Tool interrupted: The user stopped the operation before "${toolName}" could execute.]`,
+            });
+        }
+    }
+    return result;
+}
 function safeJsonParse(value) {
     if (typeof value !== 'string') {
         return value;