@llumiverse/drivers 0.25.0 → 0.25.1-dev.20260210.145152Z

package/lib/esm/openai/index.js

@@ -0,0 +1,660 @@
+import { AbstractDriver, ModelType, TrainingJobStatus, getConversationMeta, getModelCapabilities, incrementConversationTurn, modelModalitiesToArray, stripBase64ImagesFromConversation, supportsToolUse, truncateLargeTextInConversation, unwrapConversationArray, } from "@llumiverse/core";
+import { formatOpenAILikeMultimodalPrompt } from "./openai_format.js";
+// Helper function to convert string to CompletionResult[]
+function textToCompletionResult(text) {
+    return text ? [{ type: "text", value: text }] : [];
+}
+//TODO: Do we need a list?, replace with if statements and modernize?
+const supportFineTunning = new Set([
+    "gpt-3.5-turbo-1106",
+    "gpt-3.5-turbo-0613",
+    "babbage-002",
+    "davinci-002",
+    "gpt-4-0613"
+]);
+export class BaseOpenAIDriver extends AbstractDriver {
+    constructor(opts) {
+        super(opts);
+        this.formatPrompt = formatOpenAILikeMultimodalPrompt;
+    }
+    extractDataFromResponse(_options, result) {
+        const tokenInfo = mapUsage(result.usage);
+        const tools = collectTools(result.output);
+        const data = extractTextFromResponse(result);
+        if (!data && !tools) {
+            this.logger.error({ result }, "[OpenAI] Response is not valid");
+            throw new Error("Response is not valid: no data");
+        }
+        return {
+            result: textToCompletionResult(data || ''),
+            token_usage: tokenInfo,
+            finish_reason: responseFinishReason(result, tools),
+            tool_use: tools,
+        };
+    }
+    async requestTextCompletionStream(prompt, options) {
+        if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
+            this.logger.warn({ options: options.model_options }, "Invalid model options");
+        }
+        // Include conversation history (same as non-streaming)
+        const conversation = updateConversation(options.conversation, prompt);
+        const toolDefs = getToolDefinitions(options.tools);
+        const useTools = toolDefs ? supportsToolUse(options.model, this.provider, true) : false;
+        convertRoles(prompt, options.model);
+        const model_options = options.model_options;
+        insert_image_detail(prompt, model_options?.image_detail ?? "auto");
+        let parsedSchema = undefined;
+        let strictMode = false;
+        if (options.result_schema && supportsSchema(options.model)) {
+            try {
+                parsedSchema = openAISchemaFormat(options.result_schema);
+                strictMode = true;
+            }
+            catch (e) {
+                parsedSchema = limitedSchemaFormat(options.result_schema);
+                strictMode = false;
+            }
+        }
+        const reasoning = model_options?.reasoning_effort ? { effort: model_options.reasoning_effort } : undefined;
+        const stream = await this.service.responses.create({
+            stream: true,
+            model: options.model,
+            input: conversation,
+            reasoning,
+            temperature: model_options?.temperature,
+            top_p: model_options?.top_p,
+            max_output_tokens: model_options?.max_tokens,
+            tools: useTools ? toolDefs : undefined,
+            text: parsedSchema ? {
+                format: {
+                    type: "json_schema",
+                    name: "format_output",
+                    schema: parsedSchema,
+                    strict: strictMode,
+                }
+            } : undefined,
+        });
+        return mapResponseStream(stream);
+    }
+    async requestTextCompletion(prompt, options) {
+        if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
+            this.logger.warn({ options: options.model_options }, "Invalid model options");
+        }
+        convertRoles(prompt, options.model);
+        const model_options = options.model_options;
+        insert_image_detail(prompt, model_options?.image_detail ?? "auto");
+        const toolDefs = getToolDefinitions(options.tools);
+        const useTools = toolDefs ? supportsToolUse(options.model, this.provider) : false;
+        let conversation = updateConversation(options.conversation, prompt);
+        let parsedSchema = undefined;
+        let strictMode = false;
+        if (options.result_schema && supportsSchema(options.model)) {
+            try {
+                parsedSchema = openAISchemaFormat(options.result_schema);
+                strictMode = true;
+            }
+            catch (e) {
+                parsedSchema = limitedSchemaFormat(options.result_schema);
+                strictMode = false;
+            }
+        }
+        const reasoning = model_options?.reasoning_effort ? { effort: model_options.reasoning_effort } : undefined;
+        const res = await this.service.responses.create({
+            stream: false,
+            model: options.model,
+            input: conversation,
+            reasoning,
+            temperature: model_options?.temperature,
+            top_p: model_options?.top_p,
+            max_output_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
+            tools: useTools ? toolDefs : undefined,
+            text: parsedSchema ? {
+                format: {
+                    type: "json_schema",
+                    name: "format_output",
+                    schema: parsedSchema,
+                    strict: strictMode,
+                }
+            } : undefined,
+        });
+        const completion = this.extractDataFromResponse(options, res);
+        if (options.include_original_response) {
+            completion.original_response = res;
+        }
+        conversation = updateConversation(conversation, createAssistantMessageFromCompletion(completion));
+        // Increment turn counter for deferred stripping
+        conversation = incrementConversationTurn(conversation);
+        // Strip large base64 image data based on options.stripImagesAfterTurns
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        // Truncate large text content if configured
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        completion.conversation = processedConversation;
+        return completion;
+    }
+    canStream(_options) {
+        if (_options.model.includes("o1")
+            && !(_options.model.includes("mini") || _options.model.includes("preview"))) {
+            //o1 full does not support streaming
+            //TODO: Update when OpenAI adds support for streaming, last check 16/02/2025
+            return Promise.resolve(false);
+        }
+        return Promise.resolve(true);
+    }
+    /**
+     * Build conversation context after streaming completion.
+     * Reconstructs the assistant message from accumulated results and applies stripping.
+     */
+    buildStreamingConversation(prompt, result, toolUse, options) {
+        // Build assistant message from accumulated CompletionResult[]
+        const completionResults = result;
+        const textContent = completionResultsToText(completionResults);
+        // Start with the conversation from options or the prompt
+        let conversation = updateConversation(options.conversation, prompt);
+        // Add assistant message as EasyInputMessage
+        if (textContent) {
+            const assistantMessage = {
+                role: 'assistant',
+                content: textContent,
+            };
+            conversation = updateConversation(conversation, [assistantMessage]);
+        }
+        // Add function calls as separate items (Response API format)
+        if (toolUse && toolUse.length > 0) {
+            const functionCalls = toolUse.map(t => ({
+                type: 'function_call',
+                call_id: t.id,
+                name: t.tool_name,
+                arguments: typeof t.tool_input === 'string' ? t.tool_input : JSON.stringify(t.tool_input ?? {}),
+            }));
+            conversation = updateConversation(conversation, functionCalls);
+        }
+        // Increment turn counter
+        conversation = incrementConversationTurn(conversation);
+        // Apply stripping based on options
+        const currentTurn = getConversationMeta(conversation).turnNumber;
+        const stripOptions = {
+            keepForTurns: options.stripImagesAfterTurns ?? Infinity,
+            currentTurn,
+            textMaxTokens: options.stripTextMaxTokens
+        };
+        let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
+        processedConversation = truncateLargeTextInConversation(processedConversation, stripOptions);
+        return processedConversation;
+    }
+    createTrainingPrompt(options) {
+        if (options.model.includes("gpt")) {
+            return super.createTrainingPrompt(options);
+        }
+        else {
+            // babbage, davinci not yet implemented
+            throw new Error("Unsupported model for training: " + options.model);
+        }
+    }
+    async startTraining(dataset, options) {
+        const url = await dataset.getURL();
+        const file = await this.service.files.create({
+            file: await fetch(url),
+            purpose: "fine-tune",
+        });
+        const job = await this.service.fineTuning.jobs.create({
+            training_file: file.id,
+            model: options.model,
+            hyperparameters: options.params
+        });
+        return jobInfo(job);
+    }
+    async cancelTraining(jobId) {
+        const job = await this.service.fineTuning.jobs.cancel(jobId);
+        return jobInfo(job);
+    }
+    async getTrainingJob(jobId) {
+        const job = await this.service.fineTuning.jobs.retrieve(jobId);
+        return jobInfo(job);
+    }
+    // ========= management API =============
+    async validateConnection() {
+        try {
+            await this.service.models.list();
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    listTrainableModels() {
+        return this._listModels((m) => supportFineTunning.has(m.id));
+    }
+    async listModels() {
+        return this._listModels();
+    }
+    async _listModels(filter) {
+        let result = (await this.service.models.list()).data;
+        //Some of these use the completions API instead of the chat completions API.
+        //Others are for non-text input modalities. Therefore common to both.
+        const wordBlacklist = ["embed", "whisper", "transcribe", "audio", "moderation", "tts",
+            "realtime", "dall-e", "babbage", "davinci", "codex", "o1-pro", "computer-use", "sora"];
+        //OpenAI has very little information, filtering based on name.
+        result = result.filter((m) => {
+            return !wordBlacklist.some((word) => m.id.includes(word));
+        });
+        const models = filter ? result.filter(filter) : result;
+        const aiModels = models.map((m) => {
+            const modelCapability = getModelCapabilities(m.id, "openai");
+            let owner = m.owned_by;
+            if (owner == "system") {
+                owner = "openai";
+            }
+            return {
+                id: m.id,
+                name: m.id,
+                provider: this.provider,
+                owner: owner,
+                type: m.object === "model" ? ModelType.Text : ModelType.Unknown,
+                can_stream: true,
+                is_multimodal: m.id.includes("gpt-4"),
+                input_modalities: modelModalitiesToArray(modelCapability.input),
+                output_modalities: modelModalitiesToArray(modelCapability.output),
+                tool_support: modelCapability.tool_support,
+            };
+        }).sort((a, b) => a.id.localeCompare(b.id));
+        return aiModels;
+    }
+    async generateEmbeddings({ text, image, model = "text-embedding-3-small" }) {
+        if (image) {
+            throw new Error("Image embeddings not supported by OpenAI");
+        }
+        if (!text) {
+            throw new Error("No text provided");
+        }
+        const res = await this.service.embeddings.create({
+            input: text,
+            model: model,
+        });
+        const embeddings = res.data[0].embedding;
+        if (!embeddings || embeddings.length === 0) {
+            throw new Error("No embedding found");
+        }
+        return { values: embeddings, model };
+    }
+}
+function jobInfo(job) {
+    //validating_files`, `queued`, `running`, `succeeded`, `failed`, or `cancelled`.
+    const jobStatus = job.status;
+    let status = TrainingJobStatus.running;
+    let details;
+    if (jobStatus === 'succeeded') {
+        status = TrainingJobStatus.succeeded;
+    }
+    else if (jobStatus === 'failed') {
+        status = TrainingJobStatus.failed;
+        details = job.error ? `${job.error.code} - ${job.error.message} ${job.error.param ? " [" + job.error.param + "]" : ""}` : "error";
+    }
+    else if (jobStatus === 'cancelled') {
+        status = TrainingJobStatus.cancelled;
+    }
+    else {
+        status = TrainingJobStatus.running;
+        details = jobStatus;
+    }
+    return {
+        id: job.id,
+        model: job.fine_tuned_model || undefined,
+        status,
+        details
+    };
+}
+function mapUsage(usage) {
+    if (!usage) {
+        return undefined;
+    }
+    return {
+        prompt: usage.input_tokens,
+        result: usage.output_tokens,
+        total: usage.total_tokens,
+    };
+}
+function completionResultsToText(completionResults) {
+    if (!completionResults) {
+        return '';
+    }
+    return completionResults
+        .map(r => {
+        switch (r.type) {
+            case 'text':
+                return r.value;
+            case 'json':
+                return typeof r.value === 'string' ? r.value : JSON.stringify(r.value);
+            case 'image':
+                // Skip images in conversation - they're in the result
+                return '';
+            default:
+                return String(r.value || '');
+        }
+    })
+        .join('');
+}
+function createAssistantMessageFromCompletion(completion) {
+    const textContent = completionResultsToText(completion.result);
+    const result = [];
+    // Add assistant text message if present
+    if (textContent) {
+        const assistantMessage = {
+            role: 'assistant',
+            content: textContent,
+        };
+        result.push(assistantMessage);
+    }
+    // Add function calls as separate items (Response API format)
+    if (completion.tool_use && completion.tool_use.length > 0) {
+        for (const t of completion.tool_use) {
+            const functionCall = {
+                type: 'function_call',
+                call_id: t.id,
+                name: t.tool_name,
+                arguments: typeof t.tool_input === 'string'
+                    ? t.tool_input
+                    : JSON.stringify(t.tool_input ?? {}),
+            };
+            result.push(functionCall);
+        }
+    }
+    return result;
+}
+function mapResponseStream(stream) {
+    const toolCallMetadata = new Map();
+    return {
+        async *[Symbol.asyncIterator]() {
+            for await (const event of stream) {
+                if (event.type === 'response.output_item.added' && event.item.type === 'function_call') {
+                    const syntheticId = `tool_${event.output_index}`;
+                    const actualId = event.item.id ?? event.item.call_id;
+                    if (actualId) {
+                        toolCallMetadata.set(actualId, { syntheticId, name: event.item.name });
+                    }
+                    const toolUse = {
+                        id: syntheticId,
+                        _actual_id: actualId,
+                        tool_name: event.item.name,
+                        tool_input: '',
+                    };
+                    yield {
+                        result: [],
+                        tool_use: [toolUse],
+                    };
+                }
+                else if (event.type === 'response.function_call_arguments.delta') {
+                    const metadata = toolCallMetadata.get(event.item_id);
+                    const syntheticId = metadata?.syntheticId ?? `tool_${event.output_index}`;
+                    const toolUse = {
+                        id: syntheticId,
+                        _actual_id: event.item_id,
+                        tool_name: metadata?.name ?? '',
+                        tool_input: event.delta,
+                    };
+                    yield {
+                        result: [],
+                        tool_use: [toolUse],
+                    };
+                }
+                // Note: We don't emit response.function_call_arguments.done because the arguments were already
+                // streamed via delta events. Emitting it again would duplicate the tool_input content.
+                // We only update the metadata to ensure the tool name is captured.
+                else if (event.type === 'response.function_call_arguments.done') {
+                    // Just update metadata, don't yield (arguments already accumulated from delta events)
+                    const metadata = toolCallMetadata.get(event.item_id);
+                    const syntheticId = metadata?.syntheticId ?? `tool_${event.output_index}`;
+                    const tool_name = metadata?.name ?? event.name ?? '';
+                    if (event.item_id) {
+                        toolCallMetadata.set(event.item_id, { syntheticId, name: tool_name });
+                    }
+                }
+                else if (event.type === 'response.output_text.delta') {
+                    yield {
+                        result: textToCompletionResult(event.delta),
+                    };
+                }
+                // Note: We don't emit response.output_text.done because the text was already
+                // streamed via delta events. Emitting it again would duplicate the content.
+                else if (event.type === 'response.completed' || event.type === 'response.incomplete' || event.type === 'response.failed') {
+                    const finalTools = collectTools(event.response.output);
+                    yield {
+                        result: [],
+                        finish_reason: responseFinishReason(event.response, finalTools),
+                        token_usage: mapUsage(event.response.usage),
+                    };
+                }
+            }
+        }
+    };
+}
+function insert_image_detail(items, detail_level) {
+    if (detail_level === "auto" || detail_level === "low" || detail_level === "high") {
+        for (const item of items) {
+            // Check if it's an EasyInputMessage or Message with content array
+            if ('role' in item && 'content' in item && item.role !== 'assistant') {
+                const content = item.content;
+                if (Array.isArray(content)) {
+                    for (const part of content) {
+                        if (typeof part === 'object' && part.type === 'input_image') {
+                            part.detail = detail_level;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return items;
+}
+function convertRoles(items, model) {
+    //New openai models use developer role instead of system
+    if (model.includes("o1") || model.includes("o3")) {
+        if (model.includes("o1-mini") || model.includes("o1-preview")) {
+            //o1-mini and o1-preview support neither system nor developer
+            for (const item of items) {
+                if ('role' in item && item.role === 'system') {
+                    item.role = 'user';
+                }
+            }
+        }
+        else {
+            //Models newer than o1 use developer role
+            for (const item of items) {
+                if ('role' in item && item.role === 'system') {
+                    item.role = 'developer';
+                }
+            }
+        }
+    }
+    return items;
+}
+//Structured output support is typically aligned with tool use support
+//Not true for realtime models, which do not support structured output, but do support tool use.
+function supportsSchema(model) {
+    const realtimeModel = model.includes("realtime");
+    if (realtimeModel) {
+        return false;
+    }
+    return supportsToolUse(model, "openai");
+}
+function getToolDefinitions(tools) {
+    return tools ? tools.map(getToolDefinition) : undefined;
+}
+function getToolDefinition(toolDef) {
+    let parsedSchema = undefined;
+    let strictMode = false;
+    if (toolDef.input_schema) {
+        try {
+            parsedSchema = openAISchemaFormat(toolDef.input_schema);
+            strictMode = true;
+        }
+        catch (e) {
+            parsedSchema = limitedSchemaFormat(toolDef.input_schema);
+            strictMode = false;
+        }
+    }
+    return {
+        type: "function",
+        name: toolDef.name,
+        description: toolDef.description,
+        parameters: parsedSchema ?? null,
+        strict: strictMode,
+    };
+}
+function updateConversation(conversation, items) {
+    if (!items) {
+        // Unwrap array if wrapped, otherwise treat as array
+        const unwrapped = unwrapConversationArray(conversation);
+        return unwrapped ?? (conversation || []);
+    }
+    if (!conversation) {
+        return items;
+    }
+    // Unwrap array if wrapped, otherwise treat as array
+    const unwrapped = unwrapConversationArray(conversation);
+    const convArray = unwrapped ?? conversation;
+    return [...convArray, ...items];
+}
+export function collectTools(output) {
+    if (!output) {
+        return undefined;
+    }
+    const tools = [];
+    for (const item of output) {
+        if (item.type === 'function_call') {
+            const id = item.call_id || item.id;
+            if (!id) {
+                continue;
+            }
+            tools.push({
+                id,
+                tool_name: item.name ?? '',
+                tool_input: safeJsonParse(item.arguments),
+            });
+        }
+    }
+    return tools.length > 0 ? tools : undefined;
+}
+//For strict mode false
+function limitedSchemaFormat(schema) {
+    const formattedSchema = { ...schema };
+    // Defaults not supported
+    delete formattedSchema.default;
+    // OpenAI requires type field even in non-strict mode
+    // If no type is specified, default to 'object' for properties with format/editor hints,
+    // otherwise 'string' as a safe fallback
+    if (!formattedSchema.type && formattedSchema.description) {
+        // Properties with format: "document" or editor hints are typically objects
+        if (formattedSchema.format === 'document' || formattedSchema.editor) {
+            formattedSchema.type = 'object';
+        }
+        else {
+            formattedSchema.type = 'string';
+        }
+    }
+    if (formattedSchema?.properties) {
+        // Process each property recursively
+        for (const propName of Object.keys(formattedSchema.properties)) {
+            const property = formattedSchema.properties[propName];
+            // Recursively process properties
+            formattedSchema.properties[propName] = limitedSchemaFormat(property);
+            // Process arrays with items of type object
+            if (property?.type === 'array' && property.items && property.items?.type === 'object') {
+                formattedSchema.properties[propName] = {
+                    ...property,
+                    items: limitedSchemaFormat(property.items),
+                };
+            }
+        }
+    }
+    return formattedSchema;
+}
+//For strict mode true
+function openAISchemaFormat(schema, nesting = 0) {
+    if (nesting > 5) {
+        throw new Error("OpenAI schema nesting too deep");
+    }
+    const formattedSchema = { ...schema };
+    // Defaults not supported
+    delete formattedSchema.default;
+    // Additional properties not supported, required to be set.
+    if (formattedSchema?.type === "object") {
+        formattedSchema.additionalProperties = false;
+    }
+    if (formattedSchema?.properties) {
+        // Set all properties as required
+        formattedSchema.required = Object.keys(formattedSchema.properties);
+        // Process each property recursively
+        for (const propName of Object.keys(formattedSchema.properties)) {
+            const property = formattedSchema.properties[propName];
+            // OpenAI strict mode requires all properties to have a type
+            if (!property?.type) {
+                throw new Error(`Property '${propName}' is missing required 'type' field for OpenAI strict mode`);
+            }
+            // Recursively process properties
+            formattedSchema.properties[propName] = openAISchemaFormat(property, nesting + 1);
+            // Process arrays with items of type object
+            if (property?.type === 'array' && property.items && property.items?.type === 'object') {
+                formattedSchema.properties[propName] = {
+                    ...property,
+                    items: openAISchemaFormat(property.items, nesting + 1),
+                };
+            }
+        }
+    }
+    if (formattedSchema?.type === 'object' && (!formattedSchema?.properties || Object.keys(formattedSchema?.properties ?? {}).length == 0)) {
+        //If no properties are defined, then additionalProperties: true was set or the object would be empty.
+        //OpenAI does not support this on structured output/ strict mode.
+        throw new Error("OpenAI does not support empty objects or objects with additionalProperties set to true");
+    }
+    return formattedSchema;
+}
+function extractTextFromResponse(response) {
+    if (response.output_text) {
+        return response.output_text;
+    }
+    const collected = [];
+    for (const item of response.output ?? []) {
+        if (item.type === 'message') {
+            const text = item.content
+                .map(part => part.type === 'output_text' ? part.text : '')
+                .join('');
+            if (text) {
+                collected.push(text);
+            }
+        }
+    }
+    return collected.join("\n");
+}
+function responseFinishReason(response, tools) {
+    if (tools && tools.length > 0) {
+        return "tool_use";
+    }
+    if (response.status === 'incomplete') {
+        if (response.incomplete_details?.reason === 'max_output_tokens') {
+            return 'length';
+        }
+        return response.incomplete_details?.reason ?? 'incomplete';
+    }
+    if (response.status && response.status !== 'completed') {
+        return response.status;
+    }
+    return 'stop';
+}
+function safeJsonParse(value) {
+    if (typeof value !== 'string') {
+        return value;
+    }
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
+//# sourceMappingURL=index.js.map