@llumiverse/drivers 0.14.0 → 0.16.0

package/src/openai/index.ts

@@ -2,6 +2,7 @@ import {
     AIModel,
     AbstractDriver,
     Completion,
+    CompletionChunkObject,
     DataSource,
     DriverOptions,
     EmbeddingsOptions,
@@ -9,16 +10,18 @@ import {
     ExecutionOptions,
     ExecutionTokenUsage,
     ModelType,
+    ToolDefinition,
     TrainingJob,
     TrainingJobStatus,
     TrainingOptions,
-    TrainingPromptOptions
+    TrainingPromptOptions,
 } from "@llumiverse/core";
 import { asyncMap } from "@llumiverse/core/async";
 import { formatOpenAILikeMultimodalPrompt } from "@llumiverse/core/formatters";
 import OpenAI, { AzureOpenAI } from "openai";
 import { Stream } from "openai/streaming";
 
+//TODO: Do we need a list?, replace with if statements and modernise?
 const supportFineTunning = new Set([
     "gpt-3.5-turbo-1106",
     "gpt-3.5-turbo-0613",
@@ -34,8 +37,8 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
     BaseOpenAIDriverOptions,
     OpenAI.Chat.Completions.ChatCompletionMessageParam[]
 > {
-    abstract provider: "azure_openai" | "openai";
-    abstract service: OpenAI | AzureOpenAI ;
+    abstract provider: "azure_openai" | "openai" | "xai";
+    abstract service: OpenAI | AzureOpenAI;
 
     constructor(opts: BaseOpenAIDriverOptions) {
         super(opts);
@@ -54,19 +57,25 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
         };
 
         const choice = result.choices[0];
-        const finish_reason = choice.finish_reason;
 
         //if no schema, return content
         if (!options.result_schema) {
             return {
-                result: choice.message.content as string,
+                result: choice.message.content ?? undefined,
                 token_usage: tokenInfo,
-                finish_reason
+                finish_reason: choice.finish_reason, //Uses expected "stop" , "length" format
             }
         }
 
-        //we have a schema: get the content and return after validation
-        const data = choice?.message.tool_calls?.[0].function.arguments;
+        const useTools: boolean = !isNonStructureSupporting(options.model);
+        let data = undefined;
+        if (useTools) {
+            //we have a schema: get the content and return after validation
+            data = choice?.message.tool_calls?.[0].function.arguments ?? choice.message.content ?? undefined;
+        } else {
+            data = choice.message.content ?? undefined;
+        }
+
         if (!data) {
             this.logger?.error("[OpenAI] Response is not valid", result);
             throw new Error("Response is not valid: no data");
@@ -75,29 +84,57 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
         return {
             result: data,
             token_usage: tokenInfo,
-            finish_reason
+            finish_reason: choice.finish_reason,
         };
     }
 
-    async requestCompletionStream(prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[], options: ExecutionOptions): Promise<any> {
-        const mapFn = options.result_schema
-            ? (chunk: OpenAI.Chat.Completions.ChatCompletionChunk) => {
-                return (
-                    chunk.choices[0]?.delta?.tool_calls?.[0].function?.arguments ?? ""
-                );
+    async requestTextCompletionStream(prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[], options: ExecutionOptions): Promise<any> {
+        if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
+            this.logger.warn("Invalid model options", { options: options.model_options });
+        }
+
+        const useTools: boolean = !isNonStructureSupporting(options.model);
+
+        const mapFn = (chunk: OpenAI.Chat.Completions.ChatCompletionChunk) => {
+            let result = undefined
+            if (useTools && this.provider !== "xai" && options.result_schema) {
+                result = chunk.choices[0]?.delta?.tool_calls?.[0].function?.arguments ?? "";
+            } else {
+                result = chunk.choices[0]?.delta.content ?? "";
             }
-            : (chunk: OpenAI.Chat.Completions.ChatCompletionChunk) => {
-                return chunk.choices[0]?.delta?.content ?? "";
-            };
+
+            return {
+                result: result,
+                finish_reason: chunk.choices[0]?.finish_reason,         //Uses expected "stop" , "length" format
+                token_usage: {
+                    prompt: chunk.usage?.prompt_tokens,
+                    result: chunk.usage?.completion_tokens,
+                    total: (chunk.usage?.prompt_tokens ?? 0) + (chunk.usage?.completion_tokens ?? 0),
+                }
+            } as CompletionChunkObject;
+        };
+
+        convertRoles(prompt, options.model);
+
+        const model_options = options.model_options as any;
+        insert_image_detail(prompt, model_options?.image_detail ?? "auto");
 
         const stream = (await this.service.chat.completions.create({
             stream: true,
+            stream_options: { include_usage: true },
             model: options.model,
             messages: prompt,
-            temperature: options.temperature,
+            reasoning_effort: model_options?.reasoning_effort,
+            temperature: model_options?.temperature,
+            top_p: model_options?.top_p,
+            //top_logprobs: options.top_logprobs,       //Logprobs output currently not supported
+            //logprobs: options.top_logprobs ? true : false,
+            presence_penalty: model_options?.presence_penalty,
+            frequency_penalty: model_options?.frequency_penalty,
             n: 1,
-            max_tokens: options.max_tokens,
-            tools: options.result_schema
+            max_completion_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
+            //tools: getToolDefinitions(options.tools),
+            tools: useTools ? options.result_schema && this.provider.includes("openai")
                 ? [
                     {
                         function: {
@@ -107,19 +144,24 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
                         type: "function"
                     } as OpenAI.Chat.ChatCompletionTool,
                 ]
-                : undefined,
-            tool_choice: options.result_schema
+                : undefined : undefined,
+            tool_choice: useTools ? options.result_schema
                 ? {
                     type: 'function',
                     function: { name: "format_output" }
-                } : undefined,
+                } : undefined : undefined,
+            stop: model_options?.stop_sequence,
         })) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk>;
 
         return asyncMap(stream, mapFn);
     }
 
-    async requestCompletion(prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[], options: ExecutionOptions): Promise<any> {
-        const functions = options.result_schema
+    async requestTextCompletion(prompt: OpenAI.Chat.Completions.ChatCompletionMessageParam[], options: ExecutionOptions): Promise<any> {
+        if (options.model_options?._option_id !== "openai-text" && options.model_options?._option_id !== "openai-thinking") {
+            this.logger.warn("Invalid model options", { options: options.model_options });
+        }
+
+        const functions = options.result_schema && this.provider.includes("openai")
             ? [
                 {
                     function: {
@@ -131,19 +173,32 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
             ]
             : undefined;
 
+        convertRoles(prompt, options.model);
+        const useTools: boolean = !isNonStructureSupporting(options.model);
+        const model_options = options.model_options as any;
+        insert_image_detail(prompt, model_options?.image_detail ?? "auto");
+
         const res = await this.service.chat.completions.create({
             stream: false,
             model: options.model,
             messages: prompt,
-            temperature: options.temperature,
+            reasoning_effort: model_options?.reasoning_effort,
+            temperature: model_options?.temperature,
+            top_p: model_options?.top_p,
+            //top_logprobs: options.top_logprobs,       //Logprobs output currently not supported
+            //logprobs: options.top_logprobs ? true : false,
+            presence_penalty: model_options?.presence_penalty,
+            frequency_penalty: model_options?.frequency_penalty,
             n: 1,
-            max_tokens: options.max_tokens,
-            tools: functions,
-            tool_choice: options.result_schema
+            max_completion_tokens: model_options?.max_tokens, //TODO: use max_tokens for older models, currently relying on OpenAI to handle it
+            //tools: getToolDefinitions(options.tools),
+            tools: useTools ? functions : undefined,
+            tool_choice: useTools ? options.result_schema && this.provider.includes("openai")
                 ? {
                     type: 'function',
                     function: { name: "format_output" }
-                } : undefined,
+                } : undefined : undefined,
+            stop: model_options?.stop_sequence,
             // functions: functions,
             // function_call: options.result_schema
             //     ? { name: "format_output" }
@@ -157,6 +212,16 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
         return completion;
     }
 
+    protected canStream(_options: ExecutionOptions): Promise<boolean> {
+        if (_options.model.includes("o1")
+            && !(_options.model.includes("mini") || _options.model.includes("preview"))) {
+            //o1 full does not support streaming
+            //TODO: Update when OpenAI adds support for streaming, last check 16/02/2025
+            return Promise.resolve(false);
+        }
+        return Promise.resolve(true);
+    }
+
     createTrainingPrompt(options: TrainingPromptOptions): Promise<string> {
         if (options.model.includes("gpt")) {
             return super.createTrainingPrompt(options);
@@ -226,9 +291,18 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
     }
 
 
-    async generateEmbeddings({ content, model = "text-embedding-ada-002" }: EmbeddingsOptions): Promise<EmbeddingsResult> {
+    async generateEmbeddings({ text, image, model = "text-embedding-3-small" }: EmbeddingsOptions): Promise<EmbeddingsResult> {
+
+        if (image) {
+            throw new Error("Image embeddings not supported by OpenAI");
+        }
+
+        if (!text) {
+            throw new Error("No text provided");
+        }
+
         const res = await this.service.embeddings.create({
-            input: content,
+            input: text,
             model: model,
         });
 
@@ -266,4 +340,64 @@ function jobInfo(job: OpenAI.FineTuning.Jobs.FineTuningJob): TrainingJob {
         status,
         details
     }
+}
+
+function insert_image_detail(messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[], detail_level: string): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
+    if (detail_level == "auto" || detail_level == "low" || detail_level == "high") {
+        for (const message of messages) {
+            if (message.role !== 'assistant' && message.content) {
+                for (const part of message.content) {
+                    if (typeof part === "string") {
+                        continue;
+                    }
+                    if (part.type === 'image_url') {
+                        part.image_url = { ...part.image_url, detail: detail_level };
+                    }
+                }
+            }
+        }
+    }
+    return messages;
+}
+
+function convertRoles(messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[], model: string): OpenAI.Chat.Completions.ChatCompletionMessageParam[] {
+    //New openai models use developer role instead of system
+    if (model.includes("o1") || model.includes("o3")) {
+        if (model.includes("o1-mini") || model.includes("o1-preview")) {
+            //o1-mini and o1-preview support neither system nor developer
+            for (const message of messages) {
+                if (message.role === 'system') {
+                    (message.role as any) = 'user';
+                }
+            }
+        } else {
+            //Models newer than o1 use developer role
+            for (const message of messages) {
+                if (message.role === 'system') {
+                    (message.role as any) = 'developer';
+                }
+            }
+        }
+    }
+    return messages
+}
+
+function isNonStructureSupporting(model: string): boolean {
+    return model.includes("o1-mini") || model.includes("o1-preview")
+        || model.includes("chatgpt-4o");
+}
+//@ts-ignore
+function getToolDefinitions(tools: ToolDefinition[] | undefined | null): OpenAI.ChatCompletionTool[] | undefined {
+    return tools ? tools.map(getToolDefinition) : undefined;
+}
+function getToolDefinition(toolDef: ToolDefinition): OpenAI.ChatCompletionTool {
+    return {
+        type: "function",
+        function: {
+            name: toolDef.name,
+            description: toolDef.description,
+            parameters: toolDef.input_schema,
+            strict: true
+        },
+    } satisfies OpenAI.ChatCompletionTool;
 }

package/src/replicate.ts

@@ -2,14 +2,16 @@ import {
     AIModel,
     AbstractDriver,
     Completion,
+    CompletionChunk,
     DataSource,
     DriverOptions,
     EmbeddingsResult,
     ExecutionOptions,
     ModelSearchPayload,
+    TextFallbackOptions,
     TrainingJob,
     TrainingJobStatus,
-    TrainingOptions
+    TrainingOptions,
 } from "@llumiverse/core";
 import { EventStream } from "@llumiverse/core/async";
 import EventSource from "eventsource";
@@ -55,25 +57,25 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         });
     }
 
-    extractDataFromResponse(prompt: string, response: Prediction): Completion {
+    extractDataFromResponse(response: Prediction): Completion {
         const text = response.output.join("");
         return {
             result: text,
-            token_usage: {
-                result: response.output.length,
-                prompt: prompt.length,
-                total: response.output.length + prompt.length,
-            },
         };
     }
 
-    async requestCompletionStream(prompt: string, options: ExecutionOptions): Promise<AsyncIterable<string>> {
+    async requestTextCompletionStream(prompt: string, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunk>> {
+        if (options.model_options?._option_id !== "text-fallback") {
+            this.logger.warn("Invalid model options", {options: options.model_options });
+        }
+        options.model_options = options.model_options as TextFallbackOptions;
+        
         const model = ReplicateDriver.parseModelId(options.model);
         const predictionData = {
             input: {
                 prompt: prompt,
-                max_new_tokens: options.max_tokens,
-                temperature: options.temperature,
+                max_new_tokens: options.model_options?.max_tokens,
+                temperature: options.model_options?.temperature,
             },
             version: model.version,
             stream: true, //streaming described here https://replicate.com/blog/streaming
@@ -82,7 +84,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         const prediction =
             await this.service.predictions.create(predictionData);
 
-        const stream = new EventStream<string>();
+        const stream = new EventStream<CompletionChunk>();
 
         const source = new EventSource(prediction.urls.stream!);
         source.addEventListener("output", (e: any) => {
@@ -95,7 +97,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
             } catch (error) {
                 error = JSON.stringify(e);
             }
-            this.logger?.error(e, error, "Error in SSE stream");
+            this.logger?.error("Error in SSE stream", {e, error});
         });
         source.addEventListener("done", () => {
             try {
@@ -107,13 +109,17 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         return stream;
     }
 
-    async requestCompletion(prompt: string, options: ExecutionOptions) {
+    async requestTextCompletion(prompt: string, options: ExecutionOptions) {
+        if (options.model_options?._option_id !== "text-fallback") {
+            this.logger.warn("Invalid model options", {options: options.model_options });
+        }
+        options.model_options = options.model_options as TextFallbackOptions;
         const model = ReplicateDriver.parseModelId(options.model);
         const predictionData = {
             input: {
                 prompt: prompt,
-                max_new_tokens: options.max_tokens,
-                temperature: options.temperature,
+                max_new_tokens: options.model_options?.max_tokens,
+                temperature: options.model_options?.temperature,
             },
             version: model.version,
             //TODO stream
@@ -133,11 +139,6 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         const text = res.output.join("");
         return {
             result: text,
-            token_usage: {
-                result: res.output.length,
-                prompt: prompt.length,
-                total: res.output.length + prompt.length,
-            },
             original_response: options.include_original_response ? res : undefined,
         };
     }

package/src/test/TestValidationErrorCompletionStream.ts

@@ -12,9 +12,9 @@ export class TestValidationErrorCompletionStream implements CompletionStream<Pro
     async *[Symbol.asyncIterator]() {
         yield "Started TestValidationError.\n";
         await sleep(1000);
-        yield "chunk1\n"
+        yield "chunk1\n";
         await sleep(1000);
-        yield "chunk2\n"
+        yield "chunk2\n";
         await sleep(1000);
         this.completion = createValidationErrorCompletion(this.segments);
     }

package/src/test/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, AIModelStatus, CompletionStream, Driver, EmbeddingsResult, ExecutionOptions, ExecutionResponse, ModelType, PromptOptions, PromptSegment, TrainingJob } from "@llumiverse/core";
+import { AIModel, AIModelStatus, CompletionStream, Driver, EmbeddingsResult, ExecutionOptions, ExecutionResponse, ModelType, PromptSegment, TrainingJob } from "@llumiverse/core";
 import { TestErrorCompletionStream } from "./TestErrorCompletionStream.js";
 import { TestValidationErrorCompletionStream } from "./TestValidationErrorCompletionStream.js";
 import { createValidationErrorCompletion, sleep, throwError } from "./utils.js";
@@ -30,9 +30,10 @@ export class TestDriver implements Driver<PromptSegment[]> {
         throw new Error("Method not implemented.");
     }
 
-    async createPrompt(segments: PromptSegment[], _opts: PromptOptions): Promise<PromptSegment[]> {
+    async createPrompt(segments: PromptSegment[], _opts: ExecutionOptions): Promise<PromptSegment[]> {
         return segments;
     }
+    
     execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptSegment[]>> {
         switch (options.model) {
             case TestDriverModels.executionError:

package/src/togetherai/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, AbstractDriver, Completion, DriverOptions, EmbeddingsResult, ExecutionOptions } from "@llumiverse/core";
+import { AIModel, AbstractDriver, Completion, CompletionChunk, DriverOptions, EmbeddingsResult, ExecutionOptions, TextFallbackOptions } from "@llumiverse/core";
 import { transformSSEStream } from "@llumiverse/core/async";
 import { FetchClient } from "api-fetch-client";
 import { TextCompletion, TogetherModelInfo } from "./interfaces.js";
@@ -21,7 +21,7 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
         });
     }
 
-    getResponseFormat = (options: ExecutionOptions) => {
+    getResponseFormat = (options: ExecutionOptions): { type: string; schema: any } | undefined => {
         return options.result_schema ?
             {
                 type: "json_object",
@@ -29,17 +29,30 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
             } : undefined;
     }
 
-    async requestCompletion(prompt: string, options: ExecutionOptions): Promise<Completion<any>> {
+    async requestTextCompletion(prompt: string, options: ExecutionOptions): Promise<Completion<any>> {
+        if (options.model_options?._option_id !== "text-fallback") {
+            this.logger.warn("Invalid model options", {options: options.model_options });
+        }
+        options.model_options = options.model_options as TextFallbackOptions;
+
+        const stop_seq = options.model_options?.stop_sequence ?? [];
+
         const res = await this.fetchClient.post('/v1/completions', {
             payload: {
                 model: options.model,
                 prompt: prompt,
                 response_format: this.getResponseFormat(options),
-                max_tokens: options.max_tokens,
-                temperature: options.temperature,
+                max_tokens: options.model_options?.max_tokens,
+                temperature: options.model_options?.temperature,
+                top_p: options.model_options?.top_p,
+                top_k: options.model_options?.top_k,
+                //logprobs: options.top_logprobs,       //Logprobs output currently not supported
+                frequency_penalty: options.model_options?.frequency_penalty,
+                presence_penalty: options.model_options?.presence_penalty,
                 stop: [
                     "</s>",
-                    "[/INST]"
+                    "[/INST]",
+                    ...stop_seq,
                 ],
             }
         }) as TextCompletion;
@@ -53,24 +66,35 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
                 result: usage.completion_tokens,
                 total: usage.total_tokens,
             },
-            finish_reason: choice.finish_reason,
+            finish_reason: choice.finish_reason,                //Uses expected "stop" , "length" format
             original_response: options.include_original_response ? res : undefined,
         }
     }
 
-    async requestCompletionStream(prompt: string, options: ExecutionOptions): Promise<AsyncIterable<string>> {
+    async requestTextCompletionStream(prompt: string, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunk>> {
+        if (options.model_options?._option_id !== "text-fallback") {
+            this.logger.warn("Invalid model options", {options: options.model_options });
+        }
+        options.model_options = options.model_options as TextFallbackOptions;
 
+        const stop_seq = options.model_options?.stop_sequence ?? [];
         const stream = await this.fetchClient.post('/v1/completions', {
             payload: {
                 model: options.model,
                 prompt: prompt,
-                max_tokens: options.max_tokens,
-                temperature: options.temperature,
+                max_tokens: options.model_options?.max_tokens,
+                temperature: options.model_options?.temperature,
                 response_format: this.getResponseFormat(options),
+                top_p: options.model_options?.top_p,
+                top_k: options.model_options?.top_k,
+                //logprobs: options.top_logprobs,       //Logprobs output currently not supported
+                frequency_penalty: options.model_options?.frequency_penalty,
+                presence_penalty: options.model_options?.presence_penalty,
                 stream: true,
                 stop: [
                     "</s>",
-                    "[/INST]"
+                    "[/INST]",
+                    ...stop_seq,
                 ],
             },
             reader: 'sse'
@@ -78,7 +102,15 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
 
         return transformSSEStream(stream, (data: string) => {
             const json = JSON.parse(data);
-            return json.choices[0]?.text ?? '';
+            return {
+                result: json.choices[0]?.text ?? '',
+                finish_reason: json.choices[0]?.finish_reason,          //Uses expected "stop" , "length" format
+                token_usage: {
+                    prompt: json.usage?.prompt_tokens,
+                    result: json.usage?.completion_tokens,
+                    total: json.usage?.prompt_tokens + json.usage?.completion_tokens,
+                }
+            };
         });
 
     }

package/src/vertexai/embeddings/embeddings-image.ts

@@ -0,0 +1,50 @@
+
+import { EmbeddingsOptions, EmbeddingsResult } from '@llumiverse/core';
+import { VertexAIDriver } from '../index.js';
+
+export interface ImageEmbeddingsOptions {
+    model?: string;
+    image?: {bytesBase64Encoded?: string}, // the image to generate embeddings for
+    text: string,                           // the text to generate embeddings for
+}
+
+interface EmbedingsForImagePrompt {
+    instances: ImageEmbeddingsOptions[]
+}
+
+interface ImageEmbeddingsResult {
+    predictions: [
+        {
+            imageEmbedding: number[]
+            textEmbedding: number[]
+        }
+    ],
+    deployedModelId: string,
+}
+
+//Currently we are only supporting either text or images sent to the multimodal model.
+export async function getEmbeddingsForImages(driver: VertexAIDriver, options: EmbeddingsOptions): Promise<EmbeddingsResult> {
+
+    // API is returns a 400 Error if a property is empty, so you undefined and "as" to remove the property entirely.
+    const prompt = {
+        instances: [{
+            text: options.image ? undefined : options.text,
+            image: options.image ?
+                {
+                    bytesBase64Encoded: options.image
+                }
+                : undefined,
+        }]
+    } as EmbedingsForImagePrompt;
+
+    const model = options.model || "multimodalembedding@001";
+
+    const result = await driver.fetchClient.post(`/publishers/google/models/${model}:predict`, {
+        payload: prompt
+    }) as ImageEmbeddingsResult;
+
+    return {
+        values: result.predictions[0].imageEmbedding ?? result.predictions[0].textEmbedding,
+        model: model,
+    };
+}

package/src/vertexai/embeddings/embeddings-text.ts

@@ -38,7 +38,7 @@ export async function getEmbeddingsForText(driver: VertexAIDriver, options: Text
         }]
     } as EmbedingsForTextPrompt;
 
-    const model = options.model || "textembedding-gecko@latest";
+    const model = options.model || "text-embedding-004";
 
     const result = await driver.fetchClient.post(`/publishers/google/models/${model}:predict`, {
         payload: prompt