@llumiverse/drivers 0.22.0 → 0.23.0-dev-20251118

package/src/bedrock/index.ts

@@ -7,9 +7,9 @@ import { S3Client } from "@aws-sdk/client-s3";
 import { AwsCredentialIdentity, Provider } from "@aws-sdk/types";
 import {
     AbstractDriver, AIModel, Completion, CompletionChunkObject, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult,
-    ExecutionOptions, ExecutionTokenUsage, ImageGeneration, Modalities, PromptSegment,
+    ExecutionOptions, ExecutionTokenUsage, Modalities, PromptSegment,
     TextFallbackOptions, ToolDefinition, ToolUse, TrainingJob, TrainingJobStatus, TrainingOptions,
-    BedrockClaudeOptions, BedrockPalmyraOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
+    BedrockClaudeOptions, BedrockPalmyraOptions, BedrockGptOssOptions, getMaxTokensLimitBedrock, NovaCanvasOptions,
     modelModalitiesToArray, getModelCapabilities,
     StatelessExecutionOptions,
     ModelOptions
@@ -20,6 +20,12 @@ import { LRUCache } from "mnemonist";
 import { converseConcatMessages, converseJSONprefill, converseSystemToMessages, formatConversePrompt } from "./converse.js";
 import { formatNovaImageGenerationPayload, NovaImageGenerationTaskType } from "./nova-image-payload.js";
 import { forceUploadFile } from "./s3.js";
+import {
+    formatTwelvelabsPegasusPrompt,
+    TwelvelabsPegasusRequest,
+    TwelvelabsMarengoRequest,
+    TwelvelabsMarengoResponse
+} from "./twelvelabs.js";
 
 const supportStreamingCache = new LRUCache<string, boolean>(4096);
 
@@ -84,7 +90,7 @@ function maxTokenFallbackClaude(option: StatelessExecutionOptions): number {
     }
 }
 
-export type BedrockPrompt = NovaMessagesPrompt | ConverseRequest;
+export type BedrockPrompt = NovaMessagesPrompt | ConverseRequest | TwelvelabsPegasusRequest;
 
 export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockPrompt> {
 
@@ -129,6 +135,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         if (opts.model.includes("canvas")) {
             return await formatNovaPrompt(segments, opts.result_schema);
         }
+        if (opts.model.includes("twelvelabs.pegasus")) {
+            return await formatTwelvelabsPegasusPrompt(segments, opts);
+        }
         return await formatConversePrompt(segments, opts);
     }
 
@@ -160,7 +169,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                     const type = Object.keys(content).find(
                         key => key !== '$unknown' && content[key as keyof typeof content] !== undefined
                     );
-                    this.logger.info("[Bedrock] Unsupported content response type:", type);
+                    this.logger.info({ type }, "[Bedrock] Unsupported content response type:");
                 }
             }
 
@@ -171,7 +180,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completionResult: CompletionChunkObject = {
-            result: reasoning + resultText,
+            result: reasoning + resultText ? [{ type: "text", value: reasoning + resultText }] : [],
             token_usage: {
                 prompt: result.usage?.inputTokens,
                 result: result.usage?.outputTokens,
@@ -226,7 +235,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                 const type = Object.keys(delta).find(
                     key => key !== '$unknown' && (delta as any)[key] !== undefined
                 );
-                this.logger.info("[Bedrock] Unsupported content response type:", type);
+                this.logger.info({ type }, "[Bedrock] Unsupported content response type:");
             }
         }
 
@@ -252,7 +261,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completionResult: CompletionChunkObject = {
-            result: reasoning + output,
+            result: reasoning + output ? [{ type: "text", value: reasoning + output }] : [],
             token_usage: token_usage,
             finish_reason: converseFinishReason(stop_reason),
         };
@@ -320,6 +329,11 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     }
 
     protected async canStream(options: ExecutionOptions): Promise<boolean> {
+        // // TwelveLabs Pegasus supports streaming according to the documentation
+        // if (options.model.includes("twelvelabs.pegasus")) {
+        //     return true;
+        // }
+
         let canStream = supportStreamingCache.get(options.model);
         if (canStream == null) {
             let type = BedrockModelType.Unknown;
@@ -336,8 +350,15 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return canStream;
     }
 
-    async requestTextCompletion(prompt: ConverseRequest, options: ExecutionOptions): Promise<Completion> {
-        let conversation = updateConversation(options.conversation as ConverseRequest, prompt);
+    async requestTextCompletion(prompt: BedrockPrompt, options: ExecutionOptions): Promise<Completion> {
+        // Handle Twelvelabs Pegasus models
+        if (options.model.includes("twelvelabs.pegasus")) {
+            return this.requestTwelvelabsPegasusCompletion(prompt as TwelvelabsPegasusRequest, options);
+        }
+
+        // Handle other Bedrock models that use Converse API
+        const conversePrompt = prompt as ConverseRequest;
+        let conversation = updateConversation(options.conversation as ConverseRequest, conversePrompt);
 
         const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
@@ -348,7 +369,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         conversation = updateConversation(conversation, {
             messages: [res.output?.message ?? { content: [{ text: "" }], role: "assistant" }],
-            modelId: prompt.modelId,
+            modelId: conversePrompt.modelId,
         });
 
         let tool_use: ToolUse[] | undefined = undefined;
@@ -369,7 +390,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completion = {
-            ...this.getExtractedExecution(res, prompt, options),
+            ...this.getExtractedExecution(res, conversePrompt, options),
             original_response: options.include_original_response ? res : undefined,
             conversation: conversation,
             tool_use: tool_use,
@@ -378,8 +399,104 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return completion;
     }
 
-    async requestTextCompletionStream(prompt: ConverseRequest, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
-        const payload = this.preparePayload(prompt, options);
+    private async requestTwelvelabsPegasusCompletion(prompt: TwelvelabsPegasusRequest, options: ExecutionOptions): Promise<Completion> {
+        const executor = this.getExecutor();
+
+        const res = await executor.invokeModel({
+            modelId: options.model,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(prompt),
+        });
+
+        const decoder = new TextDecoder();
+        const body = decoder.decode(res.body);
+        const result = JSON.parse(body);
+
+        // Extract the response according to TwelveLabs Pegasus format
+        let finishReason: string | undefined;
+        switch (result.finishReason) {
+            case "stop":
+                finishReason = "stop";
+                break;
+            case "length":
+                finishReason = "length";
+                break;
+            default:
+                finishReason = result.finishReason;
+        }
+
+        return {
+            result: result.message ? [{ type: "text" as const, value: result.message }] : [],
+            finish_reason: finishReason,
+            original_response: options.include_original_response ? result : undefined,
+        };
+    }
+
+    private async requestTwelvelabsPegasusCompletionStream(prompt: TwelvelabsPegasusRequest, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+        const executor = this.getExecutor();
+
+        const res = await executor.invokeModelWithResponseStream({
+            modelId: options.model,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(prompt),
+        });
+
+        if (!res.body) {
+            throw new Error("[Bedrock] Stream not found in response");
+        }
+
+        return transformAsyncIterator(res.body, (chunk: any) => {
+            if (chunk.chunk?.bytes) {
+                const decoder = new TextDecoder();
+                const body = decoder.decode(chunk.chunk.bytes);
+
+                try {
+                    const result = JSON.parse(body);
+
+                    // Extract streaming response according to TwelveLabs Pegasus format
+                    let finishReason: string | undefined;
+                    if (result.finishReason) {
+                        switch (result.finishReason) {
+                            case "stop":
+                                finishReason = "stop";
+                                break;
+                            case "length":
+                                finishReason = "length";
+                                break;
+                            default:
+                                finishReason = result.finishReason;
+                        }
+                    }
+
+                    return {
+                        result: result.delta || result.message ? [{ type: "text" as const, value: result.delta || result.message || "" }] : [],
+                        finish_reason: finishReason,
+                    } satisfies CompletionChunkObject;
+                } catch (error) {
+                    // If JSON parsing fails, return empty chunk
+                    return {
+                        result: [],
+                    } satisfies CompletionChunkObject;
+                }
+            }
+
+            return {
+                result: [],
+            } satisfies CompletionChunkObject;
+        });
+    }
+
+    async requestTextCompletionStream(prompt: BedrockPrompt, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+        // Handle Twelvelabs Pegasus models
+        if (options.model.includes("twelvelabs.pegasus")) {
+            return this.requestTwelvelabsPegasusCompletionStream(prompt as TwelvelabsPegasusRequest, options);
+        }
+
+        // Handle other Bedrock models that use Converse API
+        const conversePrompt = prompt as ConverseRequest;
+        const payload = this.preparePayload(conversePrompt, options);
         const executor = this.getExecutor();
         return executor.converseStream({
             ...payload,
@@ -391,11 +508,11 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
 
             return transformAsyncIterator(stream, (streamSegment: ConverseStreamOutput) => {
-                return this.getExtractedStream(streamSegment, prompt, options);
+                return this.getExtractedStream(streamSegment, conversePrompt, options);
             });
 
         }).catch((err) => {
-            this.logger.error("[Bedrock] Failed to stream", { error: err });
+            this.logger.error({ error: err }, "[Bedrock] Failed to stream");
             throw err;
         });
     }
@@ -443,7 +560,6 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         } else if (options.model.includes("mistral")) {
             //7B instruct and 8x7B instruct
             if (options.model.includes("7b")) {
-                supportsJSONPrefill = true;
                 additionalField = { top_k: model_options.top_k };
                 //Does not support system messages
                 if (prompt.system && prompt.system?.length != 0) {
@@ -499,12 +615,17 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
         } else if (options.model.includes("deepseek")) {
             //DeepSeek models support no additional options
+        } else if (options.model.includes("gpt-oss")) {
+            const gptOssOptions = model_options as ModelOptions as BedrockGptOssOptions;
+            additionalField = {
+                reasoning_effort: gptOssOptions?.reasoning_effort,
+            };
         }
 
         //If last message is "```json", add corresponding ``` as a stop sequence.
         if (prompt.messages && prompt.messages.length > 0) {
             if (prompt.messages[prompt.messages.length - 1].content?.[0].text === "```json") {
-                let stopSeq = model_options.stop_sequence;
+                const stopSeq = model_options.stop_sequence;
                 if (!stopSeq) {
                     model_options.stop_sequence = ["```"];
                 } else if (!stopSeq.includes("```")) {
@@ -536,8 +657,8 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
         };
 
-        //Only add tools if they are defined
-        if (tool_defs) {
+        //Only add tools if they are defined and not empty
+        if (tool_defs?.length) {
             request.toolConfig = {
                 tools: tool_defs,
             }
@@ -547,12 +668,12 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     }
 
 
-    async requestImageGeneration(prompt: NovaMessagesPrompt, options: ExecutionOptions): Promise<Completion<ImageGeneration>> {
+    async requestImageGeneration(prompt: NovaMessagesPrompt, options: ExecutionOptions): Promise<Completion> {
         if (options.output_modality !== Modalities.image) {
             throw new Error(`Image generation requires image output_modality`);
         }
         if (options.model_options?._option_id !== "bedrock-nova-canvas") {
-            this.logger.warn("Invalid model options", { options: options.model_options });
+            this.logger.warn({ options: options.model_options }, "Invalid model options");
         }
         const model_options = options.model_options as NovaCanvasOptions;
 
@@ -579,13 +700,14 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         const decoder = new TextDecoder();
         const body = decoder.decode(res.body);
-        const result = JSON.parse(body);
+        const bedrockResult = JSON.parse(body);
 
         return {
-            error: result.error,
-            result: {
-                images: result.images,
-            }
+            error: bedrockResult.error,
+            result: bedrockResult.images.map((image: any) => ({
+                type: "image" as const,
+                value: image
+            }))
         }
     }
 
@@ -698,16 +820,21 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             foundationModels = foundationModels.filter(foundationFilter);
         }
 
-        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21", "mistral", "meta", "deepseek", "writer"];
+        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21",
+            "mistral", "meta", "deepseek", "writer",
+            "openai", "twelvelabs", "qwen"];
         const unsupportedModelsByPublisher = {
             amazon: ["titan-image-generator", "nova-reel", "nova-sonic", "rerank"],
             anthropic: [],
-            cohere: ["rerank"],
+            cohere: ["rerank", "embed"],
             ai21: [],
             mistral: [],
             meta: [],
             deepseek: [],
             writer: [],
+            openai: [],
+            twelvelabs: ["marengo"],
+            qwen: [],
         };
 
         // Helper function to check if model should be filtered out
@@ -833,6 +960,13 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     async generateEmbeddings({ text, image, model }: EmbeddingsOptions): Promise<EmbeddingsResult> {
 
         this.logger.info("[Bedrock] Generating embeddings with model " + model);
+
+        // Handle TwelveLabs Marengo models
+        if (model?.includes("twelvelabs.marengo")) {
+            return this.generateTwelvelabsMarengoEmbeddings({ text, image, model });
+        }
+
+        // Handle other Bedrock embedding models
         const defaultModel = image ? "amazon.titan-embed-image-v1" : "amazon.titan-embed-text-v2:0";
         const modelID = model ?? defaultModel;
 
@@ -865,6 +999,51 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             token_count: result.inputTextTokenCount
         };
     }
+
+    private async generateTwelvelabsMarengoEmbeddings({ text, image, model }: EmbeddingsOptions): Promise<EmbeddingsResult> {
+        const executor = this.getExecutor();
+
+        // Prepare the request payload for TwelveLabs Marengo
+        let invokeBody: TwelvelabsMarengoRequest = {
+            inputType: "text"
+        };
+
+        if (text) {
+            invokeBody.inputText = text;
+            invokeBody.inputType = "text";
+        }
+
+        if (image) {
+            // For the embeddings interface, image is expected to be base64
+            invokeBody.mediaSource = {
+                base64String: image
+            };
+            invokeBody.inputType = "image";
+        }
+
+        const res = await executor.invokeModel({
+            modelId: model!,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(invokeBody),
+        });
+
+        const decoder = new TextDecoder();
+        const body = decoder.decode(res.body);
+        const result: TwelvelabsMarengoResponse = JSON.parse(body);
+
+        // TwelveLabs Marengo returns embedding data
+        if (!result.embedding) {
+            throw new Error("Embeddings not found in TwelveLabs Marengo response");
+        }
+
+        return {
+            values: result.embedding,
+            model: model!,
+            // TwelveLabs Marengo doesn't return token count in the same way
+            token_count: undefined
+        };
+    }
 }
 
 function jobInfo(job: GetModelCustomizationJobCommandOutput, jobId: string): TrainingJob {

package/src/bedrock/twelvelabs.ts

@@ -0,0 +1,150 @@
+import { DataSource, ExecutionOptions, readStreamAsBase64 } from "@llumiverse/core";
+import { PromptSegment, PromptRole } from "@llumiverse/core";
+
+// TwelveLabs Pegasus Request/Response Types
+export interface TwelvelabsPegasusRequest {
+    inputPrompt: string;
+    temperature?: number;
+    responseFormat?: {
+        type: "json_schema";
+        json_schema: {
+            name: string;
+            schema: any;
+        };
+    };
+    mediaSource: {
+        base64String?: string;
+        s3Location?: {
+            uri: string;
+            bucketOwner?: string;
+        };
+    };
+    maxOutputTokens?: number;
+}
+
+export interface TwelvelabsPegasusResponse {
+    message: string;
+    finishReason: "stop" | "length";
+}
+
+// TwelveLabs Marengo Request/Response Types
+export interface TwelvelabsMarengoRequest {
+    inputType: "text" | "image" | "video" | "audio";
+    inputText?: string;
+    textTruncate?: "start" | "end";
+    mediaSource?: {
+        base64String?: string;
+        s3Location?: {
+            uri: string;
+            bucketOwner?: string;
+        };
+    };
+    embeddingOption?: "visual-text" | "visual-image" | "audio";
+    startSec?: number;
+    lengthSec?: number;
+    useFixedLengthSec?: boolean;
+    minClipSec?: number;
+}
+
+export interface TwelvelabsMarengoResponse {
+    embedding: number[];
+    embeddingOption: "visual-text" | "visual-image" | "audio";
+    startSec: number;
+    endSec: number;
+}
+
+// Convert prompt segments to TwelveLabs Pegasus request
+export async function formatTwelvelabsPegasusPrompt(
+    segments: PromptSegment[],
+    options: ExecutionOptions
+): Promise<TwelvelabsPegasusRequest> {
+    let inputPrompt = "";
+    let videoFile: DataSource | undefined;
+
+    // Extract text content and video files from segments
+    for (const segment of segments) {
+        if (segment.role === PromptRole.system || segment.role === PromptRole.user) {
+            if (segment.content) {
+                inputPrompt += segment.content + "\n";
+            }
+
+            // Look for video files
+            for (const file of segment.files ?? []) {
+                if (file.mime_type && file.mime_type.startsWith("video/")) {
+                    videoFile = file;
+                    break; // Use the first video file found
+                }
+            }
+        }
+    }
+
+    if (!videoFile) {
+        throw new Error("TwelveLabs Pegasus requires a video file input");
+    }
+
+    // Prepare media source
+    let mediaSource: TwelvelabsPegasusRequest["mediaSource"];
+
+    try {
+        // Try to get S3 URL first
+        const url = await videoFile.getURL();
+        const parsedUrl = new URL(url);
+
+        if (parsedUrl.hostname.endsWith("amazonaws.com") &&
+            (parsedUrl.hostname.startsWith("s3.") || parsedUrl.hostname.includes(".s3."))) {
+            // Convert S3 URL to s3:// format
+            const bucketMatch = parsedUrl.hostname.match(/^(?:s3\.)?([^.]+)\.s3\./);
+            const bucket = bucketMatch ? bucketMatch[1] : parsedUrl.hostname.split('.')[0];
+            const key = parsedUrl.pathname.substring(1); // Remove leading slash
+
+            mediaSource = {
+                s3Location: {
+                    uri: `s3://${bucket}/${key}`,
+                }
+            };
+        } else {
+            // Fall back to base64 encoding
+            const stream = await videoFile.getStream();
+            const base64String = await readStreamAsBase64(stream);
+
+            mediaSource = {
+                base64String
+            };
+        }
+    } catch (error) {
+        // If getting URL fails, use base64 encoding
+        const stream = await videoFile.getStream();
+        const base64String = await readStreamAsBase64(stream);
+
+        mediaSource = {
+            base64String
+        };
+    }
+
+    const request: TwelvelabsPegasusRequest = {
+        inputPrompt: inputPrompt.trim(),
+        mediaSource
+    };
+
+    // Add optional parameters from model options
+    const modelOptions = options.model_options as any;
+    if (modelOptions?.temperature !== undefined) {
+        request.temperature = modelOptions.temperature;
+    }
+    if (modelOptions?.max_tokens !== undefined) {
+        request.maxOutputTokens = modelOptions.max_tokens;
+    }
+
+    // Add response format if result schema is specified
+    if (options.result_schema) {
+        request.responseFormat = {
+            type: "json_schema",
+            json_schema: {
+                name: "response",
+                schema: options.result_schema
+            }
+        };
+    }
+
+    return request;
+}