@llumiverse/drivers 0.22.0-dev.1 → 0.22.1

package/package.json

@@ -1,87 +1,87 @@
 {
-  "name": "@llumiverse/drivers",
-  "version": "0.22.0-dev.1",
-  "type": "module",
-  "description": "LLM driver implementations. Currently supported are: openai, huggingface, bedrock, replicate.",
-  "files": [
-    "lib",
-    "src"
-  ],
-  "types": "./lib/types/index.d.ts",
-  "exports": {
+    "name": "@llumiverse/drivers",
+    "version": "0.22.1",
+    "type": "module",
+    "description": "LLM driver implementations. Currently supported are: openai, huggingface, bedrock, replicate.",
+    "files": [
+        "lib",
+        "src"
+    ],
     "types": "./lib/types/index.d.ts",
-    "import": "./lib/esm/index.js",
-    "require": "./lib/cjs/index.js"
-  },
-  "author": "Llumiverse",
-  "license": "Apache-2.0",
-  "homepage": "https://github.com/vertesia/llumiverse",
-  "repository": {
-    "type": "git",
-    "url": "git+ssh://git@github.com/vertesia/llumiverse.git"
-  },
-  "keywords": [
-    "llm",
-    "ai",
-    "prompt",
-    "prompt engineering",
-    "ml",
-    "machine learning",
-    "embeddings",
-    "training",
-    "model",
-    "universal",
-    "api",
-    "chatgpt",
-    "openai",
-    "vertexai",
-    "bedrock",
-    "replicate",
-    "huggingface",
-    "togetherai"
-  ],
-  "devDependencies": {
-    "dotenv": "^16.6.1",
-    "rimraf": "^6.0.1",
-    "ts-dual-module": "^0.6.3",
-    "typescript": "^5.9.2",
-    "vitest": "^3.2.4"
-  },
-  "dependencies": {
-    "@anthropic-ai/sdk": "^0.63.0",
-    "@anthropic-ai/vertex-sdk": "^0.14.0",
-    "@aws-sdk/client-bedrock": "^3.894.0",
-    "@aws-sdk/client-bedrock-runtime": "^3.894.0",
-    "@aws-sdk/client-s3": "^3.894.0",
-    "@aws-sdk/credential-providers": "^3.894.0",
-    "@aws-sdk/lib-storage": "^3.894.0",
-    "@aws-sdk/types": "^3.893.0",
-    "@azure-rest/ai-inference": "1.0.0-beta.6",
-    "@azure/ai-projects": "1.0.0-beta.10",
-    "@azure/core-auth": "^1.10.1",
-    "@azure/core-sse": "^2.3.0",
-    "@azure/identity": "^4.12.0",
-    "@azure/openai": "2.0.0",
-    "@google-cloud/aiplatform": "^3.35.0",
-    "@google/genai": "^1.20.0",
-    "@huggingface/inference": "^4.8.0",
-    "@vertesia/api-fetch-client": "^0.78.0",
-    "eventsource": "^4.0.0",
-    "google-auth-library": "^9.15.1",
-    "groq-sdk": "^0.32.0",
-    "mnemonist": "^0.40.3",
-    "node-web-stream-adapters": "^0.2.1",
-    "openai": "^4.104.0",
-    "replicate": "^1.2.0",
-    "@llumiverse/common": "0.22.0-dev.1",
-    "@llumiverse/core": "0.22.0-dev.1"
-  },
-  "ts_dual_module": {
-    "outDir": "lib"
-  },
-  "scripts": {
-    "test": "vitest run --retry 3",
-    "build": "pnpm exec tsmod build",
-    "clean": "rimraf ./lib tsconfig.tsbuildinfo"
-  }
-}
+    "exports": {
+        "types": "./lib/types/index.d.ts",
+        "import": "./lib/esm/index.js",
+        "require": "./lib/cjs/index.js"
+    },
+    "scripts": {
+        "test": "vitest run --retry 3",
+        "build": "pnpm exec tsmod build",
+        "clean": "rimraf ./lib tsconfig.tsbuildinfo"
+    },
+    "author": "Llumiverse",
+    "license": "Apache-2.0",
+    "homepage": "https://github.com/vertesia/llumiverse",
+    "repository": {
+        "type": "git",
+        "url": "git+ssh://git@github.com/vertesia/llumiverse.git"
+    },
+    "keywords": [
+        "llm",
+        "ai",
+        "prompt",
+        "prompt engineering",
+        "ml",
+        "machine learning",
+        "embeddings",
+        "training",
+        "model",
+        "universal",
+        "api",
+        "chatgpt",
+        "openai",
+        "vertexai",
+        "bedrock",
+        "replicate",
+        "huggingface",
+        "togetherai"
+    ],
+    "devDependencies": {
+        "dotenv": "^16.6.1",
+        "rimraf": "^6.0.1",
+        "ts-dual-module": "^0.6.3",
+        "typescript": "^5.9.2",
+        "vitest": "^3.2.4"
+    },
+    "dependencies": {
+        "@anthropic-ai/sdk": "^0.66.0",
+        "@anthropic-ai/vertex-sdk": "^0.14.0",
+        "@aws-sdk/client-bedrock": "^3.840.0",
+        "@aws-sdk/client-bedrock-runtime": "^3.840.0",
+        "@aws-sdk/client-s3": "^3.840.0",
+        "@aws-sdk/credential-providers": "^3.840.0",
+        "@aws-sdk/lib-storage": "^3.840.0",
+        "@aws-sdk/types": "^3.840.0",
+        "@azure-rest/ai-inference": "1.0.0-beta.6",
+        "@azure/ai-projects": "1.0.0-beta.10",
+        "@azure/core-auth": "^1.10.0",
+        "@azure/core-sse": "^2.3.0",
+        "@azure/identity": "^4.10.1",
+        "@azure/openai": "2.0.0",
+        "@google-cloud/aiplatform": "^3.35.0",
+        "@google/genai": "^1.25.0",
+        "@huggingface/inference": "2.6.7",
+        "@llumiverse/common": "workspace:*",
+        "@llumiverse/core": "workspace:*",
+        "@vertesia/api-fetch-client": "^0.74.0",
+        "eventsource": "^4.0.0",
+        "google-auth-library": "^9.15.1",
+        "groq-sdk": "^0.34.0",
+        "mnemonist": "^0.40.3",
+        "node-web-stream-adapters": "^0.2.1",
+        "openai": "^4.104.0",
+        "replicate": "^1.0.1"
+    },
+    "ts_dual_module": {
+        "outDir": "lib"
+    }
+}

package/src/bedrock/index.ts

@@ -20,6 +20,12 @@ import { LRUCache } from "mnemonist";
 import { converseConcatMessages, converseJSONprefill, converseSystemToMessages, formatConversePrompt } from "./converse.js";
 import { formatNovaImageGenerationPayload, NovaImageGenerationTaskType } from "./nova-image-payload.js";
 import { forceUploadFile } from "./s3.js";
+import {
+    formatTwelvelabsPegasusPrompt,
+    TwelvelabsPegasusRequest,
+    TwelvelabsMarengoRequest,
+    TwelvelabsMarengoResponse
+} from "./twelvelabs.js";
 
 const supportStreamingCache = new LRUCache<string, boolean>(4096);
 
@@ -84,7 +90,7 @@ function maxTokenFallbackClaude(option: StatelessExecutionOptions): number {
     }
 }
 
-export type BedrockPrompt = NovaMessagesPrompt | ConverseRequest;
+export type BedrockPrompt = NovaMessagesPrompt | ConverseRequest | TwelvelabsPegasusRequest;
 
 export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockPrompt> {
 
@@ -129,6 +135,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         if (opts.model.includes("canvas")) {
             return await formatNovaPrompt(segments, opts.result_schema);
         }
+        if (opts.model.includes("twelvelabs.pegasus")) {
+            return await formatTwelvelabsPegasusPrompt(segments, opts);
+        }
         return await formatConversePrompt(segments, opts);
     }
 
@@ -320,6 +329,11 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     }
 
     protected async canStream(options: ExecutionOptions): Promise<boolean> {
+        // // TwelveLabs Pegasus supports streaming according to the documentation
+        // if (options.model.includes("twelvelabs.pegasus")) {
+        //     return true;
+        // }
+
         let canStream = supportStreamingCache.get(options.model);
         if (canStream == null) {
             let type = BedrockModelType.Unknown;
@@ -336,8 +350,15 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return canStream;
     }
 
-    async requestTextCompletion(prompt: ConverseRequest, options: ExecutionOptions): Promise<Completion> {
-        let conversation = updateConversation(options.conversation as ConverseRequest, prompt);
+    async requestTextCompletion(prompt: BedrockPrompt, options: ExecutionOptions): Promise<Completion> {
+        // Handle Twelvelabs Pegasus models
+        if (options.model.includes("twelvelabs.pegasus")) {
+            return this.requestTwelvelabsPegasusCompletion(prompt as TwelvelabsPegasusRequest, options);
+        }
+
+        // Handle other Bedrock models that use Converse API
+        const conversePrompt = prompt as ConverseRequest;
+        let conversation = updateConversation(options.conversation as ConverseRequest, conversePrompt);
 
         const payload = this.preparePayload(conversation, options);
         const executor = this.getExecutor();
@@ -348,7 +369,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
 
         conversation = updateConversation(conversation, {
             messages: [res.output?.message ?? { content: [{ text: "" }], role: "assistant" }],
-            modelId: prompt.modelId,
+            modelId: conversePrompt.modelId,
         });
 
         let tool_use: ToolUse[] | undefined = undefined;
@@ -369,7 +390,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         const completion = {
-            ...this.getExtractedExecution(res, prompt, options),
+            ...this.getExtractedExecution(res, conversePrompt, options),
             original_response: options.include_original_response ? res : undefined,
             conversation: conversation,
             tool_use: tool_use,
@@ -378,8 +399,104 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return completion;
     }
 
-    async requestTextCompletionStream(prompt: ConverseRequest, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
-        const payload = this.preparePayload(prompt, options);
+    private async requestTwelvelabsPegasusCompletion(prompt: TwelvelabsPegasusRequest, options: ExecutionOptions): Promise<Completion> {
+        const executor = this.getExecutor();
+
+        const res = await executor.invokeModel({
+            modelId: options.model,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(prompt),
+        });
+
+        const decoder = new TextDecoder();
+        const body = decoder.decode(res.body);
+        const result = JSON.parse(body);
+
+        // Extract the response according to TwelveLabs Pegasus format
+        let finishReason: string | undefined;
+        switch (result.finishReason) {
+            case "stop":
+                finishReason = "stop";
+                break;
+            case "length":
+                finishReason = "length";
+                break;
+            default:
+                finishReason = result.finishReason;
+        }
+
+        return {
+            result: result.message ? [{ type: "text" as const, value: result.message }] : [],
+            finish_reason: finishReason,
+            original_response: options.include_original_response ? result : undefined,
+        };
+    }
+
+    private async requestTwelvelabsPegasusCompletionStream(prompt: TwelvelabsPegasusRequest, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+        const executor = this.getExecutor();
+
+        const res = await executor.invokeModelWithResponseStream({
+            modelId: options.model,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(prompt),
+        });
+
+        if (!res.body) {
+            throw new Error("[Bedrock] Stream not found in response");
+        }
+
+        return transformAsyncIterator(res.body, (chunk: any) => {
+            if (chunk.chunk?.bytes) {
+                const decoder = new TextDecoder();
+                const body = decoder.decode(chunk.chunk.bytes);
+
+                try {
+                    const result = JSON.parse(body);
+
+                    // Extract streaming response according to TwelveLabs Pegasus format
+                    let finishReason: string | undefined;
+                    if (result.finishReason) {
+                        switch (result.finishReason) {
+                            case "stop":
+                                finishReason = "stop";
+                                break;
+                            case "length":
+                                finishReason = "length";
+                                break;
+                            default:
+                                finishReason = result.finishReason;
+                        }
+                    }
+
+                    return {
+                        result: result.delta || result.message ? [{ type: "text" as const, value: result.delta || result.message || "" }] : [],
+                        finish_reason: finishReason,
+                    } satisfies CompletionChunkObject;
+                } catch (error) {
+                    // If JSON parsing fails, return empty chunk
+                    return {
+                        result: [],
+                    } satisfies CompletionChunkObject;
+                }
+            }
+
+            return {
+                result: [],
+            } satisfies CompletionChunkObject;
+        });
+    }
+
+    async requestTextCompletionStream(prompt: BedrockPrompt, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
+        // Handle Twelvelabs Pegasus models
+        if (options.model.includes("twelvelabs.pegasus")) {
+            return this.requestTwelvelabsPegasusCompletionStream(prompt as TwelvelabsPegasusRequest, options);
+        }
+
+        // Handle other Bedrock models that use Converse API
+        const conversePrompt = prompt as ConverseRequest;
+        const payload = this.preparePayload(conversePrompt, options);
         const executor = this.getExecutor();
         return executor.converseStream({
             ...payload,
@@ -391,7 +508,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             }
 
             return transformAsyncIterator(stream, (streamSegment: ConverseStreamOutput) => {
-                return this.getExtractedStream(streamSegment, prompt, options);
+                return this.getExtractedStream(streamSegment, conversePrompt, options);
             });
 
         }).catch((err) => {
@@ -703,17 +820,21 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             foundationModels = foundationModels.filter(foundationFilter);
         }
 
-        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21", "mistral", "meta", "deepseek", "writer", "openai"];
+        const supportedPublishers = ["amazon", "anthropic", "cohere", "ai21",
+            "mistral", "meta", "deepseek", "writer",
+            "openai", "twelvelabs", "qwen"];
         const unsupportedModelsByPublisher = {
             amazon: ["titan-image-generator", "nova-reel", "nova-sonic", "rerank"],
             anthropic: [],
-            cohere: ["rerank"],
+            cohere: ["rerank", "embed"],
             ai21: [],
             mistral: [],
             meta: [],
             deepseek: [],
             writer: [],
             openai: [],
+            twelvelabs: ["marengo"],
+            qwen: [],
         };
 
         // Helper function to check if model should be filtered out
@@ -839,6 +960,13 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
     async generateEmbeddings({ text, image, model }: EmbeddingsOptions): Promise<EmbeddingsResult> {
 
         this.logger.info("[Bedrock] Generating embeddings with model " + model);
+
+        // Handle TwelveLabs Marengo models
+        if (model?.includes("twelvelabs.marengo")) {
+            return this.generateTwelvelabsMarengoEmbeddings({ text, image, model });
+        }
+
+        // Handle other Bedrock embedding models
         const defaultModel = image ? "amazon.titan-embed-image-v1" : "amazon.titan-embed-text-v2:0";
         const modelID = model ?? defaultModel;
 
@@ -871,6 +999,51 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
             token_count: result.inputTextTokenCount
         };
     }
+
+    private async generateTwelvelabsMarengoEmbeddings({ text, image, model }: EmbeddingsOptions): Promise<EmbeddingsResult> {
+        const executor = this.getExecutor();
+
+        // Prepare the request payload for TwelveLabs Marengo
+        let invokeBody: TwelvelabsMarengoRequest = {
+            inputType: "text"
+        };
+
+        if (text) {
+            invokeBody.inputText = text;
+            invokeBody.inputType = "text";
+        }
+
+        if (image) {
+            // For the embeddings interface, image is expected to be base64
+            invokeBody.mediaSource = {
+                base64String: image
+            };
+            invokeBody.inputType = "image";
+        }
+
+        const res = await executor.invokeModel({
+            modelId: model!,
+            contentType: "application/json",
+            accept: "application/json",
+            body: JSON.stringify(invokeBody),
+        });
+
+        const decoder = new TextDecoder();
+        const body = decoder.decode(res.body);
+        const result: TwelvelabsMarengoResponse = JSON.parse(body);
+
+        // TwelveLabs Marengo returns embedding data
+        if (!result.embedding) {
+            throw new Error("Embeddings not found in TwelveLabs Marengo response");
+        }
+
+        return {
+            values: result.embedding,
+            model: model!,
+            // TwelveLabs Marengo doesn't return token count in the same way
+            token_count: undefined
+        };
+    }
 }
 
 function jobInfo(job: GetModelCustomizationJobCommandOutput, jobId: string): TrainingJob {

package/src/bedrock/twelvelabs.ts

@@ -0,0 +1,150 @@
+import { DataSource, ExecutionOptions, readStreamAsBase64 } from "@llumiverse/core";
+import { PromptSegment, PromptRole } from "@llumiverse/core";
+
+// TwelveLabs Pegasus Request/Response Types
+export interface TwelvelabsPegasusRequest {
+    inputPrompt: string;
+    temperature?: number;
+    responseFormat?: {
+        type: "json_schema";
+        json_schema: {
+            name: string;
+            schema: any;
+        };
+    };
+    mediaSource: {
+        base64String?: string;
+        s3Location?: {
+            uri: string;
+            bucketOwner?: string;
+        };
+    };
+    maxOutputTokens?: number;
+}
+
+export interface TwelvelabsPegasusResponse {
+    message: string;
+    finishReason: "stop" | "length";
+}
+
+// TwelveLabs Marengo Request/Response Types
+export interface TwelvelabsMarengoRequest {
+    inputType: "text" | "image" | "video" | "audio";
+    inputText?: string;
+    textTruncate?: "start" | "end";
+    mediaSource?: {
+        base64String?: string;
+        s3Location?: {
+            uri: string;
+            bucketOwner?: string;
+        };
+    };
+    embeddingOption?: "visual-text" | "visual-image" | "audio";
+    startSec?: number;
+    lengthSec?: number;
+    useFixedLengthSec?: boolean;
+    minClipSec?: number;
+}
+
+export interface TwelvelabsMarengoResponse {
+    embedding: number[];
+    embeddingOption: "visual-text" | "visual-image" | "audio";
+    startSec: number;
+    endSec: number;
+}
+
+// Convert prompt segments to TwelveLabs Pegasus request
+export async function formatTwelvelabsPegasusPrompt(
+    segments: PromptSegment[],
+    options: ExecutionOptions
+): Promise<TwelvelabsPegasusRequest> {
+    let inputPrompt = "";
+    let videoFile: DataSource | undefined;
+
+    // Extract text content and video files from segments
+    for (const segment of segments) {
+        if (segment.role === PromptRole.system || segment.role === PromptRole.user) {
+            if (segment.content) {
+                inputPrompt += segment.content + "\n";
+            }
+
+            // Look for video files
+            for (const file of segment.files ?? []) {
+                if (file.mime_type && file.mime_type.startsWith("video/")) {
+                    videoFile = file;
+                    break; // Use the first video file found
+                }
+            }
+        }
+    }
+
+    if (!videoFile) {
+        throw new Error("TwelveLabs Pegasus requires a video file input");
+    }
+
+    // Prepare media source
+    let mediaSource: TwelvelabsPegasusRequest["mediaSource"];
+
+    try {
+        // Try to get S3 URL first
+        const url = await videoFile.getURL();
+        const parsedUrl = new URL(url);
+
+        if (parsedUrl.hostname.endsWith("amazonaws.com") &&
+            (parsedUrl.hostname.startsWith("s3.") || parsedUrl.hostname.includes(".s3."))) {
+            // Convert S3 URL to s3:// format
+            const bucketMatch = parsedUrl.hostname.match(/^(?:s3\.)?([^.]+)\.s3\./);
+            const bucket = bucketMatch ? bucketMatch[1] : parsedUrl.hostname.split('.')[0];
+            const key = parsedUrl.pathname.substring(1); // Remove leading slash
+
+            mediaSource = {
+                s3Location: {
+                    uri: `s3://${bucket}/${key}`,
+                }
+            };
+        } else {
+            // Fall back to base64 encoding
+            const stream = await videoFile.getStream();
+            const base64String = await readStreamAsBase64(stream);
+
+            mediaSource = {
+                base64String
+            };
+        }
+    } catch (error) {
+        // If getting URL fails, use base64 encoding
+        const stream = await videoFile.getStream();
+        const base64String = await readStreamAsBase64(stream);
+
+        mediaSource = {
+            base64String
+        };
+    }
+
+    const request: TwelvelabsPegasusRequest = {
+        inputPrompt: inputPrompt.trim(),
+        mediaSource
+    };
+
+    // Add optional parameters from model options
+    const modelOptions = options.model_options as any;
+    if (modelOptions?.temperature !== undefined) {
+        request.temperature = modelOptions.temperature;
+    }
+    if (modelOptions?.max_tokens !== undefined) {
+        request.maxOutputTokens = modelOptions.max_tokens;
+    }
+
+    // Add response format if result schema is specified
+    if (options.result_schema) {
+        request.responseFormat = {
+            type: "json_schema",
+            json_schema: {
+                name: "response",
+                schema: options.result_schema
+            }
+        };
+    }
+
+    return request;
+}

package/src/groq/index.ts

@@ -4,6 +4,7 @@ import { formatOpenAILikeMultimodalPrompt } from "../openai/openai_format.js";
 
 import Groq from "groq-sdk";
 import type { ChatCompletionMessageParam, ChatCompletionTool } from "groq-sdk/resources/chat/completions";
+import type { FunctionParameters } from "groq-sdk/resources/shared";
 
 interface GroqDriverOptions extends DriverOptions {
     apiKey: string;
@@ -158,7 +159,7 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, ChatCompletion
             function: {
                 name: tool.name,
                 description: tool.description,
-                parameters: tool.input_schema as any,
+                parameters: tool.input_schema satisfies FunctionParameters,
             }
         }));
     }
@@ -287,13 +288,13 @@ export class GroqDriver extends AbstractDriver<GroqDriverOptions, ChatCompletion
 
             return {
                 result: choice.delta.content ? [{ type: "text", value: choice.delta.content }] : [],
-                finish_reason: finish_reason,
+                finish_reason: finish_reason ?? undefined,
                 token_usage: {
                     prompt: chunk.x_groq?.usage?.prompt_tokens,
                     result: chunk.x_groq?.usage?.completion_tokens,
                     total: chunk.x_groq?.usage?.total_tokens,
                 },
-            } as CompletionChunkObject;
+            } satisfies CompletionChunkObject;
         });
     }