@llumiverse/drivers 0.22.0-dev.1 → 0.22.1

package/src/huggingface_ie.ts

@@ -1,6 +1,7 @@
 import {
-    InferenceClient,
-    TextGenerationStreamOutput,
+    HfInference,
+    HfInferenceEndpoint,
+    TextGenerationStreamOutput
 } from "@huggingface/inference";
 import {
     AIModel,
@@ -24,7 +25,7 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
     static PROVIDER = "huggingface_ie";
     provider = HuggingFaceIEDriver.PROVIDER;
     service: FetchClient;
-    _executor?: InferenceClient;
+    _executor?: HfInferenceEndpoint;
 
     constructor(
         options: HuggingFaceIEDriverOptions
@@ -59,8 +60,7 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
                     `Endpoint ${model} is not running - current status: ${endpoint.status}`
                 );
 
-            // Use the new InferenceClient and bind it to the endpoint URL
-            this._executor = new InferenceClient(this.options.apiKey).endpoint(
+            this._executor = new HfInference(this.options.apiKey).endpoint(
                 endpoint.url
             );
         }
@@ -82,7 +82,8 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
             },
         });
 
-        return transformAsyncIterator(req, (val: TextGenerationStreamOutput) : CompletionChunkObject => {
+
+        return transformAsyncIterator(req, (val: TextGenerationStreamOutput) => {
             //special like <s> are not part of the result
             if (val.token.special) return { result: [] };
             let finish_reason = val.details?.finish_reason as string;
@@ -95,7 +96,7 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
                 token_usage: {
                     result: val.details?.generated_tokens ?? 0,
                 }
-            };
+            } as CompletionChunkObject;
         });
     }
 

package/src/index.ts

@@ -1,12 +1,12 @@
-export * from "./azure/azure_foundry.js";
 export * from "./bedrock/index.js";
 export * from "./groq/index.js";
 export * from "./huggingface_ie.js";
 export * from "./mistral/index.js";
 export * from "./openai/azure_openai.js";
+export * from "./azure/azure_foundry.js";
 export * from "./openai/openai.js";
 export * from "./replicate.js";
-export * from "./test-driver/index.js";
+export * from "./test/index.js";
 export * from "./togetherai/index.js";
 export * from "./vertexai/index.js";
 export * from "./watsonx/index.js";

package/src/openai/index.ts

@@ -293,7 +293,7 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
         //Some of these use the completions API instead of the chat completions API.
         //Others are for non-text input modalities. Therefore common to both.
         const wordBlacklist = ["embed", "whisper", "transcribe", "audio", "moderation", "tts",
-            "realtime", "dall-e", "babbage", "davinci", "codex", "o1-pro"];
+            "realtime", "dall-e", "babbage", "davinci", "codex", "o1-pro", "computer-use", "sora"];
 
 
         //OpenAI has very little information, filtering based on name.
@@ -304,11 +304,15 @@ export abstract class BaseOpenAIDriver extends AbstractDriver<
         const models = filter ? result.filter(filter) : result;
         const aiModels = models.map((m) => {
             const modelCapability = getModelCapabilities(m.id, "openai");
+            let owner = m.owned_by;
+            if (owner == "system") {
+                owner = "openai";
+            }
             return {
                 id: m.id,
                 name: m.id,
                 provider: this.provider,
-                owner: m.owned_by,
+                owner: owner,
                 type: m.object === "model" ? ModelType.Text : ModelType.Unknown,
                 can_stream: true,
                 is_multimodal: m.id.includes("gpt-4"),

package/src/replicate.ts

@@ -236,7 +236,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
             this.service.models.versions.list(owner, model),
         ]);
 
-        if (!rModel || !versions || (versions as any).results?.length === 0) {
+        if (!rModel || !versions || versions.length === 0) {
             throw new Error("Model not found or no versions available");
         }
 

package/src/vertexai/index.ts

@@ -23,6 +23,7 @@ import { PredictionServiceClient, v1beta1 } from "@google-cloud/aiplatform";
 import { AnthropicVertex } from "@anthropic-ai/vertex-sdk";
 import { ImagenModelDefinition, ImagenPrompt } from "./models/imagen.js";
 import { GoogleGenAI, Content, Model } from "@google/genai";
+import { NON_GLOBAL_ANTHROPIC_MODELS, ANTHROPIC_REGIONS } from "./models/claude.js";
 
 export interface VertexAIDriverOptions extends DriverOptions {
     project: string;
@@ -130,12 +131,34 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Vertex
         return this.llamaClient;
     }
 
-    public getAnthropicClient(): AnthropicVertex {
-        //Lazy initialization
+    public getAnthropicClient(region: string = this.options.region): AnthropicVertex {
+        // Extract region prefix and map if it exists in ANTHROPIC_REGIONS, otherwise use as-is
+        const getRegionPrefix = (r: string) => r.split('-')[0];
+        const regionPrefix = getRegionPrefix(region);
+        const mappedRegion = ANTHROPIC_REGIONS[regionPrefix] || region;
+
+        const defaultRegionPrefix = getRegionPrefix(this.options.region);
+        const defaultMappedRegion = ANTHROPIC_REGIONS[defaultRegionPrefix] || this.options.region;
+
+        // If mapped region is different from default mapped region, create one-off client
+        if (mappedRegion !== defaultMappedRegion) {
+            return new AnthropicVertex({
+                timeout: 20 * 60 * 10000, // Set to 20 minutes, 10 minute default, setting this disables long request error: https://github.com/anthropics/anthropic-sdk-typescript?#long-requests
+                region: mappedRegion,
+                projectId: this.options.project,
+                googleAuth: new GoogleAuth({
+                    scopes: ["https://www.googleapis.com/auth/cloud-platform"],
+                    authClient: this.authClient as JSONClient,
+                    projectId: this.options.project,
+                }),
+            });
+        }
+
+        //Lazy initialization for default region
         if (!this.anthropicClient) {
             this.anthropicClient = new AnthropicVertex({
                 timeout: 20 * 60 * 10000, // Set to 20 minutes, 10 minute default, setting this disables long request error: https://github.com/anthropics/anthropic-sdk-typescript?#long-requests
-                region: "us-east5",
+                region: mappedRegion,
                 projectId: this.options.project,
                 googleAuth: new GoogleAuth({
                     scopes: ["https://www.googleapis.com/auth/cloud-platform"],
@@ -353,6 +376,36 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Vertex
                 } satisfies AIModel<string>;
             }));
 
+            // Create global anthropic models for those not in NON_GLOBAL_ANTHROPIC_MODELS
+            if (publisher === 'anthropic') {
+                const globalAnthropicModels = response.filter((model) => {
+                    const modelName = model.name ?? "";
+                    if (retiredModels.some(retiredModel => modelName.includes(retiredModel))) {
+                        return false;
+                    }
+                    if (modelFamily.some(family => modelName.includes(family))) {
+                        if (modelName.includes("claude-3-7")) {
+                            return true;
+                        }
+                        return !NON_GLOBAL_ANTHROPIC_MODELS.some(nonGlobalModel => modelName.includes(nonGlobalModel));
+                    }
+                    return false;
+                }).map(model => {
+                    const modelCapability = getModelCapabilities(model.name ?? '', "vertexai");
+                    return {
+                        id: "locations/global/" + model.name,
+                        name: "Global " + model.name?.split('/').pop(),
+                        provider: 'vertexai',
+                        owner: publisher,
+                        input_modalities: modelModalitiesToArray(modelCapability.input),
+                        output_modalities: modelModalitiesToArray(modelCapability.output),
+                        tool_support: modelCapability.tool_support,
+                    } satisfies AIModel<string>;
+                });
+
+                models = models.concat(globalAnthropicModels);
+            }
+
             // Add additional models that are not in the listing
             for (const additionalModel of additionalModels[publisher as keyof typeof additionalModels]) {
                 const publisherModelName = `publishers/${publisher}/models/${additionalModel}`;

package/src/vertexai/models/claude.ts

@@ -9,6 +9,17 @@ import { ModelDefinition } from "../models.js";
 import { MessageCreateParamsBase, MessageCreateParamsNonStreaming, RawMessageStreamEvent } from "@anthropic-ai/sdk/resources/messages.js";
 import { MessageStreamParams } from "@anthropic-ai/sdk/resources/index.mjs";
 
+export const ANTHROPIC_REGIONS: Record<string, string> = {
+    us: "us-east5",
+    europe: "europe-west1",
+    global: "global",
+}
+
+export const NON_GLOBAL_ANTHROPIC_MODELS = [
+    "claude-3-5",
+    "claude-3",
+];
+
 interface ClaudePrompt {
     messages: MessageParam[];
     system?: TextBlockParam[];
@@ -250,7 +261,15 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
     }
 
     async requestTextCompletion(driver: VertexAIDriver, prompt: ClaudePrompt, options: ExecutionOptions): Promise<Completion> {
-        const client = driver.getAnthropicClient();
+        const splits = options.model.split("/");
+        let region: string | undefined = undefined;
+        if (splits[0] === "locations" && splits.length >= 2) {
+            region = splits[1];
+        }
+        const modelName = splits[splits.length - 1];
+        options = { ...options, model: modelName };
+
+        const client = driver.getAnthropicClient(region);
         options.model_options = options.model_options as VertexAIClaudeOptions;
 
         if (options.model_options?._option_id !== "vertexai-claude") {
@@ -287,7 +306,15 @@ export class ClaudeModelDefinition implements ModelDefinition<ClaudePrompt> {
     }
 
     async requestTextCompletionStream(driver: VertexAIDriver, prompt: ClaudePrompt, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunkObject>> {
-        const client = driver.getAnthropicClient();
+        const splits = options.model.split("/");
+        let region: string | undefined = undefined;
+        if (splits[0] === "locations" && splits.length >= 2) {
+            region = splits[1];
+        }
+        const modelName = splits[splits.length - 1];
+        options = { ...options, model: modelName };
+
+        const client = driver.getAnthropicClient(region);
         const model_options = options.model_options as VertexAIClaudeOptions | undefined;
 
         if (model_options?._option_id !== "vertexai-claude") {
@@ -399,8 +426,7 @@ interface RequestOptions {
 }
 
 function getClaudePayload(options: ExecutionOptions, prompt: ClaudePrompt): { payload: MessageCreateParamsBase, requestOptions: RequestOptions | undefined } {
-    const splits = options.model.split("/");
-    const modelName = splits[splits.length - 1];
+    const modelName = options.model; // Model name is already extracted in the calling methods
     const model_options = options.model_options as VertexAIClaudeOptions;
 
     // Add beta header for Claude 3.7 models to enable 128k output tokens

package/src/vertexai/models/gemini.ts

@@ -1,7 +1,7 @@
 import {
     Content, FinishReason, FunctionCallingConfigMode, FunctionDeclaration, GenerateContentConfig, GenerateContentParameters,
     GenerateContentResponseUsageMetadata,
-    HarmBlockThreshold, HarmCategory, Part, SafetySetting, Schema, Tool, Type
+    HarmBlockThreshold, HarmCategory, Modality, Part, SafetySetting, Schema, Tool, Type
 } from "@google/genai";
 import {
     AIModel, Completion, CompletionChunkObject, CompletionResult, ExecutionOptions,
@@ -56,7 +56,18 @@ function getGeminiPayload(options: ExecutionOptions, prompt: GenerateContentProm
         || options.model.includes("gemini-2.5");
 
     const configNanoBanana: GenerateContentConfig = {
-        responseModalities: ["TEXT", "IMAGE"]
+        systemInstruction: prompt.system,
+        safetySettings: geminiSafetySettings,
+        responseModalities: [Modality.TEXT, Modality.IMAGE], // This is an error if only Text, and Only Image just gets blank responses.
+        candidateCount: 1,
+        //Model options
+        temperature: model_options?.temperature,
+        topP: model_options?.top_p,
+        maxOutputTokens: geminiMaxTokens(options),
+        stopSequences: model_options?.stop_sequence,
+        imageConfig: {
+            aspectRatio: model_options?.image_aspect_ratio,
+        }
     }
 
     const config: GenerateContentConfig = {
@@ -502,7 +513,7 @@ export function mergeConsecutiveRole(contents: Content[] | undefined): Content[]
 const supportedFinishReasons: FinishReason[] = [
     FinishReason.MAX_TOKENS,
     FinishReason.STOP,
-    FinishReason.FINISH_REASON_UNSPECIFIED
+    FinishReason.FINISH_REASON_UNSPECIFIED,
 ]
 
 function geminiMaxTokens(option: StatelessExecutionOptions) {
@@ -716,7 +727,7 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         let region: string | undefined = undefined;
         if (splits[0] === "locations" && splits.length >= 2) {
             region = splits[1];
-        } 
+        }
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 
@@ -784,7 +795,7 @@ export class GeminiModelDefinition implements ModelDefinition<GenerateContentPro
         let region: string | undefined = undefined;
         if (splits[0] === "locations" && splits.length >= 2) {
             region = splits[1];
-        } 
+        }
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
 

package/src/vertexai/models.ts

@@ -15,8 +15,20 @@ export interface ModelDefinition<PromptT = any> {
 
 export function getModelDefinition(model: string): ModelDefinition {
     const splits = model.split("/");
-    const publisher = splits[1];
-    const modelName = trimModelName(splits[splits.length - 1]);
+
+    // Handle both formats: "publishers/anthropic/models/..." and "locations/.../publishers/anthropic/models/..."
+    let publisher: string | undefined;
+    let modelName: string;
+
+    const publisherIndex = splits.indexOf("publishers");
+    if (publisherIndex !== -1 && publisherIndex + 1 < splits.length) {
+        publisher = splits[publisherIndex + 1];
+        modelName = trimModelName(splits[splits.length - 1]);
+    } else {
+        // Fallback to old logic for backward compatibility
+        publisher = splits[1];
+        modelName = trimModelName(splits[splits.length - 1]);
+    }
 
     if (publisher?.includes("anthropic")) {
         return new ClaudeModelDefinition(modelName);