@llumiverse/drivers 0.9.2 → 0.10.0

package/lib/types/vertexai/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vertexai/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC3L,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAI/C,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,cAAe,SAAQ,cAAc,CAAC,qBAAqB,EAAE,sBAAsB,CAAC;IAC7F,QAAQ,mBAA6B;IACrC,aAAa,gBAAgC;IAG7C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,WAAW,CAAC;gBAGrB,OAAO,EAAE,qBAAqB;IAqBlC,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC;IAIzD,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,sBAAsB;IAIxF,iBAAiB,CAAC,MAAM,EAAE,sBAAsB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAGtG,uBAAuB,CAAC,MAAM,EAAE,sBAAsB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAIlH,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAiB1E,mBAAmB,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAGjD,kBAAkB,IAAI,OAAO,CAAC,OAAO,CAAC;IAGtC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;KAAE,CAAC;CAIvH"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vertexai/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,UAAU,EAAE,aAAa,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC7M,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,qBAAqB,EAAwB,MAAM,iCAAiC,CAAC;AAI9F,MAAM,WAAW,qBAAsB,SAAQ,aAAa;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,cAAe,SAAQ,cAAc,CAAC,qBAAqB,EAAE,sBAAsB,CAAC;IAC7F,QAAQ,mBAA6B;IACrC,aAAa,gBAAgC;IAG7C,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,WAAW,CAAC;gBAGrB,OAAO,EAAE,qBAAqB;IAqBlC,SAAS,CAAC,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC;IAIzD,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,sBAAsB;IAIxF,iBAAiB,CAAC,MAAM,EAAE,sBAAsB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAGtG,uBAAuB,CAAC,MAAM,EAAE,sBAAsB,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IAIlH,UAAU,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;IAkB1E,kBAAkB,IAAI,OAAO,CAAC,OAAO,CAAC;IAIhC,kBAAkB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAItF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llumiverse/drivers",
-  "version": "0.9.2",
+  "version": "0.10.0",
   "type": "module",
   "description": "LLM driver implementations. Currently supported are: openai, huggingface, bedrock, replicate.",
   "files": [
@@ -63,7 +63,7 @@
     "@google-cloud/aiplatform": "^3.10.0",
     "@google-cloud/vertexai": "^0.2.1",
     "@huggingface/inference": "^2.6.4",
-    "@llumiverse/core": "^0.9.2",
+    "@llumiverse/core": "^0.10.0",
     "api-fetch-client": "^0.8.6",
     "eventsource": "^2.0.2",
     "google-auth-library": "^9.6.1",

package/src/bedrock/index.ts

@@ -1,7 +1,7 @@
 import { Bedrock, CreateModelCustomizationJobCommand, FoundationModelSummary, GetModelCustomizationJobCommand, GetModelCustomizationJobCommandOutput, ModelCustomizationJobStatus, StopModelCustomizationJobCommand } from "@aws-sdk/client-bedrock";
 import { BedrockRuntime, InvokeModelCommandOutput, ResponseStream } from "@aws-sdk/client-bedrock-runtime";
 import { S3Client } from "@aws-sdk/client-s3";
-import { AIModel, AbstractDriver, BuiltinProviders, Completion, DataSource, DriverOptions, ExecutionOptions, PromptFormats, PromptFormatters, PromptOptions, PromptSegment, TrainingJob, TrainingJobStatus, TrainingOptions } from "@llumiverse/core";
+import { AIModel, AbstractDriver, BuiltinProviders, Completion, DataSource, DriverOptions, EmbeddingsOptions, EmbeddingsResult, ExecutionOptions, PromptFormats, PromptFormatters, PromptOptions, PromptSegment, TrainingJob, TrainingJobStatus, TrainingOptions } from "@llumiverse/core";
 import { transformAsyncIterator } from "@llumiverse/core/async";
 import { AwsCredentialIdentity, Provider } from "@smithy/types";
 import mnemonist from "mnemonist";
@@ -203,7 +203,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
                 anthropic_version: "bedrock-2023-05-31",
                 ...(prompt as ClaudeMessagesPrompt),
                 temperature: options.temperature,
-                max_tokens: options.max_tokens ?? 256,
+                max_tokens: options.max_tokens,
             } as ClaudeRequestPayload;
         } else if (contains(options.model, "ai21")) {
             return {
@@ -374,7 +374,7 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         return aimodels;
     }
 
-    async generateEmbeddings(content: string, model: string = "amazon.titan-embed-text-v1"): Promise<{ embeddings: number[], model: string; }> {
+    async generateEmbeddings({ content, model = "amazon.titan-embed-text-v1" }: EmbeddingsOptions): Promise<EmbeddingsResult> {
 
         this.logger.info("[Bedrock] Generating embeddings with model " + model);
 
@@ -401,8 +401,9 @@ export class BedrockDriver extends AbstractDriver<BedrockDriverOptions, BedrockP
         }
 
         return {
-            embeddings: result.embedding,
+            values: result.embedding,
             model: model,
+            token_count: result.inputTextTokenCount
         };
 
     }

package/src/huggingface_ie.ts

@@ -9,6 +9,7 @@ import {
     AbstractDriver,
     BuiltinProviders,
     DriverOptions,
+    EmbeddingsResult,
     ExecutionOptions,
     PromptFormats
 } from "@llumiverse/core";
@@ -106,11 +107,6 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
 
     // ============== management API ==============
 
-    // Not implemented
-    async listTrainableModels(): Promise<AIModel<string>[]> {
-        return [];
-    }
-
     async listModels(): Promise<AIModel[]> {
         const res = await this.service.get("/");
         const hfModels = res.items as HuggingFaceIEModel[];
@@ -136,8 +132,7 @@ export class HuggingFaceIEDriver extends AbstractDriver<HuggingFaceIEDriverOptio
         }
     }
 
-    async generateEmbeddings(content: string, model?: string): Promise<{ embeddings: number[], model: string; }> {
-        this.logger?.debug(`[Huggingface] Generating embeddings for ${content} on ${model}`);
+    async generateEmbeddings(): Promise<EmbeddingsResult> {
         throw new Error("Method not implemented.");
     }
 

package/src/mistral/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, AbstractDriver, Completion, DriverOptions, ExecutionOptions, PromptFormats, PromptSegment } from "@llumiverse/core";
+import { AIModel, AbstractDriver, Completion, DriverOptions, EmbeddingsResult, ExecutionOptions, PromptFormats, PromptSegment } from "@llumiverse/core";
 import { transformSSEStream } from "@llumiverse/core/async";
 import { FetchClient } from "api-fetch-client";
 import { CompletionRequestParams, ListModelsResponse, ResponseFormat } from "./types.js";
@@ -73,8 +73,8 @@ export class MistralAIDriver extends AbstractDriver<MistralAIDriverOptions, LLMM
             payload: _makeChatCompletionRequest({
                 model: options.model,
                 messages: messages,
-                maxTokens: options.max_tokens ?? 1024,
-                temperature: options.temperature ?? 0.7,
+                maxTokens: options.max_tokens,
+                temperature: options.temperature,
                 responseFormat: this.getResponseFormat(options),
             })
         })
@@ -97,8 +97,8 @@ export class MistralAIDriver extends AbstractDriver<MistralAIDriverOptions, LLMM
             payload: _makeChatCompletionRequest({
                 model: options.model,
                 messages: messages,
-                maxTokens: options.max_tokens ?? 1024,
-                temperature: options.temperature ?? 0.7,
+                maxTokens: options.max_tokens,
+                temperature: options.temperature,
                 responseFormat: this.getResponseFormat(options),
                 stream: true
             }),
@@ -128,14 +128,10 @@ export class MistralAIDriver extends AbstractDriver<MistralAIDriverOptions, LLMM
         return aimodels;
     }
 
-    listTrainableModels(): Promise<AIModel<string>[]> {
-        throw new Error("Method not implemented.");
-    }
     validateConnection(): Promise<boolean> {
         throw new Error("Method not implemented.");
     }
-    //@ts-ignore
-    generateEmbeddings(content: string, model?: string | undefined): Promise<{ embeddings: number[]; model: string; }> {
+    generateEmbeddings(): Promise<EmbeddingsResult> {
         throw new Error("Method not implemented.");
     }
 

package/src/openai.ts

@@ -5,6 +5,8 @@ import {
     Completion,
     DataSource,
     DriverOptions,
+    EmbeddingsOptions,
+    EmbeddingsResult,
     ExecutionOptions,
     ExecutionTokenUsage,
     ModelType,
@@ -211,7 +213,7 @@ export class OpenAIDriver extends AbstractDriver<
     }
 
 
-    async generateEmbeddings(content: string, model: string = "text-embedding-ada-002"): Promise<{ embeddings: number[], model: string; }> {
+    async generateEmbeddings({ content, model = "text-embedding-ada-002" }: EmbeddingsOptions): Promise<EmbeddingsResult> {
         const res = await this.service.embeddings.create({
             input: content,
             model: model,
@@ -223,7 +225,7 @@ export class OpenAIDriver extends AbstractDriver<
             throw new Error("No embedding found");
         }
 
-        return { embeddings, model };
+        return { values: embeddings, model } as EmbeddingsResult;
     }
 
 }

package/src/replicate.ts

@@ -5,6 +5,7 @@ import {
     Completion,
     DataSource,
     DriverOptions,
+    EmbeddingsResult,
     ExecutionOptions,
     ModelSearchPayload,
     PromptFormats,
@@ -73,7 +74,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         const predictionData = {
             input: {
                 prompt: prompt,
-                max_new_tokens: options.max_tokens || 1024,
+                max_new_tokens: options.max_tokens,
                 temperature: options.temperature,
             },
             version: model.version,
@@ -113,7 +114,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         const predictionData = {
             input: {
                 prompt: prompt,
-                max_new_tokens: options.max_tokens || 1024,
+                max_new_tokens: options.max_tokens,
                 temperature: options.temperature,
             },
             version: model.version,
@@ -282,8 +283,7 @@ export class ReplicateDriver extends AbstractDriver<DriverOptions, string> {
         return models;
     }
 
-    generateEmbeddings(content: string, model?: string): Promise<{ embeddings: number[], model: string; }> {
-        this.logger?.debug(`[Replicate] Generating embeddings for ${content} on ${model}`);
+    async generateEmbeddings(): Promise<EmbeddingsResult> {
         throw new Error("Method not implemented.");
     }
 

package/src/test/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, AIModelStatus, CompletionStream, Driver, ExecutionOptions, ExecutionResponse, ModelType, PromptOptions, PromptSegment, TrainingJob } from "@llumiverse/core";
+import { AIModel, AIModelStatus, CompletionStream, Driver, EmbeddingsResult, ExecutionOptions, ExecutionResponse, ModelType, PromptOptions, PromptSegment, TrainingJob } from "@llumiverse/core";
 import { TestErrorCompletionStream } from "./TestErrorCompletionStream.js";
 import { TestValidationErrorCompletionStream } from "./TestValidationErrorCompletionStream.js";
 import { createValidationErrorCompletion, sleep, throwError } from "./utils.js";
@@ -83,7 +83,7 @@ export class TestDriver implements Driver<PromptSegment[]> {
     validateConnection(): Promise<boolean> {
         throw new Error("Method not implemented.");
     }
-    generateEmbeddings(): Promise<{ embeddings: number[]; model: string; }> {
+    generateEmbeddings(): Promise<EmbeddingsResult> {
         throw new Error("Method not implemented.");
     }
 

package/src/togetherai/index.ts

@@ -1,4 +1,4 @@
-import { AIModel, AbstractDriver, Completion, DriverOptions, ExecutionOptions, PromptFormats } from "@llumiverse/core";
+import { AIModel, AbstractDriver, Completion, DriverOptions, EmbeddingsResult, ExecutionOptions, PromptFormats } from "@llumiverse/core";
 import { transformSSEStream } from "@llumiverse/core/async";
 import { FetchClient } from "api-fetch-client";
 import { TogetherModelInfo } from "./interfaces.js";
@@ -37,8 +37,8 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
                 model: options.model,
                 prompt: prompt,
                 response_format: this.getResponseFormat(options),
-                max_tokens: options.max_tokens ?? 1024,
-                temperature: options.temperature ?? 0.7,
+                max_tokens: options.max_tokens,
+                temperature: options.temperature,
                 stop: [
                     "</s>",
                     "[/INST]"
@@ -64,8 +64,8 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
             payload: {
                 model: options.model,
                 prompt: prompt,
-                max_tokens: options.max_tokens ?? 1024,
-                temperature: options.temperature ?? 0.7,
+                max_tokens: options.max_tokens,
+                temperature: options.temperature,
                 response_format: this.getResponseFormat(options),
                 stream: true,
                 stop: [
@@ -101,14 +101,10 @@ export class TogetherAIDriver extends AbstractDriver<TogetherAIDriverOptions, st
 
     }
 
-    listTrainableModels(): Promise<AIModel<string>[]> {
-        throw new Error("Method not implemented.");
-    }
     validateConnection(): Promise<boolean> {
         throw new Error("Method not implemented.");
     }
-    //@ts-ignore
-    generateEmbeddings(content: string, model?: string | undefined): Promise<{ embeddings: number[]; model: string; }> {
+    generateEmbeddings(): Promise<EmbeddingsResult> {
         throw new Error("Method not implemented.");
     }
 

package/src/vertexai/embeddings/embeddings-text.ts

@@ -0,0 +1,52 @@
+
+import { EmbeddingsResult } from '@llumiverse/core';
+import { VertexAIDriver } from '../index.js';
+
+export interface TextEmbeddingsOptions {
+    model?: string;
+    task_type?: "RETRIEVAL_QUERY" | "RETRIEVAL_DOCUMENT" | "SEMANTIC_SIMILARITY" | "CLASSIFICATION" | "CLUSTERING",
+    title?: string, // the title for the embedding
+    content: string // the text to generate embeddings for
+}
+
+interface EmbedingsForTextPrompt {
+    instances: TextEmbeddingsOptions[]
+}
+
+interface TextEmbeddingsResult {
+    predictions: [
+        {
+            embeddings: TextEmbeddings
+        }
+    ]
+}
+
+interface TextEmbeddings {
+    statistics: {
+        truncated: boolean,
+        token_count: number
+    },
+    values: [number]
+}
+
+export async function getEmbeddingsForText(driver: VertexAIDriver, options: TextEmbeddingsOptions): Promise<EmbeddingsResult> {
+    const prompt = {
+        instances: [{
+            task_type: options.task_type,
+            title: options.title,
+            content: options.content
+        }]
+    } as EmbedingsForTextPrompt;
+
+    const model = options.model || "textembedding-gecko@latest";
+
+    const result = await driver.fetchClient.post(`/publishers/google/models/${model}:predict`, {
+        payload: prompt
+    }) as TextEmbeddingsResult;
+
+    return {
+        ...result.predictions[0].embeddings,
+        model,
+        token_count: result.predictions[0].embeddings.statistics?.token_count
+    };
+}

package/src/vertexai/index.ts

@@ -1,7 +1,8 @@
 //import { v1 } from "@google-cloud/aiplatform";
 import { GenerateContentRequest, VertexAI } from "@google-cloud/vertexai";
-import { AIModel, AbstractDriver, BuiltinProviders, Completion, DriverOptions, ExecutionOptions, ModelSearchPayload, PromptFormats, PromptOptions, PromptSegment } from "@llumiverse/core";
+import { AIModel, AbstractDriver, BuiltinProviders, Completion, DriverOptions, EmbeddingsResult, ExecutionOptions, ModelSearchPayload, PromptFormats, PromptOptions, PromptSegment } from "@llumiverse/core";
 import { FetchClient } from "api-fetch-client";
+import { TextEmbeddingsOptions, getEmbeddingsForText } from "./embeddings/embeddings-text.js";
 import { BuiltinModels, getModelDefinition } from "./models.js";
 //import { GoogleAuth } from "google-auth-library";
 
@@ -72,14 +73,13 @@ export class VertexAIDriver extends AbstractDriver<VertexAIDriverOptions, Genera
 
         return []; //TODO
     }
-    listTrainableModels(): Promise<AIModel<string>[]> {
-        throw new Error("Method not implemented.");
-    }
+
     validateConnection(): Promise<boolean> {
         throw new Error("Method not implemented.");
     }
-    generateEmbeddings(_content: string, _model?: string | undefined): Promise<{ embeddings: number[]; model: string; }> {
-        throw new Error("Method not implemented.");
+
+    async generateEmbeddings(options: TextEmbeddingsOptions): Promise<EmbeddingsResult> {
+        return getEmbeddingsForText(this, options);
     }
 
 }