modelfusion 0.31.0 → 0.33.0

package/README.md

@@ -45,7 +45,9 @@ You can use [prompt formats](https://modelfusion.dev/guide/function/generate-tex
 
 ```ts
 const text = await generateText(
-  new OpenAITextGenerationModel({ model: "text-davinci-003" }),
+  new OpenAITextGenerationModel({
+    model: "gpt-3.5-turbo-instruct",
+  }),
   "Write a short story about a robot learning to love:\n\n"
 );
 ```
@@ -56,14 +58,10 @@ Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),
 
 ```ts
 const textStream = await streamText(
-  new OpenAIChatModel({
-    model: "gpt-3.5-turbo",
-    maxCompletionTokens: 1000,
+  new OpenAITextGenerationModel({
+    model: "gpt-3.5-turbo-instruct",
   }),
-  [
-    OpenAIChatMessage.system("You are a story writer."),
-    OpenAIChatMessage.user("Write a story about a robot learning to love"),
-  ]
+  "Write a short story about a robot learning to love:\n\n"
 );
 
 for await (const textFragment of textStream) {
@@ -121,7 +119,7 @@ ModelFusion model functions return rich results that include the original respon
 // the response type is specific to the model that's being used
 const { output, response, metadata } = await generateText(
   new OpenAITextGenerationModel({
-    model: "text-davinci-003",
+    model: "gpt-3.5-turbo-instruct",
     maxCompletionTokens: 1000,
     n: 2, // generate 2 completions
   }),
@@ -367,7 +365,7 @@ const reconstructedText = await tokenizer.detokenize(tokens);
 
 Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
 
-### [Upserting and Retrieving Text Chunks from Vector Indices](https://modelfusion.dev/guide/text-chunks)
+### [Upserting and Retrieving Objects from Vector Indices](https://modelfusion.dev/guide/vector-index)
 
 ```ts
 const texts = [
@@ -376,21 +374,22 @@ const texts = [
   // ...
 ];
 
-const vectorIndex = new MemoryVectorIndex<TextChunk>();
+const vectorIndex = new MemoryVectorIndex<string>();
 const embeddingModel = new OpenAITextEmbeddingModel({
   model: "text-embedding-ada-002",
 });
 
 // update an index - usually done as part of an ingestion process:
-await upsertTextChunks({
+await upsertIntoVectorIndex({
   vectorIndex,
   embeddingModel,
-  chunks: texts.map((text) => ({ text })),
+  objects: texts,
+  getValueToEmbed: (text) => text,
 });
 
 // retrieve text chunks from the vector index - usually done at query time:
-const { chunks } = await retrieveTextChunks(
-  new SimilarTextChunksFromVectorIndexRetriever({
+const retrievedTexts = await retrieve(
+  new VectorIndexRetriever({
     vectorIndex,
     embeddingModel,
     maxResults: 3,

package/index.cjs

@@ -21,6 +21,7 @@ __exportStar(require("./model-function/index.cjs"), exports);
 __exportStar(require("./model-provider/index.cjs"), exports);
 __exportStar(require("./observability/index.cjs"), exports);
 __exportStar(require("./prompt/index.cjs"), exports);
+__exportStar(require("./retriever/index.cjs"), exports);
 __exportStar(require("./text-chunk/index.cjs"), exports);
 __exportStar(require("./tool/index.cjs"), exports);
 __exportStar(require("./util/index.cjs"), exports);

package/index.d.ts

@@ -5,6 +5,7 @@ export * from "./model-function/index.js";
 export * from "./model-provider/index.js";
 export * from "./observability/index.js";
 export * from "./prompt/index.js";
+export * from "./retriever/index.js";
 export * from "./text-chunk/index.js";
 export * from "./tool/index.js";
 export * from "./util/index.js";

package/index.js

@@ -5,6 +5,7 @@ export * from "./model-function/index.js";
 export * from "./model-provider/index.js";
 export * from "./observability/index.js";
 export * from "./prompt/index.js";
+export * from "./retriever/index.js";
 export * from "./text-chunk/index.js";
 export * from "./tool/index.js";
 export * from "./util/index.js";

package/model-function/Model.d.ts

@@ -19,7 +19,7 @@ export interface Model<SETTINGS extends ModelSettings> {
      *
      * @example
      * const model = new OpenAITextGenerationModel({
-     *   model: "text-davinci-003",
+     *   model: "gpt-3.5-turbo-instruct",
      *   maxCompletionTokens: 500,
      * });
      *

package/model-provider/openai/OpenAITextGenerationModel.cjs

@@ -21,55 +21,72 @@ const TikTokenTokenizer_js_1 = require("./TikTokenTokenizer.cjs");
  * @see https://openai.com/pricing
  */
 exports.OPENAI_TEXT_GENERATION_MODELS = {
+    "gpt-3.5-turbo-instruct": {
+        contextWindowSize: 4097,
+        promptTokenCostInMillicents: 0.15,
+        completionTokenCostInMillicents: 0.2,
+    },
     "davinci-002": {
         contextWindowSize: 16384,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
         fineTunedTokenCostInMillicents: 1.2,
     },
     "babbage-002": {
         contextWindowSize: 16384,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
         fineTunedTokenCostInMillicents: 0.16,
     },
     "text-davinci-003": {
         contextWindowSize: 4096,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "text-davinci-002": {
         contextWindowSize: 4096,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "code-davinci-002": {
         contextWindowSize: 8000,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     davinci: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "text-curie-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
     },
     curie: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
     },
     "text-babbage-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.05,
+        promptTokenCostInMillicents: 0.05,
+        completionTokenCostInMillicents: 0.05,
     },
     babbage: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.05,
+        promptTokenCostInMillicents: 0.05,
+        completionTokenCostInMillicents: 0.05,
     },
     "text-ada-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
     },
     ada: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
     },
 };
 function getOpenAITextGenerationModelInformation(model) {
@@ -80,7 +97,8 @@ function getOpenAITextGenerationModelInformation(model) {
             baseModel: model,
             isFineTuned: false,
             contextWindowSize: baseModelInformation.contextWindowSize,
-            tokenCostInMillicents: baseModelInformation.tokenCostInMillicents,
+            promptTokenCostInMillicents: baseModelInformation.promptTokenCostInMillicents,
+            completionTokenCostInMillicents: baseModelInformation.completionTokenCostInMillicents,
         };
     }
     // Extract the base model from the fine-tuned model:
@@ -92,7 +110,8 @@ function getOpenAITextGenerationModelInformation(model) {
             baseModel: baseModel,
             isFineTuned: true,
             contextWindowSize: baseModelInformation.contextWindowSize,
-            tokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
+            promptTokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
+            completionTokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
         };
     }
     throw new Error(`Unknown OpenAI chat base model ${baseModel}.`);
@@ -102,8 +121,13 @@ const isOpenAITextGenerationModel = (model) => model in exports.OPENAI_TEXT_GENE
     model.startsWith("ft:davinci-002:") ||
     model.startsWith("ft:babbage-002:");
 exports.isOpenAITextGenerationModel = isOpenAITextGenerationModel;
-const calculateOpenAITextGenerationCostInMillicents = ({ model, response, }) => response.usage.total_tokens *
-    getOpenAITextGenerationModelInformation(model).tokenCostInMillicents;
+const calculateOpenAITextGenerationCostInMillicents = ({ model, response, }) => {
+    const modelInformation = getOpenAITextGenerationModelInformation(model);
+    return (response.usage.prompt_tokens *
+        modelInformation.promptTokenCostInMillicents +
+        response.usage.completion_tokens *
+            modelInformation.completionTokenCostInMillicents);
+};
 exports.calculateOpenAITextGenerationCostInMillicents = calculateOpenAITextGenerationCostInMillicents;
 /**
  * Create a text generation model that calls the OpenAI text completion API.
@@ -112,7 +136,7 @@ exports.calculateOpenAITextGenerationCostInMillicents = calculateOpenAITextGener
  *
  * @example
  * const model = new OpenAITextGenerationModel({
- *   model: "text-davinci-003",
+ *   model: "gpt-3.5-turbo-instruct",
  *   temperature: 0.7,
  *   maxCompletionTokens: 500,
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),

package/model-provider/openai/OpenAITextGenerationModel.d.ts

@@ -14,62 +14,80 @@ import { TikTokenTokenizer } from "./TikTokenTokenizer.js";
  * @see https://openai.com/pricing
  */
 export declare const OPENAI_TEXT_GENERATION_MODELS: {
+    "gpt-3.5-turbo-instruct": {
+        contextWindowSize: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
+    };
     "davinci-002": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
         fineTunedTokenCostInMillicents: number;
     };
     "babbage-002": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
         fineTunedTokenCostInMillicents: number;
     };
     "text-davinci-003": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     "text-davinci-002": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     "code-davinci-002": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     davinci: {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     "text-curie-001": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     curie: {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     "text-babbage-001": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     babbage: {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     "text-ada-001": {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
     ada: {
         contextWindowSize: number;
-        tokenCostInMillicents: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
     };
 };
 export declare function getOpenAITextGenerationModelInformation(model: OpenAITextGenerationModelType): {
     baseModel: OpenAITextGenerationBaseModelType;
     isFineTuned: boolean;
     contextWindowSize: number;
-    tokenCostInMillicents: number;
+    promptTokenCostInMillicents: number;
+    completionTokenCostInMillicents: number;
 };
 type FineTuneableOpenAITextGenerationModelType = "davinci-002" | "babbage-002";
 type FineTunedOpenAITextGenerationModelType = `ft:${FineTuneableOpenAITextGenerationModelType}:${string}:${string}:${string}`;
@@ -106,7 +124,7 @@ export interface OpenAITextGenerationModelSettings extends TextGenerationModelSe
  *
  * @example
  * const model = new OpenAITextGenerationModel({
- *   model: "text-davinci-003",
+ *   model: "gpt-3.5-turbo-instruct",
  *   temperature: 0.7,
  *   maxCompletionTokens: 500,
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),

package/model-provider/openai/OpenAITextGenerationModel.js

@@ -15,55 +15,72 @@ import { TikTokenTokenizer } from "./TikTokenTokenizer.js";
  * @see https://openai.com/pricing
  */
 export const OPENAI_TEXT_GENERATION_MODELS = {
+    "gpt-3.5-turbo-instruct": {
+        contextWindowSize: 4097,
+        promptTokenCostInMillicents: 0.15,
+        completionTokenCostInMillicents: 0.2,
+    },
     "davinci-002": {
         contextWindowSize: 16384,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
         fineTunedTokenCostInMillicents: 1.2,
     },
     "babbage-002": {
         contextWindowSize: 16384,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
         fineTunedTokenCostInMillicents: 0.16,
     },
     "text-davinci-003": {
         contextWindowSize: 4096,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "text-davinci-002": {
         contextWindowSize: 4096,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "code-davinci-002": {
         contextWindowSize: 8000,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     davinci: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 2,
+        promptTokenCostInMillicents: 2,
+        completionTokenCostInMillicents: 2,
     },
     "text-curie-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
     },
     curie: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.2,
+        promptTokenCostInMillicents: 0.2,
+        completionTokenCostInMillicents: 0.2,
     },
     "text-babbage-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.05,
+        promptTokenCostInMillicents: 0.05,
+        completionTokenCostInMillicents: 0.05,
     },
     babbage: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.05,
+        promptTokenCostInMillicents: 0.05,
+        completionTokenCostInMillicents: 0.05,
     },
     "text-ada-001": {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
     },
     ada: {
         contextWindowSize: 2048,
-        tokenCostInMillicents: 0.04,
+        promptTokenCostInMillicents: 0.04,
+        completionTokenCostInMillicents: 0.04,
     },
 };
 export function getOpenAITextGenerationModelInformation(model) {
@@ -74,7 +91,8 @@ export function getOpenAITextGenerationModelInformation(model) {
             baseModel: model,
             isFineTuned: false,
             contextWindowSize: baseModelInformation.contextWindowSize,
-            tokenCostInMillicents: baseModelInformation.tokenCostInMillicents,
+            promptTokenCostInMillicents: baseModelInformation.promptTokenCostInMillicents,
+            completionTokenCostInMillicents: baseModelInformation.completionTokenCostInMillicents,
         };
     }
     // Extract the base model from the fine-tuned model:
@@ -86,7 +104,8 @@ export function getOpenAITextGenerationModelInformation(model) {
             baseModel: baseModel,
             isFineTuned: true,
             contextWindowSize: baseModelInformation.contextWindowSize,
-            tokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
+            promptTokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
+            completionTokenCostInMillicents: baseModelInformation.fineTunedTokenCostInMillicents,
         };
     }
     throw new Error(`Unknown OpenAI chat base model ${baseModel}.`);
@@ -94,8 +113,13 @@ export function getOpenAITextGenerationModelInformation(model) {
 export const isOpenAITextGenerationModel = (model) => model in OPENAI_TEXT_GENERATION_MODELS ||
     model.startsWith("ft:davinci-002:") ||
     model.startsWith("ft:babbage-002:");
-export const calculateOpenAITextGenerationCostInMillicents = ({ model, response, }) => response.usage.total_tokens *
-    getOpenAITextGenerationModelInformation(model).tokenCostInMillicents;
+export const calculateOpenAITextGenerationCostInMillicents = ({ model, response, }) => {
+    const modelInformation = getOpenAITextGenerationModelInformation(model);
+    return (response.usage.prompt_tokens *
+        modelInformation.promptTokenCostInMillicents +
+        response.usage.completion_tokens *
+            modelInformation.completionTokenCostInMillicents);
+};
 /**
  * Create a text generation model that calls the OpenAI text completion API.
  *
@@ -103,7 +127,7 @@ export const calculateOpenAITextGenerationCostInMillicents = ({ model, response,
  *
  * @example
  * const model = new OpenAITextGenerationModel({
- *   model: "text-davinci-003",
+ *   model: "gpt-3.5-turbo-instruct",
  *   temperature: 0.7,
  *   maxCompletionTokens: 500,
  *   retry: retryWithExponentialBackoff({ maxTries: 5 }),

package/model-provider/openai/TikTokenTokenizer.cjs

@@ -57,8 +57,6 @@ function getEncodingNameForModel(model) {
         case "text-davinci-003": {
             return "p50k_base";
         }
-        case "babbage-002":
-        case "davinci-002":
         case "ada":
         case "babbage":
         case "curie":
@@ -68,11 +66,14 @@ function getEncodingNameForModel(model) {
         case "text-curie-001": {
             return "r50k_base";
         }
+        case "babbage-002":
+        case "davinci-002":
         case "gpt-3.5-turbo":
         case "gpt-3.5-turbo-0301":
         case "gpt-3.5-turbo-0613":
         case "gpt-3.5-turbo-16k":
         case "gpt-3.5-turbo-16k-0613":
+        case "gpt-3.5-turbo-instruct":
         case "gpt-4":
         case "gpt-4-0314":
         case "gpt-4-0613":

package/model-provider/openai/TikTokenTokenizer.js

@@ -53,8 +53,6 @@ function getEncodingNameForModel(model) {
         case "text-davinci-003": {
             return "p50k_base";
         }
-        case "babbage-002":
-        case "davinci-002":
         case "ada":
         case "babbage":
         case "curie":
@@ -64,11 +62,14 @@ function getEncodingNameForModel(model) {
         case "text-curie-001": {
             return "r50k_base";
         }
+        case "babbage-002":
+        case "davinci-002":
         case "gpt-3.5-turbo":
         case "gpt-3.5-turbo-0301":
         case "gpt-3.5-turbo-0613":
         case "gpt-3.5-turbo-16k":
         case "gpt-3.5-turbo-16k-0613":
+        case "gpt-3.5-turbo-instruct":
         case "gpt-4":
         case "gpt-4-0314":
         case "gpt-4-0613":

package/model-provider/stability/StabilityImageGenerationModel.cjs

@@ -86,6 +86,12 @@ class StabilityImageGenerationModel extends AbstractModel_js_1.AbstractModel {
     }
 }
 exports.StabilityImageGenerationModel = StabilityImageGenerationModel;
+const stabilityImageGenerationModels = [
+    "stable-diffusion-v1-5",
+    "stable-diffusion-512-v2-1",
+    "stable-diffusion-xl-1024-v0-9",
+    "stable-diffusion-xl-1024-v1-0",
+];
 const stabilityImageGenerationResponseSchema = zod_1.z.object({
     artifacts: zod_1.z.array(zod_1.z.object({
         base64: zod_1.z.string(),

package/model-provider/stability/StabilityImageGenerationModel.d.ts

@@ -28,7 +28,7 @@ import { ImageGenerationModel, ImageGenerationModelSettings } from "../../model-
 export declare class StabilityImageGenerationModel extends AbstractModel<StabilityImageGenerationModelSettings> implements ImageGenerationModel<StabilityImageGenerationPrompt, StabilityImageGenerationResponse, StabilityImageGenerationModelSettings> {
     constructor(settings: StabilityImageGenerationModelSettings);
     readonly provider: "stability";
-    get modelName(): string;
+    get modelName(): StabilityImageGenerationModelType;
     callAPI(input: StabilityImageGenerationPrompt, options?: ModelFunctionOptions<StabilityImageGenerationModelSettings>): Promise<StabilityImageGenerationResponse>;
     get settingsForEvent(): Partial<StabilityImageGenerationModelSettings>;
     generateImageResponse(prompt: StabilityImageGenerationPrompt, options?: ModelFunctionOptions<StabilityImageGenerationModelSettings>): Promise<{
@@ -41,9 +41,11 @@ export declare class StabilityImageGenerationModel extends AbstractModel<Stabili
     extractBase64Image(response: StabilityImageGenerationResponse): string;
     withSettings(additionalSettings: StabilityImageGenerationModelSettings): this;
 }
+declare const stabilityImageGenerationModels: readonly ["stable-diffusion-v1-5", "stable-diffusion-512-v2-1", "stable-diffusion-xl-1024-v0-9", "stable-diffusion-xl-1024-v1-0"];
+export type StabilityImageGenerationModelType = (typeof stabilityImageGenerationModels)[number] | (string & {});
 export interface StabilityImageGenerationModelSettings extends ImageGenerationModelSettings {
     api?: ApiConfiguration;
-    model: string;
+    model: StabilityImageGenerationModelType;
     height?: number;
     width?: number;
     cfgScale?: number;

package/model-provider/stability/StabilityImageGenerationModel.js

@@ -82,6 +82,12 @@ export class StabilityImageGenerationModel extends AbstractModel {
         return new StabilityImageGenerationModel(Object.assign({}, this.settings, additionalSettings));
     }
 }
+const stabilityImageGenerationModels = [
+    "stable-diffusion-v1-5",
+    "stable-diffusion-512-v2-1",
+    "stable-diffusion-xl-1024-v0-9",
+    "stable-diffusion-xl-1024-v1-0",
+];
 const stabilityImageGenerationResponseSchema = z.object({
     artifacts: z.array(z.object({
         base64: z.string(),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.31.0",
+  "version": "0.33.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [
@@ -55,7 +55,7 @@
     "js-tiktoken": "1.0.7",
     "nanoid": "3.3.6",
     "secure-json-parse": "2.7.0",
-    "zod": "3.22.2",
+    "zod": "3.21.4",
     "zod-to-json-schema": "3.21.4"
   },
   "devDependencies": {

package/retriever/Retriever.d.ts

@@ -0,0 +1,7 @@
+import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
+export interface RetrieverSettings {
+}
+export interface Retriever<OBJECT, QUERY, SETTINGS extends RetrieverSettings> {
+    retrieve(query: QUERY, options?: ModelFunctionOptions<RetrieverSettings>): Promise<OBJECT[]>;
+    withSettings(additionalSettings: Partial<SETTINGS>): this;
+}

package/retriever/index.cjs

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./Retriever.cjs"), exports);
+__exportStar(require("./retrieve.cjs"), exports);

package/retriever/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./Retriever.js";
+export * from "./retrieve.js";

package/retriever/index.js

@@ -0,0 +1,2 @@
+export * from "./Retriever.js";
+export * from "./retrieve.js";

package/retriever/retrieve.cjs

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retrieve = void 0;
+async function retrieve(retriever, query, options) {
+    // TODO add error handling, events, duration tracking, etc.
+    // TODO metadata handling
+    return retriever.retrieve(query, options);
+}
+exports.retrieve = retrieve;

package/retriever/retrieve.d.ts

@@ -0,0 +1,3 @@
+import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
+import { Retriever, RetrieverSettings } from "./Retriever.js";
+export declare function retrieve<OBJECT, QUERY, SETTINGS extends RetrieverSettings>(retriever: Retriever<OBJECT, QUERY, SETTINGS>, query: QUERY, options?: ModelFunctionOptions<SETTINGS>): Promise<OBJECT[]>;

package/retriever/retrieve.js

@@ -0,0 +1,5 @@
+export async function retrieve(retriever, query, options) {
+    // TODO add error handling, events, duration tracking, etc.
+    // TODO metadata handling
+    return retriever.retrieve(query, options);
+}

package/text-chunk/index.cjs

@@ -14,12 +14,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./SimilarTextChunksFromVectorIndexRetriever.cjs"), exports);
 __exportStar(require("./TextChunk.cjs"), exports);
-__exportStar(require("./retrieve-text-chunks/TextChunkRetriever.cjs"), exports);
-__exportStar(require("./retrieve-text-chunks/retrieveTextChunks.cjs"), exports);
 __exportStar(require("./split/SplitFunction.cjs"), exports);
 __exportStar(require("./split/splitOnSeparator.cjs"), exports);
 __exportStar(require("./split/splitRecursively.cjs"), exports);
 __exportStar(require("./split/splitTextChunks.cjs"), exports);
-__exportStar(require("./upsertTextChunks.cjs"), exports);

package/text-chunk/index.d.ts

@@ -1,9 +1,5 @@
-export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
-export * from "./retrieve-text-chunks/TextChunkRetriever.js";
-export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
 export * from "./split/splitTextChunks.js";
-export * from "./upsertTextChunks.js";

package/text-chunk/index.js

@@ -1,9 +1,5 @@
-export * from "./SimilarTextChunksFromVectorIndexRetriever.js";
 export * from "./TextChunk.js";
-export * from "./retrieve-text-chunks/TextChunkRetriever.js";
-export * from "./retrieve-text-chunks/retrieveTextChunks.js";
 export * from "./split/SplitFunction.js";
 export * from "./split/splitOnSeparator.js";
 export * from "./split/splitRecursively.js";
 export * from "./split/splitTextChunks.js";
-export * from "./upsertTextChunks.js";

package/{text-chunk/SimilarTextChunksFromVectorIndexRetriever.cjs → vector-index/VectorIndexRetriever.cjs}

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SimilarTextChunksFromVectorIndexRetriever = void 0;
+exports.VectorIndexRetriever = void 0;
 const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-class SimilarTextChunksFromVectorIndexRetriever {
+class VectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -29,9 +29,9 @@ class SimilarTextChunksFromVectorIndexRetriever {
             similarityThreshold,
         };
     }
-    async retrieveTextChunks(query, options) {
+    async retrieve(query, options) {
         if (options?.settings != null) {
-            return this.withSettings(options.settings).retrieveTextChunks(query, {
+            return this.withSettings(options.settings).retrieve(query, {
                 functionId: options.functionId,
                 observers: options.observers,
                 run: options.run,
@@ -49,10 +49,10 @@ class SimilarTextChunksFromVectorIndexRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new VectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));
     }
 }
-exports.SimilarTextChunksFromVectorIndexRetriever = SimilarTextChunksFromVectorIndexRetriever;
+exports.VectorIndexRetriever = VectorIndexRetriever;

package/vector-index/VectorIndexRetriever.d.ts

@@ -0,0 +1,19 @@
+import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
+import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
+import { Retriever, RetrieverSettings } from "../retriever/Retriever.js";
+import { VectorIndex } from "./VectorIndex.js";
+export interface VectorIndexRetrieverSettings {
+    maxResults?: number;
+    similarityThreshold?: number;
+}
+export declare class VectorIndexRetriever<OBJECT, INDEX> implements Retriever<OBJECT, string, VectorIndexRetrieverSettings> {
+    private readonly vectorIndex;
+    private readonly embeddingModel;
+    private readonly settings;
+    constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }: {
+        vectorIndex: VectorIndex<OBJECT, INDEX>;
+        embeddingModel: TextEmbeddingModel<unknown, TextEmbeddingModelSettings>;
+    } & VectorIndexRetrieverSettings);
+    retrieve(query: string, options?: ModelFunctionOptions<RetrieverSettings>): Promise<OBJECT[]>;
+    withSettings(additionalSettings: Partial<VectorIndexRetrieverSettings>): this;
+}

package/{text-chunk/SimilarTextChunksFromVectorIndexRetriever.js → vector-index/VectorIndexRetriever.js}

@@ -1,5 +1,5 @@
 import { embedText } from "../model-function/embed-text/embedText.js";
-export class SimilarTextChunksFromVectorIndexRetriever {
+export class VectorIndexRetriever {
     constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }) {
         Object.defineProperty(this, "vectorIndex", {
             enumerable: true,
@@ -26,9 +26,9 @@ export class SimilarTextChunksFromVectorIndexRetriever {
             similarityThreshold,
         };
     }
-    async retrieveTextChunks(query, options) {
+    async retrieve(query, options) {
         if (options?.settings != null) {
-            return this.withSettings(options.settings).retrieveTextChunks(query, {
+            return this.withSettings(options.settings).retrieve(query, {
                 functionId: options.functionId,
                 observers: options.observers,
                 run: options.run,
@@ -46,7 +46,7 @@ export class SimilarTextChunksFromVectorIndexRetriever {
         return queryResult.map((item) => item.data);
     }
     withSettings(additionalSettings) {
-        return new SimilarTextChunksFromVectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
+        return new VectorIndexRetriever(Object.assign({}, this.settings, additionalSettings, {
             vectorIndex: this.vectorIndex,
             embeddingModel: this.embeddingModel,
         }));

package/vector-index/index.cjs

@@ -15,5 +15,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./VectorIndex.cjs"), exports);
+__exportStar(require("./VectorIndexRetriever.cjs"), exports);
 __exportStar(require("./memory/MemoryVectorIndex.cjs"), exports);
 __exportStar(require("./pinecone/PineconeVectorIndex.cjs"), exports);
+__exportStar(require("./upsertIntoVectorIndex.cjs"), exports);

package/vector-index/index.d.ts

@@ -1,3 +1,5 @@
 export * from "./VectorIndex.js";
+export * from "./VectorIndexRetriever.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
+export * from "./upsertIntoVectorIndex.js";

package/vector-index/index.js

@@ -1,3 +1,5 @@
 export * from "./VectorIndex.js";
+export * from "./VectorIndexRetriever.js";
 export * from "./memory/MemoryVectorIndex.js";
 export * from "./pinecone/PineconeVectorIndex.js";
+export * from "./upsertIntoVectorIndex.js";

package/vector-index/upsertIntoVectorIndex.cjs

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.upsertIntoVectorIndex = void 0;
+const nanoid_1 = require("nanoid");
+const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
+async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, objects, getValueToEmbed, getId, }, options) {
+    // many embedding models support bulk embedding, so we first embed all texts:
+    const embeddings = await (0, embedText_js_1.embedTexts)(embeddingModel, objects.map(getValueToEmbed), options);
+    await vectorIndex.upsertMany(objects.map((object, i) => ({
+        id: getId?.(object, i) ?? generateId(),
+        vector: embeddings[i],
+        data: object,
+    })));
+}
+exports.upsertIntoVectorIndex = upsertIntoVectorIndex;

package/vector-index/upsertIntoVectorIndex.d.ts

@@ -0,0 +1,11 @@
+import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
+import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
+import { VectorIndex } from "./VectorIndex.js";
+export declare function upsertIntoVectorIndex<OBJECT, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, objects, getValueToEmbed, getId, }: {
+    vectorIndex: VectorIndex<OBJECT, unknown>;
+    embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
+    generateId?: () => string;
+    objects: OBJECT[];
+    getValueToEmbed: (object: OBJECT, index: number) => string;
+    getId?: (object: OBJECT, index: number) => string | undefined;
+}, options?: ModelFunctionOptions<SETTINGS>): Promise<void>;

package/vector-index/upsertIntoVectorIndex.js

@@ -0,0 +1,11 @@
+import { nanoid as createId } from "nanoid";
+import { embedTexts } from "../model-function/embed-text/embedText.js";
+export async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = createId, objects, getValueToEmbed, getId, }, options) {
+    // many embedding models support bulk embedding, so we first embed all texts:
+    const embeddings = await embedTexts(embeddingModel, objects.map(getValueToEmbed), options);
+    await vectorIndex.upsertMany(objects.map((object, i) => ({
+        id: getId?.(object, i) ?? generateId(),
+        vector: embeddings[i],
+        data: object,
+    })));
+}

package/text-chunk/SimilarTextChunksFromVectorIndexRetriever.d.ts

@@ -1,20 +0,0 @@
-import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "./TextChunk.js";
-import { TextChunkRetriever, TextChunkRetrieverSettings } from "./retrieve-text-chunks/TextChunkRetriever.js";
-import { VectorIndex } from "../vector-index/VectorIndex.js";
-export interface SimilarTextChunksFromVectorIndexRetrieverSettings {
-    maxResults?: number;
-    similarityThreshold?: number;
-}
-export declare class SimilarTextChunksFromVectorIndexRetriever<CHUNK extends TextChunk, INDEX, SETTINGS extends TextEmbeddingModelSettings> implements TextChunkRetriever<CHUNK, string, SimilarTextChunksFromVectorIndexRetrieverSettings> {
-    private readonly vectorIndex;
-    private readonly embeddingModel;
-    private readonly settings;
-    constructor({ vectorIndex, embeddingModel, maxResults, similarityThreshold, }: {
-        vectorIndex: VectorIndex<CHUNK, INDEX>;
-        embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
-    } & SimilarTextChunksFromVectorIndexRetrieverSettings);
-    retrieveTextChunks(query: string, options?: ModelFunctionOptions<TextChunkRetrieverSettings>): Promise<CHUNK[]>;
-    withSettings(additionalSettings: Partial<SimilarTextChunksFromVectorIndexRetrieverSettings>): this;
-}

package/text-chunk/retrieve-text-chunks/TextChunkRetriever.d.ts

@@ -1,8 +0,0 @@
-import { ModelFunctionOptions } from "../../model-function/ModelFunctionOptions.js";
-import { TextChunk } from "../TextChunk.js";
-export interface TextChunkRetrieverSettings {
-}
-export interface TextChunkRetriever<CHUNK extends TextChunk, QUERY, SETTINGS extends TextChunkRetrieverSettings> {
-    retrieveTextChunks(query: QUERY, options?: ModelFunctionOptions<TextChunkRetrieverSettings>): Promise<CHUNK[]>;
-    withSettings(additionalSettings: Partial<SETTINGS>): this;
-}

package/text-chunk/retrieve-text-chunks/retrieveTextChunks.cjs

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.retrieveTextChunks = void 0;
-async function retrieveTextChunks(retriever, query, options) {
-    // TODO add error handling, events, duration tracking, etc.
-    return {
-        chunks: await retriever.retrieveTextChunks(query, options),
-    };
-}
-exports.retrieveTextChunks = retrieveTextChunks;

package/text-chunk/retrieve-text-chunks/retrieveTextChunks.d.ts

@@ -1,6 +0,0 @@
-import { ModelFunctionOptions } from "../../model-function/ModelFunctionOptions.js";
-import { TextChunk } from "../TextChunk.js";
-import { TextChunkRetriever, TextChunkRetrieverSettings } from "./TextChunkRetriever.js";
-export declare function retrieveTextChunks<CHUNK extends TextChunk, QUERY, SETTINGS extends TextChunkRetrieverSettings>(retriever: TextChunkRetriever<CHUNK, QUERY, SETTINGS>, query: QUERY, options?: ModelFunctionOptions<SETTINGS>): Promise<{
-    chunks: CHUNK[];
-}>;

package/text-chunk/retrieve-text-chunks/retrieveTextChunks.js

@@ -1,6 +0,0 @@
-export async function retrieveTextChunks(retriever, query, options) {
-    // TODO add error handling, events, duration tracking, etc.
-    return {
-        chunks: await retriever.retrieveTextChunks(query, options),
-    };
-}

package/text-chunk/upsertTextChunks.cjs

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.upsertTextChunks = void 0;
-const nanoid_1 = require("nanoid");
-const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
-async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, chunks, ids, }, options) {
-    // many embedding models support bulk embedding, so we first embed all texts:
-    const embeddings = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.text), options);
-    await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
-        id: ids?.[i] ?? generateId(),
-        vector: embeddings[i],
-        data: chunk,
-    })));
-}
-exports.upsertTextChunks = upsertTextChunks;

package/text-chunk/upsertTextChunks.d.ts

@@ -1,11 +0,0 @@
-import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
-import { TextChunk } from "./TextChunk.js";
-import { VectorIndex } from "../vector-index/VectorIndex.js";
-export declare function upsertTextChunks<CHUNK extends TextChunk, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, chunks, ids, }: {
-    vectorIndex: VectorIndex<CHUNK, unknown>;
-    embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
-    generateId?: () => string;
-    chunks: CHUNK[];
-    ids?: Array<string | undefined>;
-}, options?: ModelFunctionOptions<SETTINGS>): Promise<void>;

package/text-chunk/upsertTextChunks.js

@@ -1,11 +0,0 @@
-import { nanoid as createId } from "nanoid";
-import { embedTexts } from "../model-function/embed-text/embedText.js";
-export async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = createId, chunks, ids, }, options) {
-    // many embedding models support bulk embedding, so we first embed all texts:
-    const embeddings = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.text), options);
-    await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
-        id: ids?.[i] ?? generateId(),
-        vector: embeddings[i],
-        data: chunk,
-    })));
-}