modelfusion 0.37.0 → 0.39.0

package/vector-index/memory/MemoryVectorIndex.js

@@ -1,5 +1,11 @@
+import SecureJSON from "secure-json-parse";
 import z from "zod";
 import { cosineSimilarity } from "../../util/cosineSimilarity.js";
+const jsonDataSchema = z.array(z.object({
+    id: z.string(),
+    vector: z.array(z.number()),
+    data: z.unknown(),
+}));
 /**
  * A very simple vector index that stores all entries in memory. Useful when you only have
  * a small number of entries and don't want to set up a real database, e.g. for conversational memory
@@ -15,15 +21,16 @@ export class MemoryVectorIndex {
         });
     }
     static async deserialize({ serializedData, schema, }) {
-        let json = JSON.parse(serializedData);
+        // validate the outer structure:
+        const json = jsonDataSchema.parse(SecureJSON.parse(serializedData));
         if (schema != null) {
-            json = z
-                .array(z.object({
-                id: z.string(),
-                vector: z.array(z.number()),
-                data: schema,
-            }))
-                .parse(json);
+            // when a schema is provided, validate all entries:
+            for (const entry of json) {
+                const validationResult = schema.validate(entry.data);
+                if (!validationResult.success) {
+                    throw validationResult.error;
+                }
+            }
         }
         const vectorIndex = new MemoryVectorIndex();
         vectorIndex.upsertMany(json);

package/vector-index/upsertIntoVectorIndex.cjs

@@ -2,10 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.upsertIntoVectorIndex = void 0;
 const nanoid_1 = require("nanoid");
-const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
+const embed_js_1 = require("../model-function/embed/embed.cjs");
 async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, objects, getValueToEmbed, getId, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const embeddings = await (0, embedText_js_1.embedTexts)(embeddingModel, objects.map(getValueToEmbed), options);
+    const embeddings = await (0, embed_js_1.embedMany)(embeddingModel, objects.map(getValueToEmbed), options);
     await vectorIndex.upsertMany(objects.map((object, i) => ({
         id: getId?.(object, i) ?? generateId(),
         vector: embeddings[i],

package/vector-index/upsertIntoVectorIndex.d.ts

@@ -1,11 +1,11 @@
 import { ModelFunctionOptions } from "../model-function/ModelFunctionOptions.js";
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
+import { EmbeddingModel, EmbeddingModelSettings } from "../model-function/embed/EmbeddingModel.js";
 import { VectorIndex } from "./VectorIndex.js";
-export declare function upsertIntoVectorIndex<OBJECT, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, objects, getValueToEmbed, getId, }: {
+export declare function upsertIntoVectorIndex<VALUE, OBJECT, SETTINGS extends EmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, objects, getValueToEmbed, getId, }: {
     vectorIndex: VectorIndex<OBJECT, unknown, unknown>;
-    embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
+    embeddingModel: EmbeddingModel<VALUE, unknown, SETTINGS>;
     generateId?: () => string;
     objects: OBJECT[];
-    getValueToEmbed: (object: OBJECT, index: number) => string;
+    getValueToEmbed: (object: OBJECT, index: number) => VALUE;
     getId?: (object: OBJECT, index: number) => string | undefined;
 }, options?: ModelFunctionOptions<SETTINGS>): Promise<void>;

package/vector-index/upsertIntoVectorIndex.js

@@ -1,8 +1,8 @@
 import { nanoid as createId } from "nanoid";
-import { embedTexts } from "../model-function/embed-text/embedText.js";
+import { embedMany } from "../model-function/embed/embed.js";
 export async function upsertIntoVectorIndex({ vectorIndex, embeddingModel, generateId = createId, objects, getValueToEmbed, getId, }, options) {
     // many embedding models support bulk embedding, so we first embed all texts:
-    const embeddings = await embedTexts(embeddingModel, objects.map(getValueToEmbed), options);
+    const embeddings = await embedMany(embeddingModel, objects.map(getValueToEmbed), options);
     await vectorIndex.upsertMany(objects.map((object, i) => ({
         id: getId?.(object, i) ?? generateId(),
         vector: embeddings[i],

package/model-function/embed-text/TextEmbeddingEvent.d.ts

@@ -1,21 +0,0 @@
-import { Vector } from "../../core/Vector.js";
-import { BaseModelCallFinishedEvent, BaseModelCallStartedEvent } from "../ModelCallEvent.js";
-export interface TextEmbeddingStartedEvent extends BaseModelCallStartedEvent {
-    functionType: "text-embedding";
-    input: string | Array<string>;
-}
-export type TextEmbeddingFinishedEventResult = {
-    status: "success";
-    response: unknown;
-    output: Vector | Array<Vector>;
-} | {
-    status: "error";
-    error: unknown;
-} | {
-    status: "abort";
-};
-export interface TextEmbeddingFinishedEvent extends BaseModelCallFinishedEvent {
-    functionType: "text-embedding";
-    input: string | Array<string>;
-    result: TextEmbeddingFinishedEventResult;
-}

package/model-function/embed-text/TextEmbeddingModel.d.ts

@@ -1,21 +0,0 @@
-import { Vector } from "../../core/Vector.js";
-import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
-import { Model, ModelSettings } from "../Model.js";
-export interface TextEmbeddingModelSettings extends ModelSettings {
-}
-export interface TextEmbeddingModel<RESPONSE, SETTINGS extends TextEmbeddingModelSettings> extends Model<SETTINGS> {
-    /**
-     * The limit of tokens for a single text.
-     */
-    readonly contextWindowSize: number | undefined;
-    /**
-     * The size of the embedding vector.
-     */
-    readonly embeddingDimensions: number | undefined;
-    /**
-     * Limit of how many texts can be sent in a single API call.
-     */
-    readonly maxTextsPerCall: number | undefined;
-    generateEmbeddingResponse(texts: string[], options?: ModelFunctionOptions<SETTINGS>): PromiseLike<RESPONSE>;
-    extractEmbeddings(response: RESPONSE): Vector[];
-}

package/model-function/embed-text/embedText.d.ts

@@ -1,27 +0,0 @@
-import { Vector } from "../../core/Vector.js";
-import { ModelFunctionOptions } from "../ModelFunctionOptions.js";
-import { ModelFunctionPromise } from "../executeCall.js";
-import { TextEmbeddingModel, TextEmbeddingModelSettings } from "./TextEmbeddingModel.js";
-/**
- * Generate embeddings for multiple texts.
- *
- * @example
- * const embeddings = await embedTexts(
- *   new OpenAITextEmbeddingModel(...),
- *   [
- *     "At first, Nox didn't know what to do with the pup.",
- *     "He keenly observed and absorbed everything around him, from the birds in the sky to the trees in the forest.",
- *   ]
- * );
- */
-export declare function embedTexts<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, texts: string[], options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<Vector[], RESPONSE[]>;
-/**
- * Generate an embedding for a single text.
- *
- * @example
- * const embedding = await embedText(
- *   new OpenAITextEmbeddingModel(...),
- *   "At first, Nox didn't know what to do with the pup."
- * );
- */
-export declare function embedText<RESPONSE, SETTINGS extends TextEmbeddingModelSettings>(model: TextEmbeddingModel<RESPONSE, SETTINGS>, text: string, options?: ModelFunctionOptions<SETTINGS>): ModelFunctionPromise<Vector, RESPONSE[]>;