modelfusion 0.0.44

package/vector-index/memory/MemoryVectorIndex.cjs

@@ -0,0 +1,63 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryVectorIndex = void 0;
+const zod_1 = __importDefault(require("zod"));
+const cosineSimilarity_js_1 = require("../../util/cosineSimilarity.cjs");
+/**
+ * A very simple vector index that stores all entries in memory. Useful when you only have
+ * a small number of entries and don't want to set up a real database, e.g. for conversational memory
+ * that does not need to be persisted.
+ */
+class MemoryVectorIndex {
+    constructor() {
+        Object.defineProperty(this, "entries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+    }
+    static async deserialize({ serializedData, schema, }) {
+        let json = JSON.parse(serializedData);
+        if (schema != null) {
+            json = zod_1.default
+                .array(zod_1.default.object({
+                id: zod_1.default.string(),
+                vector: zod_1.default.array(zod_1.default.number()),
+                data: schema,
+            }))
+                .parse(json);
+        }
+        const vectorIndex = new MemoryVectorIndex();
+        vectorIndex.upsertMany(json);
+        return vectorIndex;
+    }
+    async upsertMany(data) {
+        for (const entry of data) {
+            this.entries.set(entry.id, entry);
+        }
+    }
+    async queryByVector({ queryVector, similarityThreshold, maxResults, }) {
+        const results = [...this.entries.values()]
+            .map((entry) => ({
+            id: entry.id,
+            similarity: (0, cosineSimilarity_js_1.cosineSimilarity)(entry.vector, queryVector),
+            data: entry.data,
+        }))
+            .filter((entry) => similarityThreshold == undefined ||
+            entry.similarity == undefined ||
+            entry.similarity > similarityThreshold);
+        results.sort((a, b) => b.similarity - a.similarity);
+        return results.slice(0, maxResults);
+    }
+    serialize() {
+        return JSON.stringify([...this.entries.values()]);
+    }
+    asIndex() {
+        return this;
+    }
+}
+exports.MemoryVectorIndex = MemoryVectorIndex;

package/vector-index/memory/MemoryVectorIndex.d.ts

@@ -0,0 +1,31 @@
+import z from "zod";
+import { Vector } from "../../run/Vector.js";
+import { VectorIndex } from "../VectorIndex.js";
+/**
+ * A very simple vector index that stores all entries in memory. Useful when you only have
+ * a small number of entries and don't want to set up a real database, e.g. for conversational memory
+ * that does not need to be persisted.
+ */
+export declare class MemoryVectorIndex<DATA> implements VectorIndex<DATA, MemoryVectorIndex<DATA>> {
+    static deserialize<DATA>({ serializedData, schema, }: {
+        serializedData: string;
+        schema?: z.ZodSchema<DATA>;
+    }): Promise<MemoryVectorIndex<DATA>>;
+    private readonly entries;
+    upsertMany(data: Array<{
+        id: string;
+        vector: Vector;
+        data: DATA;
+    }>): Promise<void>;
+    queryByVector({ queryVector, similarityThreshold, maxResults, }: {
+        queryVector: Vector;
+        maxResults: number;
+        similarityThreshold?: number;
+    }): Promise<Array<{
+        id: string;
+        data: DATA;
+        similarity?: number;
+    }>>;
+    serialize(): string;
+    asIndex(): MemoryVectorIndex<DATA>;
+}

package/vector-index/memory/MemoryVectorIndex.js

@@ -0,0 +1,56 @@
+import z from "zod";
+import { cosineSimilarity } from "../../util/cosineSimilarity.js";
+/**
+ * A very simple vector index that stores all entries in memory. Useful when you only have
+ * a small number of entries and don't want to set up a real database, e.g. for conversational memory
+ * that does not need to be persisted.
+ */
+export class MemoryVectorIndex {
+    constructor() {
+        Object.defineProperty(this, "entries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+    }
+    static async deserialize({ serializedData, schema, }) {
+        let json = JSON.parse(serializedData);
+        if (schema != null) {
+            json = z
+                .array(z.object({
+                id: z.string(),
+                vector: z.array(z.number()),
+                data: schema,
+            }))
+                .parse(json);
+        }
+        const vectorIndex = new MemoryVectorIndex();
+        vectorIndex.upsertMany(json);
+        return vectorIndex;
+    }
+    async upsertMany(data) {
+        for (const entry of data) {
+            this.entries.set(entry.id, entry);
+        }
+    }
+    async queryByVector({ queryVector, similarityThreshold, maxResults, }) {
+        const results = [...this.entries.values()]
+            .map((entry) => ({
+            id: entry.id,
+            similarity: cosineSimilarity(entry.vector, queryVector),
+            data: entry.data,
+        }))
+            .filter((entry) => similarityThreshold == undefined ||
+            entry.similarity == undefined ||
+            entry.similarity > similarityThreshold);
+        results.sort((a, b) => b.similarity - a.similarity);
+        return results.slice(0, maxResults);
+    }
+    serialize() {
+        return JSON.stringify([...this.entries.values()]);
+    }
+    asIndex() {
+        return this;
+    }
+}

package/vector-index/pinecone/PineconeVectorIndex.cjs

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PineconeVectorIndex = void 0;
+class PineconeVectorIndex {
+    constructor({ index, namespace, schema, }) {
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.index = index;
+        this.namespace = namespace;
+        this.schema = schema;
+    }
+    async upsertMany(data) {
+        this.index.upsert({
+            upsertRequest: {
+                namespace: this.namespace,
+                vectors: data.map((entry) => ({
+                    id: entry.id,
+                    values: entry.vector,
+                    metadata: entry.data,
+                })),
+            },
+        });
+    }
+    async queryByVector({ queryVector, similarityThreshold, maxResults, }) {
+        const { matches } = await this.index.query({
+            queryRequest: {
+                namespace: this.namespace,
+                vector: queryVector,
+                topK: maxResults,
+                includeMetadata: true,
+            },
+        });
+        if (matches == undefined) {
+            return [];
+        }
+        return matches
+            .filter((match) => similarityThreshold == undefined ||
+            match.score == undefined ||
+            match.score > similarityThreshold)
+            .map((match) => ({
+            id: match.id,
+            data: this.schema.parse(match.metadata),
+            similarity: match.score,
+        }));
+    }
+    asIndex() {
+        return this;
+    }
+}
+exports.PineconeVectorIndex = PineconeVectorIndex;

package/vector-index/pinecone/PineconeVectorIndex.d.ts

@@ -0,0 +1,29 @@
+import { VectorOperationsApi } from "@pinecone-database/pinecone/dist/pinecone-generated-ts-fetch/index.js";
+import { z } from "zod";
+import { Vector } from "../../run/Vector.js";
+import { VectorIndex } from "../VectorIndex.js";
+export declare class PineconeVectorIndex<DATA extends object | undefined> implements VectorIndex<DATA, PineconeVectorIndex<DATA>> {
+    readonly index: VectorOperationsApi;
+    readonly namespace?: string;
+    readonly schema: z.ZodSchema<DATA>;
+    constructor({ index, namespace, schema, }: {
+        index: VectorOperationsApi;
+        namespace?: string;
+        schema: z.ZodSchema<DATA>;
+    });
+    upsertMany(data: Array<{
+        id: string;
+        vector: Vector;
+        data: DATA;
+    }>): Promise<void>;
+    queryByVector({ queryVector, similarityThreshold, maxResults, }: {
+        queryVector: Vector;
+        maxResults: number;
+        similarityThreshold?: number;
+    }): Promise<Array<{
+        id: string;
+        data: DATA;
+        similarity?: number;
+    }>>;
+    asIndex(): PineconeVectorIndex<DATA>;
+}

package/vector-index/pinecone/PineconeVectorIndex.js

@@ -0,0 +1,62 @@
+export class PineconeVectorIndex {
+    constructor({ index, namespace, schema, }) {
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.index = index;
+        this.namespace = namespace;
+        this.schema = schema;
+    }
+    async upsertMany(data) {
+        this.index.upsert({
+            upsertRequest: {
+                namespace: this.namespace,
+                vectors: data.map((entry) => ({
+                    id: entry.id,
+                    values: entry.vector,
+                    metadata: entry.data,
+                })),
+            },
+        });
+    }
+    async queryByVector({ queryVector, similarityThreshold, maxResults, }) {
+        const { matches } = await this.index.query({
+            queryRequest: {
+                namespace: this.namespace,
+                vector: queryVector,
+                topK: maxResults,
+                includeMetadata: true,
+            },
+        });
+        if (matches == undefined) {
+            return [];
+        }
+        return matches
+            .filter((match) => similarityThreshold == undefined ||
+            match.score == undefined ||
+            match.score > similarityThreshold)
+            .map((match) => ({
+            id: match.id,
+            data: this.schema.parse(match.metadata),
+            similarity: match.score,
+        }));
+    }
+    asIndex() {
+        return this;
+    }
+}

package/vector-index/upsertTextChunks.cjs

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.upsertTextChunks = void 0;
+const nanoid_1 = require("nanoid");
+const embedText_js_1 = require("../model-function/embed-text/embedText.cjs");
+async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = nanoid_1.nanoid, chunks, ids, }, options) {
+    // many embedding models support bulk embedding, so we first embed all texts:
+    const { embeddings } = await (0, embedText_js_1.embedTexts)(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
+        id: ids?.[i] ?? generateId(),
+        vector: embeddings[i],
+        data: chunk,
+    })));
+}
+exports.upsertTextChunks = upsertTextChunks;

package/vector-index/upsertTextChunks.d.ts

@@ -0,0 +1,11 @@
+import { FunctionOptions } from "../model-function/FunctionOptions.js";
+import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../model-function/embed-text/TextEmbeddingModel.js";
+import { TextChunk } from "../text-chunk/TextChunk.js";
+import { VectorIndex } from "./VectorIndex.js";
+export declare function upsertTextChunks<CHUNK extends TextChunk, SETTINGS extends TextEmbeddingModelSettings>({ vectorIndex, embeddingModel, generateId, chunks, ids, }: {
+    vectorIndex: VectorIndex<CHUNK, unknown>;
+    embeddingModel: TextEmbeddingModel<unknown, SETTINGS>;
+    generateId?: () => string;
+    chunks: CHUNK[];
+    ids?: Array<string | undefined>;
+}, options?: FunctionOptions<SETTINGS>): Promise<void>;

package/vector-index/upsertTextChunks.js

@@ -0,0 +1,11 @@
+import { nanoid as createId } from "nanoid";
+import { embedTexts } from "../model-function/embed-text/embedText.js";
+export async function upsertTextChunks({ vectorIndex, embeddingModel, generateId = createId, chunks, ids, }, options) {
+    // many embedding models support bulk embedding, so we first embed all texts:
+    const { embeddings } = await embedTexts(embeddingModel, chunks.map((chunk) => chunk.content), options);
+    await vectorIndex.upsertMany(chunks.map((chunk, i) => ({
+        id: ids?.[i] ?? generateId(),
+        vector: embeddings[i],
+        data: chunk,
+    })));
+}