langchain 0.0.174 → 0.0.175

package/dist/vectorstores/convex.js

@@ -0,0 +1,173 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { makeFunctionReference, } from "convex/server";
+import { Document } from "../document.js";
+import { VectorStore } from "./base.js";
+/**
+ * Class that is a wrapper around Convex storage and vector search. It is used
+ * to insert embeddings in Convex documents with a vector search index,
+ * and perform a vector search on them.
+ *
+ * ConvexVectorStore does NOT implement maxMarginalRelevanceSearch.
+ */
+export class ConvexVectorStore extends VectorStore {
+    _vectorstoreType() {
+        return "convex";
+    }
+    constructor(embeddings, config) {
+        super(embeddings, config);
+        Object.defineProperty(this, "ctx", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "table", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "textField", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "embeddingField", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataField", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "insert", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "get", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.ctx = config.ctx;
+        this.table = config.table ?? "documents";
+        this.index = config.index ?? "byEmbedding";
+        this.textField = config.textField ?? "text";
+        this.embeddingField =
+            config.embeddingField ?? "embedding";
+        this.metadataField =
+            config.metadataField ?? "metadata";
+        this.insert =
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            config.insert ?? makeFunctionReference("langchain/db:insert");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.get = config.get ?? makeFunctionReference("langchain/db:get");
+    }
+    /**
+     * Add vectors and their corresponding documents to the Convex table.
+     * @param vectors Vectors to be added.
+     * @param documents Corresponding documents to be added.
+     * @returns Promise that resolves when the vectors and documents have been added.
+     */
+    async addVectors(vectors, documents) {
+        const convexDocuments = vectors.map((embedding, idx) => ({
+            [this.textField]: documents[idx].pageContent,
+            [this.embeddingField]: embedding,
+            [this.metadataField]: documents[idx].metadata,
+        }));
+        // TODO: Remove chunking when Convex handles the concurrent requests correctly
+        const PAGE_SIZE = 16;
+        for (let i = 0; i < convexDocuments.length; i += PAGE_SIZE) {
+            await Promise.all(convexDocuments.slice(i, i + PAGE_SIZE).map((document) => this.ctx.runMutation(this.insert, {
+                table: this.table,
+                document,
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            })));
+        }
+    }
+    /**
+     * Add documents to the Convex table. It first converts
+     * the documents to vectors using the embeddings and then calls the
+     * addVectors method.
+     * @param documents Documents to be added.
+     * @returns Promise that resolves when the documents have been added.
+     */
+    async addDocuments(documents) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+    }
+    /**
+     * Similarity search on the vectors stored in the
+     * Convex table. It returns a list of documents and their
+     * corresponding similarity scores.
+     * @param query Query vector for the similarity search.
+     * @param k Number of nearest neighbors to return.
+     * @param filter Optional filter to be applied.
+     * @returns Promise that resolves to a list of documents and their corresponding similarity scores.
+     */
+    async similaritySearchVectorWithScore(query, k, filter) {
+        const idsAndScores = await this.ctx.vectorSearch(this.table, this.index, {
+            vector: query,
+            limit: k,
+            filter: filter?.filter,
+        });
+        const documents = await Promise.all(idsAndScores.map(({ _id }) => 
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.ctx.runQuery(this.get, { id: _id })));
+        return documents.map(({ [this.textField]: text, [this.embeddingField]: embedding, [this.metadataField]: metadata, }, idx) => [
+            new Document({
+                pageContent: text,
+                metadata: {
+                    ...metadata,
+                    ...(filter?.includeEmbeddings ? { embedding } : null),
+                },
+            }),
+            idsAndScores[idx]._score,
+        ]);
+    }
+    /**
+     * Static method to create an instance of ConvexVectorStore from a
+     * list of texts. It first converts the texts to vectors and then adds
+     * them to the Convex table.
+     * @param texts List of texts to be converted to vectors.
+     * @param metadatas Metadata for the texts.
+     * @param embeddings Embeddings to be used for conversion.
+     * @param dbConfig Database configuration for Convex.
+     * @returns Promise that resolves to a new instance of ConvexVectorStore.
+     */
+    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
+        const docs = texts.map((text, i) => new Document({
+            pageContent: text,
+            metadata: Array.isArray(metadatas) ? metadatas[i] : metadatas,
+        }));
+        return ConvexVectorStore.fromDocuments(docs, embeddings, dbConfig);
+    }
+    /**
+     * Static method to create an instance of ConvexVectorStore from a
+     * list of documents. It first converts the documents to vectors and then
+     * adds them to the Convex table.
+     * @param docs List of documents to be converted to vectors.
+     * @param embeddings Embeddings to be used for conversion.
+     * @param dbConfig Database configuration for Convex.
+     * @returns Promise that resolves to a new instance of ConvexVectorStore.
+     */
+    static async fromDocuments(docs, embeddings, dbConfig) {
+        const instance = new this(embeddings, dbConfig);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+}

package/dist/vectorstores/milvus.cjs

@@ -577,11 +577,13 @@ function genCollectionName() {
 }
 function getTextFieldMaxLength(documents) {
     let textMaxLength = 0;
+    const textEncoder = new TextEncoder();
     // eslint-disable-next-line no-plusplus
     for (let i = 0; i < documents.length; i++) {
         const text = documents[i].pageContent;
-        if (text.length > textMaxLength) {
-            textMaxLength = text.length;
+        const textLengthInBytes = textEncoder.encode(text).length;
+        if (textLengthInBytes > textMaxLength) {
+            textMaxLength = textLengthInBytes;
         }
     }
     return textMaxLength;

package/dist/vectorstores/milvus.js

@@ -550,11 +550,13 @@ function genCollectionName() {
 }
 function getTextFieldMaxLength(documents) {
     let textMaxLength = 0;
+    const textEncoder = new TextEncoder();
     // eslint-disable-next-line no-plusplus
     for (let i = 0; i < documents.length; i++) {
         const text = documents[i].pageContent;
-        if (text.length > textMaxLength) {
-            textMaxLength = text.length;
+        const textLengthInBytes = textEncoder.encode(text).length;
+        if (textLengthInBytes > textMaxLength) {
+            textMaxLength = textLengthInBytes;
         }
     }
     return textMaxLength;

package/dist/vectorstores/vercel_postgres.cjs

@@ -206,14 +206,36 @@ class VercelPostgres extends base_js_1.VectorStore {
      */
     async similaritySearchVectorWithScore(query, k, filter) {
         const embeddingString = `[${query.join(",")}]`;
-        const _filter = filter ?? "{}";
+        const _filter = filter ?? {};
+        const whereClauses = [];
+        const values = [embeddingString, k];
+        let paramCount = values.length;
+        for (const [key, value] of Object.entries(_filter)) {
+            if (typeof value === "object" && value !== null) {
+                const currentParamCount = paramCount;
+                const placeholders = value.in
+                    .map((_, index) => `$${currentParamCount + index + 1}`)
+                    .join(",");
+                whereClauses.push(`${this.metadataColumnName}->>'${key}' IN (${placeholders})`);
+                values.push(...value.in);
+                paramCount += value.in.length;
+            }
+            else {
+                paramCount += 1;
+                whereClauses.push(`${this.metadataColumnName}->>'${key}' = $${paramCount}`);
+                values.push(value);
+            }
+        }
+        const whereClause = whereClauses.length
+            ? `WHERE ${whereClauses.join(" AND ")}`
+            : "";
         const queryString = `
-      SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
-      FROM ${this.tableName}
-      WHERE ${this.metadataColumnName} @> $2
-      ORDER BY "_distance" ASC
-      LIMIT $3;`;
-        const documents = (await this.client.query(queryString, [embeddingString, _filter, k])).rows;
+            SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
+            FROM ${this.tableName}
+            ${whereClause}
+            ORDER BY "_distance" ASC
+            LIMIT $2;`;
+        const documents = (await this.client.query(queryString, values)).rows;
         const results = [];
         for (const doc of documents) {
             if (doc._distance != null && doc[this.contentColumnName] != null) {

package/dist/vectorstores/vercel_postgres.d.ts

@@ -2,7 +2,7 @@ import { type VercelPool, type VercelPoolClient, type VercelPostgresPoolConfig }
 import { VectorStore } from "./base.js";
 import { Embeddings } from "../embeddings/base.js";
 import { Document } from "../document.js";
-type Metadata = Record<string, unknown>;
+type Metadata = Record<string, string | number | Record<"in", string[]>>;
 /**
  * Interface that defines the arguments required to create a
  * `VercelPostgres` instance. It includes Postgres connection options,

package/dist/vectorstores/vercel_postgres.js

@@ -203,14 +203,36 @@ export class VercelPostgres extends VectorStore {
      */
     async similaritySearchVectorWithScore(query, k, filter) {
         const embeddingString = `[${query.join(",")}]`;
-        const _filter = filter ?? "{}";
+        const _filter = filter ?? {};
+        const whereClauses = [];
+        const values = [embeddingString, k];
+        let paramCount = values.length;
+        for (const [key, value] of Object.entries(_filter)) {
+            if (typeof value === "object" && value !== null) {
+                const currentParamCount = paramCount;
+                const placeholders = value.in
+                    .map((_, index) => `$${currentParamCount + index + 1}`)
+                    .join(",");
+                whereClauses.push(`${this.metadataColumnName}->>'${key}' IN (${placeholders})`);
+                values.push(...value.in);
+                paramCount += value.in.length;
+            }
+            else {
+                paramCount += 1;
+                whereClauses.push(`${this.metadataColumnName}->>'${key}' = $${paramCount}`);
+                values.push(value);
+            }
+        }
+        const whereClause = whereClauses.length
+            ? `WHERE ${whereClauses.join(" AND ")}`
+            : "";
         const queryString = `
-      SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
-      FROM ${this.tableName}
-      WHERE ${this.metadataColumnName} @> $2
-      ORDER BY "_distance" ASC
-      LIMIT $3;`;
-        const documents = (await this.client.query(queryString, [embeddingString, _filter, k])).rows;
+            SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
+            FROM ${this.tableName}
+            ${whereClause}
+            ORDER BY "_distance" ASC
+            LIMIT $2;`;
+        const documents = (await this.client.query(queryString, values)).rows;
         const results = [];
         for (const doc of documents) {
             if (doc._distance != null && doc[this.contentColumnName] != null) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.0.174",
+  "version": "0.0.175",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -223,6 +223,9 @@
     "vectorstores/cassandra.cjs",
     "vectorstores/cassandra.js",
     "vectorstores/cassandra.d.ts",
+    "vectorstores/convex.cjs",
+    "vectorstores/convex.js",
+    "vectorstores/convex.d.ts",
     "vectorstores/elasticsearch.cjs",
     "vectorstores/elasticsearch.js",
     "vectorstores/elasticsearch.d.ts",
@@ -646,6 +649,9 @@
     "stores/file/node.cjs",
     "stores/file/node.js",
     "stores/file/node.d.ts",
+    "stores/message/convex.cjs",
+    "stores/message/convex.js",
+    "stores/message/convex.d.ts",
     "stores/message/cloudflare_d1.cjs",
     "stores/message/cloudflare_d1.js",
     "stores/message/cloudflare_d1.d.ts",
@@ -679,6 +685,9 @@
     "stores/message/xata.cjs",
     "stores/message/xata.js",
     "stores/message/xata.d.ts",
+    "storage/convex.cjs",
+    "storage/convex.js",
+    "storage/convex.d.ts",
     "storage/encoder_backed.cjs",
     "storage/encoder_backed.js",
     "storage/encoder_backed.d.ts",
@@ -703,6 +712,9 @@
     "hub.cjs",
     "hub.js",
     "hub.d.ts",
+    "util/convex.cjs",
+    "util/convex.js",
+    "util/convex.d.ts",
     "util/document.cjs",
     "util/document.js",
     "util/document.d.ts",
@@ -853,6 +865,7 @@
     "closevector-node": "0.1.0-alpha.10",
     "closevector-web": "0.1.0-alpha.15",
     "cohere-ai": ">=6.0.0",
+    "convex": "^1.3.1",
     "d3-dsv": "^2.0.0",
     "dotenv": "^16.0.3",
     "dpdm": "^3.12.0",
@@ -969,6 +982,7 @@
     "closevector-node": "0.1.0-alpha.10",
     "closevector-web": "0.1.0-alpha.16",
     "cohere-ai": ">=6.0.0",
+    "convex": "^1.3.1",
     "d3-dsv": "^2.0.0",
     "epub2": "^3.0.1",
     "faiss-node": "^0.5.1",
@@ -1174,6 +1188,9 @@
     "cohere-ai": {
       "optional": true
     },
+    "convex": {
+      "optional": true
+    },
     "d3-dsv": {
       "optional": true
     },
@@ -1695,6 +1712,11 @@
       "import": "./vectorstores/cassandra.js",
       "require": "./vectorstores/cassandra.cjs"
     },
+    "./vectorstores/convex": {
+      "types": "./vectorstores/convex.d.ts",
+      "import": "./vectorstores/convex.js",
+      "require": "./vectorstores/convex.cjs"
+    },
     "./vectorstores/elasticsearch": {
       "types": "./vectorstores/elasticsearch.d.ts",
       "import": "./vectorstores/elasticsearch.js",
@@ -2400,6 +2422,11 @@
       "import": "./stores/file/node.js",
       "require": "./stores/file/node.cjs"
     },
+    "./stores/message/convex": {
+      "types": "./stores/message/convex.d.ts",
+      "import": "./stores/message/convex.js",
+      "require": "./stores/message/convex.cjs"
+    },
     "./stores/message/cloudflare_d1": {
       "types": "./stores/message/cloudflare_d1.d.ts",
       "import": "./stores/message/cloudflare_d1.js",
@@ -2455,6 +2482,11 @@
       "import": "./stores/message/xata.js",
       "require": "./stores/message/xata.cjs"
     },
+    "./storage/convex": {
+      "types": "./storage/convex.d.ts",
+      "import": "./storage/convex.js",
+      "require": "./storage/convex.cjs"
+    },
     "./storage/encoder_backed": {
       "types": "./storage/encoder_backed.d.ts",
       "import": "./storage/encoder_backed.js",
@@ -2495,6 +2527,11 @@
       "import": "./hub.js",
       "require": "./hub.cjs"
     },
+    "./util/convex": {
+      "types": "./util/convex.d.ts",
+      "import": "./util/convex.js",
+      "require": "./util/convex.cjs"
+    },
     "./util/document": {
       "types": "./util/document.d.ts",
       "import": "./util/document.js",

package/storage/convex.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/storage/convex.cjs');

package/storage/convex.d.ts

@@ -0,0 +1 @@
+export * from '../dist/storage/convex.js'

package/storage/convex.js

@@ -0,0 +1 @@
+export * from '../dist/storage/convex.js'

package/stores/message/convex.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/stores/message/convex.cjs');

package/stores/message/convex.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/stores/message/convex.js'

package/stores/message/convex.js

@@ -0,0 +1 @@
+export * from '../../dist/stores/message/convex.js'

package/util/convex.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/util/convex.cjs');

package/util/convex.d.ts

@@ -0,0 +1 @@
+export * from '../dist/util/convex.js'

package/util/convex.js

@@ -0,0 +1 @@
+export * from '../dist/util/convex.js'

package/vectorstores/convex.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/vectorstores/convex.cjs');

package/vectorstores/convex.d.ts

@@ -0,0 +1 @@
+export * from '../dist/vectorstores/convex.js'

package/vectorstores/convex.js

@@ -0,0 +1 @@
+export * from '../dist/vectorstores/convex.js'