langchain 0.0.197 → 0.0.199

package/dist/vectorstores/pgvector.js

@@ -20,6 +20,24 @@ export class PGVectorStore extends VectorStore {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "collectionTableName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "collectionName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "langchain"
+        });
+        Object.defineProperty(this, "collectionMetadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "idColumnName", {
             enumerable: true,
             configurable: true,
@@ -75,6 +93,9 @@ export class PGVectorStore extends VectorStore {
             value: 500
         });
         this.tableName = config.tableName;
+        this.collectionTableName = config.collectionTableName;
+        this.collectionName = config.collectionName ?? "langchain";
+        this.collectionMetadata = config.collectionMetadata ?? null;
         this.filter = config.filter;
         this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
         this.contentColumnName = config.columns?.contentColumnName ?? "text";
@@ -100,6 +121,9 @@ export class PGVectorStore extends VectorStore {
         const postgresqlVectorStore = new PGVectorStore(embeddings, config);
         await postgresqlVectorStore._initializeClient();
         await postgresqlVectorStore.ensureTableInDatabase();
+        if (postgresqlVectorStore.collectionTableName) {
+            await postgresqlVectorStore.ensureCollectionTableInDatabase();
+        }
         return postgresqlVectorStore;
     }
     async _initializeClient() {
@@ -116,15 +140,56 @@ export class PGVectorStore extends VectorStore {
         const texts = documents.map(({ pageContent }) => pageContent);
         return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
     }
+    /**
+     * Inserts a row for the collectionName provided at initialization if it does not
+     * exist and returns the collectionId.
+     *
+     * @returns The collectionId for the given collectionName.
+     */
+    async getOrCreateCollection() {
+        const queryString = `
+      SELECT uuid from ${this.collectionTableName}
+      WHERE name = $1;
+    `;
+        const queryResult = await this.pool.query(queryString, [
+            this.collectionName,
+        ]);
+        let collectionId = queryResult.rows[0]?.uuid;
+        if (!collectionId) {
+            const insertString = `
+        INSERT INTO ${this.collectionTableName}(
+          uuid,
+          name,
+          cmetadata
+        )
+        VALUES (
+          uuid_generate_v4(),
+          $1,
+          $2
+        )
+        RETURNING uuid;
+      `;
+            const insertResult = await this.pool.query(insertString, [
+                this.collectionName,
+                this.collectionMetadata,
+            ]);
+            collectionId = insertResult.rows[0]?.uuid;
+        }
+        return collectionId;
+    }
     /**
      * Generates the SQL placeholders for a specific row at the provided index.
      *
      * @param index - The index of the row for which placeholders need to be generated.
+     * @param numOfColumns - The number of columns we are inserting data into.
      * @returns The SQL placeholders for the row values.
      */
-    generatePlaceholderForRowAt(index) {
-        const base = index * 3;
-        return `($${base + 1}, $${base + 2}, $${base + 3})`;
+    generatePlaceholderForRowAt(index, numOfColumns) {
+        const placeholders = [];
+        for (let i = 0; i < numOfColumns; i += 1) {
+            placeholders.push(`$${index * numOfColumns + i + 1}`);
+        }
+        return `(${placeholders.join(", ")})`;
     }
     /**
      * Constructs the SQL query for inserting rows into the specified table.
@@ -133,15 +198,25 @@ export class PGVectorStore extends VectorStore {
      * @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
      * @returns The complete SQL INSERT INTO query string.
      */
-    buildInsertQuery(rows) {
+    async buildInsertQuery(rows) {
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        const columns = [
+            this.contentColumnName,
+            this.vectorColumnName,
+            this.metadataColumnName,
+        ];
+        if (collectionId) {
+            columns.push("collection_id");
+        }
         const valuesPlaceholders = rows
-            .map((_, j) => this.generatePlaceholderForRowAt(j))
+            .map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
             .join(", ");
         const text = `
       INSERT INTO ${this.tableName}(
-        ${this.contentColumnName},
-        ${this.vectorColumnName},
-        ${this.metadataColumnName}
+        ${columns}
       )
       VALUES ${valuesPlaceholders}
     `;
@@ -156,17 +231,24 @@ export class PGVectorStore extends VectorStore {
      * @returns Promise that resolves when the vectors have been added.
      */
     async addVectors(vectors, documents) {
-        const rows = vectors.map((embedding, idx) => {
+        const rows = [];
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        for (let i = 0; i < vectors.length; i += 1) {
+            const values = [];
+            const embedding = vectors[i];
             const embeddingString = `[${embedding.join(",")}]`;
-            return [
-                documents[idx].pageContent,
-                embeddingString,
-                documents[idx].metadata,
-            ];
-        });
+            values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
+            if (collectionId) {
+                values.push(collectionId);
+            }
+            rows.push(values);
+        }
         for (let i = 0; i < rows.length; i += this.chunkSize) {
             const chunk = rows.slice(i, i + this.chunkSize);
-            const insertQuery = this.buildInsertQuery(chunk);
+            const insertQuery = await this.buildInsertQuery(chunk);
             const flatValues = chunk.flat();
             try {
                 await this.pool.query(insertQuery, flatValues);
@@ -190,13 +272,23 @@ export class PGVectorStore extends VectorStore {
     async similaritySearchVectorWithScore(query, k, filter) {
         const embeddingString = `[${query.join(",")}]`;
         const _filter = filter ?? "{}";
+        let collectionId;
+        if (this.collectionTableName) {
+            collectionId = await this.getOrCreateCollection();
+        }
+        const parameters = [embeddingString, _filter, k];
+        if (collectionId) {
+            parameters.push(collectionId);
+        }
         const queryString = `
       SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
       FROM ${this.tableName}
       WHERE ${this.metadataColumnName}::jsonb @> $2
+      ${collectionId ? "AND collection_id = $4" : ""}
       ORDER BY "_distance" ASC
-      LIMIT $3;`;
-        const documents = (await this.pool.query(queryString, [embeddingString, _filter, k])).rows;
+      LIMIT $3;
+    `;
+        const documents = (await this.pool.query(queryString, parameters)).rows;
         const results = [];
         for (const doc of documents) {
             if (doc._distance != null && doc[this.contentColumnName] != null) {
@@ -227,6 +319,38 @@ export class PGVectorStore extends VectorStore {
       );
     `);
     }
+    /**
+     * Method to ensure the existence of the collection table in the database.
+     * It creates the table if it does not already exist.
+     *
+     * @returns Promise that resolves when the collection table has been ensured.
+     */
+    async ensureCollectionTableInDatabase() {
+        try {
+            await this.pool.query(`
+        CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
+          uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
+          name character varying,
+          cmetadata jsonb
+        );
+
+        ALTER TABLE ${this.tableName}
+          ADD COLUMN collection_id uuid;
+
+        ALTER TABLE ${this.tableName}
+          ADD CONSTRAINT ${this.tableName}_collection_id_fkey
+          FOREIGN KEY (collection_id)
+          REFERENCES ${this.collectionTableName}(uuid)
+          ON DELETE CASCADE;
+      `);
+        }
+        catch (e) {
+            if (!e.message.includes("already exists")) {
+                console.error(e);
+                throw new Error(`Error adding column: ${e.message}`);
+            }
+        }
+    }
     /**
      * Static method to create a new `PGVectorStore` instance from an
      * array of texts and their metadata. It converts the texts into

package/dist/vectorstores/prisma.cjs

@@ -249,7 +249,7 @@ class PrismaVectorStore extends base_js_1.VectorStore {
                         !value.every((v) => typeof v === "string")) {
                         throw new Error(`Invalid filter: IN operator requires an array of strings. Received: ${JSON.stringify(value, null, 2)}`);
                     }
-                    return this.Prisma.sql `${colRaw} ${opRaw} (${value.join(",")})`;
+                    return this.Prisma.sql `${colRaw} ${opRaw} (${this.Prisma.join(value)})`;
                 }
                 case OpMap.isNull:
                 case OpMap.isNotNull:

package/dist/vectorstores/prisma.js

@@ -246,7 +246,7 @@ class PrismaVectorStore extends VectorStore {
                         !value.every((v) => typeof v === "string")) {
                         throw new Error(`Invalid filter: IN operator requires an array of strings. Received: ${JSON.stringify(value, null, 2)}`);
                     }
-                    return this.Prisma.sql `${colRaw} ${opRaw} (${value.join(",")})`;
+                    return this.Prisma.sql `${colRaw} ${opRaw} (${this.Prisma.join(value)})`;
                 }
                 case OpMap.isNull:
                 case OpMap.isNotNull:

package/dist/vectorstores/weaviate.cjs

@@ -27,6 +27,7 @@ exports.WeaviateStore = exports.flattenObjectForWeaviate = void 0;
 const uuid = __importStar(require("uuid"));
 const base_js_1 = require("./base.cjs");
 const document_js_1 = require("../document.cjs");
+const math_js_1 = require("../util/math.cjs");
 // Note this function is not generic, it is designed specifically for Weaviate
 // https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
 const flattenObjectForWeaviate = (
@@ -235,11 +236,27 @@ class WeaviateStore extends base_js_1.VectorStore {
      * @returns An array of tuples, where each tuple contains a document and its similarity score.
      */
     async similaritySearchVectorWithScore(query, k, filter) {
+        const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
+        return resultsWithEmbedding.map(([document, score, _embedding]) => [
+            document,
+            score,
+        ]);
+    }
+    /**
+     * Method to perform a similarity search on the stored vectors in the
+     * Weaviate index. It returns the top k most similar documents, their
+     * similarity scores and embedding vectors.
+     * @param query The query vector.
+     * @param k The number of most similar documents to return.
+     * @param filter Optional filter to apply to the search.
+     * @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
+     */
+    async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
         try {
-            let builder = await this.client.graphql
+            let builder = this.client.graphql
                 .get()
                 .withClassName(this.indexName)
-                .withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
+                .withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
                 .withNearVector({
                 vector: query,
                 distance: filter?.distance,
@@ -261,6 +278,7 @@ class WeaviateStore extends base_js_1.VectorStore {
                         metadata: rest,
                     }),
                     _additional.distance,
+                    _additional.vector,
                 ]);
             }
             return documents;
@@ -269,6 +287,31 @@ class WeaviateStore extends base_js_1.VectorStore {
             throw Error(`'Error in similaritySearch' ${e}`);
         }
     }
+    /**
+     * Return documents selected using the maximal marginal relevance.
+     * Maximal marginal relevance optimizes for similarity to the query AND diversity
+     * among selected documents.
+     *
+     * @param {string} query - Text to look up documents similar to.
+     * @param {number} options.k - Number of documents to return.
+     * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
+     * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
+     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
+     * @param {this["FilterType"]} options.filter - Optional filter
+     * @param _callbacks
+     *
+     * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
+     */
+    async maxMarginalRelevanceSearch(query, options, _callbacks) {
+        const { k, fetchK = 20, lambda = 0.5, filter } = options;
+        const queryEmbedding = await this.embeddings.embedQuery(query);
+        const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
+        const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
+        const mmrIndexes = (0, math_js_1.maximalMarginalRelevance)(queryEmbedding, embeddingList, lambda, k);
+        return mmrIndexes
+            .filter((idx) => idx !== -1)
+            .map((idx) => allResults[idx][0]);
+    }
     /**
      * Static method to create a new `WeaviateStore` instance from a list of
      * texts. It first creates documents from the texts and metadata, then

package/dist/vectorstores/weaviate.d.ts

@@ -1,5 +1,5 @@
 import type { WeaviateClient, WhereFilter } from "weaviate-ts-client";
-import { VectorStore } from "./base.js";
+import { MaxMarginalRelevanceSearchOptions, VectorStore } from "./base.js";
 import { Embeddings } from "../embeddings/base.js";
 import { Document } from "../document.js";
 export declare const flattenObjectForWeaviate: (obj: Record<string, any>) => Record<string, any>;
@@ -83,6 +83,32 @@ export declare class WeaviateStore extends VectorStore {
      * @returns An array of tuples, where each tuple contains a document and its similarity score.
      */
     similaritySearchVectorWithScore(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number][]>;
+    /**
+     * Method to perform a similarity search on the stored vectors in the
+     * Weaviate index. It returns the top k most similar documents, their
+     * similarity scores and embedding vectors.
+     * @param query The query vector.
+     * @param k The number of most similar documents to return.
+     * @param filter Optional filter to apply to the search.
+     * @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
+     */
+    similaritySearchVectorWithScoreAndEmbedding(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number, number[]][]>;
+    /**
+     * Return documents selected using the maximal marginal relevance.
+     * Maximal marginal relevance optimizes for similarity to the query AND diversity
+     * among selected documents.
+     *
+     * @param {string} query - Text to look up documents similar to.
+     * @param {number} options.k - Number of documents to return.
+     * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
+     * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
+     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
+     * @param {this["FilterType"]} options.filter - Optional filter
+     * @param _callbacks
+     *
+     * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
+     */
+    maxMarginalRelevanceSearch(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>, _callbacks?: undefined): Promise<Document[]>;
     /**
      * Static method to create a new `WeaviateStore` instance from a list of
      * texts. It first creates documents from the texts and metadata, then

package/dist/vectorstores/weaviate.js

@@ -1,6 +1,7 @@
 import * as uuid from "uuid";
 import { VectorStore } from "./base.js";
 import { Document } from "../document.js";
+import { maximalMarginalRelevance } from "../util/math.js";
 // Note this function is not generic, it is designed specifically for Weaviate
 // https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
 export const flattenObjectForWeaviate = (
@@ -208,11 +209,27 @@ export class WeaviateStore extends VectorStore {
      * @returns An array of tuples, where each tuple contains a document and its similarity score.
      */
     async similaritySearchVectorWithScore(query, k, filter) {
+        const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
+        return resultsWithEmbedding.map(([document, score, _embedding]) => [
+            document,
+            score,
+        ]);
+    }
+    /**
+     * Method to perform a similarity search on the stored vectors in the
+     * Weaviate index. It returns the top k most similar documents, their
+     * similarity scores and embedding vectors.
+     * @param query The query vector.
+     * @param k The number of most similar documents to return.
+     * @param filter Optional filter to apply to the search.
+     * @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
+     */
+    async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
         try {
-            let builder = await this.client.graphql
+            let builder = this.client.graphql
                 .get()
                 .withClassName(this.indexName)
-                .withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
+                .withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
                 .withNearVector({
                 vector: query,
                 distance: filter?.distance,
@@ -234,6 +251,7 @@ export class WeaviateStore extends VectorStore {
                         metadata: rest,
                     }),
                     _additional.distance,
+                    _additional.vector,
                 ]);
             }
             return documents;
@@ -242,6 +260,31 @@ export class WeaviateStore extends VectorStore {
             throw Error(`'Error in similaritySearch' ${e}`);
         }
     }
+    /**
+     * Return documents selected using the maximal marginal relevance.
+     * Maximal marginal relevance optimizes for similarity to the query AND diversity
+     * among selected documents.
+     *
+     * @param {string} query - Text to look up documents similar to.
+     * @param {number} options.k - Number of documents to return.
+     * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
+     * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
+     *                 where 0 corresponds to maximum diversity and 1 to minimum diversity.
+     * @param {this["FilterType"]} options.filter - Optional filter
+     * @param _callbacks
+     *
+     * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
+     */
+    async maxMarginalRelevanceSearch(query, options, _callbacks) {
+        const { k, fetchK = 20, lambda = 0.5, filter } = options;
+        const queryEmbedding = await this.embeddings.embedQuery(query);
+        const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
+        const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
+        const mmrIndexes = maximalMarginalRelevance(queryEmbedding, embeddingList, lambda, k);
+        return mmrIndexes
+            .filter((idx) => idx !== -1)
+            .map((idx) => allResults[idx][0]);
+    }
     /**
      * Static method to create a new `WeaviateStore` instance from a list of
      * texts. It first creates documents from the texts and metadata, then

package/dist/vectorstores/xata.cjs

@@ -93,8 +93,9 @@ class XataVectorSearch extends base_js_1.VectorStore {
             size: k,
             filter,
         });
+        return (
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return records.map((record) => [
+        records?.map((record) => [
             new document_js_1.Document({
                 pageContent: record.content,
                 metadata: Object.fromEntries(Object.entries(record).filter(([key]) => key !== "content" &&
@@ -103,7 +104,7 @@ class XataVectorSearch extends base_js_1.VectorStore {
                     key !== "id")),
             }),
             record.xata.score,
-        ]);
+        ]) ?? []);
     }
 }
 exports.XataVectorSearch = XataVectorSearch;

package/dist/vectorstores/xata.js

@@ -90,8 +90,9 @@ export class XataVectorSearch extends VectorStore {
             size: k,
             filter,
         });
+        return (
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return records.map((record) => [
+        records?.map((record) => [
             new Document({
                 pageContent: record.content,
                 metadata: Object.fromEntries(Object.entries(record).filter(([key]) => key !== "content" &&
@@ -100,6 +101,6 @@ export class XataVectorSearch extends VectorStore {
                     key !== "id")),
             }),
             record.xata.score,
-        ]);
+        ]) ?? []);
     }
 }

package/document_loaders/fs/chatgpt.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/fs/chatgpt.cjs');

package/document_loaders/fs/chatgpt.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/chatgpt.js'

package/document_loaders/fs/chatgpt.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/chatgpt.js'

package/document_loaders/fs/pptx.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/fs/pptx.cjs');

package/document_loaders/fs/pptx.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/pptx.js'

package/document_loaders/fs/pptx.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/pptx.js'

package/experimental/tools/pyinterpreter.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/experimental/tools/pyinterpreter.cjs');

package/experimental/tools/pyinterpreter.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/experimental/tools/pyinterpreter.js'

package/experimental/tools/pyinterpreter.js

@@ -0,0 +1 @@
+export * from '../../dist/experimental/tools/pyinterpreter.js'

package/llms/gradient_ai.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/llms/gradient_ai.cjs');

package/llms/gradient_ai.d.ts

@@ -0,0 +1 @@
+export * from '../dist/llms/gradient_ai.js'

package/llms/gradient_ai.js

@@ -0,0 +1 @@
+export * from '../dist/llms/gradient_ai.js'

package/llms/watsonx_ai.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/llms/watsonx_ai.cjs');

package/llms/watsonx_ai.d.ts

@@ -0,0 +1 @@
+export * from '../dist/llms/watsonx_ai.js'

package/llms/watsonx_ai.js

@@ -0,0 +1 @@
+export * from '../dist/llms/watsonx_ai.js'