langchain 0.0.127 → 0.0.128

package/dist/vectorstores/faiss.d.ts

@@ -15,6 +15,8 @@ export declare class FaissStore extends SaveableVectorStore {
     docstore: SynchronousInMemoryDocstore;
     args: FaissLibArgs;
     _vectorstoreType(): string;
+    getMapping(): Record<number, string>;
+    getDocstore(): SynchronousInMemoryDocstore;
     constructor(embeddings: Embeddings, args: FaissLibArgs);
     addDocuments(documents: Document[]): Promise<string[]>;
     get index(): IndexFlatL2;
@@ -22,6 +24,7 @@ export declare class FaissStore extends SaveableVectorStore {
     addVectors(vectors: number[][], documents: Document[]): Promise<string[]>;
     similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document<Record<string, any>>, number][]>;
     save(directory: string): Promise<void>;
+    mergeFrom(targetIndex: FaissStore): Promise<string[]>;
     static load(directory: string, embeddings: Embeddings): Promise<FaissStore>;
     static loadFromPython(directory: string, embeddings: Embeddings): Promise<FaissStore>;
     static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, dbConfig?: {
@@ -30,6 +33,9 @@ export declare class FaissStore extends SaveableVectorStore {
     static fromDocuments(docs: Document[], embeddings: Embeddings, dbConfig?: {
         docstore?: SynchronousInMemoryDocstore;
     }): Promise<FaissStore>;
+    static fromIndex(targetIndex: FaissStore, embeddings: Embeddings, dbConfig?: {
+        docstore?: SynchronousInMemoryDocstore;
+    }): Promise<FaissStore>;
     static importFaiss(): Promise<{
         IndexFlatL2: typeof IndexFlatL2;
     }>;

package/dist/vectorstores/faiss.js

@@ -6,6 +6,12 @@ export class FaissStore extends SaveableVectorStore {
     _vectorstoreType() {
         return "faiss";
     }
+    getMapping() {
+        return this._mapping;
+    }
+    getDocstore() {
+        return this.docstore;
+    }
     constructor(embeddings, args) {
         super(embeddings, args);
         Object.defineProperty(this, "_index", {
@@ -44,7 +50,7 @@ export class FaissStore extends SaveableVectorStore {
     }
     get index() {
         if (!this._index) {
-            throw new Error("Vector store not initialised yet. Try calling `fromTexts` or `fromDocuments` first.");
+            throw new Error("Vector store not initialised yet. Try calling `fromTexts`, `fromDocuments` or `fromIndex` first.");
         }
         return this._index;
     }
@@ -108,6 +114,32 @@ export class FaissStore extends SaveableVectorStore {
             ])),
         ]);
     }
+    async mergeFrom(targetIndex) {
+        const targetIndexDimensions = targetIndex.index.getDimension();
+        if (!this._index) {
+            const { IndexFlatL2 } = await FaissStore.importFaiss();
+            this._index = new IndexFlatL2(targetIndexDimensions);
+        }
+        const d = this.index.getDimension();
+        if (targetIndexDimensions !== d) {
+            throw new Error("Cannot merge indexes with different dimensions.");
+        }
+        const targetMapping = targetIndex.getMapping();
+        const targetDocstore = targetIndex.getDocstore();
+        const targetSize = targetIndex.index.ntotal();
+        const documentIds = [];
+        const currentDocstoreSize = this.index.ntotal();
+        for (let i = 0; i < targetSize; i += 1) {
+            const targetId = targetMapping[i];
+            documentIds.push(targetId);
+            const targetDocument = targetDocstore.search(targetId);
+            const id = currentDocstoreSize + i;
+            this._mapping[id] = targetId;
+            this.docstore.add({ [targetId]: targetDocument });
+        }
+        this.index.mergeFrom(targetIndex.index);
+        return documentIds;
+    }
     static async load(directory, embeddings) {
         const fs = await import("node:fs/promises");
         const path = await import("node:path");
@@ -205,6 +237,14 @@ export class FaissStore extends SaveableVectorStore {
         await instance.addDocuments(docs);
         return instance;
     }
+    static async fromIndex(targetIndex, embeddings, dbConfig) {
+        const args = {
+            docstore: dbConfig?.docstore,
+        };
+        const instance = new this(embeddings, args);
+        await instance.mergeFrom(targetIndex);
+        return instance;
+    }
     static async importFaiss() {
         try {
             const { default: { IndexFlatL2 }, } = await import("faiss-node");

package/dist/vectorstores/supabase.cjs

@@ -33,10 +33,17 @@ class SupabaseVectorStore extends base_js_1.VectorStore {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "upsertBatchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 500
+        });
         this.client = args.client;
         this.tableName = args.tableName || "documents";
         this.queryName = args.queryName || "match_documents";
         this.filter = args.filter;
+        this.upsertBatchSize = args.upsertBatchSize ?? this.upsertBatchSize;
     }
     async addDocuments(documents, options) {
         const texts = documents.map(({ pageContent }) => pageContent);
@@ -50,12 +57,11 @@ class SupabaseVectorStore extends base_js_1.VectorStore {
         }));
         // upsert returns 500/502/504 (yes really any of them) if given too many rows/characters
         // ~2000 trips it, but my data is probably smaller than average pageContent and metadata
-        const chunkSize = 500;
         let returnedIds = [];
-        for (let i = 0; i < rows.length; i += chunkSize) {
-            const chunk = rows.slice(i, i + chunkSize).map((row) => {
+        for (let i = 0; i < rows.length; i += this.upsertBatchSize) {
+            const chunk = rows.slice(i, i + this.upsertBatchSize).map((row, j) => {
                 if (options?.ids) {
-                    return { id: options.ids[i], ...row };
+                    return { id: options.ids[i + j], ...row };
                 }
                 return row;
             });

package/dist/vectorstores/supabase.d.ts

@@ -11,6 +11,7 @@ export interface SupabaseLibArgs {
     tableName?: string;
     queryName?: string;
     filter?: SupabaseMetadata | SupabaseFilterRPCCall;
+    upsertBatchSize?: number;
 }
 export declare class SupabaseVectorStore extends VectorStore {
     FilterType: SupabaseMetadata | SupabaseFilterRPCCall;
@@ -18,13 +19,14 @@ export declare class SupabaseVectorStore extends VectorStore {
     tableName: string;
     queryName: string;
     filter?: SupabaseMetadata | SupabaseFilterRPCCall;
+    upsertBatchSize: number;
     _vectorstoreType(): string;
     constructor(embeddings: Embeddings, args: SupabaseLibArgs);
     addDocuments(documents: Document[], options?: {
-        ids?: string[];
+        ids?: string[] | number[];
     }): Promise<string[]>;
     addVectors(vectors: number[][], documents: Document[], options?: {
-        ids?: string[];
+        ids?: string[] | number[];
     }): Promise<string[]>;
     delete(params: {
         ids: string[];

package/dist/vectorstores/supabase.js

@@ -30,10 +30,17 @@ export class SupabaseVectorStore extends VectorStore {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "upsertBatchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 500
+        });
         this.client = args.client;
         this.tableName = args.tableName || "documents";
         this.queryName = args.queryName || "match_documents";
         this.filter = args.filter;
+        this.upsertBatchSize = args.upsertBatchSize ?? this.upsertBatchSize;
     }
     async addDocuments(documents, options) {
         const texts = documents.map(({ pageContent }) => pageContent);
@@ -47,12 +54,11 @@ export class SupabaseVectorStore extends VectorStore {
         }));
         // upsert returns 500/502/504 (yes really any of them) if given too many rows/characters
         // ~2000 trips it, but my data is probably smaller than average pageContent and metadata
-        const chunkSize = 500;
         let returnedIds = [];
-        for (let i = 0; i < rows.length; i += chunkSize) {
-            const chunk = rows.slice(i, i + chunkSize).map((row) => {
+        for (let i = 0; i < rows.length; i += this.upsertBatchSize) {
+            const chunk = rows.slice(i, i + this.upsertBatchSize).map((row, j) => {
                 if (options?.ids) {
-                    return { id: options.ids[i], ...row };
+                    return { id: options.ids[i + j], ...row };
                 }
                 return row;
             });

package/document_transformers/mozilla_readability.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/document_transformers/mozilla_readability.cjs');

package/document_transformers/mozilla_readability.d.ts

@@ -0,0 +1 @@
+export * from '../dist/document_transformers/mozilla_readability.js'

package/document_transformers/mozilla_readability.js

@@ -0,0 +1 @@
+export * from '../dist/document_transformers/mozilla_readability.js'

package/llms/bedrock.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/llms/bedrock.cjs');

package/llms/bedrock.d.ts

@@ -0,0 +1 @@
+export * from '../dist/llms/bedrock.js'

package/llms/bedrock.js

@@ -0,0 +1 @@
+export * from '../dist/llms/bedrock.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.0.127",
+  "version": "0.0.128",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -139,6 +139,9 @@
     "llms/sagemaker_endpoint.cjs",
     "llms/sagemaker_endpoint.js",
     "llms/sagemaker_endpoint.d.ts",
+    "llms/bedrock.cjs",
+    "llms/bedrock.js",
+    "llms/bedrock.d.ts",
     "prompts.cjs",
     "prompts.js",
     "prompts.d.ts",
@@ -343,6 +346,9 @@
     "document_transformers/html_to_text.cjs",
     "document_transformers/html_to_text.js",
     "document_transformers/html_to_text.d.ts",
+    "document_transformers/mozilla_readability.cjs",
+    "document_transformers/mozilla_readability.js",
+    "document_transformers/mozilla_readability.d.ts",
     "document_transformers/openai_functions.cjs",
     "document_transformers/openai_functions.js",
     "document_transformers/openai_functions.d.ts",
@@ -561,12 +567,17 @@
   "author": "LangChain",
   "license": "MIT",
   "devDependencies": {
+    "@aws-crypto/sha256-js": "^5.0.0",
     "@aws-sdk/client-dynamodb": "^3.310.0",
     "@aws-sdk/client-kendra": "^3.352.0",
     "@aws-sdk/client-lambda": "^3.310.0",
     "@aws-sdk/client-s3": "^3.310.0",
     "@aws-sdk/client-sagemaker-runtime": "^3.310.0",
     "@aws-sdk/client-sfn": "^3.362.0",
+    "@aws-sdk/credential-provider-node": "^3.388.0",
+    "@aws-sdk/protocol-http": "^3.374.0",
+    "@aws-sdk/signature-v4": "^3.374.0",
+    "@aws-sdk/types": "^3.357.0",
     "@azure/storage-blob": "^12.15.0",
     "@clickhouse/client": "^0.0.14",
     "@elastic/elasticsearch": "^8.4.0",
@@ -578,6 +589,7 @@
     "@google-cloud/storage": "^6.10.1",
     "@huggingface/inference": "^1.5.1",
     "@jest/globals": "^29.5.0",
+    "@mozilla/readability": "^0.4.4",
     "@notionhq/client": "^2.2.5",
     "@opensearch-project/opensearch": "^2.2.0",
     "@pinecone-database/pinecone": "^0.0.14",
@@ -597,6 +609,8 @@
     "@types/flat": "^5.0.2",
     "@types/html-to-text": "^9",
     "@types/js-yaml": "^4",
+    "@types/jsdom": "^21.1.1",
+    "@types/mozilla-readability": "^0.2.1",
     "@types/object-hash": "^3.0.2",
     "@types/pdf-parse": "^1.1.1",
     "@types/pg": "^8",
@@ -622,7 +636,7 @@
     "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-no-instanceof": "^1.0.1",
     "eslint-plugin-prettier": "^4.2.1",
-    "faiss-node": "^0.2.1",
+    "faiss-node": "^0.3.0",
     "firebase-admin": "^11.9.0",
     "google-auth-library": "^8.9.0",
     "graphql": "^16.6.0",
@@ -631,6 +645,7 @@
     "ignore": "^5.2.0",
     "ioredis": "^5.3.2",
     "jest": "^29.5.0",
+    "jsdom": "^22.1.0",
     "mammoth": "^1.5.1",
     "ml-matrix": "^6.10.4",
     "mongodb": "^5.2.0",
@@ -661,12 +676,16 @@
     "weaviate-ts-client": "^1.4.0"
   },
   "peerDependencies": {
+    "@aws-crypto/sha256-js": "^5.0.0",
     "@aws-sdk/client-dynamodb": "^3.310.0",
     "@aws-sdk/client-kendra": "^3.352.0",
     "@aws-sdk/client-lambda": "^3.310.0",
     "@aws-sdk/client-s3": "^3.310.0",
     "@aws-sdk/client-sagemaker-runtime": "^3.310.0",
     "@aws-sdk/client-sfn": "^3.310.0",
+    "@aws-sdk/credential-provider-node": "^3.388.0",
+    "@aws-sdk/protocol-http": "^3.374.0",
+    "@aws-sdk/signature-v4": "^3.374.0",
     "@azure/storage-blob": "^12.15.0",
     "@clickhouse/client": "^0.0.14",
     "@elastic/elasticsearch": "^8.4.0",
@@ -676,6 +695,7 @@
     "@google-ai/generativelanguage": "^0.2.1",
     "@google-cloud/storage": "^6.10.1",
     "@huggingface/inference": "^1.5.1",
+    "@mozilla/readability": "*",
     "@notionhq/client": "^2.2.5",
     "@opensearch-project/opensearch": "*",
     "@pinecone-database/pinecone": "*",
@@ -698,13 +718,14 @@
     "cohere-ai": "^5.0.2",
     "d3-dsv": "^2.0.0",
     "epub2": "^3.0.1",
-    "faiss-node": "^0.2.1",
+    "faiss-node": "^0.3.0",
     "firebase-admin": "^11.9.0",
     "google-auth-library": "^8.9.0",
     "hnswlib-node": "^1.4.2",
     "html-to-text": "^9.0.5",
     "ignore": "^5.2.0",
     "ioredis": "^5.3.2",
+    "jsdom": "*",
     "mammoth": "*",
     "mongodb": "^5.2.0",
     "mysql2": "^3.3.3",
@@ -727,6 +748,9 @@
     "weaviate-ts-client": "^1.4.0"
   },
   "peerDependenciesMeta": {
+    "@aws-crypto/sha256-js": {
+      "optional": true
+    },
     "@aws-sdk/client-dynamodb": {
       "optional": true
     },
@@ -745,6 +769,15 @@
     "@aws-sdk/client-sfn": {
       "optional": true
     },
+    "@aws-sdk/credential-provider-node": {
+      "optional": true
+    },
+    "@aws-sdk/protocol-http": {
+      "optional": true
+    },
+    "@aws-sdk/signature-v4": {
+      "optional": true
+    },
     "@azure/storage-blob": {
       "optional": true
     },
@@ -1185,6 +1218,11 @@
       "import": "./llms/sagemaker_endpoint.js",
       "require": "./llms/sagemaker_endpoint.cjs"
     },
+    "./llms/bedrock": {
+      "types": "./llms/bedrock.d.ts",
+      "import": "./llms/bedrock.js",
+      "require": "./llms/bedrock.cjs"
+    },
     "./prompts": {
       "types": "./prompts.d.ts",
       "import": "./prompts.js",
@@ -1529,6 +1567,11 @@
       "import": "./document_transformers/html_to_text.js",
       "require": "./document_transformers/html_to_text.cjs"
     },
+    "./document_transformers/mozilla_readability": {
+      "types": "./document_transformers/mozilla_readability.d.ts",
+      "import": "./document_transformers/mozilla_readability.js",
+      "require": "./document_transformers/mozilla_readability.cjs"
+    },
     "./document_transformers/openai_functions": {
       "types": "./document_transformers/openai_functions.d.ts",
       "import": "./document_transformers/openai_functions.js",