langchain 0.0.75 → 0.0.76

package/dist/vectorstores/redis.js

@@ -0,0 +1,232 @@
+import { SchemaFieldTypes, VectorAlgorithms } from "redis";
+import { VectorStore } from "./base.js";
+import { Document } from "../document.js";
+export class RedisVectorStore extends VectorStore {
+    constructor(embeddings, _dbConfig) {
+        super(embeddings, _dbConfig);
+        Object.defineProperty(this, "redisClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "keyPrefix", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contentKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "metadataKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "vectorKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "filter", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.redisClient = _dbConfig.redisClient;
+        this.indexName = _dbConfig.indexName;
+        this.indexOptions = _dbConfig.indexOptions ?? {
+            ALGORITHM: VectorAlgorithms.HNSW,
+            DISTANCE_METRIC: "COSINE",
+        };
+        this.keyPrefix = _dbConfig.keyPrefix ?? `doc:${this.indexName}:`;
+        this.contentKey = _dbConfig.contentKey ?? "content";
+        this.metadataKey = _dbConfig.metadataKey ?? "metadata";
+        this.vectorKey = _dbConfig.vectorKey ?? "content_vector";
+        this.filter = _dbConfig.filter;
+    }
+    async addDocuments(documents, options) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        await this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
+    }
+    async addVectors(vectors, documents, { keys, batchSize = 1000 } = {}) {
+        // check if the index exists and create it if it doesn't
+        await this.createIndex(vectors[0].length);
+        const multi = this.redisClient.multi();
+        vectors.map(async (vector, idx) => {
+            const key = keys && keys.length ? keys[idx] : `${this.keyPrefix}${idx}`;
+            const metadata = documents[idx] && documents[idx].metadata
+                ? documents[idx].metadata
+                : {};
+            multi.hSet(key, {
+                [this.vectorKey]: this.getFloat32Buffer(vector),
+                [this.contentKey]: documents[idx].pageContent,
+                [this.metadataKey]: this.escapeSpecialChars(JSON.stringify(metadata)),
+            });
+            // write batch
+            if (idx % batchSize === 0) {
+                await multi.exec();
+            }
+        });
+        // insert final batch
+        await multi.exec();
+    }
+    async similaritySearchVectorWithScore(query, k, filter) {
+        if (filter && this.filter) {
+            throw new Error("cannot provide both `filter` and `this.filter`");
+        }
+        const _filter = filter ?? this.filter;
+        const results = await this.redisClient.ft.search(this.indexName, ...this.buildQuery(query, k, _filter));
+        const result = [];
+        if (results.total) {
+            for (const res of results.documents) {
+                if (res.value) {
+                    const document = res.value;
+                    if (document.vector_score) {
+                        result.push([
+                            new Document({
+                                pageContent: document[this.contentKey],
+                                metadata: JSON.parse(this.unEscapeSpecialChars(document.metadata)),
+                            }),
+                            Number(document.vector_score),
+                        ]);
+                    }
+                }
+            }
+        }
+        return result;
+    }
+    static fromTexts(texts, metadatas, embeddings, dbConfig) {
+        const docs = [];
+        for (let i = 0; i < texts.length; i += 1) {
+            const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
+            const newDoc = new Document({
+                pageContent: texts[i],
+                metadata,
+            });
+            docs.push(newDoc);
+        }
+        return RedisVectorStore.fromDocuments(docs, embeddings, dbConfig);
+    }
+    static async fromDocuments(docs, embeddings, dbConfig) {
+        const instance = new this(embeddings, dbConfig);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+    async checkIndexExists() {
+        try {
+            await this.redisClient.ft.info(this.indexName);
+        }
+        catch (err) {
+            // index doesn't exist
+            return false;
+        }
+        return true;
+    }
+    async createIndex(dimensions = 1536) {
+        if (await this.checkIndexExists()) {
+            return;
+        }
+        const schema = {
+            [this.vectorKey]: {
+                type: SchemaFieldTypes.VECTOR,
+                TYPE: "FLOAT32",
+                DIM: dimensions,
+                ...this.indexOptions,
+            },
+            [this.contentKey]: SchemaFieldTypes.TEXT,
+            [this.metadataKey]: SchemaFieldTypes.TEXT,
+        };
+        await this.redisClient.ft.create(this.indexName, schema, {
+            ON: "HASH",
+            PREFIX: this.keyPrefix,
+        });
+    }
+    async dropIndex() {
+        try {
+            await this.redisClient.ft.dropIndex(this.indexName);
+            return true;
+        }
+        catch (err) {
+            return false;
+        }
+    }
+    buildQuery(query, k, filter) {
+        const vectorScoreField = "vector_score";
+        let hybridFields = "*";
+        // if a filter is set, modify the hybrid query
+        if (filter && filter.length) {
+            // `filter` is a list of strings, then it's applied using the OR operator in the metadata key
+            // for example: filter = ['foo', 'bar'] => this will filter all metadata containing either 'foo' OR 'bar'
+            hybridFields = `@${this.metadataKey}:(${this.prepareFilter(filter)})`;
+        }
+        const baseQuery = `${hybridFields} => [KNN ${k} @${this.vectorKey} $vector AS ${vectorScoreField}]`;
+        const returnFields = [this.metadataKey, this.contentKey, vectorScoreField];
+        const options = {
+            PARAMS: {
+                vector: this.getFloat32Buffer(query),
+            },
+            RETURN: returnFields,
+            SORTBY: vectorScoreField,
+            DIALECT: 2,
+            LIMIT: {
+                from: 0,
+                size: k,
+            },
+        };
+        return [baseQuery, options];
+    }
+    prepareFilter(filter) {
+        return filter.map(this.escapeSpecialChars).join("|");
+    }
+    /**
+     * Escapes all '-' characters.
+     * RediSearch considers '-' as a negative operator, hence we need
+     * to escape it
+     * @see https://redis.io/docs/stack/search/reference/query_syntax
+     *
+     * @param str
+     * @returns
+     */
+    escapeSpecialChars(str) {
+        return str.replaceAll("-", "\\-");
+    }
+    /**
+     * Unescapes all '-' characters, returning the original string
+     *
+     * @param str
+     * @returns
+     */
+    unEscapeSpecialChars(str) {
+        return str.replaceAll("\\-", "-");
+    }
+    /**
+     * Converts the vector to the buffer Redis needs to
+     * correctly store an embedding
+     *
+     * @param vector
+     * @returns Buffer
+     */
+    getFloat32Buffer(vector) {
+        return Buffer.from(new Float32Array(vector).buffer);
+    }
+}

package/document_loaders/web/apify_dataset.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/web/apify_dataset.cjs');

package/document_loaders/web/apify_dataset.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/apify_dataset.js'

package/document_loaders/web/apify_dataset.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/apify_dataset.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.0.75",
+  "version": "0.0.76",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -124,6 +124,9 @@
     "vectorstores/myscale.cjs",
     "vectorstores/myscale.js",
     "vectorstores/myscale.d.ts",
+    "vectorstores/redis.cjs",
+    "vectorstores/redis.js",
+    "vectorstores/redis.d.ts",
     "text_splitter.cjs",
     "text_splitter.js",
     "text_splitter.d.ts",
@@ -142,6 +145,9 @@
     "document_loaders/base.cjs",
     "document_loaders/base.js",
     "document_loaders/base.d.ts",
+    "document_loaders/web/apify_dataset.cjs",
+    "document_loaders/web/apify_dataset.js",
+    "document_loaders/web/apify_dataset.d.ts",
     "document_loaders/web/cheerio.cjs",
     "document_loaders/web/cheerio.js",
     "document_loaders/web/cheerio.d.ts",
@@ -311,7 +317,7 @@
     "clean": "rm -rf dist/ && node scripts/create-entrypoints.js clean",
     "prepack": "yarn build",
     "release": "release-it --only-version --config .release-it.json",
-    "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --testPathIgnorePatterns=\\.int\\.test.ts --testTimeout 30000",
     "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --watch --testPathIgnorePatterns=\\.int\\.test.ts",
     "test:integration": "NODE_OPTIONS=--experimental-vm-modules jest --testPathPattern=\\.int\\.test.ts --testTimeout 50000",
     "test:single": "NODE_OPTIONS=--experimental-vm-modules yarn run jest --config jest.config.cjs --testTimeout 50000",
@@ -345,6 +351,7 @@
     "@types/uuid": "^9",
     "@typescript-eslint/eslint-plugin": "^5.58.0",
     "@typescript-eslint/parser": "^5.58.0",
+    "apify-client": "^2.7.1",
     "axios": "^0.26.0",
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "^1.4.0",
@@ -369,7 +376,7 @@
     "playwright": "^1.32.1",
     "prettier": "^2.8.3",
     "puppeteer": "^19.7.2",
-    "redis": "^4.6.4",
+    "redis": "^4.6.6",
     "release-it": "^15.10.1",
     "replicate": "^0.9.0",
     "rollup": "^3.19.1",
@@ -394,6 +401,7 @@
     "@tensorflow/tfjs-converter": "*",
     "@tensorflow/tfjs-core": "*",
     "@zilliz/milvus2-sdk-node": "^2.2.0",
+    "apify-client": "^2.7.1",
     "axios": "*",
     "cheerio": "^1.0.0-rc.12",
     "chromadb": "^1.4.0",
@@ -453,6 +461,9 @@
     "@zilliz/milvus2-sdk-node": {
       "optional": true
     },
+    "apify-client": {
+      "optional": true
+    },
     "axios": {
       "optional": true
     },
@@ -510,12 +521,11 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.4.3",
-    "@dqbd/tiktoken": "^1.0.7",
     "ansi-styles": "^5.0.0",
     "binary-extensions": "^2.2.0",
-    "browser-or-node": "^2.1.1",
     "expr-eval": "^2.0.2",
     "flat": "^5.0.2",
+    "js-tiktoken": "^1.0.6",
     "jsonpointer": "^5.0.1",
     "ml-distance": "^4.0.0",
     "object-hash": "^3.0.0",
@@ -746,6 +756,11 @@
       "import": "./vectorstores/myscale.js",
       "require": "./vectorstores/myscale.cjs"
     },
+    "./vectorstores/redis": {
+      "types": "./vectorstores/redis.d.ts",
+      "import": "./vectorstores/redis.js",
+      "require": "./vectorstores/redis.cjs"
+    },
     "./text_splitter": {
       "types": "./text_splitter.d.ts",
       "import": "./text_splitter.js",
@@ -778,6 +793,11 @@
       "import": "./document_loaders/base.js",
       "require": "./document_loaders/base.cjs"
     },
+    "./document_loaders/web/apify_dataset": {
+      "types": "./document_loaders/web/apify_dataset.d.ts",
+      "import": "./document_loaders/web/apify_dataset.js",
+      "require": "./document_loaders/web/apify_dataset.cjs"
+    },
     "./document_loaders/web/cheerio": {
       "types": "./document_loaders/web/cheerio.d.ts",
       "import": "./document_loaders/web/cheerio.js",

package/vectorstores/redis.cjs

@@ -0,0 +1 @@
+module.exports = require('../dist/vectorstores/redis.cjs');

package/vectorstores/redis.d.ts

@@ -0,0 +1 @@
+export * from '../dist/vectorstores/redis.js'

package/vectorstores/redis.js

@@ -0,0 +1 @@
+export * from '../dist/vectorstores/redis.js'