langchain 0.0.7 → 0.0.9

package/dist/memory/buffer_window_memory.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferWindowMemory = void 0;
+const base_1 = require("./base");
+const getInputValue = (inputValues) => {
+    const keys = Object.keys(inputValues);
+    if (keys.length === 1) {
+        return inputValues[keys[0]];
+    }
+    throw new Error("input values have multiple keys, memory only supported when one key currently");
+};
+class BufferWindowMemory extends base_1.BaseMemory {
+    constructor(fields) {
+        var _a, _b, _c, _d;
+        super();
+        Object.defineProperty(this, "humanPrefix", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "Human"
+        });
+        Object.defineProperty(this, "aiPrefix", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "AI"
+        });
+        Object.defineProperty(this, "memoryKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "history"
+        });
+        Object.defineProperty(this, "buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "k", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 5
+        });
+        this.humanPrefix = (_a = fields === null || fields === void 0 ? void 0 : fields.humanPrefix) !== null && _a !== void 0 ? _a : this.humanPrefix;
+        this.aiPrefix = (_b = fields === null || fields === void 0 ? void 0 : fields.aiPrefix) !== null && _b !== void 0 ? _b : this.aiPrefix;
+        this.memoryKey = (_c = fields === null || fields === void 0 ? void 0 : fields.memoryKey) !== null && _c !== void 0 ? _c : this.memoryKey;
+        this.k = (_d = fields === null || fields === void 0 ? void 0 : fields.k) !== null && _d !== void 0 ? _d : this.k;
+    }
+    async loadMemoryVariables(_values) {
+        const result = {
+            [this.memoryKey]: this.buffer.slice(-this.k).join("\n\n"),
+        };
+        return result;
+    }
+    async saveContext(inputValues, outputValues) {
+        const values = await outputValues;
+        const human = `${this.humanPrefix}: ${getInputValue(inputValues)}`;
+        const ai = `${this.aiPrefix}: ${getInputValue(values)}`;
+        const newlines = [human, ai];
+        this.buffer.push(`\n${newlines.join("\n")}`);
+    }
+}
+exports.BufferWindowMemory = BufferWindowMemory;
+//# sourceMappingURL=buffer_window_memory.js.map

package/dist/memory/buffer_window_memory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer_window_memory.js","sourceRoot":"","sources":["../../memory/buffer_window_memory.ts"],"names":[],"mappings":";;;AAAA,iCAAgF;AAShF,MAAM,aAAa,GAAG,CAAC,WAAwB,EAAE,EAAE;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACtC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;QACrB,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;KAC7B;IACD,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAa,kBACX,SAAQ,iBAAU;IAalB,YAAY,MAAyC;;QACnD,KAAK,EAAE,CAAC;QAXV;;;;mBAAc,OAAO;WAAC;QAEtB;;;;mBAAW,IAAI;WAAC;QAEhB;;;;mBAAY,SAAS;WAAC;QAEtB;;;;mBAAmB,EAAE;WAAC;QAEtB;;;;mBAAI,CAAC;WAAC;QAIJ,IAAI,CAAC,WAAW,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,WAAW,mCAAI,IAAI,CAAC,WAAW,CAAC;QAC3D,IAAI,CAAC,QAAQ,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,QAAQ,mCAAI,IAAI,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,CAAC,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,CAAC,mCAAI,IAAI,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAAoB;QAC5C,MAAM,MAAM,GAAG;YACb,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;SAC1D,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,WAAW,CACf,WAAwB,EACxB,YAAmC;QAEnC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;QAClC,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,WAAW,KAAK,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QACnE,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC,QAAQ,KAAK,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;CACF;AAvCD,gDAuCC"}

package/dist/memory/index.d.ts

@@ -1,2 +1,3 @@
 export { BufferMemory } from "./buffer_memory";
 export { BaseMemory } from "./base";
+export { BufferWindowMemory } from "./buffer_window_memory";

package/dist/memory/index.js

@@ -1,8 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BaseMemory = exports.BufferMemory = void 0;
+exports.BufferWindowMemory = exports.BaseMemory = exports.BufferMemory = void 0;
 var buffer_memory_1 = require("./buffer_memory");
 Object.defineProperty(exports, "BufferMemory", { enumerable: true, get: function () { return buffer_memory_1.BufferMemory; } });
 var base_1 = require("./base");
 Object.defineProperty(exports, "BaseMemory", { enumerable: true, get: function () { return base_1.BaseMemory; } });
+var buffer_window_memory_1 = require("./buffer_window_memory");
+Object.defineProperty(exports, "BufferWindowMemory", { enumerable: true, get: function () { return buffer_window_memory_1.BufferWindowMemory; } });
 //# sourceMappingURL=index.js.map

package/dist/memory/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../memory/index.ts"],"names":[],"mappings":";;;AAAA,iDAA+C;AAAtC,6GAAA,YAAY,OAAA;AACrB,+BAAoC;AAA3B,kGAAA,UAAU,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../memory/index.ts"],"names":[],"mappings":";;;AAAA,iDAA+C;AAAtC,6GAAA,YAAY,OAAA;AACrB,+BAAoC;AAA3B,kGAAA,UAAU,OAAA;AACnB,+DAA4D;AAAnD,0HAAA,kBAAkB,OAAA"}

package/dist/memory/tests/buffer_window_memory.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory/tests/buffer_window_memory.test.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+const buffer_window_memory_1 = require("../buffer_window_memory");
+(0, globals_1.test)("Test buffer memory", async () => {
+    const memory = new buffer_window_memory_1.BufferWindowMemory({ k: 1 });
+    const result1 = await memory.loadMemoryVariables({});
+    (0, globals_1.expect)(result1).toStrictEqual({ history: "" });
+    const llmResult = new Promise((resolve, _reject) => {
+        resolve({ bar: "foo" });
+    });
+    await memory.saveContext({ foo: "bar" }, llmResult);
+    const expectedString = "\nHuman: bar\nAI: foo";
+    const result2 = await memory.loadMemoryVariables({});
+    (0, globals_1.expect)(result2).toStrictEqual({ history: expectedString });
+    const llmResult2 = new Promise((resolve, _reject) => {
+        resolve({ bar: "foo" });
+    });
+    await memory.saveContext({ foo: "bar1" }, llmResult2);
+    const expectedString3 = "\nHuman: bar1\nAI: foo";
+    const result3 = await memory.loadMemoryVariables({});
+    (0, globals_1.expect)(result3).toStrictEqual({ history: expectedString3 });
+});
+//# sourceMappingURL=buffer_window_memory.test.js.map

package/dist/memory/tests/buffer_window_memory.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer_window_memory.test.js","sourceRoot":"","sources":["../../../memory/tests/buffer_window_memory.test.ts"],"names":[],"mappings":";;AAAA,2CAA6C;AAC7C,kEAA6D;AAG7D,IAAA,cAAI,EAAC,oBAAoB,EAAE,KAAK,IAAI,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,yCAAkB,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IACrD,IAAA,gBAAM,EAAC,OAAO,CAAC,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAE/C,MAAM,SAAS,GAAG,IAAI,OAAO,CAAe,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAC/D,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IACpD,MAAM,cAAc,GAAG,uBAAuB,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IACrD,IAAA,gBAAM,EAAC,OAAO,CAAC,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;IAE3D,MAAM,UAAU,GAAG,IAAI,OAAO,CAAe,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAChE,OAAO,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC;IACtD,MAAM,eAAe,GAAG,wBAAwB,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,EAAE,CAAC,CAAC;IACrD,IAAA,gBAAM,EAAC,OAAO,CAAC,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;AAC9D,CAAC,CAAC,CAAC"}

package/dist/tests/text_splitter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tests/text_splitter.test.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+const document_1 = require("../document");
+const text_splitter_1 = require("../text_splitter");
+(0, globals_1.test)("Test splitting by character count.", () => {
+    const text = "foo bar baz 123";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 7,
+        chunkOverlap: 3,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo bar", "bar baz", "baz 123"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count doesn't create empty documents.", () => {
+    const text = "foo  bar";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 2,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count on long words.", () => {
+    const text = "foo bar baz a a";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar", "baz", "a a"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count when shorter words are first.", () => {
+    const text = "a a foo bar baz";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["a a", "foo", "bar", "baz"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by characters when splits not found easily.", () => {
+    const text = "foo bar baz 123";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 1,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar", "baz", "123"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test invalid arguments.", () => {
+    (0, globals_1.expect)(() => {
+        const res = new text_splitter_1.CharacterTextSplitter({ chunkSize: 2, chunkOverlap: 4 });
+        console.log(res);
+    }).toThrow();
+});
+(0, globals_1.test)("Test create documents method.", () => {
+    const texts = ["foo bar", "baz"];
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const docs = splitter.createDocuments(texts);
+    const expectedDocs = [
+        new document_1.Document({ pageContent: "foo" }),
+        new document_1.Document({ pageContent: "bar" }),
+        new document_1.Document({ pageContent: "baz" }),
+    ];
+    (0, globals_1.expect)(docs).toEqual(expectedDocs);
+});
+(0, globals_1.test)("Test create documents with metadata method.", () => {
+    const texts = ["foo bar", "baz"];
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const docs = splitter.createDocuments(texts, [
+        { source: "1" },
+        { source: "2" },
+    ]);
+    const expectedDocs = [
+        new document_1.Document({ pageContent: "foo", metadata: { source: "1" } }),
+        new document_1.Document({ pageContent: "bar", metadata: { source: "1" } }),
+        new document_1.Document({ pageContent: "baz", metadata: { source: "2" } }),
+    ];
+    (0, globals_1.expect)(docs).toEqual(expectedDocs);
+});
+(0, globals_1.test)("Test iterative text splitter.", () => {
+    const text = `Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.
+This is a weird text to write, but gotta test the splittingggg some how.\n\n
+Bye!\n\n-H.`;
+    const splitter = new text_splitter_1.RecursiveCharacterTextSplitter({
+        chunkSize: 10,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = [
+        "Hi.",
+        "I'm",
+        "Harrison.",
+        "How? Are?",
+        "You?",
+        "Okay then f",
+        "f f f f.",
+        "This is a",
+        "a weird",
+        "text to",
+        "write, but",
+        "gotta test",
+        "the",
+        "splitting",
+        "gggg",
+        "some how.",
+        "Bye!\n\n-H.",
+    ];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Token text splitter", () => {
+    const text = "foo bar baz a a";
+    const splitter = new text_splitter_1.TokenTextSplitter({
+        encodingName: "r50k_base",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo bar b", "az a a"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+//# sourceMappingURL=text_splitter.test.js.map

package/dist/tests/text_splitter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text_splitter.test.js","sourceRoot":"","sources":["../../tests/text_splitter.test.ts"],"names":[],"mappings":";;AAAA,2CAA6C;AAC7C,0CAAuC;AACvC,oDAI0B;AAE1B,IAAA,cAAI,EAAC,oCAAoC,EAAE,GAAG,EAAE;IAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACtC,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,kDAAkD,EAAE,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,iEAAiE,EAAE,GAAG,EAAE;IAC3E,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,yBAAyB,EAAE,GAAG,EAAE;IACnC,IAAA,gBAAM,EAAC,GAAG,EAAE;QACV,MAAM,GAAG,GAAG,IAAI,qCAAqB,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;AACf,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,+BAA+B,EAAE,GAAG,EAAE;IACzC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG;QACnB,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACpC,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACpC,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;KACrC,CAAC;IACF,IAAA,gBAAM,EAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,6CAA6C,EAAE,GAAG,EAAE;IACvD,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,EAAE;QAC3C,EAAE,MAAM,EAAE,GAAG,EAAE;QACf,EAAE,MAAM,EAAE,GAAG,EAAE;KAChB,CAAC,CAAC;IACH,MAAM,YAAY,GAAG;QACnB,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/D,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/D,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;KAChE,CAAC;IACF,IAAA,gBAAM,EAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,+BAA+B,EAAE,GAAG,EAAE;IACzC,MAAM,IAAI,GAAG;;YAEH,CAAC;IACX,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;QAClD,SAAS,EAAE,EAAE;QACb,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG;QACrB,KAAK;QACL,KAAK;QACL,WAAW;QACX,WAAW;QACX,MAAM;QACN,aAAa;QACb,UAAU;QACV,WAAW;QACX,SAAS;QACT,SAAS;QACT,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,WAAW;QACX,MAAM;QACN,WAAW;QACX,aAAa;KACd,CAAC;IACF,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,qBAAqB,EAAE,GAAG,EAAE;IAC/B,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,iCAAiB,CAAC;QACrC,YAAY,EAAE,WAAW;QACzB,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAE/C,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC"}

package/dist/text_splitter.d.ts

@@ -1,9 +1,10 @@
+import type * as tiktoken from "@dqbd/tiktoken";
 import { Document } from "./document";
 interface TextSplitterParams {
     chunkSize: number;
     chunkOverlap: number;
 }
-declare abstract class TextSplitter implements TextSplitterParams {
+export declare abstract class TextSplitter implements TextSplitterParams {
     chunkSize: number;
     chunkOverlap: number;
     constructor(fields?: Partial<TextSplitterParams>);
@@ -29,4 +30,20 @@ export declare class RecursiveCharacterTextSplitter extends TextSplitter impleme
     constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
     splitText(text: string): string[];
 }
+export interface TokenTextSplitterParams extends TextSplitterParams {
+    encodingName: tiktoken.TiktokenEmbedding;
+    allowedSpecial: "all" | Set<string>;
+    disallowedSpecial: "all" | Array<string>;
+}
+/**
+ * Implementation of splitter which looks at tokens.
+ */
+export declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
+    encodingName: tiktoken.TiktokenEmbedding;
+    allowedSpecial: "all" | Set<string>;
+    disallowedSpecial: "all" | Array<string>;
+    private tokenizer;
+    constructor(fields?: Partial<TokenTextSplitterParams>);
+    splitText(text: string): string[];
+}
 export {};

package/dist/text_splitter.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RecursiveCharacterTextSplitter = exports.CharacterTextSplitter = void 0;
+exports.TokenTextSplitter = exports.RecursiveCharacterTextSplitter = exports.CharacterTextSplitter = exports.TextSplitter = void 0;
 const document_1 = require("./document");
 class TextSplitter {
     constructor(fields) {
@@ -81,6 +81,7 @@ which is longer than the specified ${this.chunkSize}`);
         return docs;
     }
 }
+exports.TextSplitter = TextSplitter;
 class CharacterTextSplitter extends TextSplitter {
     constructor(fields) {
         var _a;
@@ -164,4 +165,72 @@ class RecursiveCharacterTextSplitter extends TextSplitter {
     }
 }
 exports.RecursiveCharacterTextSplitter = RecursiveCharacterTextSplitter;
+/**
+ * Implementation of splitter which looks at tokens.
+ */
+class TokenTextSplitter extends TextSplitter {
+    constructor(fields) {
+        var _a, _b, _c;
+        super(fields);
+        Object.defineProperty(this, "encodingName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "allowedSpecial", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "disallowedSpecial", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.encodingName = (_a = fields === null || fields === void 0 ? void 0 : fields.encodingName) !== null && _a !== void 0 ? _a : "gpt2";
+        this.allowedSpecial = (_b = fields === null || fields === void 0 ? void 0 : fields.allowedSpecial) !== null && _b !== void 0 ? _b : new Set();
+        this.disallowedSpecial = (_c = fields === null || fields === void 0 ? void 0 : fields.disallowedSpecial) !== null && _c !== void 0 ? _c : "all";
+        if ((fields === null || fields === void 0 ? void 0 : fields.allowedSpecial) != null) {
+            throw new Error("allowedSpecial is not implemented yet.");
+        }
+        if ((fields === null || fields === void 0 ? void 0 : fields.disallowedSpecial) != null) {
+            throw new Error("disallowedSpecial is not implemented yet.");
+        }
+        try {
+            const tiktoken = 
+            // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require
+            require("@dqbd/tiktoken");
+            this.tokenizer = tiktoken.get_encoding(this.encodingName);
+        }
+        catch (err) {
+            console.error(err);
+            throw new Error("Please install @dqbd/tiktoken as a dependency with, e.g. `npm install -S @dqbd/tiktoken`");
+        }
+    }
+    splitText(text) {
+        const splits = [];
+        const input_ids = this.tokenizer.encode(text);
+        let start_idx = 0;
+        let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+        let chunk_ids = input_ids.slice(start_idx, cur_idx);
+        const decoder = new TextDecoder();
+        while (start_idx < input_ids.length) {
+            splits.push(decoder.decode(this.tokenizer.decode(chunk_ids)));
+            start_idx += this.chunkSize - this.chunkOverlap;
+            cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+            chunk_ids = input_ids.slice(start_idx, cur_idx);
+        }
+        return splits;
+    }
+}
+exports.TokenTextSplitter = TokenTextSplitter;
 //# sourceMappingURL=text_splitter.js.map

package/dist/text_splitter.js.map

@@ -1 +1 @@
-{"version":3,"file":"text_splitter.js","sourceRoot":"","sources":["../text_splitter.ts"],"names":[],"mappings":";;;AAAA,yCAAsC;AAQtC,MAAe,YAAY;IAKzB,YAAY,MAAoC;;QAJhD;;;;mBAAY,IAAI;WAAC;QAEjB;;;;mBAAe,GAAG;WAAC;QAGjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,IAAI,CAAC,YAAY,CAAC;QAC9D,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;SAC1D;IACH,CAAC;IAID,eAAe,CACb,KAAe;IACf,8DAA8D;IAC9D,YAAmC,EAAE;QAErC,MAAM,UAAU,GACd,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,KAAK,EAAY,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACxC,SAAS,CAAC,IAAI,CACZ,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAC9D,CAAC;aACH;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,cAAc,CAAC,SAAqB;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAChD,CAAC;IAEO,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC;IAED,WAAW,CAAC,MAAgB,EAAE,SAAiB;QAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE;oBAC1B,OAAO,CAAC,IAAI,CACV,2BAA2B,KAAK;qCACP,IAAI,CAAC,SAAS,EAAE,CAC1C,CAAC;iBACH;gBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;oBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBACjD,IAAI,GAAG,KAAK,IAAI,EAAE;wBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAChB;oBACD,sBAAsB;oBACtB,qDAAqD;oBACrD,0DAA0D;oBAC1D,OACE,KAAK,GAAG,IAAI,CAAC,YAAY;wBACzB,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,CAAC,EAC5C;wBACA,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;qBACpB;iBACF;aACF;YACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC;SACf;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAChB;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAMD,MAAa,qBACX,SAAQ,YAAY;IAKpB,YAAY,MAA6C;;QACvD,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAY,MAAM;WAAC;QAIjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;IACvD,CAAC;IAEM,SAAS,CAAC,IAAY;QAC3B,uEAAuE;QACvE,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;CACF;AArBD,sDAqBC;AAOD,MAAa,8BACX,SAAQ,YAAY;IAKpB,YAAY,MAAsD;;QAChE,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAuB,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;WAAC;QAI7C,IAAI,CAAC,UAAU,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,mCAAI,IAAI,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,mCAAmC;QACnC,IAAI,SAAS,GAAW,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,KAAK,EAAE,EAAE;gBACZ,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;gBACpB,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;SACF;QAED,iDAAiD;QACjD,IAAI,MAAgB,CAAC;QACrB,IAAI,SAAS,EAAE;YACb,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SAChC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QAED,6DAA6D;QAC7D,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC7B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACpB;iBAAM;gBACL,IAAI,UAAU,CAAC,MAAM,EAAE;oBACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;oBAChC,UAAU,GAAG,EAAE,CAAC;iBACjB;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;aAChC;SACF;QACD,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;SACjC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAxDD,wEAwDC"}
+{"version":3,"file":"text_splitter.js","sourceRoot":"","sources":["../text_splitter.ts"],"names":[],"mappings":";;;AACA,yCAAsC;AAQtC,MAAsB,YAAY;IAKhC,YAAY,MAAoC;;QAJhD;;;;mBAAY,IAAI;WAAC;QAEjB;;;;mBAAe,GAAG;WAAC;QAGjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,IAAI,CAAC,YAAY,CAAC;QAC9D,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;SAC1D;IACH,CAAC;IAID,eAAe,CACb,KAAe;IACf,8DAA8D;IAC9D,YAAmC,EAAE;QAErC,MAAM,UAAU,GACd,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,KAAK,EAAY,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACxC,SAAS,CAAC,IAAI,CACZ,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAC9D,CAAC;aACH;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,cAAc,CAAC,SAAqB;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAChD,CAAC;IAEO,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC;IAED,WAAW,CAAC,MAAgB,EAAE,SAAiB;QAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE;oBAC1B,OAAO,CAAC,IAAI,CACV,2BAA2B,KAAK;qCACP,IAAI,CAAC,SAAS,EAAE,CAC1C,CAAC;iBACH;gBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;oBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBACjD,IAAI,GAAG,KAAK,IAAI,EAAE;wBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAChB;oBACD,sBAAsB;oBACtB,qDAAqD;oBACrD,0DAA0D;oBAC1D,OACE,KAAK,GAAG,IAAI,CAAC,YAAY;wBACzB,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,CAAC,EAC5C;wBACA,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;qBACpB;iBACF;aACF;YACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC;SACf;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAChB;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AApFD,oCAoFC;AAMD,MAAa,qBACX,SAAQ,YAAY;IAKpB,YAAY,MAA6C;;QACvD,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAY,MAAM;WAAC;QAIjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;IACvD,CAAC;IAEM,SAAS,CAAC,IAAY;QAC3B,uEAAuE;QACvE,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;CACF;AArBD,sDAqBC;AAOD,MAAa,8BACX,SAAQ,YAAY;IAKpB,YAAY,MAAsD;;QAChE,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAuB,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;WAAC;QAI7C,IAAI,CAAC,UAAU,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,mCAAI,IAAI,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,mCAAmC;QACnC,IAAI,SAAS,GAAW,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,KAAK,EAAE,EAAE;gBACZ,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;gBACpB,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;SACF;QAED,iDAAiD;QACjD,IAAI,MAAgB,CAAC;QACrB,IAAI,SAAS,EAAE;YACb,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SAChC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QAED,6DAA6D;QAC7D,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC7B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACpB;iBAAM;gBACL,IAAI,UAAU,CAAC,MAAM,EAAE;oBACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;oBAChC,UAAU,GAAG,EAAE,CAAC;iBACjB;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;aAChC;SACF;QACD,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;SACjC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAxDD,wEAwDC;AAQD;;GAEG;AACH,MAAa,iBACX,SAAQ,YAAY;IAWpB,YAAY,MAAyC;;QACnD,KAAK,CAAC,MAAM,CAAC,CAAC;QAThB;;;;;WAAyC;QAEzC;;;;;WAAoC;QAEpC;;;;;WAAyC;QAEzC;;;;;WAAqC;QAKnC,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,MAAM,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,cAAc,mCAAI,IAAI,GAAG,EAAE,CAAC;QAC1D,IAAI,CAAC,iBAAiB,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,mCAAI,KAAK,CAAC;QAE5D,IAAI,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,cAAc,KAAI,IAAI,EAAE;YAClC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QAED,IAAI,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,KAAI,IAAI,EAAE;YACrC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;QAED,IAAI;YACF,MAAM,QAAQ;YACZ,8EAA8E;YAC9E,OAAO,CAAC,gBAAgB,CAAoC,CAAC;YAC/D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SAC3D;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,0FAA0F,CAC3F,CAAC;SACH;IACH,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEpD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAElC,OAAO,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE;YACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAE9D,SAAS,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;YAChD,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACjE,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACjD;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA7DD,8CA6DC"}

package/dist/util/hub.js

@@ -12,6 +12,7 @@ const index_1 = require("./index");
 const HUB_PATH_REGEX = /lc(@[^:]+)?:\/\/(.*)/;
 const DEFAULT_REF = (_a = process.env.LANGCHAIN_HUB_DEFAULT_REF) !== null && _a !== void 0 ? _a : "master";
 const URL_BASE = (_b = process.env.LANGCHAIN_HUB_URL_BASE) !== null && _b !== void 0 ? _b : "https://raw.githubusercontent.com/hwchase17/langchain-hub/";
+const URL_PATH_SEPARATOR = "/";
 const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {}) => {
     const match = uri.match(HUB_PATH_REGEX);
     if (!match) {
@@ -19,7 +20,7 @@ const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {})
     }
     const [rawRef, remotePath] = match.slice(1);
     const ref = rawRef ? rawRef.slice(1) : DEFAULT_REF;
-    const parts = remotePath.split(path_1.default.sep);
+    const parts = remotePath.split(URL_PATH_SEPARATOR);
     if (parts[0] !== validPrefix) {
         return undefined;
     }
@@ -33,7 +34,7 @@ const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {})
     }
     const text = await res.text();
     const tmpdir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), "langchain"));
-    const file = path_1.default.join(tmpdir, path_1.default.basename(remotePath));
+    const file = path_1.default.join(tmpdir, path_1.default.basename(remotePath.replace(URL_PATH_SEPARATOR, path_1.default.sep)));
     fs_1.default.writeFileSync(file, text);
     return loader(file, values);
 };

package/dist/util/hub.js.map

@@ -1 +1 @@
-{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../util/hub.ts"],"names":[],"mappings":";;;;;;;AAAA,gDAAwB;AACxB,4CAAoB;AACpB,4CAAoB;AACpB,mCAA2C;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAIxD,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,MAA4C,EAC5C,WAAmB,EACnB,aAA0B,EAC1B,SAAqB,EAAE,EACC,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,cAAI,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,YAAE,CAAC,WAAW,CAAC,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1D,YAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC,CAAC;AAjCW,QAAA,WAAW,eAiCtB"}
+{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../util/hub.ts"],"names":[],"mappings":";;;;;;;AAAA,gDAAwB;AACxB,4CAAoB;AACpB,4CAAoB;AACpB,mCAA2C;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAI/D,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAExB,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,MAA4C,EAC5C,WAAmB,EACnB,aAA0B,EAC1B,SAAqB,EAAE,EACC,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACnD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,YAAE,CAAC,WAAW,CAAC,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CACpB,MAAM,EACN,cAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAI,CAAC,GAAG,CAAC,CAAC,CAChE,CAAC;IACF,YAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC,CAAC;AApCW,QAAA,WAAW,eAoCtB"}

package/dist/vectorstores/base.d.ts

@@ -1,14 +1,10 @@
 import { Embeddings } from "../embeddings/base";
 import { Document } from "../document";
-export interface DocStore {
-    [key: number]: object;
-}
 export declare abstract class VectorStore {
     embeddings: Embeddings;
-    docstore: DocStore;
+    constructor(embeddings: Embeddings);
     abstract addVectors(vectors: number[][], documents: Document[]): Promise<void>;
     abstract similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
-    addDocuments(documents: Document[]): Promise<void>;
     similaritySearch(query: string, k?: number): Promise<Document[]>;
     similaritySearchWithScore(query: string, k?: number): Promise<[object, number][]>;
 }

package/dist/vectorstores/base.js

@@ -2,23 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SaveableVectorStore = exports.VectorStore = void 0;
 class VectorStore {
-    constructor() {
+    constructor(embeddings) {
         Object.defineProperty(this, "embeddings", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "docstore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-    }
-    async addDocuments(documents) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+        this.embeddings = embeddings;
     }
     async similaritySearch(query, k = 4) {
         const results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k);

package/dist/vectorstores/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../../vectorstores/base.ts"],"names":[],"mappings":";;;AAQA,MAAsB,WAAW;IAAjC;QACE;;;;;WAAuB;QAEvB;;;;;WAAmB;IAmCrB,CAAC;IAvBC,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,KAAa,EAAE,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,CACxD,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC7B,KAAa,EACb,CAAC,GAAG,CAAC;QAEL,OAAO,IAAI,CAAC,+BAA+B,CACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAtCD,kCAsCC;AAED,MAAsB,mBAAoB,SAAQ,WAAW;IAG3D,MAAM,CAAC,IAAI,CACT,UAAkB,EAClB,WAAuB;QAEvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AATD,kDASC"}
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../vectorstores/base.ts"],"names":[],"mappings":";;;AAGA,MAAsB,WAAW;IAG/B,YAAY,UAAsB;QAFlC;;;;;WAAuB;QAGrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAYD,KAAK,CAAC,gBAAgB,CAAC,KAAa,EAAE,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,CACxD,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC7B,KAAa,EACb,CAAC,GAAG,CAAC;QAEL,OAAO,IAAI,CAAC,+BAA+B,CACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAnCD,kCAmCC;AAED,MAAsB,mBAAoB,SAAQ,WAAW;IAG3D,MAAM,CAAC,IAAI,CACT,UAAkB,EAClB,WAAuB;QAEvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AATD,kDASC"}

package/dist/vectorstores/hnswlib.d.ts

@@ -1,19 +1,22 @@
 import type { HierarchicalNSW as HierarchicalNSWT, SpaceName } from "hnswlib-node";
 import { Embeddings } from "../embeddings/base";
-import { DocStore, SaveableVectorStore } from "./base";
+import { SaveableVectorStore } from "./base";
 import { Document } from "../document";
+import { InMemoryDocstore } from "../docstore";
 export interface HNSWLibArgs {
     space: SpaceName;
     numDimensions?: number;
 }
 export declare class HNSWLib extends SaveableVectorStore {
     index?: HierarchicalNSWT;
+    docstore: InMemoryDocstore;
     args: HNSWLibArgs;
-    constructor(args: HNSWLibArgs, embeddings: Embeddings, docstore: DocStore, index?: HierarchicalNSWT);
+    constructor(args: HNSWLibArgs, embeddings: Embeddings, docstore: InMemoryDocstore, index?: HierarchicalNSWT);
+    addDocuments(documents: Document[]): Promise<void>;
     addVectors(vectors: number[][], documents: Document[]): Promise<void>;
     similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
     save(directory: string): Promise<void>;
     static load(directory: string, embeddings: Embeddings): Promise<HNSWLib>;
-    static fromTexts(texts: string[], metadatas: object[], embeddings: Embeddings): Promise<HNSWLib>;
-    static fromDocuments(docs: Document[], embeddings: Embeddings): Promise<HNSWLib>;
+    static fromTexts(texts: string[], metadatas: object[], embeddings: Embeddings, docstore?: InMemoryDocstore): Promise<HNSWLib>;
+    static fromDocuments(docs: Document[], embeddings: Embeddings, docstore?: InMemoryDocstore): Promise<HNSWLib>;
 }