langchain 0.0.7 → 0.0.8

package/dist/tests/text_splitter.test.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+const document_1 = require("../document");
+const text_splitter_1 = require("../text_splitter");
+(0, globals_1.test)("Test splitting by character count.", () => {
+    const text = "foo bar baz 123";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 7,
+        chunkOverlap: 3,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo bar", "bar baz", "baz 123"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count doesn't create empty documents.", () => {
+    const text = "foo  bar";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 2,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count on long words.", () => {
+    const text = "foo bar baz a a";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar", "baz", "a a"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by character count when shorter words are first.", () => {
+    const text = "a a foo bar baz";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["a a", "foo", "bar", "baz"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test splitting by characters when splits not found easily.", () => {
+    const text = "foo bar baz 123";
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 1,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo", "bar", "baz", "123"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Test invalid arguments.", () => {
+    (0, globals_1.expect)(() => {
+        const res = new text_splitter_1.CharacterTextSplitter({ chunkSize: 2, chunkOverlap: 4 });
+        console.log(res);
+    }).toThrow();
+});
+(0, globals_1.test)("Test create documents method.", () => {
+    const texts = ["foo bar", "baz"];
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const docs = splitter.createDocuments(texts);
+    const expectedDocs = [
+        new document_1.Document({ pageContent: "foo" }),
+        new document_1.Document({ pageContent: "bar" }),
+        new document_1.Document({ pageContent: "baz" }),
+    ];
+    (0, globals_1.expect)(docs).toEqual(expectedDocs);
+});
+(0, globals_1.test)("Test create documents with metadata method.", () => {
+    const texts = ["foo bar", "baz"];
+    const splitter = new text_splitter_1.CharacterTextSplitter({
+        separator: " ",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const docs = splitter.createDocuments(texts, [
+        { source: "1" },
+        { source: "2" },
+    ]);
+    const expectedDocs = [
+        new document_1.Document({ pageContent: "foo", metadata: { source: "1" } }),
+        new document_1.Document({ pageContent: "bar", metadata: { source: "1" } }),
+        new document_1.Document({ pageContent: "baz", metadata: { source: "2" } }),
+    ];
+    (0, globals_1.expect)(docs).toEqual(expectedDocs);
+});
+(0, globals_1.test)("Test iterative text splitter.", () => {
+    const text = `Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.
+This is a weird text to write, but gotta test the splittingggg some how.\n\n
+Bye!\n\n-H.`;
+    const splitter = new text_splitter_1.RecursiveCharacterTextSplitter({
+        chunkSize: 10,
+        chunkOverlap: 1,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = [
+        "Hi.",
+        "I'm",
+        "Harrison.",
+        "How? Are?",
+        "You?",
+        "Okay then f",
+        "f f f f.",
+        "This is a",
+        "a weird",
+        "text to",
+        "write, but",
+        "gotta test",
+        "the",
+        "splitting",
+        "gggg",
+        "some how.",
+        "Bye!\n\n-H.",
+    ];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+(0, globals_1.test)("Token text splitter", () => {
+    const text = "foo bar baz a a";
+    const splitter = new text_splitter_1.TokenTextSplitter({
+        encodingName: "r50k_base",
+        chunkSize: 3,
+        chunkOverlap: 0,
+    });
+    const output = splitter.splitText(text);
+    const expectedOutput = ["foo bar b", "az a a"];
+    (0, globals_1.expect)(output).toEqual(expectedOutput);
+});
+//# sourceMappingURL=text_splitter.test.js.map

package/dist/tests/text_splitter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text_splitter.test.js","sourceRoot":"","sources":["../../tests/text_splitter.test.ts"],"names":[],"mappings":";;AAAA,2CAA6C;AAC7C,0CAAuC;AACvC,oDAI0B;AAE1B,IAAA,cAAI,EAAC,oCAAoC,EAAE,GAAG,EAAE;IAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,mEAAmE,EAAE,GAAG,EAAE;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACtC,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,kDAAkD,EAAE,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,iEAAiE,EAAE,GAAG,EAAE;IAC3E,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,yBAAyB,EAAE,GAAG,EAAE;IACnC,IAAA,gBAAM,EAAC,GAAG,EAAE;QACV,MAAM,GAAG,GAAG,IAAI,qCAAqB,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;AACf,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,+BAA+B,EAAE,GAAG,EAAE;IACzC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG;QACnB,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACpC,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACpC,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;KACrC,CAAC;IACF,IAAA,gBAAM,EAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,6CAA6C,EAAE,GAAG,EAAE;IACvD,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qCAAqB,CAAC;QACzC,SAAS,EAAE,GAAG;QACd,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,EAAE;QAC3C,EAAE,MAAM,EAAE,GAAG,EAAE;QACf,EAAE,MAAM,EAAE,GAAG,EAAE;KAChB,CAAC,CAAC;IACH,MAAM,YAAY,GAAG;QACnB,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/D,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;QAC/D,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;KAChE,CAAC;IACF,IAAA,gBAAM,EAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,+BAA+B,EAAE,GAAG,EAAE;IACzC,MAAM,IAAI,GAAG;;YAEH,CAAC;IACX,MAAM,QAAQ,GAAG,IAAI,8CAA8B,CAAC;QAClD,SAAS,EAAE,EAAE;QACb,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG;QACrB,KAAK;QACL,KAAK;QACL,WAAW;QACX,WAAW;QACX,MAAM;QACN,aAAa;QACb,UAAU;QACV,WAAW;QACX,SAAS;QACT,SAAS;QACT,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,WAAW;QACX,MAAM;QACN,WAAW;QACX,aAAa;KACd,CAAC;IACF,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,qBAAqB,EAAE,GAAG,EAAE;IAC/B,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,iCAAiB,CAAC;QACrC,YAAY,EAAE,WAAW;QACzB,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAE/C,IAAA,gBAAM,EAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC"}

package/dist/text_splitter.d.ts

@@ -1,3 +1,4 @@
+import type * as tiktoken from "@dqbd/tiktoken";
 import { Document } from "./document";
 interface TextSplitterParams {
     chunkSize: number;
@@ -29,4 +30,20 @@ export declare class RecursiveCharacterTextSplitter extends TextSplitter impleme
     constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>);
     splitText(text: string): string[];
 }
+export interface TokenTextSplitterParams extends TextSplitterParams {
+    encodingName: tiktoken.TiktokenEmbedding;
+    allowedSpecial: "all" | Set<string>;
+    disallowedSpecial: "all" | Array<string>;
+}
+/**
+ * Implementation of splitter which looks at tokens.
+ */
+export declare class TokenTextSplitter extends TextSplitter implements TokenTextSplitterParams {
+    encodingName: tiktoken.TiktokenEmbedding;
+    allowedSpecial: "all" | Set<string>;
+    disallowedSpecial: "all" | Array<string>;
+    private tokenizer;
+    constructor(fields?: Partial<TokenTextSplitterParams>);
+    splitText(text: string): string[];
+}
 export {};

package/dist/text_splitter.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.RecursiveCharacterTextSplitter = exports.CharacterTextSplitter = void 0;
+exports.TokenTextSplitter = exports.RecursiveCharacterTextSplitter = exports.CharacterTextSplitter = void 0;
 const document_1 = require("./document");
 class TextSplitter {
     constructor(fields) {
@@ -164,4 +164,72 @@ class RecursiveCharacterTextSplitter extends TextSplitter {
     }
 }
 exports.RecursiveCharacterTextSplitter = RecursiveCharacterTextSplitter;
+/**
+ * Implementation of splitter which looks at tokens.
+ */
+class TokenTextSplitter extends TextSplitter {
+    constructor(fields) {
+        var _a, _b, _c;
+        super(fields);
+        Object.defineProperty(this, "encodingName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "allowedSpecial", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "disallowedSpecial", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.encodingName = (_a = fields === null || fields === void 0 ? void 0 : fields.encodingName) !== null && _a !== void 0 ? _a : "gpt2";
+        this.allowedSpecial = (_b = fields === null || fields === void 0 ? void 0 : fields.allowedSpecial) !== null && _b !== void 0 ? _b : new Set();
+        this.disallowedSpecial = (_c = fields === null || fields === void 0 ? void 0 : fields.disallowedSpecial) !== null && _c !== void 0 ? _c : "all";
+        if ((fields === null || fields === void 0 ? void 0 : fields.allowedSpecial) != null) {
+            throw new Error("allowedSpecial is not implemented yet.");
+        }
+        if ((fields === null || fields === void 0 ? void 0 : fields.disallowedSpecial) != null) {
+            throw new Error("disallowedSpecial is not implemented yet.");
+        }
+        try {
+            const tiktoken = 
+            // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require
+            require("@dqbd/tiktoken");
+            this.tokenizer = tiktoken.get_encoding(this.encodingName);
+        }
+        catch (err) {
+            console.error(err);
+            throw new Error("Please install @dqbd/tiktoken as a dependency with, e.g. `npm install -S @dqbd/tiktoken`");
+        }
+    }
+    splitText(text) {
+        const splits = [];
+        const input_ids = this.tokenizer.encode(text);
+        let start_idx = 0;
+        let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+        let chunk_ids = input_ids.slice(start_idx, cur_idx);
+        const decoder = new TextDecoder();
+        while (start_idx < input_ids.length) {
+            splits.push(decoder.decode(this.tokenizer.decode(chunk_ids)));
+            start_idx += this.chunkSize - this.chunkOverlap;
+            cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
+            chunk_ids = input_ids.slice(start_idx, cur_idx);
+        }
+        return splits;
+    }
+}
+exports.TokenTextSplitter = TokenTextSplitter;
 //# sourceMappingURL=text_splitter.js.map

package/dist/text_splitter.js.map

@@ -1 +1 @@
-{"version":3,"file":"text_splitter.js","sourceRoot":"","sources":["../text_splitter.ts"],"names":[],"mappings":";;;AAAA,yCAAsC;AAQtC,MAAe,YAAY;IAKzB,YAAY,MAAoC;;QAJhD;;;;mBAAY,IAAI;WAAC;QAEjB;;;;mBAAe,GAAG;WAAC;QAGjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,IAAI,CAAC,YAAY,CAAC;QAC9D,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;SAC1D;IACH,CAAC;IAID,eAAe,CACb,KAAe;IACf,8DAA8D;IAC9D,YAAmC,EAAE;QAErC,MAAM,UAAU,GACd,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,KAAK,EAAY,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACxC,SAAS,CAAC,IAAI,CACZ,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAC9D,CAAC;aACH;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,cAAc,CAAC,SAAqB;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAChD,CAAC;IAEO,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC;IAED,WAAW,CAAC,MAAgB,EAAE,SAAiB;QAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE;oBAC1B,OAAO,CAAC,IAAI,CACV,2BAA2B,KAAK;qCACP,IAAI,CAAC,SAAS,EAAE,CAC1C,CAAC;iBACH;gBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;oBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBACjD,IAAI,GAAG,KAAK,IAAI,EAAE;wBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAChB;oBACD,sBAAsB;oBACtB,qDAAqD;oBACrD,0DAA0D;oBAC1D,OACE,KAAK,GAAG,IAAI,CAAC,YAAY;wBACzB,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,CAAC,EAC5C;wBACA,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;qBACpB;iBACF;aACF;YACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC;SACf;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAChB;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAMD,MAAa,qBACX,SAAQ,YAAY;IAKpB,YAAY,MAA6C;;QACvD,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAY,MAAM;WAAC;QAIjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;IACvD,CAAC;IAEM,SAAS,CAAC,IAAY;QAC3B,uEAAuE;QACvE,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;CACF;AArBD,sDAqBC;AAOD,MAAa,8BACX,SAAQ,YAAY;IAKpB,YAAY,MAAsD;;QAChE,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAuB,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;WAAC;QAI7C,IAAI,CAAC,UAAU,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,mCAAI,IAAI,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,mCAAmC;QACnC,IAAI,SAAS,GAAW,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,KAAK,EAAE,EAAE;gBACZ,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;gBACpB,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;SACF;QAED,iDAAiD;QACjD,IAAI,MAAgB,CAAC;QACrB,IAAI,SAAS,EAAE;YACb,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SAChC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QAED,6DAA6D;QAC7D,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC7B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACpB;iBAAM;gBACL,IAAI,UAAU,CAAC,MAAM,EAAE;oBACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;oBAChC,UAAU,GAAG,EAAE,CAAC;iBACjB;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;aAChC;SACF;QACD,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;SACjC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAxDD,wEAwDC"}
+{"version":3,"file":"text_splitter.js","sourceRoot":"","sources":["../text_splitter.ts"],"names":[],"mappings":";;;AACA,yCAAsC;AAQtC,MAAe,YAAY;IAKzB,YAAY,MAAoC;;QAJhD;;;;mBAAY,IAAI;WAAC;QAEjB;;;;mBAAe,GAAG;WAAC;QAGjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;QACrD,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,IAAI,CAAC,YAAY,CAAC;QAC9D,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;SAC1D;IACH,CAAC;IAID,eAAe,CACb,KAAe;IACf,8DAA8D;IAC9D,YAAmC,EAAE;QAErC,MAAM,UAAU,GACd,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtE,MAAM,SAAS,GAAG,IAAI,KAAK,EAAY,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBACxC,SAAS,CAAC,IAAI,CACZ,IAAI,mBAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAC9D,CAAC;aACH;SACF;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,cAAc,CAAC,SAAqB;QAClC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvD,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAChD,CAAC;IAEO,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACnC,CAAC;IAED,WAAW,CAAC,MAAgB,EAAE,SAAiB;QAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE;gBAClC,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE;oBAC1B,OAAO,CAAC,IAAI,CACV,2BAA2B,KAAK;qCACP,IAAI,CAAC,SAAS,EAAE,CAC1C,CAAC;iBACH;gBACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;oBACzB,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBACjD,IAAI,GAAG,KAAK,IAAI,EAAE;wBAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;qBAChB;oBACD,sBAAsB;oBACtB,qDAAqD;oBACrD,0DAA0D;oBAC1D,OACE,KAAK,GAAG,IAAI,CAAC,YAAY;wBACzB,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,CAAC,EAC5C;wBACA,KAAK,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;wBAC9B,UAAU,CAAC,KAAK,EAAE,CAAC;qBACpB;iBACF;aACF;YACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACnB,KAAK,IAAI,IAAI,CAAC;SACf;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjD,IAAI,GAAG,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;SAChB;QACD,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAMD,MAAa,qBACX,SAAQ,YAAY;IAKpB,YAAY,MAA6C;;QACvD,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAY,MAAM;WAAC;QAIjB,IAAI,CAAC,SAAS,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,SAAS,mCAAI,IAAI,CAAC,SAAS,CAAC;IACvD,CAAC;IAEM,SAAS,CAAC,IAAY;QAC3B,uEAAuE;QACvE,IAAI,MAAgB,CAAC;QACrB,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QACD,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAClD,CAAC;CACF;AArBD,sDAqBC;AAOD,MAAa,8BACX,SAAQ,YAAY;IAKpB,YAAY,MAAsD;;QAChE,KAAK,CAAC,MAAM,CAAC,CAAC;QAHhB;;;;mBAAuB,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;WAAC;QAI7C,IAAI,CAAC,UAAU,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,UAAU,mCAAI,IAAI,CAAC,UAAU,CAAC;IAC1D,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,mCAAmC;QACnC,IAAI,SAAS,GAAW,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE;YAC/B,IAAI,CAAC,KAAK,EAAE,EAAE;gBACZ,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;YACD,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;gBACpB,SAAS,GAAG,CAAC,CAAC;gBACd,MAAM;aACP;SACF;QAED,iDAAiD;QACjD,IAAI,MAAgB,CAAC;QACrB,IAAI,SAAS,EAAE;YACb,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;SAChC;aAAM;YACL,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACzB;QAED,6DAA6D;QAC7D,IAAI,UAAU,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE;YACtB,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE;gBAC7B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACpB;iBAAM;gBACL,IAAI,UAAU,CAAC,MAAM,EAAE;oBACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;oBAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;oBAChC,UAAU,GAAG,EAAE,CAAC;iBACjB;gBACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;aAChC;SACF;QACD,IAAI,UAAU,CAAC,MAAM,EAAE;YACrB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC3D,WAAW,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC;SACjC;QACD,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAxDD,wEAwDC;AAQD;;GAEG;AACH,MAAa,iBACX,SAAQ,YAAY;IAWpB,YAAY,MAAyC;;QACnD,KAAK,CAAC,MAAM,CAAC,CAAC;QAThB;;;;;WAAyC;QAEzC;;;;;WAAoC;QAEpC;;;;;WAAyC;QAEzC;;;;;WAAqC;QAKnC,IAAI,CAAC,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,YAAY,mCAAI,MAAM,CAAC;QACnD,IAAI,CAAC,cAAc,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,cAAc,mCAAI,IAAI,GAAG,EAAE,CAAC;QAC1D,IAAI,CAAC,iBAAiB,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,mCAAI,KAAK,CAAC;QAE5D,IAAI,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,cAAc,KAAI,IAAI,EAAE;YAClC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;SAC3D;QAED,IAAI,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,KAAI,IAAI,EAAE;YACrC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;SAC9D;QAED,IAAI;YACF,MAAM,QAAQ;YACZ,8EAA8E;YAC9E,OAAO,CAAC,gBAAgB,CAAoC,CAAC;YAC/D,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;SAC3D;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,IAAI,KAAK,CACb,0FAA0F,CAC3F,CAAC;SACH;IACH,CAAC;IAED,SAAS,CAAC,IAAY;QACpB,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE9C,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QACrE,IAAI,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAEpD,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;QAElC,OAAO,SAAS,GAAG,SAAS,CAAC,MAAM,EAAE;YACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YAE9D,SAAS,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;YAChD,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACjE,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;SACjD;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AA7DD,8CA6DC"}

package/dist/util/hub.js

@@ -12,6 +12,7 @@ const index_1 = require("./index");
 const HUB_PATH_REGEX = /lc(@[^:]+)?:\/\/(.*)/;
 const DEFAULT_REF = (_a = process.env.LANGCHAIN_HUB_DEFAULT_REF) !== null && _a !== void 0 ? _a : "master";
 const URL_BASE = (_b = process.env.LANGCHAIN_HUB_URL_BASE) !== null && _b !== void 0 ? _b : "https://raw.githubusercontent.com/hwchase17/langchain-hub/";
+const URL_PATH_SEPARATOR = "/";
 const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {}) => {
     const match = uri.match(HUB_PATH_REGEX);
     if (!match) {
@@ -19,7 +20,7 @@ const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {})
     }
     const [rawRef, remotePath] = match.slice(1);
     const ref = rawRef ? rawRef.slice(1) : DEFAULT_REF;
-    const parts = remotePath.split(path_1.default.sep);
+    const parts = remotePath.split(URL_PATH_SEPARATOR);
     if (parts[0] !== validPrefix) {
         return undefined;
     }
@@ -33,7 +34,7 @@ const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {})
     }
     const text = await res.text();
     const tmpdir = fs_1.default.mkdtempSync(path_1.default.join(os_1.default.tmpdir(), "langchain"));
-    const file = path_1.default.join(tmpdir, path_1.default.basename(remotePath));
+    const file = path_1.default.join(tmpdir, path_1.default.basename(remotePath.replace(URL_PATH_SEPARATOR, path_1.default.sep)));
     fs_1.default.writeFileSync(file, text);
     return loader(file, values);
 };

package/dist/util/hub.js.map

@@ -1 +1 @@
-{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../util/hub.ts"],"names":[],"mappings":";;;;;;;AAAA,gDAAwB;AACxB,4CAAoB;AACpB,4CAAoB;AACpB,mCAA2C;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAIxD,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,MAA4C,EAC5C,WAAmB,EACnB,aAA0B,EAC1B,SAAqB,EAAE,EACC,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,cAAI,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,YAAE,CAAC,WAAW,CAAC,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1D,YAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC,CAAC;AAjCW,QAAA,WAAW,eAiCtB"}
+{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../util/hub.ts"],"names":[],"mappings":";;;;;;;AAAA,gDAAwB;AACxB,4CAAoB;AACpB,4CAAoB;AACpB,mCAA2C;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAI/D,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAExB,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,MAA4C,EAC5C,WAAmB,EACnB,aAA0B,EAC1B,SAAqB,EAAE,EACC,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACnD,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,cAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,YAAE,CAAC,WAAW,CAAC,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,cAAI,CAAC,IAAI,CACpB,MAAM,EACN,cAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,kBAAkB,EAAE,cAAI,CAAC,GAAG,CAAC,CAAC,CAChE,CAAC;IACF,YAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC,CAAC;AApCW,QAAA,WAAW,eAoCtB"}

package/dist/vectorstores/base.d.ts

@@ -1,14 +1,10 @@
 import { Embeddings } from "../embeddings/base";
 import { Document } from "../document";
-export interface DocStore {
-    [key: number]: object;
-}
 export declare abstract class VectorStore {
     embeddings: Embeddings;
-    docstore: DocStore;
+    constructor(embeddings: Embeddings);
     abstract addVectors(vectors: number[][], documents: Document[]): Promise<void>;
     abstract similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
-    addDocuments(documents: Document[]): Promise<void>;
     similaritySearch(query: string, k?: number): Promise<Document[]>;
     similaritySearchWithScore(query: string, k?: number): Promise<[object, number][]>;
 }

package/dist/vectorstores/base.js

@@ -2,23 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SaveableVectorStore = exports.VectorStore = void 0;
 class VectorStore {
-    constructor() {
+    constructor(embeddings) {
         Object.defineProperty(this, "embeddings", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "docstore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-    }
-    async addDocuments(documents) {
-        const texts = documents.map(({ pageContent }) => pageContent);
-        this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+        this.embeddings = embeddings;
     }
     async similaritySearch(query, k = 4) {
         const results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k);

package/dist/vectorstores/base.js.map

@@ -1 +1 @@
-{"version":3,"file":"base.js","sourceRoot":"","sources":["../../vectorstores/base.ts"],"names":[],"mappings":";;;AAQA,MAAsB,WAAW;IAAjC;QACE;;;;;WAAuB;QAEvB;;;;;WAAmB;IAmCrB,CAAC;IAvBC,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1E,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,KAAa,EAAE,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,CACxD,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC7B,KAAa,EACb,CAAC,GAAG,CAAC;QAEL,OAAO,IAAI,CAAC,+BAA+B,CACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAtCD,kCAsCC;AAED,MAAsB,mBAAoB,SAAQ,WAAW;IAG3D,MAAM,CAAC,IAAI,CACT,UAAkB,EAClB,WAAuB;QAEvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AATD,kDASC"}
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../vectorstores/base.ts"],"names":[],"mappings":";;;AAGA,MAAsB,WAAW;IAG/B,YAAY,UAAsB;QAFlC;;;;;WAAuB;QAGrB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAYD,KAAK,CAAC,gBAAgB,CAAC,KAAa,EAAE,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,CACxD,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC7B,KAAa,EACb,CAAC,GAAG,CAAC;QAEL,OAAO,IAAI,CAAC,+BAA+B,CACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAnCD,kCAmCC;AAED,MAAsB,mBAAoB,SAAQ,WAAW;IAG3D,MAAM,CAAC,IAAI,CACT,UAAkB,EAClB,WAAuB;QAEvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF;AATD,kDASC"}

package/dist/vectorstores/hnswlib.d.ts

@@ -1,19 +1,22 @@
 import type { HierarchicalNSW as HierarchicalNSWT, SpaceName } from "hnswlib-node";
 import { Embeddings } from "../embeddings/base";
-import { DocStore, SaveableVectorStore } from "./base";
+import { SaveableVectorStore } from "./base";
 import { Document } from "../document";
+import { InMemoryDocstore } from "../docstore";
 export interface HNSWLibArgs {
     space: SpaceName;
     numDimensions?: number;
 }
 export declare class HNSWLib extends SaveableVectorStore {
     index?: HierarchicalNSWT;
+    docstore: InMemoryDocstore;
     args: HNSWLibArgs;
-    constructor(args: HNSWLibArgs, embeddings: Embeddings, docstore: DocStore, index?: HierarchicalNSWT);
+    constructor(args: HNSWLibArgs, embeddings: Embeddings, docstore: InMemoryDocstore, index?: HierarchicalNSWT);
+    addDocuments(documents: Document[]): Promise<void>;
     addVectors(vectors: number[][], documents: Document[]): Promise<void>;
     similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
     save(directory: string): Promise<void>;
     static load(directory: string, embeddings: Embeddings): Promise<HNSWLib>;
-    static fromTexts(texts: string[], metadatas: object[], embeddings: Embeddings): Promise<HNSWLib>;
-    static fromDocuments(docs: Document[], embeddings: Embeddings): Promise<HNSWLib>;
+    static fromTexts(texts: string[], metadatas: object[], embeddings: Embeddings, docstore?: InMemoryDocstore): Promise<HNSWLib>;
+    static fromDocuments(docs: Document[], embeddings: Embeddings, docstore?: InMemoryDocstore): Promise<HNSWLib>;
 }

package/dist/vectorstores/hnswlib.js

@@ -8,6 +8,7 @@ const promises_1 = __importDefault(require("fs/promises"));
 const path_1 = __importDefault(require("path"));
 const base_1 = require("./base");
 const document_1 = require("../document");
+const docstore_1 = require("../docstore");
 let HierarchicalNSW = null;
 try {
     // eslint-disable-next-line global-require,import/no-extraneous-dependencies
@@ -18,13 +19,19 @@ catch (_a) {
 }
 class HNSWLib extends base_1.SaveableVectorStore {
     constructor(args, embeddings, docstore, index) {
-        super();
+        super(embeddings);
         Object.defineProperty(this, "index", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "docstore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "args", {
             enumerable: true,
             configurable: true,
@@ -36,6 +43,10 @@ class HNSWLib extends base_1.SaveableVectorStore {
         this.embeddings = embeddings;
         this.docstore = docstore;
     }
+    async addDocuments(documents) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+    }
     async addVectors(vectors, documents) {
         if (vectors.length === 0) {
             return;
@@ -63,11 +74,12 @@ class HNSWLib extends base_1.SaveableVectorStore {
         const capacity = this.index.getMaxElements();
         const needed = this.index.getCurrentCount() + vectors.length;
         if (needed > capacity) {
-            this.index.resizeIndex(needed - capacity);
+            this.index.resizeIndex(needed);
         }
+        const docstoreSize = this.docstore.count;
         for (let i = 0; i < vectors.length; i += 1) {
-            this.index.addPoint(vectors[i], i);
-            this.docstore[i] = documents[i];
+            this.index.addPoint(vectors[i], docstoreSize + i);
+            this.docstore.add({ [docstoreSize + i]: documents[i] });
         }
     }
     async similaritySearchVectorWithScore(query, k) {
@@ -75,7 +87,10 @@ class HNSWLib extends base_1.SaveableVectorStore {
             throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
         }
         const result = this.index.searchKnn(query, k);
-        return result.neighbors.map((docIndex, resultIndex) => [this.docstore[docIndex], result.distances[resultIndex]]);
+        return result.neighbors.map((docIndex, resultIndex) => [
+            this.docstore.search(String(docIndex)),
+            result.distances[resultIndex],
+        ]);
     }
     async save(directory) {
         if (!this.index) {
@@ -102,7 +117,7 @@ class HNSWLib extends base_1.SaveableVectorStore {
         ]);
         return new HNSWLib(args, embeddings, docstore, index);
     }
-    static async fromTexts(texts, metadatas, embeddings) {
+    static async fromTexts(texts, metadatas, embeddings, docstore = new docstore_1.InMemoryDocstore()) {
         const docs = [];
         for (let i = 0; i < texts.length; i += 1) {
             const newDoc = new document_1.Document({
@@ -111,16 +126,16 @@ class HNSWLib extends base_1.SaveableVectorStore {
             });
             docs.push(newDoc);
         }
-        return HNSWLib.fromDocuments(docs, embeddings);
+        return HNSWLib.fromDocuments(docs, embeddings, docstore);
     }
-    static async fromDocuments(docs, embeddings) {
+    static async fromDocuments(docs, embeddings, docstore = new docstore_1.InMemoryDocstore()) {
         if (HierarchicalNSW === null) {
             throw new Error("Please install hnswlib-node as a dependency with, e.g. `npm install -S hnswlib-node`");
         }
         const args = {
             space: "ip", // dot product
         };
-        const instance = new this(args, embeddings, {});
+        const instance = new this(args, embeddings, docstore);
         await instance.addDocuments(docs);
         return instance;
     }

package/dist/vectorstores/hnswlib.js.map

@@ -1 +1 @@
-{"version":3,"file":"hnswlib.js","sourceRoot":"","sources":["../../vectorstores/hnswlib.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA6B;AAC7B,gDAAwB;AAQxB,iCAAuD;AACvD,0CAAuC;AAEvC,IAAI,eAAe,GAAmC,IAAI,CAAC;AAE3D,IAAI;IACF,4EAA4E;IAC5E,CAAC,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;CACjD;AAAC,WAAM;IACN,eAAe;CAChB;AAOD,MAAa,OAAQ,SAAQ,0BAAmB;IAK9C,YACE,IAAiB,EACjB,UAAsB,EACtB,QAAkB,EAClB,KAAwB;QAExB,KAAK,EAAE,CAAC;QAVV;;;;;WAAyB;QAEzB;;;;;WAAkB;QAShB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAmB,EAAE,SAAqB;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE;gBACzC,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;aAC7C;YACD,IAAI,eAAe,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;aACH;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,aAAa,CACxB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACtC;QACD,qEAAqE;QACrE,oEAAoE;QACpE,uEAAuE;QACvE,+DAA+D;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACjD,MAAM,IAAI,KAAK,CACb,kEAAkE,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAC7F,CAAC;SACH;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,MAAM,GAAG,QAAQ,EAAE;YACrB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;SAC3C;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YAC1C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;SACjC;IACH,CAAC;IAED,KAAK,CAAC,+BAA+B,CAAC,KAAe,EAAE,CAAS;QAC9D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CACzB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,CACxB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAGtD,CACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,kBAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC5D,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1B;YACD,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9B;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,UAAsB;QACzD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,MAAM,kBAAE,CAAC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAC7D,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACnC,kBAAE;iBACC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC;iBACvD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACnB,KAAK,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACvD,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CACpB,KAAe,EACf,SAAmB,EACnB,UAAsB;QAEtB,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,mBAAQ,CAAC;gBAC1B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;gBACrB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnB;QACD,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAgB,EAChB,UAAsB;QAEtB,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAgB;YACxB,KAAK,EAAE,IAAI,EAAE,cAAc;SAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AApJD,0BAoJC"}
+{"version":3,"file":"hnswlib.js","sourceRoot":"","sources":["../../vectorstores/hnswlib.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA6B;AAC7B,gDAAwB;AAMxB,iCAA6C;AAC7C,0CAAuC;AACvC,0CAA+C;AAE/C,IAAI,eAAe,GAAmC,IAAI,CAAC;AAE3D,IAAI;IACF,4EAA4E;IAC5E,CAAC,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;CACjD;AAAC,WAAM;IACN,eAAe;CAChB;AAOD,MAAa,OAAQ,SAAQ,0BAAmB;IAO9C,YACE,IAAiB,EACjB,UAAsB,EACtB,QAA0B,EAC1B,KAAwB;QAExB,KAAK,CAAC,UAAU,CAAC,CAAC;QAZpB;;;;;WAAyB;QAEzB;;;;;WAA2B;QAE3B;;;;;WAAkB;QAShB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,UAAU,CACpB,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAC3C,SAAS,CACV,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAmB,EAAE,SAAqB;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE;gBACzC,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;aAC7C;YACD,IAAI,eAAe,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;aACH;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,aAAa,CACxB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACtC;QACD,qEAAqE;QACrE,oEAAoE;QACpE,uEAAuE;QACvE,+DAA+D;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACjD,MAAM,IAAI,KAAK,CACb,kEAAkE,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAC7F,CAAC;SACH;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,MAAM,GAAG,QAAQ,EAAE;YACrB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;SAChC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YAC1C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACzD;IACH,CAAC;IAED,KAAK,CAAC,+BAA+B,CAAC,KAAe,EAAE,CAAS;QAC9D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CACzB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,CACxB;YACE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC;SACR,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,kBAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC5D,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1B;YACD,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9B;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,UAAsB;QACzD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,MAAM,kBAAE,CAAC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAC7D,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACnC,kBAAE;iBACC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC;iBACvD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACnB,KAAK,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACvD,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CACpB,KAAe,EACf,SAAmB,EACnB,UAAsB,EACtB,WAA6B,IAAI,2BAAgB,EAAE;QAEnD,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,mBAAQ,CAAC;gBAC1B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;gBACrB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnB;QACD,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAgB,EAChB,UAAsB,EACtB,WAA6B,IAAI,2BAAgB,EAAE;QAEnD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAgB;YACxB,KAAK,EAAE,IAAI,EAAE,cAAc;SAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAjKD,0BAiKC"}

package/dist/vectorstores/index.d.ts

@@ -1 +1,2 @@
 export { HNSWLib } from "./hnswlib";
+export { PineconeStore } from "./pinecone";

package/dist/vectorstores/index.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HNSWLib = void 0;
+exports.PineconeStore = exports.HNSWLib = void 0;
 var hnswlib_1 = require("./hnswlib");
 Object.defineProperty(exports, "HNSWLib", { enumerable: true, get: function () { return hnswlib_1.HNSWLib; } });
+var pinecone_1 = require("./pinecone");
+Object.defineProperty(exports, "PineconeStore", { enumerable: true, get: function () { return pinecone_1.PineconeStore; } });
 //# sourceMappingURL=index.js.map

package/dist/vectorstores/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../vectorstores/index.ts"],"names":[],"mappings":";;;AAAA,qCAAoC;AAA3B,kGAAA,OAAO,OAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../vectorstores/index.ts"],"names":[],"mappings":";;;AAAA,qCAAoC;AAA3B,kGAAA,OAAO,OAAA;AAChB,uCAA2C;AAAlC,yGAAA,aAAa,OAAA"}

package/dist/vectorstores/pinecone.d.ts

@@ -0,0 +1,17 @@
+import type { PineconeClient } from "pinecone-client";
+import { VectorStore } from "./base";
+import { Embeddings } from "../embeddings/base";
+import { Document } from "../document";
+type PineconeMetadata = Record<string, any>;
+export declare class PineconeStore extends VectorStore {
+    textKey: string;
+    pineconeClient: PineconeClient<PineconeMetadata>;
+    constructor(pineconeClient: PineconeClient<PineconeMetadata>, embeddings: Embeddings, textKey?: string);
+    addDocuments(documents: Document[], ids?: string[]): Promise<void>;
+    addVectors(vectors: number[][], documents: Document[], ids?: string[]): Promise<void>;
+    similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
+    static fromTexts(pineconeClient: PineconeClient<PineconeMetadata>, texts: string[], metadatas: object[], embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
+    static fromDocuments(pineconeClient: PineconeClient<PineconeMetadata>, docs: Document[], embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
+    static fromExistingIndex(pineconeClient: PineconeClient<PineconeMetadata>, embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
+}
+export {};

package/dist/vectorstores/pinecone.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PineconeStore = void 0;
+const uuid_1 = require("uuid");
+const base_1 = require("./base");
+const document_1 = require("../document");
+class PineconeStore extends base_1.VectorStore {
+    constructor(pineconeClient, embeddings, textKey = "text") {
+        super(embeddings);
+        Object.defineProperty(this, "textKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pineconeClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.pineconeClient = pineconeClient;
+        this.embeddings = embeddings;
+        this.textKey = textKey;
+    }
+    async addDocuments(documents, ids) {
+        const texts = documents.map(({ pageContent }) => pageContent);
+        return this.addVectors(await this.embeddings.embedDocuments(texts), documents, ids);
+    }
+    async addVectors(vectors, documents, ids) {
+        const documentIds = ids == null ? documents.map(() => (0, uuid_1.v4)()) : ids;
+        return this.pineconeClient.upsert({
+            vectors: vectors.map((values, idx) => ({
+                id: documentIds[idx],
+                metadata: {
+                    ...documents[idx].metadata,
+                    [this.textKey]: documents[idx].pageContent,
+                },
+                values,
+            })),
+        });
+    }
+    async similaritySearchVectorWithScore(query, k) {
+        const results = await this.pineconeClient.query({
+            topK: k,
+            includeMetadata: true,
+            vector: query,
+        });
+        const result = [];
+        for (const res of results.matches) {
+            const { [this.textKey]: pageContent, ...metadata } = res.metadata;
+            result.push([new document_1.Document({ metadata, pageContent }), res.score]);
+        }
+        return result;
+    }
+    static async fromTexts(pineconeClient, texts, metadatas, embeddings, textKey = "text") {
+        const docs = [];
+        for (let i = 0; i < texts.length; i += 1) {
+            const newDoc = new document_1.Document({
+                pageContent: texts[i],
+                metadata: metadatas[i],
+            });
+            docs.push(newDoc);
+        }
+        return PineconeStore.fromDocuments(pineconeClient, docs, embeddings, textKey);
+    }
+    static async fromDocuments(pineconeClient, docs, embeddings, textKey = "text") {
+        const instance = new this(pineconeClient, embeddings, textKey);
+        await instance.addDocuments(docs);
+        return instance;
+    }
+    static async fromExistingIndex(pineconeClient, embeddings, textKey = "text") {
+        const instance = new this(pineconeClient, embeddings, textKey);
+        return instance;
+    }
+}
+exports.PineconeStore = PineconeStore;
+//# sourceMappingURL=pinecone.js.map