langchain 0.0.4 → 0.0.5

package/dist/esm/text_splitter.test.js

@@ -0,0 +1,85 @@
+import { test, expect } from "@jest/globals";
+import { Document } from './document';
+import { CharacterTextSplitter, RecursiveCharacterTextSplitter } from './text_splitter';
+test('Test splitting by character count.', () => {
+    const text = 'foo bar baz 123';
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 7, chunkOverlap: 3 });
+    const output = splitter.splitText(text);
+    const expectedOutput = ['foo bar', 'bar baz', 'baz 123'];
+    expect(output).toEqual(expectedOutput);
+});
+test('Test splitting by character count doesn\'t create empty documents.', () => {
+    const text = 'foo  bar';
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 2, chunkOverlap: 0 });
+    const output = splitter.splitText(text);
+    const expectedOutput = ['foo', 'bar'];
+    expect(output).toEqual(expectedOutput);
+});
+test('Test splitting by character count on long words.', () => {
+    const text = 'foo bar baz a a';
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 3, chunkOverlap: 1 });
+    const output = splitter.splitText(text);
+    const expectedOutput = ['foo', 'bar', 'baz', 'a a'];
+    expect(output).toEqual(expectedOutput);
+});
+test('Test splitting by character count when shorter words are first.', () => {
+    const text = 'a a foo bar baz';
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 3, chunkOverlap: 1 });
+    const output = splitter.splitText(text);
+    const expectedOutput = ['a a', 'foo', 'bar', 'baz'];
+    expect(output).toEqual(expectedOutput);
+});
+test('Test splitting by characters when splits not found easily.', () => {
+    const text = 'foo bar baz 123';
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 1, chunkOverlap: 0 });
+    const output = splitter.splitText(text);
+    const expectedOutput = ['foo', 'bar', 'baz', '123'];
+    expect(output).toEqual(expectedOutput);
+});
+test('Test invalid arguments.', () => {
+    expect(() => {
+        new CharacterTextSplitter({ chunkSize: 2, chunkOverlap: 4 });
+    }).toThrow();
+});
+test('Test create documents method.', () => {
+    const texts = ['foo bar', 'baz'];
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 3, chunkOverlap: 0 });
+    const docs = splitter.createDocuments(texts);
+    const expectedDocs = [new Document({ pageContent: 'foo' }), new Document({ pageContent: 'bar' }), new Document({ pageContent: 'baz' }),];
+    expect(docs).toEqual(expectedDocs);
+});
+test('Test create documents with metadata method.', () => {
+    const texts = ['foo bar', 'baz'];
+    const splitter = new CharacterTextSplitter({ separator: ' ', chunkSize: 3, chunkOverlap: 0 });
+    const docs = splitter.createDocuments(texts, [{ source: '1' }, { source: '2' }]);
+    const expectedDocs = [new Document({ pageContent: 'foo', metadata: { source: '1' } }), new Document({ pageContent: 'bar', metadata: { source: '1' } }), new Document({ pageContent: 'baz', metadata: { source: '2' } }),];
+    expect(docs).toEqual(expectedDocs);
+});
+test('Test iterative text splitter.', () => {
+    const text = `Hi.\n\nI'm Harrison.\n\nHow? Are? You?\nOkay then f f f f.
+This is a weird text to write, but gotta test the splittingggg some how.\n\n
+Bye!\n\n-H.`;
+    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 10, chunkOverlap: 1 });
+    const output = splitter.splitText(text);
+    const expectedOutput = [
+        "Hi.",
+        "I'm",
+        "Harrison.",
+        "How? Are?",
+        "You?",
+        "Okay then f",
+        "f f f f.",
+        "This is a",
+        "a weird",
+        "text to",
+        "write, but",
+        "gotta test",
+        "the",
+        "splitting",
+        "gggg",
+        "some how.",
+        "Bye!\n\n-H.",
+    ];
+    expect(output).toEqual(expectedOutput);
+});
+//# sourceMappingURL=text_splitter.test.js.map

package/dist/esm/text_splitter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text_splitter.test.js","sourceRoot":"","sources":["../../text_splitter.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,qBAAqB,EAAE,8BAA8B,EAAE,MAAM,iBAAiB,CAAC;AAExF,IAAI,CAAC,oCAAoC,EAAE,GAAG,EAAE;IAC9C,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IACzD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oEAAoE,EAAE,GAAG,EAAE;IAC9E,MAAM,IAAI,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iEAAiE,EAAE,GAAG,EAAE;IAC3E,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4DAA4D,EAAE,GAAG,EAAE;IACtE,MAAM,IAAI,GAAG,iBAAiB,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACpD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACnC,MAAM,CAAC,GAAG,EAAE;QACV,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;AACf,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+BAA+B,EAAE,GAAG,EAAE;IACzC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,CAAK,IAAI,QAAQ,CAAC,EAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAK,IAAI,QAAQ,CAAC,EAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAK,IAAI,QAAQ,CAAC,EAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAI,CAAC;IAClJ,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,GAAG,EAAE;IACvD,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,qBAAqB,CAAC,EAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,KAAK,EAAE,CAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAClF,MAAM,YAAY,GAAG,CAAK,IAAI,QAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,EAAK,IAAI,QAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,EAAK,IAAI,QAAQ,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,EAAI,CAAC;IACtO,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;AACrC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+BAA+B,EAAE,GAAG,EAAE;IACvC,MAAM,IAAI,GAAG;;YAEL,CAAC;IACT,MAAM,QAAQ,GAAG,IAAI,8BAA8B,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;IACxF,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,cAAc,GAAG;QACnB,KAAK;QACL,KAAK;QACL,WAAW;QACX,WAAW;QACX,MAAM;QACN,aAAa;QACb,UAAU;QACV,WAAW;QACX,SAAS;QACT,SAAS;QACT,YAAY;QACZ,YAAY;QACZ,KAAK;QACL,WAAW;QACX,MAAM;QACN,WAAW;QACX,aAAa;KAChB,CAAC;IACF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC"}

package/dist/esm/util/hub.d.ts

@@ -1 +1,2 @@
-export declare const loadFromHub: <T>(uri: string, loader: (a: string) => T, validPrefix: string, validSuffixes: Set<string>) => Promise<T | undefined>;
+export type LoadValues = Record<string, any>;
+export declare const loadFromHub: <T>(uri: string, loader: (a: string, values: LoadValues) => T, validPrefix: string, validSuffixes: Set<string>, values?: LoadValues) => Promise<T | undefined>;

package/dist/esm/util/hub.js

@@ -1,12 +1,3 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 var _a, _b;
 import path from "path";
 import os from "os";
@@ -15,7 +6,7 @@ import { fetchWithTimeout } from "./index";
 const HUB_PATH_REGEX = /lc(@[^:]+)?:\/\/(.*)/;
 const DEFAULT_REF = (_a = process.env.LANGCHAIN_HUB_DEFAULT_REF) !== null && _a !== void 0 ? _a : "master";
 const URL_BASE = (_b = process.env.LANGCHAIN_HUB_URL_BASE) !== null && _b !== void 0 ? _b : "https://raw.githubusercontent.com/hwchase17/langchain-hub/";
-export const loadFromHub = (uri, loader, validPrefix, validSuffixes) => __awaiter(void 0, void 0, void 0, function* () {
+export const loadFromHub = async (uri, loader, validPrefix, validSuffixes, values = {}) => {
     const match = uri.match(HUB_PATH_REGEX);
     if (!match) {
         return undefined;
@@ -30,14 +21,14 @@ export const loadFromHub = (uri, loader, validPrefix, validSuffixes) => __awaite
         throw new Error("Unsupported file type.");
     }
     const url = [URL_BASE, ref, remotePath].join("/");
-    const res = yield fetchWithTimeout(url, { timeout: 5000 });
+    const res = await fetchWithTimeout(url, { timeout: 5000 });
     if (res.status !== 200) {
         throw new Error(`Could not find file at ${url}`);
     }
-    const text = yield res.text();
+    const text = await res.text();
     const tmpdir = fs.mkdtempSync(path.join(os.tmpdir(), "langchain"));
     const file = path.join(tmpdir, path.basename(remotePath));
     fs.writeFileSync(file, text);
-    return loader(file);
-});
+    return loader(file, values);
+};
 //# sourceMappingURL=hub.js.map

package/dist/esm/util/hub.js.map

@@ -1 +1 @@
-{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../../util/hub.ts"],"names":[],"mappings":";;;;;;;;;;AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAE/D,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,GAAW,EACX,MAAwB,EACxB,WAAmB,EACnB,aAA0B,EACF,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC,CAAA,CAAC"}
+{"version":3,"file":"hub.js","sourceRoot":"","sources":["../../../util/hub.ts"],"names":[],"mappings":";AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAC9C,MAAM,WAAW,GAAG,MAAA,OAAO,CAAC,GAAG,CAAC,yBAAyB,mCAAI,QAAQ,CAAC;AACtE,MAAM,QAAQ,GACZ,MAAA,OAAO,CAAC,GAAG,CAAC,sBAAsB,mCAClC,4DAA4D,CAAC;AAI/D,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAW,EACX,MAA4C,EAC5C,WAAmB,EACnB,aAA0B,EAC1B,SAAqB,EAAE,EACC,EAAE;IAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,SAAS,CAAC;KAClB;IACD,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IACnD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,WAAW,EAAE;QAC5B,OAAO,SAAS,CAAC;KAClB;IAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;QACzD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;KAC3C;IAED,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE;QACtB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;KAClD;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1D,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;AAC9B,CAAC,CAAC"}

package/dist/esm/util/index.d.ts

@@ -10,3 +10,4 @@ export type FromPath<key extends string, T> = {
 export declare const resolveTemplateFromFile: <K extends string>(fieldName: K, config: FromPath<K, string>) => string;
 export declare const resolveConfigFromFile: <K extends string, T>(fieldName: K, config: FromPath<K, T>) => T;
 export declare const parseFileConfig: (file: string, supportedTypes?: string[]) => any;
+export declare const chunkArray: <T>(arr: T[], chunkSize: number) => T[][];

package/dist/esm/util/index.js

@@ -1,36 +1,16 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
 import path from "path";
 import fetch from "node-fetch";
 import fs from "fs";
 import * as yaml from "yaml";
-export const fetchWithTimeout = (url, init) => __awaiter(void 0, void 0, void 0, function* () {
-    const { timeout } = init, rest = __rest(init, ["timeout"]);
+export const fetchWithTimeout = async (url, init) => {
+    const { timeout, ...rest } = init;
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const res = yield fetch(url, Object.assign(Object.assign({}, rest), { signal: controller.signal }));
+    const res = await fetch(url, { ...rest, signal: controller.signal });
     clearTimeout(timeoutId);
     return res;
-});
+};
 const loadFileContents = (contents, format) => {
     switch (format) {
         case ".json":
@@ -71,4 +51,11 @@ export const parseFileConfig = (file, supportedTypes) => {
     const contents = fs.readFileSync(file).toString();
     return loadFileContents(contents, suffix);
 };
+export const chunkArray = (arr, chunkSize) => arr.reduce((chunks, elem, index) => {
+    const chunkIndex = Math.floor(index / chunkSize);
+    const chunk = chunks[chunkIndex] || [];
+    // eslint-disable-next-line no-param-reassign
+    chunks[chunkIndex] = chunk.concat([elem]);
+    return chunks;
+}, []);
 //# sourceMappingURL=index.js.map

package/dist/esm/util/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../util/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAsB,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,GAAW,EACX,IAAuD,EACvD,EAAE;IACF,MAAM,EAAE,OAAO,KAAc,IAAI,EAAb,IAAI,UAAK,IAAI,EAA3B,WAAoB,CAAO,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAChE,8DAA8D;IAC9D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,kCAAO,IAAI,KAAE,MAAM,EAAE,UAAU,CAAC,MAAa,IAAG,CAAC;IAC5E,YAAY,CAAC,SAAS,CAAC,CAAC;IACxB,OAAO,GAAG,CAAC;AACb,CAAC,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAE,EAAE;IAC5D,QAAQ,MAAM,EAAE;QACd,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B;YACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CAAC;AAiBF,MAAM,oBAAoB,GAAG,CAC3B,SAAY,EACZ,MAAsB,EACtB,IAA6C,EAC7C,eAA0B,EACvB,EAAE;IACL,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,SAAS,OAAO,CAAuB,CAAC;IACpE,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAkB,CAAC;IACjD,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE;QAClD,MAAM,IAAI,KAAK,CACb,SAAS,SAAS,eAAe,SAAS,uBAAuB,CAClE,CAAC;KACH;IAED,IAAI,KAAK,KAAK,SAAS,EAAE;QACvB,OAAO,KAAK,CAAC;KACd;IAED,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;SACtC;QAED,OAAO,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;KAC5D;IAED,MAAM,IAAI,KAAK,CACb,WAAW,SAAS,eAAe,SAAS,qBAAqB,CAClE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAAY,EACZ,MAA2B,EAC3B,EAAE,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;AAE/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,SAAY,EACZ,MAAsB,EACnB,EAAE,CACL,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAE,cAAyB,EAAE,EAAE;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC,IACE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,CAAC,cAAc,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EACpD;QACA,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;KACnD;IAED,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;IAClD,OAAO,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../util/index.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,KAAsB,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,GAAW,EACX,IAAuD,EACvD,EAAE;IACF,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAChE,8DAA8D;IAC9D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,MAAa,EAAE,CAAC,CAAC;IAC5E,YAAY,CAAC,SAAS,CAAC,CAAC;IACxB,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,QAAgB,EAAE,MAAc,EAAE,EAAE;IAC5D,QAAQ,MAAM,EAAE;QACd,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO;YACV,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9B;YACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CAAC;AAiBF,MAAM,oBAAoB,GAAG,CAC3B,SAAY,EACZ,MAAsB,EACtB,IAA6C,EAC7C,eAA0B,EACvB,EAAE;IACL,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,SAAS,OAAO,CAAuB,CAAC;IACpE,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAkB,CAAC;IACjD,IAAI,SAAS,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE;QAClD,MAAM,IAAI,KAAK,CACb,SAAS,SAAS,eAAe,SAAS,uBAAuB,CAClE,CAAC;KACH;IAED,IAAI,KAAK,KAAK,SAAS,EAAE;QACvB,OAAO,KAAK,CAAC;KACd;IAED,IAAI,SAAS,KAAK,SAAS,EAAE;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,eAAe,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YACxD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;SACtC;QAED,OAAO,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,CAAC;KAC5D;IAED,MAAM,IAAI,KAAK,CACb,WAAW,SAAS,eAAe,SAAS,qBAAqB,CAClE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAAY,EACZ,MAA2B,EAC3B,EAAE,CAAC,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;AAE/E,MAAM,CAAC,MAAM,qBAAqB,GAAG,CACnC,SAAY,EACZ,MAAsB,EACnB,EAAE,CACL,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAE,cAAyB,EAAE,EAAE;IACzE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAElC,IACE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,CAAC,cAAc,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EACpD;QACA,MAAM,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC;KACnD;IAED,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;IAClD,OAAO,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAI,GAAQ,EAAE,SAAiB,EAAE,EAAE,CAC3D,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IACvC,6CAA6C;IAC7C,MAAM,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC;AAChB,CAAC,EAAE,EAAW,CAAC,CAAC"}

package/dist/esm/vectorstores/base.d.ts

@@ -0,0 +1,18 @@
+import { Embeddings } from "embeddings/base";
+import { Document } from "../document";
+export interface DocStore {
+    [key: number]: object;
+}
+export declare abstract class VectorStore {
+    embeddings: Embeddings;
+    docstore: DocStore;
+    abstract addVectors(vectors: number[][], documents: Document[]): Promise<void>;
+    abstract similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
+    addDocuments(documents: Document[]): Promise<void>;
+    similaritySearch(query: string, k?: number): Promise<Document[]>;
+    similaritySearchWithScore(query: string, k?: number): Promise<[object, number][]>;
+}
+export declare abstract class SaveableVectorStore extends VectorStore {
+    abstract save(directory: string): Promise<void>;
+    static load(_directory: string, _embeddings: Embeddings): Promise<SaveableVectorStore>;
+}

package/dist/esm/vectorstores/base.js

@@ -0,0 +1,33 @@
+export class VectorStore {
+    constructor() {
+        Object.defineProperty(this, "embeddings", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "docstore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+    }
+    async addDocuments(documents) {
+        const texts = documents.map(({ pageContent }) => (pageContent));
+        this.addVectors(await this.embeddings.embedDocuments(texts), documents);
+    }
+    async similaritySearch(query, k = 4) {
+        const results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k);
+        return results.map((result) => result[0]);
+    }
+    async similaritySearchWithScore(query, k = 4) {
+        return this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k);
+    }
+}
+export class SaveableVectorStore extends VectorStore {
+    static load(_directory, _embeddings) {
+        throw new Error("Not implemented");
+    }
+}
+//# sourceMappingURL=base.js.map

package/dist/esm/vectorstores/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../vectorstores/base.ts"],"names":[],"mappings":"AAQA,MAAM,OAAgB,WAAW;IAAjC;QACE;;;;;WAAuB;QAEvB;;;;;WAAmB;IAmCrB,CAAC;IA1BC,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAE,CAAC,EAAC,WAAW,EAAC,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,UAAU,CACb,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAC3C,SAAS,CACV,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,KAAa,EAAE,CAAC,GAAG,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,+BAA+B,CACxD,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,yBAAyB,CAC7B,KAAa,EACb,CAAC,GAAG,CAAC;QAEL,OAAO,IAAI,CAAC,+BAA+B,CACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,EACvC,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAED,MAAM,OAAgB,mBAAoB,SAAQ,WAAW;IAG3D,MAAM,CAAC,IAAI,CACT,UAAkB,EAClB,WAAuB;QAEvB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrC,CAAC;CACF"}

package/dist/esm/vectorstores/hnswlib.d.ts

@@ -0,0 +1,19 @@
+import type { HierarchicalNSW as HierarchicalNSWT, SpaceName } from "hnswlib-node";
+import { Embeddings } from "../embeddings/base";
+import { DocStore, SaveableVectorStore } from "./base";
+import { Document } from "../document";
+export interface HNSWLibArgs {
+    space: SpaceName;
+    numDimensions?: number;
+}
+export declare class HNSWLib extends SaveableVectorStore {
+    index?: HierarchicalNSWT;
+    args: HNSWLibArgs;
+    constructor(args: HNSWLibArgs, embeddings: Embeddings, docstore: DocStore, index?: HierarchicalNSWT);
+    addVectors(vectors: number[][], documents: Document[]): Promise<void>;
+    similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
+    save(directory: string): Promise<void>;
+    static load(directory: string, embeddings: Embeddings): Promise<HNSWLib>;
+    static fromTexts(texts: string[], metadatas: object[], embeddings: Embeddings): Promise<HNSWLib>;
+    static fromDocuments(docs: Document[], embeddings: Embeddings): Promise<HNSWLib>;
+}

package/dist/esm/vectorstores/hnswlib.js

@@ -0,0 +1,119 @@
+import fs from "fs/promises";
+import path from "path";
+import { SaveableVectorStore } from "./base";
+import { Document } from "../document";
+let HierarchicalNSW = null;
+try {
+    // eslint-disable-next-line global-require
+    ({ HierarchicalNSW } = require("hnswlib-node"));
+}
+catch (_a) {
+    // ignore error
+}
+export class HNSWLib extends SaveableVectorStore {
+    constructor(args, embeddings, docstore, index) {
+        super();
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "args", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.index = index;
+        this.args = args;
+        this.embeddings = embeddings;
+        this.docstore = docstore;
+    }
+    async addVectors(vectors, documents) {
+        if (vectors.length === 0) {
+            return;
+        }
+        if (!this.index) {
+            if (this.args.numDimensions === undefined) {
+                this.args.numDimensions = vectors[0].length;
+            }
+            if (HierarchicalNSW === null) {
+                throw new Error("Please install hnswlib-node as a dependency with, e.g. `npm install -S hnswlib-node`");
+            }
+            this.index = new HierarchicalNSW(this.args.space, this.args.numDimensions);
+            this.index.initIndex(vectors.length);
+        }
+        // TODO here we could optionally normalise the vectors to unit length
+        // so that dot product is equivalent to cosine similarity, like this
+        // https://github.com/nmslib/hnswlib/issues/384#issuecomment-1155737730
+        // While we only support OpenAI embeddings this isn't necessary
+        if (vectors.length !== documents.length) {
+            throw new Error(`Vectors and metadatas must have the same length`);
+        }
+        if (vectors[0].length !== this.args.numDimensions) {
+            throw new Error(`Vectors must have the same length as the number of dimensions (${this.args.numDimensions})`);
+        }
+        const capacity = this.index.getMaxElements();
+        const needed = this.index.getCurrentCount() + vectors.length;
+        if (needed > capacity) {
+            this.index.resizeIndex(needed - capacity);
+        }
+        for (let i = 0; i < vectors.length; i += 1) {
+            this.index.addPoint(vectors[i], i);
+            this.docstore[i] = documents[i];
+        }
+    }
+    async similaritySearchVectorWithScore(query, k) {
+        if (!this.index) {
+            throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
+        }
+        const result = this.index.searchKnn(query, k);
+        return result.neighbors.map((docIndex, resultIndex) => [this.docstore[docIndex], result.distances[resultIndex]]);
+    }
+    async save(directory) {
+        if (!this.index) {
+            throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
+        }
+        await fs.mkdir(directory, { recursive: true });
+        await Promise.all([
+            this.index.writeIndex(path.join(directory, "hnswlib.index")),
+            await fs.writeFile(path.join(directory, "args.json"), JSON.stringify(this.args)),
+            await fs.writeFile(path.join(directory, "docstore.json"), JSON.stringify(this.docstore)),
+        ]);
+    }
+    static async load(directory, embeddings) {
+        if (HierarchicalNSW === null) {
+            throw new Error("Please install hnswlib-node as a dependency with, e.g. `npm install -S hnswlib-node`");
+        }
+        const args = JSON.parse(await fs.readFile(path.join(directory, "args.json"), "utf8"));
+        const index = new HierarchicalNSW(args.space, args.numDimensions);
+        const [docstore] = await Promise.all([
+            fs
+                .readFile(path.join(directory, "docstore.json"), "utf8")
+                .then(JSON.parse),
+            index.readIndex(path.join(directory, "hnswlib.index")),
+        ]);
+        return new HNSWLib(args, embeddings, docstore, index);
+    }
+    static async fromTexts(texts, metadatas, embeddings) {
+        const docs = [];
+        for (let i = 0; i < texts.length; i++) {
+            const newDoc = new Document({ pageContent: texts[i], metadata: metadatas[i] });
+            docs.push(newDoc);
+        }
+        return HNSWLib.fromDocuments(docs, embeddings);
+    }
+    static async fromDocuments(docs, embeddings) {
+        if (HierarchicalNSW === null) {
+            throw new Error("Please install hnswlib-node as a dependency with, e.g. `npm install -S hnswlib-node`");
+        }
+        const args = {
+            space: "ip", // dot product
+        };
+        const instance = new this(args, embeddings, {});
+        await instance.addDocuments(docs);
+        return instance;
+    }
+}
+//# sourceMappingURL=hnswlib.js.map

package/dist/esm/vectorstores/hnswlib.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hnswlib.js","sourceRoot":"","sources":["../../../vectorstores/hnswlib.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB,OAAO,EAAY,mBAAmB,EAAE,MAAM,QAAQ,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,IAAI,eAAe,GAAmC,IAAI,CAAC;AAE3D,IAAI;IACF,0CAA0C;IAC1C,CAAC,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;CACjD;AAAC,WAAM;IACN,eAAe;CAChB;AAOD,MAAM,OAAO,OAAQ,SAAQ,mBAAmB;IAK9C,YACE,IAAiB,EACjB,UAAsB,EACtB,QAAkB,EAClB,KAAwB;QAExB,KAAK,EAAE,CAAC;QAVV;;;;;WAAyB;QAEzB;;;;;WAAkB;QAShB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAmB,EAAE,SAAqB;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE;gBACzC,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;aAC7C;YACD,IAAI,eAAe,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;aACH;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,aAAa,CACxB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACtC;QACD,qEAAqE;QACrE,oEAAoE;QACpE,uEAAuE;QACvE,+DAA+D;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACjD,MAAM,IAAI,KAAK,CACb,kEAAkE,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAC7F,CAAC;SACH;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,MAAM,GAAG,QAAQ,EAAE;YACrB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC;SAC3C;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YAC1C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;SACjC;IACH,CAAC;IAED,KAAK,CAAC,+BAA+B,CAAC,KAAe,EAAE,CAAS;QAC9D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CACzB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,CACxB,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,CAGtD,CACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC5D,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1B;YACD,MAAM,EAAE,CAAC,SAAS,CAChB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9B;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,UAAsB;QACzD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAC7D,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACnC,EAAE;iBACC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC;iBACvD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACnB,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACvD,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CACpB,KAAe,EACf,SAAmB,EACnB,UAAsB;QAEtB,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC,EAAC,CAAC,CAAC;YAC7E,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACrB;QACD,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAgB,EAChB,UAAsB;QAEtB,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAgB;YACxB,KAAK,EAAE,IAAI,EAAE,cAAc;SAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;QAChD,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/esm/vectorstores/index.d.ts

@@ -0,0 +1 @@
+export { HNSWLib } from "./hnswlib";

package/dist/esm/vectorstores/index.js

@@ -0,0 +1,2 @@
+export { HNSWLib } from "./hnswlib";
+//# sourceMappingURL=index.js.map

package/dist/esm/vectorstores/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../vectorstores/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC"}

package/dist/esm/vectorstores/tests/hnswlib.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/esm/vectorstores/tests/hnswlib.test.js

@@ -0,0 +1,39 @@
+import { test, expect } from "@jest/globals";
+import fs from "fs/promises";
+import path from "path";
+import os from "os";
+import { HNSWLib } from "../hnswlib";
+import { OpenAIEmbeddings } from "../../embeddings";
+test("Test HNSWLib.fromTexts", async () => {
+    var _a;
+    const vectorStore = await HNSWLib.fromTexts(["Hello world", "Bye bye", "hello nice world"], [{ id: 2 }, { id: 1 }, { id: 3 }], new OpenAIEmbeddings());
+    expect((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getCurrentCount()).toBe(3);
+    const resultOne = await vectorStore.similaritySearch("hello world", 1);
+    const resultOneMetadatas = resultOne.map(({ metadata }) => (metadata));
+    expect(resultOneMetadatas).toEqual([{ id: 2 }]);
+    const resultTwo = await vectorStore.similaritySearch("hello world", 3);
+    const resultTwoMetadatas = resultTwo.map(({ metadata }) => (metadata));
+    expect(resultTwoMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
+});
+test("Test HNSWLib.load and HNSWLib.save", async () => {
+    var _a;
+    const vectorStore = await HNSWLib.fromTexts(["Hello world", "Bye bye", "hello nice world"], [{ id: 2 }, { id: 1 }, { id: 3 }], new OpenAIEmbeddings());
+    expect((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getCurrentCount()).toBe(3);
+    const resultOne = await vectorStore.similaritySearch("hello world", 1);
+    const resultOneMetadatas = resultOne.map(({ metadata }) => (metadata));
+    expect(resultOneMetadatas).toEqual([{ id: 2 }]);
+    const resultTwo = await vectorStore.similaritySearch("hello world", 3);
+    const resultTwoMetadatas = resultTwo.map(({ metadata }) => (metadata));
+    expect(resultTwoMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
+    const tempDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "lcjs-"));
+    console.log(tempDirectory);
+    await vectorStore.save(tempDirectory);
+    const loadedVectorStore = await HNSWLib.load(tempDirectory, new OpenAIEmbeddings());
+    const resultThree = await loadedVectorStore.similaritySearch("hello world", 1);
+    const resultThreeMetadatas = resultThree.map(({ metadata }) => (metadata));
+    expect(resultThreeMetadatas).toEqual([{ id: 2 }]);
+    const resultFour = await loadedVectorStore.similaritySearch("hello world", 3);
+    const resultFourMetadatas = resultFour.map(({ metadata }) => (metadata));
+    expect(resultFourMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
+});
+//# sourceMappingURL=hnswlib.test.js.map

package/dist/esm/vectorstores/tests/hnswlib.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hnswlib.test.js","sourceRoot":"","sources":["../../../../vectorstores/tests/hnswlib.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,IAAI,CAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;;IACxC,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,SAAS,CACzC,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,CAAC,EAC9C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACjC,IAAI,gBAAgB,EAAE,CACvB,CAAC;IACF,MAAM,CAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,MAAM,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,MAAM,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;;IACpD,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,SAAS,CACzC,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,CAAC,EAC9C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACjC,IAAI,gBAAgB,EAAE,CACvB,CAAC;IACF,MAAM,CAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,MAAM,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,MAAM,CAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,aAAa,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAE3B,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAEtC,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,IAAI,CAC1C,aAAa,EACb,IAAI,gBAAgB,EAAE,CACvB,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,CAC1D,aAAa,EACb,CAAC,CACF,CAAC;IAEF,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzE,MAAM,CAAC,oBAAoB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAElD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IAE9E,MAAM,mBAAmB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAC,QAAQ,EAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,MAAM,CAAC,mBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC,CAAC,CAAC"}