langchain 0.0.167 → 0.0.168
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/chat_models/cloudflare_workersai.cjs +1 -0
- package/chat_models/cloudflare_workersai.d.ts +1 -0
- package/chat_models/cloudflare_workersai.js +1 -0
- package/chat_models/fake.cjs +1 -0
- package/chat_models/fake.d.ts +1 -0
- package/chat_models/fake.js +1 -0
- package/dist/agents/chat/index.cjs +3 -2
- package/dist/agents/chat/index.d.ts +3 -0
- package/dist/agents/chat/index.js +3 -2
- package/dist/chat_models/cloudflare_workersai.cjs +140 -0
- package/dist/chat_models/cloudflare_workersai.d.ts +46 -0
- package/dist/chat_models/cloudflare_workersai.js +136 -0
- package/dist/chat_models/fake.cjs +101 -0
- package/dist/chat_models/fake.d.ts +36 -0
- package/dist/chat_models/fake.js +97 -0
- package/dist/llms/cloudflare_workersai.cjs +117 -0
- package/dist/llms/cloudflare_workersai.d.ts +49 -0
- package/dist/llms/cloudflare_workersai.js +113 -0
- package/dist/llms/fake.cjs +82 -0
- package/dist/llms/fake.d.ts +31 -0
- package/dist/llms/fake.js +78 -0
- package/dist/llms/sagemaker_endpoint.cjs +9 -7
- package/dist/llms/sagemaker_endpoint.d.ts +3 -3
- package/dist/llms/sagemaker_endpoint.js +9 -7
- package/dist/load/import_constants.cjs +2 -0
- package/dist/load/import_constants.js +2 -0
- package/dist/load/import_map.cjs +6 -2
- package/dist/load/import_map.d.ts +4 -0
- package/dist/load/import_map.js +4 -0
- package/dist/util/axios-fetch-adapter.cjs +1 -1
- package/dist/util/axios-fetch-adapter.js +1 -1
- package/dist/util/env.cjs +1 -1
- package/dist/util/env.js +1 -1
- package/dist/util/event-source-parse.cjs +1 -1
- package/dist/util/event-source-parse.js +1 -1
- package/dist/vectorstores/closevector/common.cjs +128 -0
- package/dist/vectorstores/closevector/common.d.ts +82 -0
- package/dist/vectorstores/closevector/common.js +124 -0
- package/dist/vectorstores/closevector/node.cjs +109 -0
- package/dist/vectorstores/closevector/node.d.ts +83 -0
- package/dist/vectorstores/closevector/node.js +105 -0
- package/dist/vectorstores/closevector/web.cjs +109 -0
- package/dist/vectorstores/closevector/web.d.ts +80 -0
- package/dist/vectorstores/closevector/web.js +105 -0
- package/llms/cloudflare_workersai.cjs +1 -0
- package/llms/cloudflare_workersai.d.ts +1 -0
- package/llms/cloudflare_workersai.js +1 -0
- package/llms/fake.cjs +1 -0
- package/llms/fake.d.ts +1 -0
- package/llms/fake.js +1 -0
- package/package.json +68 -5
- package/vectorstores/closevector/node.cjs +1 -0
- package/vectorstores/closevector/node.d.ts +1 -0
- package/vectorstores/closevector/node.js +1 -0
- package/vectorstores/closevector/web.cjs +1 -0
- package/vectorstores/closevector/web.d.ts +1 -0
- package/vectorstores/closevector/web.js +1 -0
|
@@ -15,9 +15,11 @@ export * as llms__base from "../llms/base.js";
|
|
|
15
15
|
export * as llms__openai from "../llms/openai.js";
|
|
16
16
|
export * as llms__ai21 from "../llms/ai21.js";
|
|
17
17
|
export * as llms__aleph_alpha from "../llms/aleph_alpha.js";
|
|
18
|
+
export * as llms__cloudflare_workersai from "../llms/cloudflare_workersai.js";
|
|
18
19
|
export * as llms__ollama from "../llms/ollama.js";
|
|
19
20
|
export * as llms__fireworks from "../llms/fireworks.js";
|
|
20
21
|
export * as llms__yandex from "../llms/yandex.js";
|
|
22
|
+
export * as llms__fake from "../llms/fake.js";
|
|
21
23
|
export * as prompts from "../prompts/index.js";
|
|
22
24
|
export * as vectorstores__base from "../vectorstores/base.js";
|
|
23
25
|
export * as vectorstores__memory from "../vectorstores/memory.js";
|
|
@@ -35,10 +37,12 @@ export * as document_transformers__openai_functions from "../document_transforme
|
|
|
35
37
|
export * as chat_models__base from "../chat_models/base.js";
|
|
36
38
|
export * as chat_models__openai from "../chat_models/openai.js";
|
|
37
39
|
export * as chat_models__anthropic from "../chat_models/anthropic.js";
|
|
40
|
+
export * as chat_models__cloudflare_workersai from "../chat_models/cloudflare_workersai.js";
|
|
38
41
|
export * as chat_models__fireworks from "../chat_models/fireworks.js";
|
|
39
42
|
export * as chat_models__baiduwenxin from "../chat_models/baiduwenxin.js";
|
|
40
43
|
export * as chat_models__ollama from "../chat_models/ollama.js";
|
|
41
44
|
export * as chat_models__minimax from "../chat_models/minimax.js";
|
|
45
|
+
export * as chat_models__fake from "../chat_models/fake.js";
|
|
42
46
|
export * as schema from "../schema/index.js";
|
|
43
47
|
export * as schema__document from "../schema/document.js";
|
|
44
48
|
export * as schema__output_parser from "../schema/output_parser.js";
|
package/dist/load/import_map.js
CHANGED
|
@@ -16,9 +16,11 @@ export * as llms__base from "../llms/base.js";
|
|
|
16
16
|
export * as llms__openai from "../llms/openai.js";
|
|
17
17
|
export * as llms__ai21 from "../llms/ai21.js";
|
|
18
18
|
export * as llms__aleph_alpha from "../llms/aleph_alpha.js";
|
|
19
|
+
export * as llms__cloudflare_workersai from "../llms/cloudflare_workersai.js";
|
|
19
20
|
export * as llms__ollama from "../llms/ollama.js";
|
|
20
21
|
export * as llms__fireworks from "../llms/fireworks.js";
|
|
21
22
|
export * as llms__yandex from "../llms/yandex.js";
|
|
23
|
+
export * as llms__fake from "../llms/fake.js";
|
|
22
24
|
export * as prompts from "../prompts/index.js";
|
|
23
25
|
export * as vectorstores__base from "../vectorstores/base.js";
|
|
24
26
|
export * as vectorstores__memory from "../vectorstores/memory.js";
|
|
@@ -36,10 +38,12 @@ export * as document_transformers__openai_functions from "../document_transforme
|
|
|
36
38
|
export * as chat_models__base from "../chat_models/base.js";
|
|
37
39
|
export * as chat_models__openai from "../chat_models/openai.js";
|
|
38
40
|
export * as chat_models__anthropic from "../chat_models/anthropic.js";
|
|
41
|
+
export * as chat_models__cloudflare_workersai from "../chat_models/cloudflare_workersai.js";
|
|
39
42
|
export * as chat_models__fireworks from "../chat_models/fireworks.js";
|
|
40
43
|
export * as chat_models__baiduwenxin from "../chat_models/baiduwenxin.js";
|
|
41
44
|
export * as chat_models__ollama from "../chat_models/ollama.js";
|
|
42
45
|
export * as chat_models__minimax from "../chat_models/minimax.js";
|
|
46
|
+
export * as chat_models__fake from "../chat_models/fake.js";
|
|
43
47
|
export * as schema from "../schema/index.js";
|
|
44
48
|
export * as schema__document from "../schema/document.js";
|
|
45
49
|
export * as schema__output_parser from "../schema/output_parser.js";
|
|
@@ -293,7 +293,7 @@ function createRequest(config) {
|
|
|
293
293
|
}
|
|
294
294
|
// Some `fetch` implementations will override the Content-Type to text/plain
|
|
295
295
|
// when body is a string.
|
|
296
|
-
// See https://github.com/
|
|
296
|
+
// See https://github.com/langchain-ai/langchainjs/issues/1010
|
|
297
297
|
if (typeof options.body === "string") {
|
|
298
298
|
options.body = new TextEncoder().encode(options.body);
|
|
299
299
|
}
|
|
@@ -287,7 +287,7 @@ function createRequest(config) {
|
|
|
287
287
|
}
|
|
288
288
|
// Some `fetch` implementations will override the Content-Type to text/plain
|
|
289
289
|
// when body is a string.
|
|
290
|
-
// See https://github.com/
|
|
290
|
+
// See https://github.com/langchain-ai/langchainjs/issues/1010
|
|
291
291
|
if (typeof options.body === "string") {
|
|
292
292
|
options.body = new TextEncoder().encode(options.body);
|
|
293
293
|
}
|
package/dist/util/env.cjs
CHANGED
|
@@ -59,7 +59,7 @@ async function getRuntimeEnvironment() {
|
|
|
59
59
|
exports.getRuntimeEnvironment = getRuntimeEnvironment;
|
|
60
60
|
function getEnvironmentVariable(name) {
|
|
61
61
|
// Certain Deno setups will throw an error if you try to access environment variables
|
|
62
|
-
// https://github.com/
|
|
62
|
+
// https://github.com/langchain-ai/langchainjs/issues/1412
|
|
63
63
|
try {
|
|
64
64
|
return typeof process !== "undefined"
|
|
65
65
|
? // eslint-disable-next-line no-process-env
|
package/dist/util/env.js
CHANGED
|
@@ -49,7 +49,7 @@ export async function getRuntimeEnvironment() {
|
|
|
49
49
|
}
|
|
50
50
|
export function getEnvironmentVariable(name) {
|
|
51
51
|
// Certain Deno setups will throw an error if you try to access environment variables
|
|
52
|
-
// https://github.com/
|
|
52
|
+
// https://github.com/langchain-ai/langchainjs/issues/1412
|
|
53
53
|
try {
|
|
54
54
|
return typeof process !== "undefined"
|
|
55
55
|
? // eslint-disable-next-line no-process-env
|
|
@@ -36,7 +36,7 @@ async function getBytes(stream, onChunk) {
|
|
|
36
36
|
// This change is essential to ensure that we capture every last piece of information from streams,
|
|
37
37
|
// such as those from Azure OpenAI, which may not terminate with a blank line. Without this
|
|
38
38
|
// mechanism, we risk ignoring a possibly significant last message.
|
|
39
|
-
// See https://github.com/
|
|
39
|
+
// See https://github.com/langchain-ai/langchainjs/issues/1299 for details.
|
|
40
40
|
// eslint-disable-next-line no-constant-condition
|
|
41
41
|
while (true) {
|
|
42
42
|
const result = await reader.read();
|
|
@@ -33,7 +33,7 @@ export async function getBytes(stream, onChunk) {
|
|
|
33
33
|
// This change is essential to ensure that we capture every last piece of information from streams,
|
|
34
34
|
// such as those from Azure OpenAI, which may not terminate with a blank line. Without this
|
|
35
35
|
// mechanism, we risk ignoring a possibly significant last message.
|
|
36
|
-
// See https://github.com/
|
|
36
|
+
// See https://github.com/langchain-ai/langchainjs/issues/1299 for details.
|
|
37
37
|
// eslint-disable-next-line no-constant-condition
|
|
38
38
|
while (true) {
|
|
39
39
|
const result = await reader.read();
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CloseVector = void 0;
|
|
4
|
+
const document_js_1 = require("../../document.cjs");
|
|
5
|
+
const base_js_1 = require("../base.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* package closevector is largely based on hnswlib.ts in the current folder with the following exceptions:
|
|
8
|
+
* 1. It uses a modified version of hnswlib-node to ensure the generated index can be loaded by closevector_web.ts.
|
|
9
|
+
* 2. It adds features to upload and download the index to/from the CDN provided by CloseVector.
|
|
10
|
+
*
|
|
11
|
+
* For more information, check out https://closevector-docs.getmegaportal.com/
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* Class that implements a vector store using Hierarchical Navigable Small
|
|
15
|
+
* World (HNSW) graphs. It extends the SaveableVectorStore class and
|
|
16
|
+
* provides methods for adding documents and vectors, performing
|
|
17
|
+
* similarity searches, and saving and loading the vector store.
|
|
18
|
+
*/
|
|
19
|
+
class CloseVector extends base_js_1.SaveableVectorStore {
|
|
20
|
+
_vectorstoreType() {
|
|
21
|
+
return "closevector";
|
|
22
|
+
}
|
|
23
|
+
constructor(embeddings, args, credentials) {
|
|
24
|
+
super(embeddings, args);
|
|
25
|
+
Object.defineProperty(this, "_instance", {
|
|
26
|
+
enumerable: true,
|
|
27
|
+
configurable: true,
|
|
28
|
+
writable: true,
|
|
29
|
+
value: void 0
|
|
30
|
+
});
|
|
31
|
+
// credentials will not be saved to disk
|
|
32
|
+
Object.defineProperty(this, "credentials", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
writable: true,
|
|
36
|
+
value: void 0
|
|
37
|
+
});
|
|
38
|
+
this.credentials = credentials;
|
|
39
|
+
}
|
|
40
|
+
get instance() {
|
|
41
|
+
if (!this._instance) {
|
|
42
|
+
throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
|
|
43
|
+
}
|
|
44
|
+
return this._instance;
|
|
45
|
+
}
|
|
46
|
+
set instance(instance) {
|
|
47
|
+
this._instance = instance;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Method to add documents to the vector store. It first converts the
|
|
51
|
+
* documents to vectors using the embeddings, then adds the vectors to the
|
|
52
|
+
* vector store.
|
|
53
|
+
* @param documents The documents to be added to the vector store.
|
|
54
|
+
* @returns A Promise that resolves when the documents have been added.
|
|
55
|
+
*/
|
|
56
|
+
async addDocuments(documents) {
|
|
57
|
+
await this.instance.addDocuments(documents);
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Method to save the vector store to a directory. It saves the HNSW
|
|
61
|
+
* index, the arguments, and the document store to the directory.
|
|
62
|
+
* @param directory The directory to which to save the vector store. In CloseVector, we use IndexedDB to mock the file system. Therefore, this parameter is can be treated as a key to the contents stored.
|
|
63
|
+
* @returns A Promise that resolves when the vector store has been saved.
|
|
64
|
+
*/
|
|
65
|
+
async save(directory) {
|
|
66
|
+
await this.instance.save(directory);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Method to add vectors to the vector store. It first initializes the
|
|
70
|
+
* index if it hasn't been initialized yet, then adds the vectors to the
|
|
71
|
+
* index and the documents to the document store.
|
|
72
|
+
* @param vectors The vectors to be added to the vector store.
|
|
73
|
+
* @param documents The documents corresponding to the vectors.
|
|
74
|
+
* @returns A Promise that resolves when the vectors and documents have been added.
|
|
75
|
+
*/
|
|
76
|
+
async addVectors(vectors, documents) {
|
|
77
|
+
await this.instance.addVectors(vectors, documents);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Method to perform a similarity search in the vector store using a query
|
|
81
|
+
* vector. It returns the k most similar documents along with their
|
|
82
|
+
* similarity scores. An optional filter function can be provided to
|
|
83
|
+
* filter the documents.
|
|
84
|
+
* @param query The query vector.
|
|
85
|
+
* @param k The number of most similar documents to return.
|
|
86
|
+
* @param filter An optional filter function to filter the documents.
|
|
87
|
+
* @returns A Promise that resolves to an array of tuples, where each tuple contains a document and its similarity score.
|
|
88
|
+
*/
|
|
89
|
+
async similaritySearchVectorWithScore(query, k, filter) {
|
|
90
|
+
const resp = await this.instance.similaritySearchVectorWithScore(query, k, filter
|
|
91
|
+
? (x) => filter?.({
|
|
92
|
+
pageContent: x.pageContent,
|
|
93
|
+
metadata: x.metadata || {},
|
|
94
|
+
}) || false
|
|
95
|
+
: undefined);
|
|
96
|
+
const mapped = resp.map((x) => [
|
|
97
|
+
new document_js_1.Document({
|
|
98
|
+
pageContent: x[0].pageContent,
|
|
99
|
+
metadata: x[0].metadata || {},
|
|
100
|
+
}),
|
|
101
|
+
x[1],
|
|
102
|
+
]);
|
|
103
|
+
return mapped;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Method to delete the vector store from a directory. It deletes the
|
|
107
|
+
* hnswlib.index file, the docstore.json file, and the args.json file from
|
|
108
|
+
* the directory.
|
|
109
|
+
* @param params An object with a directory property that specifies the directory from which to delete the vector store.
|
|
110
|
+
* @returns A Promise that resolves when the vector store has been deleted.
|
|
111
|
+
*/
|
|
112
|
+
async delete(params) {
|
|
113
|
+
return await this.instance.delete(params);
|
|
114
|
+
}
|
|
115
|
+
static textsToDocuments(texts, metadatas) {
|
|
116
|
+
const docs = [];
|
|
117
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
118
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
119
|
+
const newDoc = new document_js_1.Document({
|
|
120
|
+
pageContent: texts[i],
|
|
121
|
+
metadata,
|
|
122
|
+
});
|
|
123
|
+
docs.push(newDoc);
|
|
124
|
+
}
|
|
125
|
+
return docs;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
exports.CloseVector = CloseVector;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import type { CloseVectorSaveableVectorStore } from "closevector-common";
|
|
2
|
+
import { Embeddings } from "../../embeddings/base.js";
|
|
3
|
+
import { Document } from "../../document.js";
|
|
4
|
+
import { SaveableVectorStore } from "../base.js";
|
|
5
|
+
type CloseVectorCredentials = {
|
|
6
|
+
key?: string;
|
|
7
|
+
secret?: string;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* package closevector is largely based on hnswlib.ts in the current folder with the following exceptions:
|
|
11
|
+
* 1. It uses a modified version of hnswlib-node to ensure the generated index can be loaded by closevector_web.ts.
|
|
12
|
+
* 2. It adds features to upload and download the index to/from the CDN provided by CloseVector.
|
|
13
|
+
*
|
|
14
|
+
* For more information, check out https://closevector-docs.getmegaportal.com/
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Class that implements a vector store using Hierarchical Navigable Small
|
|
18
|
+
* World (HNSW) graphs. It extends the SaveableVectorStore class and
|
|
19
|
+
* provides methods for adding documents and vectors, performing
|
|
20
|
+
* similarity searches, and saving and loading the vector store.
|
|
21
|
+
*/
|
|
22
|
+
export declare abstract class CloseVector<CloseVectorHNSWImplementation extends CloseVectorSaveableVectorStore> extends SaveableVectorStore {
|
|
23
|
+
FilterType: (doc: Document) => boolean;
|
|
24
|
+
_instance?: CloseVectorHNSWImplementation;
|
|
25
|
+
credentials?: CloseVectorCredentials;
|
|
26
|
+
_vectorstoreType(): string;
|
|
27
|
+
constructor(embeddings: Embeddings, args: {
|
|
28
|
+
space: "l2" | "ip" | "cosine";
|
|
29
|
+
numDimensions?: number;
|
|
30
|
+
maxElements?: number;
|
|
31
|
+
}, credentials?: CloseVectorCredentials);
|
|
32
|
+
get instance(): CloseVectorHNSWImplementation;
|
|
33
|
+
protected set instance(instance: CloseVectorHNSWImplementation);
|
|
34
|
+
/**
|
|
35
|
+
* Method to add documents to the vector store. It first converts the
|
|
36
|
+
* documents to vectors using the embeddings, then adds the vectors to the
|
|
37
|
+
* vector store.
|
|
38
|
+
* @param documents The documents to be added to the vector store.
|
|
39
|
+
* @returns A Promise that resolves when the documents have been added.
|
|
40
|
+
*/
|
|
41
|
+
addDocuments(documents: Document[]): Promise<void>;
|
|
42
|
+
abstract saveToCloud(_options: Record<string, unknown>): Promise<void>;
|
|
43
|
+
/**
|
|
44
|
+
* Method to save the vector store to a directory. It saves the HNSW
|
|
45
|
+
* index, the arguments, and the document store to the directory.
|
|
46
|
+
* @param directory The directory to which to save the vector store. In CloseVector, we use IndexedDB to mock the file system. Therefore, this parameter is can be treated as a key to the contents stored.
|
|
47
|
+
* @returns A Promise that resolves when the vector store has been saved.
|
|
48
|
+
*/
|
|
49
|
+
save(directory: string): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Method to add vectors to the vector store. It first initializes the
|
|
52
|
+
* index if it hasn't been initialized yet, then adds the vectors to the
|
|
53
|
+
* index and the documents to the document store.
|
|
54
|
+
* @param vectors The vectors to be added to the vector store.
|
|
55
|
+
* @param documents The documents corresponding to the vectors.
|
|
56
|
+
* @returns A Promise that resolves when the vectors and documents have been added.
|
|
57
|
+
*/
|
|
58
|
+
addVectors(vectors: number[][], documents: Document[]): Promise<void>;
|
|
59
|
+
/**
|
|
60
|
+
* Method to perform a similarity search in the vector store using a query
|
|
61
|
+
* vector. It returns the k most similar documents along with their
|
|
62
|
+
* similarity scores. An optional filter function can be provided to
|
|
63
|
+
* filter the documents.
|
|
64
|
+
* @param query The query vector.
|
|
65
|
+
* @param k The number of most similar documents to return.
|
|
66
|
+
* @param filter An optional filter function to filter the documents.
|
|
67
|
+
* @returns A Promise that resolves to an array of tuples, where each tuple contains a document and its similarity score.
|
|
68
|
+
*/
|
|
69
|
+
similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document<Record<string, unknown>>, number][]>;
|
|
70
|
+
/**
|
|
71
|
+
* Method to delete the vector store from a directory. It deletes the
|
|
72
|
+
* hnswlib.index file, the docstore.json file, and the args.json file from
|
|
73
|
+
* the directory.
|
|
74
|
+
* @param params An object with a directory property that specifies the directory from which to delete the vector store.
|
|
75
|
+
* @returns A Promise that resolves when the vector store has been deleted.
|
|
76
|
+
*/
|
|
77
|
+
delete(params: {
|
|
78
|
+
directory: string;
|
|
79
|
+
}): Promise<void>;
|
|
80
|
+
static textsToDocuments(texts: string[], metadatas: object[] | object): Document<Record<string, any>>[];
|
|
81
|
+
}
|
|
82
|
+
export {};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { Document } from "../../document.js";
|
|
2
|
+
import { SaveableVectorStore } from "../base.js";
|
|
3
|
+
/**
|
|
4
|
+
* package closevector is largely based on hnswlib.ts in the current folder with the following exceptions:
|
|
5
|
+
* 1. It uses a modified version of hnswlib-node to ensure the generated index can be loaded by closevector_web.ts.
|
|
6
|
+
* 2. It adds features to upload and download the index to/from the CDN provided by CloseVector.
|
|
7
|
+
*
|
|
8
|
+
* For more information, check out https://closevector-docs.getmegaportal.com/
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Class that implements a vector store using Hierarchical Navigable Small
|
|
12
|
+
* World (HNSW) graphs. It extends the SaveableVectorStore class and
|
|
13
|
+
* provides methods for adding documents and vectors, performing
|
|
14
|
+
* similarity searches, and saving and loading the vector store.
|
|
15
|
+
*/
|
|
16
|
+
export class CloseVector extends SaveableVectorStore {
|
|
17
|
+
_vectorstoreType() {
|
|
18
|
+
return "closevector";
|
|
19
|
+
}
|
|
20
|
+
constructor(embeddings, args, credentials) {
|
|
21
|
+
super(embeddings, args);
|
|
22
|
+
Object.defineProperty(this, "_instance", {
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
writable: true,
|
|
26
|
+
value: void 0
|
|
27
|
+
});
|
|
28
|
+
// credentials will not be saved to disk
|
|
29
|
+
Object.defineProperty(this, "credentials", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: void 0
|
|
34
|
+
});
|
|
35
|
+
this.credentials = credentials;
|
|
36
|
+
}
|
|
37
|
+
get instance() {
|
|
38
|
+
if (!this._instance) {
|
|
39
|
+
throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
|
|
40
|
+
}
|
|
41
|
+
return this._instance;
|
|
42
|
+
}
|
|
43
|
+
set instance(instance) {
|
|
44
|
+
this._instance = instance;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Method to add documents to the vector store. It first converts the
|
|
48
|
+
* documents to vectors using the embeddings, then adds the vectors to the
|
|
49
|
+
* vector store.
|
|
50
|
+
* @param documents The documents to be added to the vector store.
|
|
51
|
+
* @returns A Promise that resolves when the documents have been added.
|
|
52
|
+
*/
|
|
53
|
+
async addDocuments(documents) {
|
|
54
|
+
await this.instance.addDocuments(documents);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Method to save the vector store to a directory. It saves the HNSW
|
|
58
|
+
* index, the arguments, and the document store to the directory.
|
|
59
|
+
* @param directory The directory to which to save the vector store. In CloseVector, we use IndexedDB to mock the file system. Therefore, this parameter is can be treated as a key to the contents stored.
|
|
60
|
+
* @returns A Promise that resolves when the vector store has been saved.
|
|
61
|
+
*/
|
|
62
|
+
async save(directory) {
|
|
63
|
+
await this.instance.save(directory);
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Method to add vectors to the vector store. It first initializes the
|
|
67
|
+
* index if it hasn't been initialized yet, then adds the vectors to the
|
|
68
|
+
* index and the documents to the document store.
|
|
69
|
+
* @param vectors The vectors to be added to the vector store.
|
|
70
|
+
* @param documents The documents corresponding to the vectors.
|
|
71
|
+
* @returns A Promise that resolves when the vectors and documents have been added.
|
|
72
|
+
*/
|
|
73
|
+
async addVectors(vectors, documents) {
|
|
74
|
+
await this.instance.addVectors(vectors, documents);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Method to perform a similarity search in the vector store using a query
|
|
78
|
+
* vector. It returns the k most similar documents along with their
|
|
79
|
+
* similarity scores. An optional filter function can be provided to
|
|
80
|
+
* filter the documents.
|
|
81
|
+
* @param query The query vector.
|
|
82
|
+
* @param k The number of most similar documents to return.
|
|
83
|
+
* @param filter An optional filter function to filter the documents.
|
|
84
|
+
* @returns A Promise that resolves to an array of tuples, where each tuple contains a document and its similarity score.
|
|
85
|
+
*/
|
|
86
|
+
async similaritySearchVectorWithScore(query, k, filter) {
|
|
87
|
+
const resp = await this.instance.similaritySearchVectorWithScore(query, k, filter
|
|
88
|
+
? (x) => filter?.({
|
|
89
|
+
pageContent: x.pageContent,
|
|
90
|
+
metadata: x.metadata || {},
|
|
91
|
+
}) || false
|
|
92
|
+
: undefined);
|
|
93
|
+
const mapped = resp.map((x) => [
|
|
94
|
+
new Document({
|
|
95
|
+
pageContent: x[0].pageContent,
|
|
96
|
+
metadata: x[0].metadata || {},
|
|
97
|
+
}),
|
|
98
|
+
x[1],
|
|
99
|
+
]);
|
|
100
|
+
return mapped;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Method to delete the vector store from a directory. It deletes the
|
|
104
|
+
* hnswlib.index file, the docstore.json file, and the args.json file from
|
|
105
|
+
* the directory.
|
|
106
|
+
* @param params An object with a directory property that specifies the directory from which to delete the vector store.
|
|
107
|
+
* @returns A Promise that resolves when the vector store has been deleted.
|
|
108
|
+
*/
|
|
109
|
+
async delete(params) {
|
|
110
|
+
return await this.instance.delete(params);
|
|
111
|
+
}
|
|
112
|
+
static textsToDocuments(texts, metadatas) {
|
|
113
|
+
const docs = [];
|
|
114
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
115
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
116
|
+
const newDoc = new Document({
|
|
117
|
+
pageContent: texts[i],
|
|
118
|
+
metadata,
|
|
119
|
+
});
|
|
120
|
+
docs.push(newDoc);
|
|
121
|
+
}
|
|
122
|
+
return docs;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CloseVectorNode = void 0;
|
|
4
|
+
const closevector_node_1 = require("closevector-node");
|
|
5
|
+
const common_js_1 = require("./common.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* Class that implements a vector store using Hierarchical Navigable Small
|
|
8
|
+
* World (HNSW) graphs. It extends the SaveableVectorStore class and
|
|
9
|
+
* provides methods for adding documents and vectors, performing
|
|
10
|
+
* similarity searches, and saving and loading the vector store.
|
|
11
|
+
*/
|
|
12
|
+
class CloseVectorNode extends common_js_1.CloseVector {
|
|
13
|
+
constructor(embeddings, args, credentials) {
|
|
14
|
+
super(embeddings, args, credentials);
|
|
15
|
+
if (args.instance) {
|
|
16
|
+
this.instance = args.instance;
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
this.instance = new closevector_node_1.CloseVectorHNSWNode(embeddings, args);
|
|
20
|
+
}
|
|
21
|
+
if (this.credentials?.key) {
|
|
22
|
+
this.instance.accessKey = this.credentials.key;
|
|
23
|
+
}
|
|
24
|
+
if (this.credentials?.secret) {
|
|
25
|
+
this.instance.secret = this.credentials.secret;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Method to save the index to the CloseVector CDN.
|
|
30
|
+
* @param options
|
|
31
|
+
* @param options.description A description of the index.
|
|
32
|
+
* @param options.public Whether the index should be public or private. Defaults to false.
|
|
33
|
+
* @param options.uuid A UUID for the index. If not provided, a new index will be created.
|
|
34
|
+
* @param options.onProgress A callback function that will be called with the progress of the upload.
|
|
35
|
+
*/
|
|
36
|
+
async saveToCloud(options) {
|
|
37
|
+
await this.instance.saveToCloud(options);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Method to load the index from the CloseVector CDN.
|
|
41
|
+
* @param options
|
|
42
|
+
* @param options.uuid The UUID of the index to be downloaded.
|
|
43
|
+
* @param options.credentials The credentials to be used by the CloseVectorNode instance.
|
|
44
|
+
* @param options.embeddings The embeddings to be used by the CloseVectorNode instance.
|
|
45
|
+
* @param options.onProgress A callback function that will be called with the progress of the download.
|
|
46
|
+
*/
|
|
47
|
+
static async loadFromCloud(options) {
|
|
48
|
+
if (!options.credentials.key || !options.credentials.secret) {
|
|
49
|
+
throw new Error("key and secret must be provided");
|
|
50
|
+
}
|
|
51
|
+
const instance = await closevector_node_1.CloseVectorHNSWNode.loadFromCloud({
|
|
52
|
+
...options,
|
|
53
|
+
accessKey: options.credentials.key,
|
|
54
|
+
secret: options.credentials.secret,
|
|
55
|
+
});
|
|
56
|
+
const vectorstore = new this(options.embeddings, instance.args, options.credentials);
|
|
57
|
+
return vectorstore;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Static method to load a vector store from a directory. It reads the
|
|
61
|
+
* HNSW index, the arguments, and the document store from the directory,
|
|
62
|
+
* then creates a new HNSWLib instance with these values.
|
|
63
|
+
* @param directory The directory from which to load the vector store.
|
|
64
|
+
* @param embeddings The embeddings to be used by the CloseVectorNode instance.
|
|
65
|
+
* @returns A Promise that resolves to a new CloseVectorNode instance.
|
|
66
|
+
*/
|
|
67
|
+
static async load(directory, embeddings, credentials) {
|
|
68
|
+
const instance = await closevector_node_1.CloseVectorHNSWNode.load(directory, embeddings);
|
|
69
|
+
const vectorstore = new this(embeddings, instance.args, credentials);
|
|
70
|
+
return vectorstore;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Static method to create a new CloseVectorWeb instance from texts and metadata.
|
|
74
|
+
* It creates a new Document instance for each text and metadata, then
|
|
75
|
+
* calls the fromDocuments method to create the CloseVectorWeb instance.
|
|
76
|
+
* @param texts The texts to be used to create the documents.
|
|
77
|
+
* @param metadatas The metadata to be used to create the documents.
|
|
78
|
+
* @param embeddings The embeddings to be used by the CloseVectorWeb instance.
|
|
79
|
+
* @param args An optional configuration object for the CloseVectorWeb instance.
|
|
80
|
+
* @param credential An optional credential object for the CloseVector API.
|
|
81
|
+
* @returns A Promise that resolves to a new CloseVectorWeb instance.
|
|
82
|
+
*/
|
|
83
|
+
static async fromTexts(texts, metadatas, embeddings, args, credential) {
|
|
84
|
+
const docs = common_js_1.CloseVector.textsToDocuments(texts, metadatas);
|
|
85
|
+
return await CloseVectorNode.fromDocuments(docs, embeddings, args, credential);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Static method to create a new CloseVectorNode instance from documents. It
|
|
89
|
+
* creates a new CloseVectorNode instance, adds the documents to it, then returns
|
|
90
|
+
* the instance.
|
|
91
|
+
* @param docs The documents to be added to the HNSWLib instance.
|
|
92
|
+
* @param embeddings The embeddings to be used by the HNSWLib instance.
|
|
93
|
+
* @param args An optional configuration object for the HNSWLib instance.
|
|
94
|
+
* @param credentials An optional credential object for the CloseVector API.
|
|
95
|
+
* @returns A Promise that resolves to a new CloseVectorNode instance.
|
|
96
|
+
*/
|
|
97
|
+
static async fromDocuments(docs, embeddings, args, credentials) {
|
|
98
|
+
const _args = args || {
|
|
99
|
+
space: "cosine",
|
|
100
|
+
};
|
|
101
|
+
const instance = new this(embeddings, _args, credentials);
|
|
102
|
+
await instance.addDocuments(docs);
|
|
103
|
+
return instance;
|
|
104
|
+
}
|
|
105
|
+
static async imports() {
|
|
106
|
+
return closevector_node_1.CloseVectorHNSWNode.imports();
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
exports.CloseVectorNode = CloseVectorNode;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { CloseVectorHNSWNode, HierarchicalNSWT, CloseVectorHNSWLibArgs, CloseVectorCredentials } from "closevector-node";
|
|
2
|
+
import { CloseVector } from "./common.js";
|
|
3
|
+
import { Embeddings } from "../../embeddings/base.js";
|
|
4
|
+
import { Document } from "../../document.js";
|
|
5
|
+
/**
|
|
6
|
+
* package closevector-node is largely based on hnswlib.ts in the current folder with the following exceptions:
|
|
7
|
+
* 1. It uses a modified version of hnswlib-node to ensure the generated index can be loaded by closevector_web.ts.
|
|
8
|
+
* 2. It adds features to upload and download the index to/from the CDN provided by CloseVector.
|
|
9
|
+
*
|
|
10
|
+
* For more information, check out https://closevector-docs.getmegaportal.com/
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Arguments for creating a CloseVectorNode instance, extending CloseVectorHNSWLibArgs.
|
|
14
|
+
*/
|
|
15
|
+
export interface CloseVectorNodeArgs extends CloseVectorHNSWLibArgs<HierarchicalNSWT> {
|
|
16
|
+
instance?: CloseVectorHNSWNode;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Class that implements a vector store using Hierarchical Navigable Small
|
|
20
|
+
* World (HNSW) graphs. It extends the SaveableVectorStore class and
|
|
21
|
+
* provides methods for adding documents and vectors, performing
|
|
22
|
+
* similarity searches, and saving and loading the vector store.
|
|
23
|
+
*/
|
|
24
|
+
export declare class CloseVectorNode extends CloseVector<CloseVectorHNSWNode> {
|
|
25
|
+
FilterType: (doc: Document) => boolean;
|
|
26
|
+
constructor(embeddings: Embeddings, args: CloseVectorNodeArgs, credentials?: CloseVectorCredentials);
|
|
27
|
+
/**
|
|
28
|
+
* Method to save the index to the CloseVector CDN.
|
|
29
|
+
* @param options
|
|
30
|
+
* @param options.description A description of the index.
|
|
31
|
+
* @param options.public Whether the index should be public or private. Defaults to false.
|
|
32
|
+
* @param options.uuid A UUID for the index. If not provided, a new index will be created.
|
|
33
|
+
* @param options.onProgress A callback function that will be called with the progress of the upload.
|
|
34
|
+
*/
|
|
35
|
+
saveToCloud(options: Parameters<CloseVectorHNSWNode["saveToCloud"]>[0]): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Method to load the index from the CloseVector CDN.
|
|
38
|
+
* @param options
|
|
39
|
+
* @param options.uuid The UUID of the index to be downloaded.
|
|
40
|
+
* @param options.credentials The credentials to be used by the CloseVectorNode instance.
|
|
41
|
+
* @param options.embeddings The embeddings to be used by the CloseVectorNode instance.
|
|
42
|
+
* @param options.onProgress A callback function that will be called with the progress of the download.
|
|
43
|
+
*/
|
|
44
|
+
static loadFromCloud(options: Omit<Parameters<(typeof CloseVectorHNSWNode)["loadFromCloud"]>[0] & {
|
|
45
|
+
embeddings: Embeddings;
|
|
46
|
+
credentials: CloseVectorCredentials;
|
|
47
|
+
}, "accessKey" | "secret">): Promise<CloseVectorNode>;
|
|
48
|
+
/**
|
|
49
|
+
* Static method to load a vector store from a directory. It reads the
|
|
50
|
+
* HNSW index, the arguments, and the document store from the directory,
|
|
51
|
+
* then creates a new HNSWLib instance with these values.
|
|
52
|
+
* @param directory The directory from which to load the vector store.
|
|
53
|
+
* @param embeddings The embeddings to be used by the CloseVectorNode instance.
|
|
54
|
+
* @returns A Promise that resolves to a new CloseVectorNode instance.
|
|
55
|
+
*/
|
|
56
|
+
static load(directory: string, embeddings: Embeddings, credentials?: CloseVectorCredentials): Promise<CloseVectorNode>;
|
|
57
|
+
/**
|
|
58
|
+
* Static method to create a new CloseVectorWeb instance from texts and metadata.
|
|
59
|
+
* It creates a new Document instance for each text and metadata, then
|
|
60
|
+
* calls the fromDocuments method to create the CloseVectorWeb instance.
|
|
61
|
+
* @param texts The texts to be used to create the documents.
|
|
62
|
+
* @param metadatas The metadata to be used to create the documents.
|
|
63
|
+
* @param embeddings The embeddings to be used by the CloseVectorWeb instance.
|
|
64
|
+
* @param args An optional configuration object for the CloseVectorWeb instance.
|
|
65
|
+
* @param credential An optional credential object for the CloseVector API.
|
|
66
|
+
* @returns A Promise that resolves to a new CloseVectorWeb instance.
|
|
67
|
+
*/
|
|
68
|
+
static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, args?: Record<string, unknown>, credential?: CloseVectorCredentials): Promise<CloseVectorNode>;
|
|
69
|
+
/**
|
|
70
|
+
* Static method to create a new CloseVectorNode instance from documents. It
|
|
71
|
+
* creates a new CloseVectorNode instance, adds the documents to it, then returns
|
|
72
|
+
* the instance.
|
|
73
|
+
* @param docs The documents to be added to the HNSWLib instance.
|
|
74
|
+
* @param embeddings The embeddings to be used by the HNSWLib instance.
|
|
75
|
+
* @param args An optional configuration object for the HNSWLib instance.
|
|
76
|
+
* @param credentials An optional credential object for the CloseVector API.
|
|
77
|
+
* @returns A Promise that resolves to a new CloseVectorNode instance.
|
|
78
|
+
*/
|
|
79
|
+
static fromDocuments(docs: Document[], embeddings: Embeddings, args?: Record<string, unknown>, credentials?: CloseVectorCredentials): Promise<CloseVectorNode>;
|
|
80
|
+
static imports(): Promise<{
|
|
81
|
+
HierarchicalNSW: typeof HierarchicalNSWT;
|
|
82
|
+
}>;
|
|
83
|
+
}
|