langchain 0.0.145 → 0.0.146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chains/api/api_chain.d.ts +1 -1
- package/dist/chains/openai_functions/extraction.d.ts +1 -1
- package/dist/chains/openai_functions/openapi.d.ts +1 -1
- package/dist/chains/openai_functions/structured_output.d.ts +1 -1
- package/dist/chains/openai_functions/tagging.d.ts +1 -1
- package/dist/chat_models/anthropic.d.ts +2 -2
- package/dist/hub.cjs +14 -0
- package/dist/hub.d.ts +14 -0
- package/dist/hub.js +14 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/dist/schema/index.cjs +5 -22
- package/dist/schema/index.d.ts +1 -5
- package/dist/schema/index.js +5 -22
- package/dist/types/googlevertexai-types.d.ts +1 -1
- package/dist/vectorstores/voy.cjs +158 -0
- package/dist/vectorstores/voy.d.ts +73 -0
- package/dist/vectorstores/voy.js +154 -0
- package/package.json +16 -68
- package/vectorstores/voy.cjs +1 -0
- package/vectorstores/voy.d.ts +1 -0
- package/vectorstores/voy.js +1 -0
- package/chat_models.cjs +0 -1
- package/chat_models.d.ts +0 -1
- package/chat_models.js +0 -1
- package/dist/chat_models/index.cjs +0 -11
- package/dist/chat_models/index.d.ts +0 -3
- package/dist/chat_models/index.js +0 -4
- package/dist/document_loaders/index.cjs +0 -40
- package/dist/document_loaders/index.d.ts +0 -18
- package/dist/document_loaders/index.js +0 -18
- package/dist/embeddings/index.cjs +0 -12
- package/dist/embeddings/index.d.ts +0 -4
- package/dist/embeddings/index.js +0 -5
- package/dist/index.cjs +0 -12
- package/dist/index.d.ts +0 -3
- package/dist/index.js +0 -4
- package/dist/llms/index.cjs +0 -18
- package/dist/llms/index.d.ts +0 -6
- package/dist/llms/index.js +0 -7
- package/dist/retrievers/index.cjs +0 -14
- package/dist/retrievers/index.d.ts +0 -5
- package/dist/retrievers/index.js +0 -6
- package/dist/vectorstores/index.cjs +0 -17
- package/dist/vectorstores/index.d.ts +0 -6
- package/dist/vectorstores/index.js +0 -7
- package/document_loaders.cjs +0 -1
- package/document_loaders.d.ts +0 -1
- package/document_loaders.js +0 -1
- package/embeddings.cjs +0 -1
- package/embeddings.d.ts +0 -1
- package/embeddings.js +0 -1
- package/llms.cjs +0 -1
- package/llms.d.ts +0 -1
- package/llms.js +0 -1
- package/retrievers.cjs +0 -1
- package/retrievers.d.ts +0 -1
- package/retrievers.js +0 -1
- package/vectorstores.cjs +0 -1
- package/vectorstores.d.ts +0 -1
- package/vectorstores.js +0 -1
|
@@ -4,7 +4,7 @@ import { LLMChain } from "../llm_chain.js";
|
|
|
4
4
|
import { BaseLanguageModel } from "../../base_language/index.js";
|
|
5
5
|
import { CallbackManagerForChainRun } from "../../callbacks/manager.js";
|
|
6
6
|
import { ChainValues } from "../../schema/index.js";
|
|
7
|
-
import { BasePromptTemplate } from "../../
|
|
7
|
+
import { BasePromptTemplate } from "../../prompts/base.js";
|
|
8
8
|
/**
|
|
9
9
|
* Interface that extends ChainInputs and defines additional input
|
|
10
10
|
* parameters specific to an APIChain.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { FunctionParameters } from "../../output_parsers/openai_functions.js";
|
|
3
3
|
import { LLMChain } from "../llm_chain.js";
|
|
4
|
-
import { BaseChatModel } from "../../chat_models/
|
|
4
|
+
import { BaseChatModel } from "../../chat_models/base.js";
|
|
5
5
|
import { BaseFunctionCallOptions } from "../../base_language/index.js";
|
|
6
6
|
/**
|
|
7
7
|
* Function that creates an extraction chain using the provided JSON schema.
|
|
@@ -3,7 +3,7 @@ import { BaseChain } from "../base.js";
|
|
|
3
3
|
import { LLMChainInput } from "../llm_chain.js";
|
|
4
4
|
import { BasePromptTemplate } from "../../prompts/base.js";
|
|
5
5
|
import { SequentialChain } from "../sequential_chain.js";
|
|
6
|
-
import { BaseChatModel } from "../../chat_models/
|
|
6
|
+
import { BaseChatModel } from "../../chat_models/base.js";
|
|
7
7
|
import { BaseFunctionCallOptions } from "../../base_language/index.js";
|
|
8
8
|
/**
|
|
9
9
|
* Type representing the options for creating an OpenAPI chain.
|
|
@@ -7,7 +7,7 @@ import { BasePromptTemplate } from "../../prompts/index.js";
|
|
|
7
7
|
import { BaseLLMOutputParser } from "../../schema/output_parser.js";
|
|
8
8
|
import { OutputFunctionsParser } from "../../output_parsers/openai_functions.js";
|
|
9
9
|
import { ChatGeneration } from "../../schema/index.js";
|
|
10
|
-
import { BaseChatModel } from "../../chat_models/
|
|
10
|
+
import { BaseChatModel } from "../../chat_models/base.js";
|
|
11
11
|
import { BaseFunctionCallOptions } from "../../base_language/index.js";
|
|
12
12
|
/**
|
|
13
13
|
* Type representing the input for creating a structured output chain. It
|
|
@@ -2,7 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { PromptTemplate } from "../../prompts/prompt.js";
|
|
3
3
|
import { FunctionParameters } from "../../output_parsers/openai_functions.js";
|
|
4
4
|
import { LLMChain, LLMChainInput } from "../llm_chain.js";
|
|
5
|
-
import { BaseChatModel } from "../../chat_models/
|
|
5
|
+
import { BaseChatModel } from "../../chat_models/base.js";
|
|
6
6
|
import { BaseFunctionCallOptions } from "../../base_language/index.js";
|
|
7
7
|
/**
|
|
8
8
|
* Type representing the options for creating a tagging chain.
|
|
@@ -103,9 +103,9 @@ export declare class ChatAnthropic<CallOptions extends BaseLanguageModelCallOpti
|
|
|
103
103
|
model: (string & {}) | "claude-2" | "claude-instant-1";
|
|
104
104
|
temperature?: number | undefined;
|
|
105
105
|
top_p?: number | undefined;
|
|
106
|
+
top_k?: number | undefined;
|
|
106
107
|
max_tokens_to_sample: number;
|
|
107
108
|
stop_sequences?: string[] | undefined;
|
|
108
|
-
top_k?: number | undefined;
|
|
109
109
|
model_name: string;
|
|
110
110
|
};
|
|
111
111
|
/**
|
|
@@ -117,9 +117,9 @@ export declare class ChatAnthropic<CallOptions extends BaseLanguageModelCallOpti
|
|
|
117
117
|
model: (string & {}) | "claude-2" | "claude-instant-1";
|
|
118
118
|
temperature?: number | undefined;
|
|
119
119
|
top_p?: number | undefined;
|
|
120
|
+
top_k?: number | undefined;
|
|
120
121
|
max_tokens_to_sample: number;
|
|
121
122
|
stop_sequences?: string[] | undefined;
|
|
122
|
-
top_k?: number | undefined;
|
|
123
123
|
model_name: string;
|
|
124
124
|
};
|
|
125
125
|
_streamResponseChunks(messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatGenerationChunk>;
|
package/dist/hub.cjs
CHANGED
|
@@ -3,11 +3,25 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.pull = exports.push = void 0;
|
|
4
4
|
const langchainhub_1 = require("langchainhub");
|
|
5
5
|
const index_js_1 = require("./load/index.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* Push a prompt to the hub.
|
|
8
|
+
* If the specified repo doesn't already exist, it will be created.
|
|
9
|
+
* @param repoFullName The full name of the repo.
|
|
10
|
+
* @param runnable The prompt to push.
|
|
11
|
+
* @param options
|
|
12
|
+
* @returns The URL of the newly pushed prompt in the hub.
|
|
13
|
+
*/
|
|
6
14
|
async function push(repoFullName, runnable, options) {
|
|
7
15
|
const client = new langchainhub_1.Client(options);
|
|
8
16
|
return client.push(repoFullName, JSON.stringify(runnable), options);
|
|
9
17
|
}
|
|
10
18
|
exports.push = push;
|
|
19
|
+
/**
|
|
20
|
+
* Pull a prompt from the hub.
|
|
21
|
+
* @param ownerRepoCommit The name of the repo containing the prompt, as well as an optional commit hash separated by a slash.
|
|
22
|
+
* @param options
|
|
23
|
+
* @returns
|
|
24
|
+
*/
|
|
11
25
|
async function pull(ownerRepoCommit, options) {
|
|
12
26
|
const client = new langchainhub_1.Client(options);
|
|
13
27
|
const result = await client.pull(ownerRepoCommit);
|
package/dist/hub.d.ts
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
import { ClientConfiguration, HubPushOptions } from "langchainhub";
|
|
2
2
|
import { Runnable } from "./schema/runnable.js";
|
|
3
|
+
/**
|
|
4
|
+
* Push a prompt to the hub.
|
|
5
|
+
* If the specified repo doesn't already exist, it will be created.
|
|
6
|
+
* @param repoFullName The full name of the repo.
|
|
7
|
+
* @param runnable The prompt to push.
|
|
8
|
+
* @param options
|
|
9
|
+
* @returns The URL of the newly pushed prompt in the hub.
|
|
10
|
+
*/
|
|
3
11
|
export declare function push(repoFullName: string, runnable: Runnable, options?: HubPushOptions & ClientConfiguration): Promise<string>;
|
|
12
|
+
/**
|
|
13
|
+
* Pull a prompt from the hub.
|
|
14
|
+
* @param ownerRepoCommit The name of the repo containing the prompt, as well as an optional commit hash separated by a slash.
|
|
15
|
+
* @param options
|
|
16
|
+
* @returns
|
|
17
|
+
*/
|
|
4
18
|
export declare function pull<T extends Runnable>(ownerRepoCommit: string, options?: ClientConfiguration): Promise<T>;
|
package/dist/hub.js
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
import { Client } from "langchainhub";
|
|
2
2
|
import { load } from "./load/index.js";
|
|
3
|
+
/**
|
|
4
|
+
* Push a prompt to the hub.
|
|
5
|
+
* If the specified repo doesn't already exist, it will be created.
|
|
6
|
+
* @param repoFullName The full name of the repo.
|
|
7
|
+
* @param runnable The prompt to push.
|
|
8
|
+
* @param options
|
|
9
|
+
* @returns The URL of the newly pushed prompt in the hub.
|
|
10
|
+
*/
|
|
3
11
|
export async function push(repoFullName, runnable, options) {
|
|
4
12
|
const client = new Client(options);
|
|
5
13
|
return client.push(repoFullName, JSON.stringify(runnable), options);
|
|
6
14
|
}
|
|
15
|
+
/**
|
|
16
|
+
* Pull a prompt from the hub.
|
|
17
|
+
* @param ownerRepoCommit The name of the repo containing the prompt, as well as an optional commit hash separated by a slash.
|
|
18
|
+
* @param options
|
|
19
|
+
* @returns
|
|
20
|
+
*/
|
|
7
21
|
export async function pull(ownerRepoCommit, options) {
|
|
8
22
|
const client = new Client(options);
|
|
9
23
|
const result = await client.pull(ownerRepoCommit);
|
|
@@ -55,6 +55,7 @@ exports.optionalImportEntrypoints = [
|
|
|
55
55
|
"langchain/vectorstores/singlestore",
|
|
56
56
|
"langchain/vectorstores/tigris",
|
|
57
57
|
"langchain/vectorstores/usearch",
|
|
58
|
+
"langchain/vectorstores/voy",
|
|
58
59
|
"langchain/vectorstores/zep",
|
|
59
60
|
"langchain/memory/zep",
|
|
60
61
|
"langchain/document_loaders/web/apify_dataset",
|
|
@@ -52,6 +52,7 @@ export const optionalImportEntrypoints = [
|
|
|
52
52
|
"langchain/vectorstores/singlestore",
|
|
53
53
|
"langchain/vectorstores/tigris",
|
|
54
54
|
"langchain/vectorstores/usearch",
|
|
55
|
+
"langchain/vectorstores/voy",
|
|
55
56
|
"langchain/vectorstores/zep",
|
|
56
57
|
"langchain/memory/zep",
|
|
57
58
|
"langchain/document_loaders/web/apify_dataset",
|
package/dist/schema/index.cjs
CHANGED
|
@@ -333,35 +333,18 @@ function coerceMessageLikeToMessage(messageLike) {
|
|
|
333
333
|
else if (isBaseMessage(messageLike)) {
|
|
334
334
|
return messageLike;
|
|
335
335
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
let name;
|
|
339
|
-
if (Array.isArray(messageLike)) {
|
|
340
|
-
[role, content] = messageLike;
|
|
341
|
-
name = "";
|
|
342
|
-
}
|
|
343
|
-
else {
|
|
344
|
-
role = messageLike.role;
|
|
345
|
-
content = messageLike.content;
|
|
346
|
-
name = messageLike.name;
|
|
347
|
-
}
|
|
348
|
-
if (role === "human" || role === "user") {
|
|
336
|
+
const [type, content] = messageLike;
|
|
337
|
+
if (type === "human" || type === "user") {
|
|
349
338
|
return new HumanMessage({ content });
|
|
350
339
|
}
|
|
351
|
-
else if (
|
|
340
|
+
else if (type === "ai" || type === "assistant") {
|
|
352
341
|
return new AIMessage({ content });
|
|
353
342
|
}
|
|
354
|
-
else if (
|
|
343
|
+
else if (type === "system") {
|
|
355
344
|
return new SystemMessage({ content });
|
|
356
345
|
}
|
|
357
|
-
else if (role === "function") {
|
|
358
|
-
if (!name) {
|
|
359
|
-
throw new Error(`Unable to coerce function message from object: no "name" field provided.`);
|
|
360
|
-
}
|
|
361
|
-
return new FunctionMessage({ content, name });
|
|
362
|
-
}
|
|
363
346
|
else {
|
|
364
|
-
|
|
347
|
+
throw new Error(`Unable to coerce message from array: only human, AI, or system message coercion is currently supported.`);
|
|
365
348
|
}
|
|
366
349
|
}
|
|
367
350
|
exports.coerceMessageLikeToMessage = coerceMessageLikeToMessage;
|
package/dist/schema/index.d.ts
CHANGED
|
@@ -210,11 +210,7 @@ export declare class ChatMessage extends BaseMessage implements ChatMessageField
|
|
|
210
210
|
_getType(): MessageType;
|
|
211
211
|
static isInstance(message: BaseMessage): message is ChatMessage;
|
|
212
212
|
}
|
|
213
|
-
export type BaseMessageLike = BaseMessage |
|
|
214
|
-
role: MessageType | "user" | "assistant" | (string & Record<never, never>);
|
|
215
|
-
content: string;
|
|
216
|
-
name?: string;
|
|
217
|
-
} | [
|
|
213
|
+
export type BaseMessageLike = BaseMessage | [
|
|
218
214
|
MessageType | "user" | "assistant" | (string & Record<never, never>),
|
|
219
215
|
string
|
|
220
216
|
] | string;
|
package/dist/schema/index.js
CHANGED
|
@@ -317,35 +317,18 @@ export function coerceMessageLikeToMessage(messageLike) {
|
|
|
317
317
|
else if (isBaseMessage(messageLike)) {
|
|
318
318
|
return messageLike;
|
|
319
319
|
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
let name;
|
|
323
|
-
if (Array.isArray(messageLike)) {
|
|
324
|
-
[role, content] = messageLike;
|
|
325
|
-
name = "";
|
|
326
|
-
}
|
|
327
|
-
else {
|
|
328
|
-
role = messageLike.role;
|
|
329
|
-
content = messageLike.content;
|
|
330
|
-
name = messageLike.name;
|
|
331
|
-
}
|
|
332
|
-
if (role === "human" || role === "user") {
|
|
320
|
+
const [type, content] = messageLike;
|
|
321
|
+
if (type === "human" || type === "user") {
|
|
333
322
|
return new HumanMessage({ content });
|
|
334
323
|
}
|
|
335
|
-
else if (
|
|
324
|
+
else if (type === "ai" || type === "assistant") {
|
|
336
325
|
return new AIMessage({ content });
|
|
337
326
|
}
|
|
338
|
-
else if (
|
|
327
|
+
else if (type === "system") {
|
|
339
328
|
return new SystemMessage({ content });
|
|
340
329
|
}
|
|
341
|
-
else if (role === "function") {
|
|
342
|
-
if (!name) {
|
|
343
|
-
throw new Error(`Unable to coerce function message from object: no "name" field provided.`);
|
|
344
|
-
}
|
|
345
|
-
return new FunctionMessage({ content, name });
|
|
346
|
-
}
|
|
347
330
|
else {
|
|
348
|
-
|
|
331
|
+
throw new Error(`Unable to coerce message from array: only human, AI, or system message coercion is currently supported.`);
|
|
349
332
|
}
|
|
350
333
|
}
|
|
351
334
|
/**
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.VoyVectorStore = void 0;
|
|
4
|
+
const base_js_1 = require("./base.cjs");
|
|
5
|
+
const document_js_1 = require("../document.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* Class that extends `VectorStore`. It allows to perform similarity search using
|
|
8
|
+
* Voi similarity search engine. The class requires passing Voy Client as an input parameter.
|
|
9
|
+
*/
|
|
10
|
+
class VoyVectorStore extends base_js_1.VectorStore {
|
|
11
|
+
_vectorstoreType() {
|
|
12
|
+
return "voi";
|
|
13
|
+
}
|
|
14
|
+
constructor(client, embeddings) {
|
|
15
|
+
super(embeddings, {});
|
|
16
|
+
Object.defineProperty(this, "client", {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true,
|
|
20
|
+
value: void 0
|
|
21
|
+
});
|
|
22
|
+
Object.defineProperty(this, "numDimensions", {
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
writable: true,
|
|
26
|
+
value: null
|
|
27
|
+
});
|
|
28
|
+
Object.defineProperty(this, "docstore", {
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true,
|
|
32
|
+
value: []
|
|
33
|
+
});
|
|
34
|
+
this.client = client;
|
|
35
|
+
this.embeddings = embeddings;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Adds documents to the Voy database. The documents are embedded using embeddings provided while instantiating the class.
|
|
39
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
40
|
+
*/
|
|
41
|
+
async addDocuments(documents) {
|
|
42
|
+
const texts = documents.map(({ pageContent }) => pageContent);
|
|
43
|
+
if (documents.length === 0) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
const firstVector = (await this.embeddings.embedDocuments(texts.slice(0, 1)))[0];
|
|
47
|
+
if (this.numDimensions === null) {
|
|
48
|
+
this.numDimensions = firstVector.length;
|
|
49
|
+
}
|
|
50
|
+
else if (this.numDimensions !== firstVector.length) {
|
|
51
|
+
throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
52
|
+
}
|
|
53
|
+
const restResults = await this.embeddings.embedDocuments(texts.slice(1));
|
|
54
|
+
await this.addVectors([firstVector, ...restResults], documents);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Adds vectors to the Voy database. The vectors are associated with
|
|
58
|
+
* the provided documents.
|
|
59
|
+
* @param vectors An array of vectors to be added to the database.
|
|
60
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
61
|
+
*/
|
|
62
|
+
async addVectors(vectors, documents) {
|
|
63
|
+
if (vectors.length === 0) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (this.numDimensions === null) {
|
|
67
|
+
this.numDimensions = vectors[0].length;
|
|
68
|
+
}
|
|
69
|
+
if (vectors.length !== documents.length) {
|
|
70
|
+
throw new Error(`Vectors and metadata must have the same length`);
|
|
71
|
+
}
|
|
72
|
+
if (!vectors.every((v) => v.length === this.numDimensions)) {
|
|
73
|
+
throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
74
|
+
}
|
|
75
|
+
vectors.forEach((item, idx) => {
|
|
76
|
+
const doc = documents[idx];
|
|
77
|
+
this.docstore.push({ embeddings: item, document: doc });
|
|
78
|
+
});
|
|
79
|
+
const embeddings = this.docstore.map((item, idx) => ({
|
|
80
|
+
id: String(idx),
|
|
81
|
+
embeddings: item.embeddings,
|
|
82
|
+
title: "",
|
|
83
|
+
url: "",
|
|
84
|
+
}));
|
|
85
|
+
this.client.index({ embeddings });
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Searches for vectors in the Voy database that are similar to the
|
|
89
|
+
* provided query vector.
|
|
90
|
+
* @param query The query vector.
|
|
91
|
+
* @param k The number of similar vectors to return.
|
|
92
|
+
* @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
|
|
93
|
+
*/
|
|
94
|
+
async similaritySearchVectorWithScore(query, k) {
|
|
95
|
+
if (this.numDimensions === null) {
|
|
96
|
+
throw new Error("There aren't any elements in the index yet.");
|
|
97
|
+
}
|
|
98
|
+
if (query.length !== this.numDimensions) {
|
|
99
|
+
throw new Error(`Query vector must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
100
|
+
}
|
|
101
|
+
const itemsToQuery = Math.min(this.docstore.length, k);
|
|
102
|
+
if (itemsToQuery > this.docstore.length) {
|
|
103
|
+
console.warn(`k (${k}) is greater than the number of elements in the index (${this.docstore.length}), setting k to ${itemsToQuery}`);
|
|
104
|
+
}
|
|
105
|
+
const results = this.client.search(new Float32Array(query), itemsToQuery);
|
|
106
|
+
return results.neighbors.map(({ id }, idx) => [this.docstore[parseInt(id, 10)].document, idx]);
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Method to delete data from the Voy index. It can delete data based
|
|
110
|
+
* on specific IDs or a filter.
|
|
111
|
+
* @param params Object that includes either an array of IDs or a filter for the data to be deleted.
|
|
112
|
+
* @returns Promise that resolves when the deletion is complete.
|
|
113
|
+
*/
|
|
114
|
+
async delete(params) {
|
|
115
|
+
if (params.deleteAll === true) {
|
|
116
|
+
await this.client.clear();
|
|
117
|
+
}
|
|
118
|
+
else {
|
|
119
|
+
throw new Error(`You must provide a "deleteAll" parameter.`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Creates a new `VoyVectorStore` instance from an array of text strings. The text
|
|
124
|
+
* strings are converted to `Document` instances and added to the Voy
|
|
125
|
+
* database.
|
|
126
|
+
* @param texts An array of text strings.
|
|
127
|
+
* @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
|
|
128
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
129
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
130
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
131
|
+
*/
|
|
132
|
+
static async fromTexts(texts, metadatas, embeddings, client) {
|
|
133
|
+
const docs = [];
|
|
134
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
135
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
136
|
+
const newDoc = new document_js_1.Document({
|
|
137
|
+
pageContent: texts[i],
|
|
138
|
+
metadata,
|
|
139
|
+
});
|
|
140
|
+
docs.push(newDoc);
|
|
141
|
+
}
|
|
142
|
+
return VoyVectorStore.fromDocuments(docs, embeddings, client);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Creates a new `VoyVectorStore` instance from an array of `Document` instances.
|
|
146
|
+
* The documents are added to the Voy database.
|
|
147
|
+
* @param docs An array of `Document` instances.
|
|
148
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
149
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
150
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
151
|
+
*/
|
|
152
|
+
static async fromDocuments(docs, embeddings, client) {
|
|
153
|
+
const instance = new VoyVectorStore(client, embeddings);
|
|
154
|
+
await instance.addDocuments(docs);
|
|
155
|
+
return instance;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
exports.VoyVectorStore = VoyVectorStore;
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { Voy as VoyOriginClient } from "voy-search";
|
|
2
|
+
import { Embeddings } from "../embeddings/base.js";
|
|
3
|
+
import { VectorStore } from "./base.js";
|
|
4
|
+
import { Document } from "../document.js";
|
|
5
|
+
export type VoyClient = Omit<VoyOriginClient, "remove" | "size" | "serialize" | "free">;
|
|
6
|
+
/**
|
|
7
|
+
* Internal interface for storing documents mappings.
|
|
8
|
+
*/
|
|
9
|
+
interface InternalDoc {
|
|
10
|
+
embeddings: number[];
|
|
11
|
+
document: Document;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Class that extends `VectorStore`. It allows to perform similarity search using
|
|
15
|
+
* Voi similarity search engine. The class requires passing Voy Client as an input parameter.
|
|
16
|
+
*/
|
|
17
|
+
export declare class VoyVectorStore extends VectorStore {
|
|
18
|
+
client: VoyClient;
|
|
19
|
+
numDimensions: number | null;
|
|
20
|
+
docstore: InternalDoc[];
|
|
21
|
+
_vectorstoreType(): string;
|
|
22
|
+
constructor(client: VoyClient, embeddings: Embeddings);
|
|
23
|
+
/**
|
|
24
|
+
* Adds documents to the Voy database. The documents are embedded using embeddings provided while instantiating the class.
|
|
25
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
26
|
+
*/
|
|
27
|
+
addDocuments(documents: Document[]): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Adds vectors to the Voy database. The vectors are associated with
|
|
30
|
+
* the provided documents.
|
|
31
|
+
* @param vectors An array of vectors to be added to the database.
|
|
32
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
33
|
+
*/
|
|
34
|
+
addVectors(vectors: number[][], documents: Document[]): Promise<void>;
|
|
35
|
+
/**
|
|
36
|
+
* Searches for vectors in the Voy database that are similar to the
|
|
37
|
+
* provided query vector.
|
|
38
|
+
* @param query The query vector.
|
|
39
|
+
* @param k The number of similar vectors to return.
|
|
40
|
+
* @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
|
|
41
|
+
*/
|
|
42
|
+
similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document<Record<string, any>>, number][]>;
|
|
43
|
+
/**
|
|
44
|
+
* Method to delete data from the Voy index. It can delete data based
|
|
45
|
+
* on specific IDs or a filter.
|
|
46
|
+
* @param params Object that includes either an array of IDs or a filter for the data to be deleted.
|
|
47
|
+
* @returns Promise that resolves when the deletion is complete.
|
|
48
|
+
*/
|
|
49
|
+
delete(params: {
|
|
50
|
+
deleteAll?: boolean;
|
|
51
|
+
}): Promise<void>;
|
|
52
|
+
/**
|
|
53
|
+
* Creates a new `VoyVectorStore` instance from an array of text strings. The text
|
|
54
|
+
* strings are converted to `Document` instances and added to the Voy
|
|
55
|
+
* database.
|
|
56
|
+
* @param texts An array of text strings.
|
|
57
|
+
* @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
|
|
58
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
59
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
60
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
61
|
+
*/
|
|
62
|
+
static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, client: VoyClient): Promise<VoyVectorStore>;
|
|
63
|
+
/**
|
|
64
|
+
* Creates a new `VoyVectorStore` instance from an array of `Document` instances.
|
|
65
|
+
* The documents are added to the Voy database.
|
|
66
|
+
* @param docs An array of `Document` instances.
|
|
67
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
68
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
69
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
70
|
+
*/
|
|
71
|
+
static fromDocuments(docs: Document[], embeddings: Embeddings, client: VoyClient): Promise<VoyVectorStore>;
|
|
72
|
+
}
|
|
73
|
+
export {};
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { VectorStore } from "./base.js";
|
|
2
|
+
import { Document } from "../document.js";
|
|
3
|
+
/**
|
|
4
|
+
* Class that extends `VectorStore`. It allows to perform similarity search using
|
|
5
|
+
* Voi similarity search engine. The class requires passing Voy Client as an input parameter.
|
|
6
|
+
*/
|
|
7
|
+
export class VoyVectorStore extends VectorStore {
|
|
8
|
+
_vectorstoreType() {
|
|
9
|
+
return "voi";
|
|
10
|
+
}
|
|
11
|
+
constructor(client, embeddings) {
|
|
12
|
+
super(embeddings, {});
|
|
13
|
+
Object.defineProperty(this, "client", {
|
|
14
|
+
enumerable: true,
|
|
15
|
+
configurable: true,
|
|
16
|
+
writable: true,
|
|
17
|
+
value: void 0
|
|
18
|
+
});
|
|
19
|
+
Object.defineProperty(this, "numDimensions", {
|
|
20
|
+
enumerable: true,
|
|
21
|
+
configurable: true,
|
|
22
|
+
writable: true,
|
|
23
|
+
value: null
|
|
24
|
+
});
|
|
25
|
+
Object.defineProperty(this, "docstore", {
|
|
26
|
+
enumerable: true,
|
|
27
|
+
configurable: true,
|
|
28
|
+
writable: true,
|
|
29
|
+
value: []
|
|
30
|
+
});
|
|
31
|
+
this.client = client;
|
|
32
|
+
this.embeddings = embeddings;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Adds documents to the Voy database. The documents are embedded using embeddings provided while instantiating the class.
|
|
36
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
37
|
+
*/
|
|
38
|
+
async addDocuments(documents) {
|
|
39
|
+
const texts = documents.map(({ pageContent }) => pageContent);
|
|
40
|
+
if (documents.length === 0) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const firstVector = (await this.embeddings.embedDocuments(texts.slice(0, 1)))[0];
|
|
44
|
+
if (this.numDimensions === null) {
|
|
45
|
+
this.numDimensions = firstVector.length;
|
|
46
|
+
}
|
|
47
|
+
else if (this.numDimensions !== firstVector.length) {
|
|
48
|
+
throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
49
|
+
}
|
|
50
|
+
const restResults = await this.embeddings.embedDocuments(texts.slice(1));
|
|
51
|
+
await this.addVectors([firstVector, ...restResults], documents);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Adds vectors to the Voy database. The vectors are associated with
|
|
55
|
+
* the provided documents.
|
|
56
|
+
* @param vectors An array of vectors to be added to the database.
|
|
57
|
+
* @param documents An array of `Document` instances associated with the vectors.
|
|
58
|
+
*/
|
|
59
|
+
async addVectors(vectors, documents) {
|
|
60
|
+
if (vectors.length === 0) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
if (this.numDimensions === null) {
|
|
64
|
+
this.numDimensions = vectors[0].length;
|
|
65
|
+
}
|
|
66
|
+
if (vectors.length !== documents.length) {
|
|
67
|
+
throw new Error(`Vectors and metadata must have the same length`);
|
|
68
|
+
}
|
|
69
|
+
if (!vectors.every((v) => v.length === this.numDimensions)) {
|
|
70
|
+
throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
71
|
+
}
|
|
72
|
+
vectors.forEach((item, idx) => {
|
|
73
|
+
const doc = documents[idx];
|
|
74
|
+
this.docstore.push({ embeddings: item, document: doc });
|
|
75
|
+
});
|
|
76
|
+
const embeddings = this.docstore.map((item, idx) => ({
|
|
77
|
+
id: String(idx),
|
|
78
|
+
embeddings: item.embeddings,
|
|
79
|
+
title: "",
|
|
80
|
+
url: "",
|
|
81
|
+
}));
|
|
82
|
+
this.client.index({ embeddings });
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Searches for vectors in the Voy database that are similar to the
|
|
86
|
+
* provided query vector.
|
|
87
|
+
* @param query The query vector.
|
|
88
|
+
* @param k The number of similar vectors to return.
|
|
89
|
+
* @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
|
|
90
|
+
*/
|
|
91
|
+
async similaritySearchVectorWithScore(query, k) {
|
|
92
|
+
if (this.numDimensions === null) {
|
|
93
|
+
throw new Error("There aren't any elements in the index yet.");
|
|
94
|
+
}
|
|
95
|
+
if (query.length !== this.numDimensions) {
|
|
96
|
+
throw new Error(`Query vector must have the same length as the number of dimensions (${this.numDimensions})`);
|
|
97
|
+
}
|
|
98
|
+
const itemsToQuery = Math.min(this.docstore.length, k);
|
|
99
|
+
if (itemsToQuery > this.docstore.length) {
|
|
100
|
+
console.warn(`k (${k}) is greater than the number of elements in the index (${this.docstore.length}), setting k to ${itemsToQuery}`);
|
|
101
|
+
}
|
|
102
|
+
const results = this.client.search(new Float32Array(query), itemsToQuery);
|
|
103
|
+
return results.neighbors.map(({ id }, idx) => [this.docstore[parseInt(id, 10)].document, idx]);
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Method to delete data from the Voy index. It can delete data based
|
|
107
|
+
* on specific IDs or a filter.
|
|
108
|
+
* @param params Object that includes either an array of IDs or a filter for the data to be deleted.
|
|
109
|
+
* @returns Promise that resolves when the deletion is complete.
|
|
110
|
+
*/
|
|
111
|
+
async delete(params) {
|
|
112
|
+
if (params.deleteAll === true) {
|
|
113
|
+
await this.client.clear();
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
throw new Error(`You must provide a "deleteAll" parameter.`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Creates a new `VoyVectorStore` instance from an array of text strings. The text
|
|
121
|
+
* strings are converted to `Document` instances and added to the Voy
|
|
122
|
+
* database.
|
|
123
|
+
* @param texts An array of text strings.
|
|
124
|
+
* @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
|
|
125
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
126
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
127
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
128
|
+
*/
|
|
129
|
+
static async fromTexts(texts, metadatas, embeddings, client) {
|
|
130
|
+
const docs = [];
|
|
131
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
132
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
133
|
+
const newDoc = new Document({
|
|
134
|
+
pageContent: texts[i],
|
|
135
|
+
metadata,
|
|
136
|
+
});
|
|
137
|
+
docs.push(newDoc);
|
|
138
|
+
}
|
|
139
|
+
return VoyVectorStore.fromDocuments(docs, embeddings, client);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Creates a new `VoyVectorStore` instance from an array of `Document` instances.
|
|
143
|
+
* The documents are added to the Voy database.
|
|
144
|
+
* @param docs An array of `Document` instances.
|
|
145
|
+
* @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
|
|
146
|
+
* @param client An instance of Voy client to use in the underlying operations.
|
|
147
|
+
* @returns A promise that resolves with a new `VoyVectorStore` instance.
|
|
148
|
+
*/
|
|
149
|
+
static async fromDocuments(docs, embeddings, client) {
|
|
150
|
+
const instance = new VoyVectorStore(client, embeddings);
|
|
151
|
+
await instance.addDocuments(docs);
|
|
152
|
+
return instance;
|
|
153
|
+
}
|
|
154
|
+
}
|