langchain 0.0.145 → 0.0.147

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/base_language/index.cjs +2 -2
  2. package/dist/base_language/index.d.ts +2 -1
  3. package/dist/base_language/index.js +1 -1
  4. package/dist/chains/api/api_chain.d.ts +1 -1
  5. package/dist/chains/base.d.ts +1 -1
  6. package/dist/chains/openai_functions/extraction.d.ts +1 -1
  7. package/dist/chains/openai_functions/openapi.cjs +32 -27
  8. package/dist/chains/openai_functions/openapi.d.ts +10 -1
  9. package/dist/chains/openai_functions/openapi.js +31 -27
  10. package/dist/chains/openai_functions/structured_output.d.ts +1 -1
  11. package/dist/chains/openai_functions/tagging.d.ts +1 -1
  12. package/dist/chat_models/anthropic.d.ts +2 -2
  13. package/dist/chat_models/base.d.ts +1 -1
  14. package/dist/chat_models/openai.cjs +1 -1
  15. package/dist/chat_models/openai.js +1 -1
  16. package/dist/experimental/llms/bittensor.cjs +141 -0
  17. package/dist/experimental/llms/bittensor.d.ts +33 -0
  18. package/dist/experimental/llms/bittensor.js +137 -0
  19. package/dist/hub.cjs +14 -0
  20. package/dist/hub.d.ts +15 -1
  21. package/dist/hub.js +14 -0
  22. package/dist/llms/base.d.ts +1 -1
  23. package/dist/llms/openai-chat.cjs +1 -1
  24. package/dist/llms/openai-chat.js +1 -1
  25. package/dist/llms/openai.cjs +1 -1
  26. package/dist/llms/openai.js +1 -1
  27. package/dist/load/import_constants.cjs +2 -0
  28. package/dist/load/import_constants.js +2 -0
  29. package/dist/load/import_map.cjs +1 -1
  30. package/dist/load/import_map.d.ts +1 -1
  31. package/dist/load/import_map.js +1 -1
  32. package/dist/load/index.cjs +2 -1
  33. package/dist/load/index.js +2 -1
  34. package/dist/prompts/base.cjs +2 -2
  35. package/dist/prompts/base.d.ts +1 -1
  36. package/dist/prompts/base.js +1 -1
  37. package/dist/prompts/chat.cjs +2 -2
  38. package/dist/prompts/chat.d.ts +1 -1
  39. package/dist/prompts/chat.js +1 -1
  40. package/dist/schema/document.cjs +2 -2
  41. package/dist/schema/document.d.ts +1 -1
  42. package/dist/schema/document.js +1 -1
  43. package/dist/schema/index.cjs +5 -22
  44. package/dist/schema/index.d.ts +1 -5
  45. package/dist/schema/index.js +5 -22
  46. package/dist/schema/output_parser.cjs +2 -2
  47. package/dist/schema/output_parser.d.ts +2 -1
  48. package/dist/schema/output_parser.js +1 -1
  49. package/dist/schema/retriever.cjs +2 -2
  50. package/dist/schema/retriever.d.ts +2 -1
  51. package/dist/schema/retriever.js +1 -1
  52. package/dist/schema/runnable/config.cjs +8 -0
  53. package/dist/schema/runnable/config.d.ts +3 -0
  54. package/dist/schema/runnable/config.js +4 -0
  55. package/dist/schema/{runnable.cjs → runnable/index.cjs} +290 -101
  56. package/dist/schema/{runnable.d.ts → runnable/index.d.ts} +127 -41
  57. package/dist/schema/{runnable.js → runnable/index.js} +284 -99
  58. package/dist/tools/base.d.ts +1 -1
  59. package/dist/types/googlevertexai-types.d.ts +1 -1
  60. package/dist/util/async_caller.cjs +35 -25
  61. package/dist/util/async_caller.d.ts +8 -0
  62. package/dist/util/async_caller.js +35 -25
  63. package/dist/vectorstores/pinecone.cjs +30 -22
  64. package/dist/vectorstores/pinecone.d.ts +3 -1
  65. package/dist/vectorstores/pinecone.js +30 -22
  66. package/dist/vectorstores/vectara.cjs +20 -23
  67. package/dist/vectorstores/vectara.d.ts +9 -2
  68. package/dist/vectorstores/vectara.js +20 -23
  69. package/dist/vectorstores/voy.cjs +158 -0
  70. package/dist/vectorstores/voy.d.ts +73 -0
  71. package/dist/vectorstores/voy.js +154 -0
  72. package/experimental/llms/bittensor.cjs +1 -0
  73. package/experimental/llms/bittensor.d.ts +1 -0
  74. package/experimental/llms/bittensor.js +1 -0
  75. package/package.json +24 -68
  76. package/schema/runnable.cjs +1 -1
  77. package/schema/runnable.d.ts +1 -1
  78. package/schema/runnable.js +1 -1
  79. package/vectorstores/voy.cjs +1 -0
  80. package/vectorstores/voy.d.ts +1 -0
  81. package/vectorstores/voy.js +1 -0
  82. package/chat_models.cjs +0 -1
  83. package/chat_models.d.ts +0 -1
  84. package/chat_models.js +0 -1
  85. package/dist/chat_models/index.cjs +0 -11
  86. package/dist/chat_models/index.d.ts +0 -3
  87. package/dist/chat_models/index.js +0 -4
  88. package/dist/document_loaders/index.cjs +0 -40
  89. package/dist/document_loaders/index.d.ts +0 -18
  90. package/dist/document_loaders/index.js +0 -18
  91. package/dist/embeddings/index.cjs +0 -12
  92. package/dist/embeddings/index.d.ts +0 -4
  93. package/dist/embeddings/index.js +0 -5
  94. package/dist/index.cjs +0 -12
  95. package/dist/index.d.ts +0 -3
  96. package/dist/index.js +0 -4
  97. package/dist/llms/index.cjs +0 -18
  98. package/dist/llms/index.d.ts +0 -6
  99. package/dist/llms/index.js +0 -7
  100. package/dist/retrievers/index.cjs +0 -14
  101. package/dist/retrievers/index.d.ts +0 -5
  102. package/dist/retrievers/index.js +0 -6
  103. package/dist/vectorstores/index.cjs +0 -17
  104. package/dist/vectorstores/index.d.ts +0 -6
  105. package/dist/vectorstores/index.js +0 -7
  106. package/document_loaders.cjs +0 -1
  107. package/document_loaders.d.ts +0 -1
  108. package/document_loaders.js +0 -1
  109. package/embeddings.cjs +0 -1
  110. package/embeddings.d.ts +0 -1
  111. package/embeddings.js +0 -1
  112. package/llms.cjs +0 -1
  113. package/llms.d.ts +0 -1
  114. package/llms.js +0 -1
  115. package/retrievers.cjs +0 -1
  116. package/retrievers.d.ts +0 -1
  117. package/retrievers.js +0 -1
  118. package/vectorstores.cjs +0 -1
  119. package/vectorstores.d.ts +0 -1
  120. package/vectorstores.js +0 -1
@@ -12,6 +12,32 @@ const STATUS_NO_RETRY = [
12
12
  408,
13
13
  409, // Conflict
14
14
  ];
15
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
16
+ const defaultFailedAttemptHandler = (error) => {
17
+ if (error.message.startsWith("Cancel") ||
18
+ error.message.startsWith("TimeoutError") ||
19
+ error.name === "TimeoutError" ||
20
+ error.message.startsWith("AbortError") ||
21
+ error.name === "AbortError") {
22
+ throw error;
23
+ }
24
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
25
+ if (error?.code === "ECONNABORTED") {
26
+ throw error;
27
+ }
28
+ const status =
29
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
30
+ error?.response?.status ?? error?.status;
31
+ if (status && STATUS_NO_RETRY.includes(+status)) {
32
+ throw error;
33
+ }
34
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
35
+ if (error?.error?.code === "insufficient_quota") {
36
+ const err = new Error(error?.message);
37
+ err.name = "InsufficientQuotaError";
38
+ throw err;
39
+ }
40
+ };
15
41
  /**
16
42
  * A class that can be used to make async calls with concurrency and retry logic.
17
43
  *
@@ -39,6 +65,12 @@ export class AsyncCaller {
39
65
  writable: true,
40
66
  value: void 0
41
67
  });
68
+ Object.defineProperty(this, "onFailedAttempt", {
69
+ enumerable: true,
70
+ configurable: true,
71
+ writable: true,
72
+ value: void 0
73
+ });
42
74
  Object.defineProperty(this, "queue", {
43
75
  enumerable: true,
44
76
  configurable: true,
@@ -47,6 +79,8 @@ export class AsyncCaller {
47
79
  });
48
80
  this.maxConcurrency = params.maxConcurrency ?? Infinity;
49
81
  this.maxRetries = params.maxRetries ?? 6;
82
+ this.onFailedAttempt =
83
+ params.onFailedAttempt ?? defaultFailedAttemptHandler;
50
84
  const PQueue = "default" in PQueueMod ? PQueueMod.default : PQueueMod;
51
85
  this.queue = new PQueue({ concurrency: this.maxConcurrency });
52
86
  }
@@ -61,31 +95,7 @@ export class AsyncCaller {
61
95
  throw new Error(error);
62
96
  }
63
97
  }), {
64
- onFailedAttempt(error) {
65
- if (error.message.startsWith("Cancel") ||
66
- error.message.startsWith("TimeoutError") ||
67
- error.name === "TimeoutError" ||
68
- error.message.startsWith("AbortError") ||
69
- error.name === "AbortError") {
70
- throw error;
71
- }
72
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
73
- if (error?.code === "ECONNABORTED") {
74
- throw error;
75
- }
76
- const status =
77
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
78
- error?.response?.status ?? error?.status;
79
- if (status && STATUS_NO_RETRY.includes(+status)) {
80
- throw error;
81
- }
82
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
83
- if (error?.error?.code === "insufficient_quota") {
84
- const err = new Error(error?.message);
85
- err.name = "InsufficientQuotaError";
86
- throw err;
87
- }
88
- },
98
+ onFailedAttempt: this.onFailedAttempt,
89
99
  retries: this.maxRetries,
90
100
  randomize: true,
91
101
  // If needed we can change some of the defaults here,
@@ -31,6 +31,8 @@ const uuid = __importStar(require("uuid"));
31
31
  const flat_1 = __importDefault(require("flat"));
32
32
  const base_js_1 = require("./base.cjs");
33
33
  const document_js_1 = require("../document.cjs");
34
+ const chunk_js_1 = require("../util/chunk.cjs");
35
+ const async_caller_js_1 = require("../util/async_caller.cjs");
34
36
  /**
35
37
  * Class that extends the VectorStore class and provides methods to
36
38
  * interact with the Pinecone vector database.
@@ -65,11 +67,19 @@ class PineconeStore extends base_js_1.VectorStore {
65
67
  writable: true,
66
68
  value: void 0
67
69
  });
70
+ Object.defineProperty(this, "caller", {
71
+ enumerable: true,
72
+ configurable: true,
73
+ writable: true,
74
+ value: void 0
75
+ });
68
76
  this.embeddings = embeddings;
69
- this.namespace = args.namespace;
70
- this.pineconeIndex = args.pineconeIndex;
71
- this.textKey = args.textKey ?? "text";
72
- this.filter = args.filter;
77
+ const { namespace, pineconeIndex, textKey, filter, ...asyncCallerArgs } = args;
78
+ this.namespace = namespace;
79
+ this.pineconeIndex = pineconeIndex;
80
+ this.textKey = textKey ?? "text";
81
+ this.filter = filter;
82
+ this.caller = new async_caller_js_1.AsyncCaller(asyncCallerArgs);
73
83
  }
74
84
  /**
75
85
  * Method that adds documents to the Pinecone database.
@@ -126,16 +136,15 @@ class PineconeStore extends base_js_1.VectorStore {
126
136
  };
127
137
  });
128
138
  // Pinecone recommends a limit of 100 vectors per upsert request
129
- const chunkSize = 50;
130
- for (let i = 0; i < pineconeVectors.length; i += chunkSize) {
131
- const chunk = pineconeVectors.slice(i, i + chunkSize);
132
- await this.pineconeIndex.upsert({
133
- upsertRequest: {
134
- vectors: chunk,
135
- namespace: this.namespace,
136
- },
137
- });
138
- }
139
+ const chunkSize = 100;
140
+ const chunkedVectors = (0, chunk_js_1.chunkArray)(pineconeVectors, chunkSize);
141
+ const batchRequests = chunkedVectors.map((chunk) => this.caller.call(async () => this.pineconeIndex.upsert({
142
+ upsertRequest: {
143
+ vectors: chunk,
144
+ namespace: this.namespace,
145
+ },
146
+ })));
147
+ await Promise.all(batchRequests);
139
148
  return documentIds;
140
149
  }
141
150
  /**
@@ -154,14 +163,13 @@ class PineconeStore extends base_js_1.VectorStore {
154
163
  }
155
164
  else if (ids) {
156
165
  const batchSize = 1000;
157
- for (let i = 0; i < ids.length; i += batchSize) {
158
- const batchIds = ids.slice(i, i + batchSize);
159
- await this.pineconeIndex.delete1({
160
- ids: batchIds,
161
- namespace,
162
- ...rest,
163
- });
164
- }
166
+ const batchedIds = (0, chunk_js_1.chunkArray)(ids, batchSize);
167
+ const batchRequests = batchedIds.map((batchIds) => this.caller.call(async () => this.pineconeIndex.delete1({
168
+ ids: batchIds,
169
+ namespace,
170
+ ...rest,
171
+ })));
172
+ await Promise.all(batchRequests);
165
173
  }
166
174
  else {
167
175
  throw new Error("Either ids or delete_all must be provided.");
@@ -1,9 +1,10 @@
1
1
  import { VectorStore } from "./base.js";
2
2
  import { Embeddings } from "../embeddings/base.js";
3
3
  import { Document } from "../document.js";
4
+ import { AsyncCaller, type AsyncCallerParams } from "../util/async_caller.js";
4
5
  type PineconeMetadata = Record<string, any>;
5
6
  type VectorOperationsApi = ReturnType<import("@pinecone-database/pinecone").PineconeClient["Index"]>;
6
- export interface PineconeLibArgs {
7
+ export interface PineconeLibArgs extends AsyncCallerParams {
7
8
  pineconeIndex: VectorOperationsApi;
8
9
  textKey?: string;
9
10
  namespace?: string;
@@ -28,6 +29,7 @@ export declare class PineconeStore extends VectorStore {
28
29
  namespace?: string;
29
30
  pineconeIndex: VectorOperationsApi;
30
31
  filter?: PineconeMetadata;
32
+ caller: AsyncCaller;
31
33
  _vectorstoreType(): string;
32
34
  constructor(embeddings: Embeddings, args: PineconeLibArgs);
33
35
  /**
@@ -2,6 +2,8 @@ import * as uuid from "uuid";
2
2
  import flatten from "flat";
3
3
  import { VectorStore } from "./base.js";
4
4
  import { Document } from "../document.js";
5
+ import { chunkArray } from "../util/chunk.js";
6
+ import { AsyncCaller } from "../util/async_caller.js";
5
7
  /**
6
8
  * Class that extends the VectorStore class and provides methods to
7
9
  * interact with the Pinecone vector database.
@@ -36,11 +38,19 @@ export class PineconeStore extends VectorStore {
36
38
  writable: true,
37
39
  value: void 0
38
40
  });
41
+ Object.defineProperty(this, "caller", {
42
+ enumerable: true,
43
+ configurable: true,
44
+ writable: true,
45
+ value: void 0
46
+ });
39
47
  this.embeddings = embeddings;
40
- this.namespace = args.namespace;
41
- this.pineconeIndex = args.pineconeIndex;
42
- this.textKey = args.textKey ?? "text";
43
- this.filter = args.filter;
48
+ const { namespace, pineconeIndex, textKey, filter, ...asyncCallerArgs } = args;
49
+ this.namespace = namespace;
50
+ this.pineconeIndex = pineconeIndex;
51
+ this.textKey = textKey ?? "text";
52
+ this.filter = filter;
53
+ this.caller = new AsyncCaller(asyncCallerArgs);
44
54
  }
45
55
  /**
46
56
  * Method that adds documents to the Pinecone database.
@@ -97,16 +107,15 @@ export class PineconeStore extends VectorStore {
97
107
  };
98
108
  });
99
109
  // Pinecone recommends a limit of 100 vectors per upsert request
100
- const chunkSize = 50;
101
- for (let i = 0; i < pineconeVectors.length; i += chunkSize) {
102
- const chunk = pineconeVectors.slice(i, i + chunkSize);
103
- await this.pineconeIndex.upsert({
104
- upsertRequest: {
105
- vectors: chunk,
106
- namespace: this.namespace,
107
- },
108
- });
109
- }
110
+ const chunkSize = 100;
111
+ const chunkedVectors = chunkArray(pineconeVectors, chunkSize);
112
+ const batchRequests = chunkedVectors.map((chunk) => this.caller.call(async () => this.pineconeIndex.upsert({
113
+ upsertRequest: {
114
+ vectors: chunk,
115
+ namespace: this.namespace,
116
+ },
117
+ })));
118
+ await Promise.all(batchRequests);
110
119
  return documentIds;
111
120
  }
112
121
  /**
@@ -125,14 +134,13 @@ export class PineconeStore extends VectorStore {
125
134
  }
126
135
  else if (ids) {
127
136
  const batchSize = 1000;
128
- for (let i = 0; i < ids.length; i += batchSize) {
129
- const batchIds = ids.slice(i, i + batchSize);
130
- await this.pineconeIndex.delete1({
131
- ids: batchIds,
132
- namespace,
133
- ...rest,
134
- });
135
- }
137
+ const batchedIds = chunkArray(ids, batchSize);
138
+ const batchRequests = batchedIds.map((batchIds) => this.caller.call(async () => this.pineconeIndex.delete1({
139
+ ids: batchIds,
140
+ namespace,
141
+ ...rest,
142
+ })));
143
+ await Promise.all(batchRequests);
136
144
  }
137
145
  else {
138
146
  throw new Error("Either ids or delete_all must be provided.");
@@ -185,42 +185,39 @@ class VectaraStore extends base_js_1.VectorStore {
185
185
  * pre-processing and chunking internally in an optimal manner. This method is a wrapper
186
186
  * to utilize that API within LangChain.
187
187
  *
188
- * @param filePaths An array of Blob objects representing the files to be uploaded to Vectara.
188
+ * @param files An array of VectaraFile objects representing the files and their respective file names to be uploaded to Vectara.
189
189
  * @param metadata Optional. An array of metadata objects corresponding to each file in the `filePaths` array.
190
190
  * @returns A Promise that resolves to the number of successfully uploaded files.
191
191
  */
192
- async addFiles(filePaths, metadatas = undefined) {
192
+ async addFiles(files, metadatas = undefined) {
193
193
  if (this.corpusId.length > 1)
194
194
  throw new Error("addFiles does not support multiple corpus ids");
195
195
  let numDocs = 0;
196
- for (const [index, fileBlob] of filePaths.entries()) {
196
+ for (const [index, file] of files.entries()) {
197
197
  const md = metadatas ? metadatas[index] : {};
198
198
  const data = new FormData();
199
- data.append("file", fileBlob, `file_${index}`);
199
+ data.append("file", file.blob, file.fileName);
200
200
  data.append("doc-metadata", JSON.stringify(md));
201
- try {
202
- const response = await fetch(`https://api.vectara.io/v1/upload?c=${this.customerId}&o=${this.corpusId[0]}`, {
203
- method: "POST",
204
- headers: {
205
- "x-api-key": this.apiKey,
206
- },
207
- body: data,
208
- });
209
- const result = await response.json();
210
- const { status } = response;
211
- if (status !== 200 && status !== 409) {
212
- throw new Error(`Vectara API returned status code ${status}: ${result}`);
213
- }
214
- else {
215
- numDocs += 1;
216
- }
201
+ const response = await fetch(`https://api.vectara.io/v1/upload?c=${this.customerId}&o=${this.corpusId[0]}`, {
202
+ method: "POST",
203
+ headers: {
204
+ "x-api-key": this.apiKey,
205
+ },
206
+ body: data,
207
+ });
208
+ const { status } = response;
209
+ if (status === 409) {
210
+ throw new Error(`File at index ${index} already exists in Vectara`);
211
+ }
212
+ else if (status !== 200) {
213
+ throw new Error(`Vectara API returned status code ${status}`);
217
214
  }
218
- catch (err) {
219
- console.error(`Failed to upload file at index ${index}:`, err);
215
+ else {
216
+ numDocs += 1;
220
217
  }
221
218
  }
222
219
  if (this.verbose) {
223
- console.log(`Uploaded ${filePaths.length} files to Vectara`);
220
+ console.log(`Uploaded ${files.length} files to Vectara`);
224
221
  }
225
222
  return numDocs;
226
223
  }
@@ -21,6 +21,13 @@ interface VectaraCallHeader {
21
21
  "customer-id": string;
22
22
  };
23
23
  }
24
+ /**
25
+ * Interface for the file objects to be uploaded to Vectara.
26
+ */
27
+ export interface VectaraFile {
28
+ blob: Blob;
29
+ fileName: string;
30
+ }
24
31
  /**
25
32
  * Interface for the filter options used in Vectara API calls.
26
33
  */
@@ -80,11 +87,11 @@ export declare class VectaraStore extends VectorStore {
80
87
  * pre-processing and chunking internally in an optimal manner. This method is a wrapper
81
88
  * to utilize that API within LangChain.
82
89
  *
83
- * @param filePaths An array of Blob objects representing the files to be uploaded to Vectara.
90
+ * @param files An array of VectaraFile objects representing the files and their respective file names to be uploaded to Vectara.
84
91
  * @param metadata Optional. An array of metadata objects corresponding to each file in the `filePaths` array.
85
92
  * @returns A Promise that resolves to the number of successfully uploaded files.
86
93
  */
87
- addFiles(filePaths: Blob[], metadatas?: Record<string, unknown> | undefined): Promise<number>;
94
+ addFiles(files: VectaraFile[], metadatas?: Record<string, unknown> | undefined): Promise<number>;
88
95
  /**
89
96
  * Performs a similarity search and returns documents along with their
90
97
  * scores.
@@ -182,42 +182,39 @@ export class VectaraStore extends VectorStore {
182
182
  * pre-processing and chunking internally in an optimal manner. This method is a wrapper
183
183
  * to utilize that API within LangChain.
184
184
  *
185
- * @param filePaths An array of Blob objects representing the files to be uploaded to Vectara.
185
+ * @param files An array of VectaraFile objects representing the files and their respective file names to be uploaded to Vectara.
186
186
  * @param metadata Optional. An array of metadata objects corresponding to each file in the `filePaths` array.
187
187
  * @returns A Promise that resolves to the number of successfully uploaded files.
188
188
  */
189
- async addFiles(filePaths, metadatas = undefined) {
189
+ async addFiles(files, metadatas = undefined) {
190
190
  if (this.corpusId.length > 1)
191
191
  throw new Error("addFiles does not support multiple corpus ids");
192
192
  let numDocs = 0;
193
- for (const [index, fileBlob] of filePaths.entries()) {
193
+ for (const [index, file] of files.entries()) {
194
194
  const md = metadatas ? metadatas[index] : {};
195
195
  const data = new FormData();
196
- data.append("file", fileBlob, `file_${index}`);
196
+ data.append("file", file.blob, file.fileName);
197
197
  data.append("doc-metadata", JSON.stringify(md));
198
- try {
199
- const response = await fetch(`https://api.vectara.io/v1/upload?c=${this.customerId}&o=${this.corpusId[0]}`, {
200
- method: "POST",
201
- headers: {
202
- "x-api-key": this.apiKey,
203
- },
204
- body: data,
205
- });
206
- const result = await response.json();
207
- const { status } = response;
208
- if (status !== 200 && status !== 409) {
209
- throw new Error(`Vectara API returned status code ${status}: ${result}`);
210
- }
211
- else {
212
- numDocs += 1;
213
- }
198
+ const response = await fetch(`https://api.vectara.io/v1/upload?c=${this.customerId}&o=${this.corpusId[0]}`, {
199
+ method: "POST",
200
+ headers: {
201
+ "x-api-key": this.apiKey,
202
+ },
203
+ body: data,
204
+ });
205
+ const { status } = response;
206
+ if (status === 409) {
207
+ throw new Error(`File at index ${index} already exists in Vectara`);
208
+ }
209
+ else if (status !== 200) {
210
+ throw new Error(`Vectara API returned status code ${status}`);
214
211
  }
215
- catch (err) {
216
- console.error(`Failed to upload file at index ${index}:`, err);
212
+ else {
213
+ numDocs += 1;
217
214
  }
218
215
  }
219
216
  if (this.verbose) {
220
- console.log(`Uploaded ${filePaths.length} files to Vectara`);
217
+ console.log(`Uploaded ${files.length} files to Vectara`);
221
218
  }
222
219
  return numDocs;
223
220
  }
@@ -0,0 +1,158 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.VoyVectorStore = void 0;
4
+ const base_js_1 = require("./base.cjs");
5
+ const document_js_1 = require("../document.cjs");
6
+ /**
7
+ * Class that extends `VectorStore`. It allows to perform similarity search using
8
+ * Voi similarity search engine. The class requires passing Voy Client as an input parameter.
9
+ */
10
+ class VoyVectorStore extends base_js_1.VectorStore {
11
+ _vectorstoreType() {
12
+ return "voi";
13
+ }
14
+ constructor(client, embeddings) {
15
+ super(embeddings, {});
16
+ Object.defineProperty(this, "client", {
17
+ enumerable: true,
18
+ configurable: true,
19
+ writable: true,
20
+ value: void 0
21
+ });
22
+ Object.defineProperty(this, "numDimensions", {
23
+ enumerable: true,
24
+ configurable: true,
25
+ writable: true,
26
+ value: null
27
+ });
28
+ Object.defineProperty(this, "docstore", {
29
+ enumerable: true,
30
+ configurable: true,
31
+ writable: true,
32
+ value: []
33
+ });
34
+ this.client = client;
35
+ this.embeddings = embeddings;
36
+ }
37
+ /**
38
+ * Adds documents to the Voy database. The documents are embedded using embeddings provided while instantiating the class.
39
+ * @param documents An array of `Document` instances associated with the vectors.
40
+ */
41
+ async addDocuments(documents) {
42
+ const texts = documents.map(({ pageContent }) => pageContent);
43
+ if (documents.length === 0) {
44
+ return;
45
+ }
46
+ const firstVector = (await this.embeddings.embedDocuments(texts.slice(0, 1)))[0];
47
+ if (this.numDimensions === null) {
48
+ this.numDimensions = firstVector.length;
49
+ }
50
+ else if (this.numDimensions !== firstVector.length) {
51
+ throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
52
+ }
53
+ const restResults = await this.embeddings.embedDocuments(texts.slice(1));
54
+ await this.addVectors([firstVector, ...restResults], documents);
55
+ }
56
+ /**
57
+ * Adds vectors to the Voy database. The vectors are associated with
58
+ * the provided documents.
59
+ * @param vectors An array of vectors to be added to the database.
60
+ * @param documents An array of `Document` instances associated with the vectors.
61
+ */
62
+ async addVectors(vectors, documents) {
63
+ if (vectors.length === 0) {
64
+ return;
65
+ }
66
+ if (this.numDimensions === null) {
67
+ this.numDimensions = vectors[0].length;
68
+ }
69
+ if (vectors.length !== documents.length) {
70
+ throw new Error(`Vectors and metadata must have the same length`);
71
+ }
72
+ if (!vectors.every((v) => v.length === this.numDimensions)) {
73
+ throw new Error(`Vectors must have the same length as the number of dimensions (${this.numDimensions})`);
74
+ }
75
+ vectors.forEach((item, idx) => {
76
+ const doc = documents[idx];
77
+ this.docstore.push({ embeddings: item, document: doc });
78
+ });
79
+ const embeddings = this.docstore.map((item, idx) => ({
80
+ id: String(idx),
81
+ embeddings: item.embeddings,
82
+ title: "",
83
+ url: "",
84
+ }));
85
+ this.client.index({ embeddings });
86
+ }
87
+ /**
88
+ * Searches for vectors in the Voy database that are similar to the
89
+ * provided query vector.
90
+ * @param query The query vector.
91
+ * @param k The number of similar vectors to return.
92
+ * @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
93
+ */
94
+ async similaritySearchVectorWithScore(query, k) {
95
+ if (this.numDimensions === null) {
96
+ throw new Error("There aren't any elements in the index yet.");
97
+ }
98
+ if (query.length !== this.numDimensions) {
99
+ throw new Error(`Query vector must have the same length as the number of dimensions (${this.numDimensions})`);
100
+ }
101
+ const itemsToQuery = Math.min(this.docstore.length, k);
102
+ if (itemsToQuery > this.docstore.length) {
103
+ console.warn(`k (${k}) is greater than the number of elements in the index (${this.docstore.length}), setting k to ${itemsToQuery}`);
104
+ }
105
+ const results = this.client.search(new Float32Array(query), itemsToQuery);
106
+ return results.neighbors.map(({ id }, idx) => [this.docstore[parseInt(id, 10)].document, idx]);
107
+ }
108
+ /**
109
+ * Method to delete data from the Voy index. It can delete data based
110
+ * on specific IDs or a filter.
111
+ * @param params Object that includes either an array of IDs or a filter for the data to be deleted.
112
+ * @returns Promise that resolves when the deletion is complete.
113
+ */
114
+ async delete(params) {
115
+ if (params.deleteAll === true) {
116
+ await this.client.clear();
117
+ }
118
+ else {
119
+ throw new Error(`You must provide a "deleteAll" parameter.`);
120
+ }
121
+ }
122
+ /**
123
+ * Creates a new `VoyVectorStore` instance from an array of text strings. The text
124
+ * strings are converted to `Document` instances and added to the Voy
125
+ * database.
126
+ * @param texts An array of text strings.
127
+ * @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
128
+ * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
129
+ * @param client An instance of Voy client to use in the underlying operations.
130
+ * @returns A promise that resolves with a new `VoyVectorStore` instance.
131
+ */
132
+ static async fromTexts(texts, metadatas, embeddings, client) {
133
+ const docs = [];
134
+ for (let i = 0; i < texts.length; i += 1) {
135
+ const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
136
+ const newDoc = new document_js_1.Document({
137
+ pageContent: texts[i],
138
+ metadata,
139
+ });
140
+ docs.push(newDoc);
141
+ }
142
+ return VoyVectorStore.fromDocuments(docs, embeddings, client);
143
+ }
144
+ /**
145
+ * Creates a new `VoyVectorStore` instance from an array of `Document` instances.
146
+ * The documents are added to the Voy database.
147
+ * @param docs An array of `Document` instances.
148
+ * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
149
+ * @param client An instance of Voy client to use in the underlying operations.
150
+ * @returns A promise that resolves with a new `VoyVectorStore` instance.
151
+ */
152
+ static async fromDocuments(docs, embeddings, client) {
153
+ const instance = new VoyVectorStore(client, embeddings);
154
+ await instance.addDocuments(docs);
155
+ return instance;
156
+ }
157
+ }
158
+ exports.VoyVectorStore = VoyVectorStore;
@@ -0,0 +1,73 @@
1
+ import type { Voy as VoyOriginClient } from "voy-search";
2
+ import { Embeddings } from "../embeddings/base.js";
3
+ import { VectorStore } from "./base.js";
4
+ import { Document } from "../document.js";
5
+ export type VoyClient = Omit<VoyOriginClient, "remove" | "size" | "serialize" | "free">;
6
+ /**
7
+ * Internal interface for storing documents mappings.
8
+ */
9
+ interface InternalDoc {
10
+ embeddings: number[];
11
+ document: Document;
12
+ }
13
+ /**
14
+ * Class that extends `VectorStore`. It allows to perform similarity search using
15
+ * Voi similarity search engine. The class requires passing Voy Client as an input parameter.
16
+ */
17
+ export declare class VoyVectorStore extends VectorStore {
18
+ client: VoyClient;
19
+ numDimensions: number | null;
20
+ docstore: InternalDoc[];
21
+ _vectorstoreType(): string;
22
+ constructor(client: VoyClient, embeddings: Embeddings);
23
+ /**
24
+ * Adds documents to the Voy database. The documents are embedded using embeddings provided while instantiating the class.
25
+ * @param documents An array of `Document` instances associated with the vectors.
26
+ */
27
+ addDocuments(documents: Document[]): Promise<void>;
28
+ /**
29
+ * Adds vectors to the Voy database. The vectors are associated with
30
+ * the provided documents.
31
+ * @param vectors An array of vectors to be added to the database.
32
+ * @param documents An array of `Document` instances associated with the vectors.
33
+ */
34
+ addVectors(vectors: number[][], documents: Document[]): Promise<void>;
35
+ /**
36
+ * Searches for vectors in the Voy database that are similar to the
37
+ * provided query vector.
38
+ * @param query The query vector.
39
+ * @param k The number of similar vectors to return.
40
+ * @returns A promise that resolves with an array of tuples, each containing a `Document` instance and a similarity score.
41
+ */
42
+ similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document<Record<string, any>>, number][]>;
43
+ /**
44
+ * Method to delete data from the Voy index. It can delete data based
45
+ * on specific IDs or a filter.
46
+ * @param params Object that includes either an array of IDs or a filter for the data to be deleted.
47
+ * @returns Promise that resolves when the deletion is complete.
48
+ */
49
+ delete(params: {
50
+ deleteAll?: boolean;
51
+ }): Promise<void>;
52
+ /**
53
+ * Creates a new `VoyVectorStore` instance from an array of text strings. The text
54
+ * strings are converted to `Document` instances and added to the Voy
55
+ * database.
56
+ * @param texts An array of text strings.
57
+ * @param metadatas An array of metadata objects or a single metadata object. If an array is provided, it must have the same length as the `texts` array.
58
+ * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
59
+ * @param client An instance of Voy client to use in the underlying operations.
60
+ * @returns A promise that resolves with a new `VoyVectorStore` instance.
61
+ */
62
+ static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, client: VoyClient): Promise<VoyVectorStore>;
63
+ /**
64
+ * Creates a new `VoyVectorStore` instance from an array of `Document` instances.
65
+ * The documents are added to the Voy database.
66
+ * @param docs An array of `Document` instances.
67
+ * @param embeddings An `Embeddings` instance used to generate embeddings for the documents.
68
+ * @param client An instance of Voy client to use in the underlying operations.
69
+ * @returns A promise that resolves with a new `VoyVectorStore` instance.
70
+ */
71
+ static fromDocuments(docs: Document[], embeddings: Embeddings, client: VoyClient): Promise<VoyVectorStore>;
72
+ }
73
+ export {};