langchain 0.0.154 → 0.0.156
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/chat_models/bedrock.cjs +1 -0
- package/chat_models/bedrock.d.ts +1 -0
- package/chat_models/bedrock.js +1 -0
- package/dist/callbacks/base.d.ts +42 -28
- package/dist/callbacks/handlers/log_stream.cjs +283 -0
- package/dist/callbacks/handlers/log_stream.d.ts +99 -0
- package/dist/callbacks/handlers/log_stream.js +277 -0
- package/dist/callbacks/handlers/tracer.cjs +34 -18
- package/dist/callbacks/handlers/tracer.d.ts +18 -16
- package/dist/callbacks/handlers/tracer.js +34 -18
- package/dist/chat_models/bedrock.cjs +260 -0
- package/dist/chat_models/bedrock.d.ts +58 -0
- package/dist/chat_models/bedrock.js +254 -0
- package/dist/document_loaders/web/notionapi.cjs +8 -4
- package/dist/document_loaders/web/notionapi.js +8 -4
- package/dist/document_loaders/web/searchapi.cjs +134 -0
- package/dist/document_loaders/web/searchapi.d.ts +65 -0
- package/dist/document_loaders/web/searchapi.js +130 -0
- package/dist/embeddings/cloudflare_workersai.cjs +69 -0
- package/dist/embeddings/cloudflare_workersai.d.ts +28 -0
- package/dist/embeddings/cloudflare_workersai.js +65 -0
- package/dist/llms/bedrock.cjs +57 -67
- package/dist/llms/bedrock.d.ts +8 -35
- package/dist/llms/bedrock.js +57 -67
- package/dist/load/import_constants.cjs +4 -0
- package/dist/load/import_constants.js +4 -0
- package/dist/load/import_map.cjs +3 -2
- package/dist/load/import_map.d.ts +1 -0
- package/dist/load/import_map.js +1 -0
- package/dist/schema/runnable/base.cjs +64 -5
- package/dist/schema/runnable/base.d.ts +13 -0
- package/dist/schema/runnable/base.js +64 -5
- package/dist/tools/index.cjs +3 -1
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.js +1 -0
- package/dist/tools/searchapi.cjs +139 -0
- package/dist/tools/searchapi.d.ts +64 -0
- package/dist/tools/searchapi.js +135 -0
- package/dist/util/bedrock.cjs +54 -0
- package/dist/util/bedrock.d.ts +59 -0
- package/dist/util/bedrock.js +50 -0
- package/dist/util/fast-json-patch/index.cjs +48 -0
- package/dist/util/fast-json-patch/index.d.ts +21 -0
- package/dist/util/fast-json-patch/index.js +15 -0
- package/dist/util/fast-json-patch/src/core.cjs +469 -0
- package/dist/util/fast-json-patch/src/core.d.ts +111 -0
- package/dist/util/fast-json-patch/src/core.js +459 -0
- package/dist/util/fast-json-patch/src/helpers.cjs +194 -0
- package/dist/util/fast-json-patch/src/helpers.d.ts +36 -0
- package/dist/util/fast-json-patch/src/helpers.js +181 -0
- package/dist/util/googlevertexai-webauth.cjs +6 -2
- package/dist/util/googlevertexai-webauth.d.ts +1 -0
- package/dist/util/googlevertexai-webauth.js +6 -2
- package/dist/util/stream.cjs +2 -40
- package/dist/util/stream.d.ts +1 -2
- package/dist/util/stream.js +1 -38
- package/dist/vectorstores/cloudflare_vectorize.cjs +200 -0
- package/dist/vectorstores/cloudflare_vectorize.d.ts +90 -0
- package/dist/vectorstores/cloudflare_vectorize.js +173 -0
- package/dist/vectorstores/pgvector.cjs +1 -1
- package/dist/vectorstores/pgvector.js +1 -1
- package/dist/vectorstores/supabase.d.ts +1 -1
- package/dist/vectorstores/vercel_postgres.cjs +300 -0
- package/dist/vectorstores/vercel_postgres.d.ts +145 -0
- package/dist/vectorstores/vercel_postgres.js +296 -0
- package/document_loaders/web/searchapi.cjs +1 -0
- package/document_loaders/web/searchapi.d.ts +1 -0
- package/document_loaders/web/searchapi.js +1 -0
- package/embeddings/cloudflare_workersai.cjs +1 -0
- package/embeddings/cloudflare_workersai.d.ts +1 -0
- package/embeddings/cloudflare_workersai.js +1 -0
- package/package.json +60 -14
- package/vectorstores/cloudflare_vectorize.cjs +1 -0
- package/vectorstores/cloudflare_vectorize.d.ts +1 -0
- package/vectorstores/cloudflare_vectorize.js +1 -0
- package/vectorstores/vercel_postgres.cjs +1 -0
- package/vectorstores/vercel_postgres.d.ts +1 -0
- package/vectorstores/vercel_postgres.js +1 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
2
|
+
// Inlined because of ESM import issues
|
|
3
|
+
/*!
|
|
4
|
+
* https://github.com/Starcounter-Jack/JSON-Patch
|
|
5
|
+
* (c) 2017-2022 Joachim Wester
|
|
6
|
+
* MIT licensed
|
|
7
|
+
*/
|
|
8
|
+
const _hasOwnProperty = Object.prototype.hasOwnProperty;
|
|
9
|
+
export function hasOwnProperty(obj, key) {
|
|
10
|
+
return _hasOwnProperty.call(obj, key);
|
|
11
|
+
}
|
|
12
|
+
export function _objectKeys(obj) {
|
|
13
|
+
if (Array.isArray(obj)) {
|
|
14
|
+
const keys = new Array(obj.length);
|
|
15
|
+
for (let k = 0; k < keys.length; k++) {
|
|
16
|
+
keys[k] = "" + k;
|
|
17
|
+
}
|
|
18
|
+
return keys;
|
|
19
|
+
}
|
|
20
|
+
if (Object.keys) {
|
|
21
|
+
return Object.keys(obj);
|
|
22
|
+
}
|
|
23
|
+
let keys = [];
|
|
24
|
+
for (let i in obj) {
|
|
25
|
+
if (hasOwnProperty(obj, i)) {
|
|
26
|
+
keys.push(i);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return keys;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Deeply clone the object.
|
|
33
|
+
* https://jsperf.com/deep-copy-vs-json-stringify-json-parse/25 (recursiveDeepCopy)
|
|
34
|
+
* @param {any} obj value to clone
|
|
35
|
+
* @return {any} cloned obj
|
|
36
|
+
*/
|
|
37
|
+
export function _deepClone(obj) {
|
|
38
|
+
switch (typeof obj) {
|
|
39
|
+
case "object":
|
|
40
|
+
return JSON.parse(JSON.stringify(obj)); //Faster than ES5 clone - http://jsperf.com/deep-cloning-of-objects/5
|
|
41
|
+
case "undefined":
|
|
42
|
+
return null; //this is how JSON.stringify behaves for array items
|
|
43
|
+
default:
|
|
44
|
+
return obj; //no need to clone primitives
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
//3x faster than cached /^\d+$/.test(str)
|
|
48
|
+
export function isInteger(str) {
|
|
49
|
+
let i = 0;
|
|
50
|
+
const len = str.length;
|
|
51
|
+
let charCode;
|
|
52
|
+
while (i < len) {
|
|
53
|
+
charCode = str.charCodeAt(i);
|
|
54
|
+
if (charCode >= 48 && charCode <= 57) {
|
|
55
|
+
i++;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Escapes a json pointer path
|
|
64
|
+
* @param path The raw pointer
|
|
65
|
+
* @return the Escaped path
|
|
66
|
+
*/
|
|
67
|
+
export function escapePathComponent(path) {
|
|
68
|
+
if (path.indexOf("/") === -1 && path.indexOf("~") === -1)
|
|
69
|
+
return path;
|
|
70
|
+
return path.replace(/~/g, "~0").replace(/\//g, "~1");
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Unescapes a json pointer path
|
|
74
|
+
* @param path The escaped pointer
|
|
75
|
+
* @return The unescaped path
|
|
76
|
+
*/
|
|
77
|
+
export function unescapePathComponent(path) {
|
|
78
|
+
return path.replace(/~1/g, "/").replace(/~0/g, "~");
|
|
79
|
+
}
|
|
80
|
+
export function _getPathRecursive(root, obj) {
|
|
81
|
+
let found;
|
|
82
|
+
for (let key in root) {
|
|
83
|
+
if (hasOwnProperty(root, key)) {
|
|
84
|
+
if (root[key] === obj) {
|
|
85
|
+
return escapePathComponent(key) + "/";
|
|
86
|
+
}
|
|
87
|
+
else if (typeof root[key] === "object") {
|
|
88
|
+
found = _getPathRecursive(root[key], obj);
|
|
89
|
+
if (found != "") {
|
|
90
|
+
return escapePathComponent(key) + "/" + found;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return "";
|
|
96
|
+
}
|
|
97
|
+
export function getPath(root, obj) {
|
|
98
|
+
if (root === obj) {
|
|
99
|
+
return "/";
|
|
100
|
+
}
|
|
101
|
+
const path = _getPathRecursive(root, obj);
|
|
102
|
+
if (path === "") {
|
|
103
|
+
throw new Error("Object not found in root");
|
|
104
|
+
}
|
|
105
|
+
return `/${path}`;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Recursively checks whether an object has any undefined values inside.
|
|
109
|
+
*/
|
|
110
|
+
export function hasUndefined(obj) {
|
|
111
|
+
if (obj === undefined) {
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
if (obj) {
|
|
115
|
+
if (Array.isArray(obj)) {
|
|
116
|
+
for (let i = 0, len = obj.length; i < len; i++) {
|
|
117
|
+
if (hasUndefined(obj[i])) {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
else if (typeof obj === "object") {
|
|
123
|
+
const objKeys = _objectKeys(obj);
|
|
124
|
+
const objKeysLength = objKeys.length;
|
|
125
|
+
for (var i = 0; i < objKeysLength; i++) {
|
|
126
|
+
if (hasUndefined(obj[objKeys[i]])) {
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
function patchErrorMessageFormatter(message, args) {
|
|
135
|
+
const messageParts = [message];
|
|
136
|
+
for (const key in args) {
|
|
137
|
+
const value = typeof args[key] === "object"
|
|
138
|
+
? JSON.stringify(args[key], null, 2)
|
|
139
|
+
: args[key]; // pretty print
|
|
140
|
+
if (typeof value !== "undefined") {
|
|
141
|
+
messageParts.push(`${key}: ${value}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return messageParts.join("\n");
|
|
145
|
+
}
|
|
146
|
+
export class PatchError extends Error {
|
|
147
|
+
constructor(message, name, index, operation, tree) {
|
|
148
|
+
super(patchErrorMessageFormatter(message, { name, index, operation, tree }));
|
|
149
|
+
Object.defineProperty(this, "name", {
|
|
150
|
+
enumerable: true,
|
|
151
|
+
configurable: true,
|
|
152
|
+
writable: true,
|
|
153
|
+
value: name
|
|
154
|
+
});
|
|
155
|
+
Object.defineProperty(this, "index", {
|
|
156
|
+
enumerable: true,
|
|
157
|
+
configurable: true,
|
|
158
|
+
writable: true,
|
|
159
|
+
value: index
|
|
160
|
+
});
|
|
161
|
+
Object.defineProperty(this, "operation", {
|
|
162
|
+
enumerable: true,
|
|
163
|
+
configurable: true,
|
|
164
|
+
writable: true,
|
|
165
|
+
value: operation
|
|
166
|
+
});
|
|
167
|
+
Object.defineProperty(this, "tree", {
|
|
168
|
+
enumerable: true,
|
|
169
|
+
configurable: true,
|
|
170
|
+
writable: true,
|
|
171
|
+
value: tree
|
|
172
|
+
});
|
|
173
|
+
Object.setPrototypeOf(this, new.target.prototype); // restore prototype chain, see https://stackoverflow.com/a/48342359
|
|
174
|
+
this.message = patchErrorMessageFormatter(message, {
|
|
175
|
+
name,
|
|
176
|
+
index,
|
|
177
|
+
operation,
|
|
178
|
+
tree,
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
}
|
|
@@ -11,19 +11,23 @@ class WebGoogleAuth {
|
|
|
11
11
|
writable: true,
|
|
12
12
|
value: void 0
|
|
13
13
|
});
|
|
14
|
+
const accessToken = options?.accessToken;
|
|
14
15
|
const credentials = options?.credentials ??
|
|
15
16
|
(0, env_js_1.getEnvironmentVariable)("GOOGLE_VERTEX_AI_WEB_CREDENTIALS");
|
|
16
17
|
if (credentials === undefined)
|
|
17
18
|
throw new Error(`Credentials not found. Please set the GOOGLE_VERTEX_AI_WEB_CREDENTIALS or pass credentials into "authOptions.credentials".`);
|
|
18
19
|
const scope = options?.scope ?? "https://www.googleapis.com/auth/cloud-platform";
|
|
19
|
-
this.options = { ...options, credentials, scope };
|
|
20
|
+
this.options = { ...options, accessToken, credentials, scope };
|
|
20
21
|
}
|
|
21
22
|
async getProjectId() {
|
|
22
23
|
const credentials = (0, google_1.getCredentials)(this.options.credentials);
|
|
23
24
|
return credentials.project_id;
|
|
24
25
|
}
|
|
25
26
|
async request(opts) {
|
|
26
|
-
|
|
27
|
+
let { accessToken } = this.options;
|
|
28
|
+
if (accessToken === undefined) {
|
|
29
|
+
accessToken = await (0, google_1.getAccessToken)(this.options);
|
|
30
|
+
}
|
|
27
31
|
if (opts.url == null)
|
|
28
32
|
throw new Error("Missing URL");
|
|
29
33
|
const fetchOptions = {
|
|
@@ -3,6 +3,7 @@ import type { GoogleVertexAIAbstractedClient } from "../types/googlevertexai-typ
|
|
|
3
3
|
export type WebGoogleAuthOptions = {
|
|
4
4
|
credentials: string | Credentials;
|
|
5
5
|
scope?: string | string[];
|
|
6
|
+
accessToken?: string;
|
|
6
7
|
};
|
|
7
8
|
export declare class WebGoogleAuth implements GoogleVertexAIAbstractedClient {
|
|
8
9
|
options: WebGoogleAuthOptions;
|
|
@@ -8,19 +8,23 @@ export class WebGoogleAuth {
|
|
|
8
8
|
writable: true,
|
|
9
9
|
value: void 0
|
|
10
10
|
});
|
|
11
|
+
const accessToken = options?.accessToken;
|
|
11
12
|
const credentials = options?.credentials ??
|
|
12
13
|
getEnvironmentVariable("GOOGLE_VERTEX_AI_WEB_CREDENTIALS");
|
|
13
14
|
if (credentials === undefined)
|
|
14
15
|
throw new Error(`Credentials not found. Please set the GOOGLE_VERTEX_AI_WEB_CREDENTIALS or pass credentials into "authOptions.credentials".`);
|
|
15
16
|
const scope = options?.scope ?? "https://www.googleapis.com/auth/cloud-platform";
|
|
16
|
-
this.options = { ...options, credentials, scope };
|
|
17
|
+
this.options = { ...options, accessToken, credentials, scope };
|
|
17
18
|
}
|
|
18
19
|
async getProjectId() {
|
|
19
20
|
const credentials = getCredentials(this.options.credentials);
|
|
20
21
|
return credentials.project_id;
|
|
21
22
|
}
|
|
22
23
|
async request(opts) {
|
|
23
|
-
|
|
24
|
+
let { accessToken } = this.options;
|
|
25
|
+
if (accessToken === undefined) {
|
|
26
|
+
accessToken = await getAccessToken(this.options);
|
|
27
|
+
}
|
|
24
28
|
if (opts.url == null)
|
|
25
29
|
throw new Error("Missing URL");
|
|
26
30
|
const fetchOptions = {
|
package/dist/util/stream.cjs
CHANGED
|
@@ -1,47 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.IterableReadableStream =
|
|
3
|
+
exports.IterableReadableStream = void 0;
|
|
4
4
|
/*
|
|
5
5
|
* Support async iterator syntax for ReadableStreams in all environments.
|
|
6
6
|
* Source: https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490
|
|
7
7
|
*/
|
|
8
|
-
function readableStreamToAsyncIterable(
|
|
9
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
10
|
-
stream, preventCancel = false) {
|
|
11
|
-
if (stream[Symbol.asyncIterator]) {
|
|
12
|
-
return stream[Symbol.asyncIterator]();
|
|
13
|
-
}
|
|
14
|
-
const reader = stream.getReader();
|
|
15
|
-
return {
|
|
16
|
-
async next() {
|
|
17
|
-
try {
|
|
18
|
-
const result = await reader.read();
|
|
19
|
-
if (result.done)
|
|
20
|
-
reader.releaseLock(); // release lock when stream becomes closed
|
|
21
|
-
return result;
|
|
22
|
-
}
|
|
23
|
-
catch (e) {
|
|
24
|
-
reader.releaseLock(); // release lock when stream becomes errored
|
|
25
|
-
throw e;
|
|
26
|
-
}
|
|
27
|
-
},
|
|
28
|
-
async return() {
|
|
29
|
-
if (!preventCancel) {
|
|
30
|
-
const cancelPromise = reader.cancel(); // cancel first, but don't await yet
|
|
31
|
-
reader.releaseLock(); // release lock first
|
|
32
|
-
await cancelPromise; // now await it
|
|
33
|
-
}
|
|
34
|
-
else {
|
|
35
|
-
reader.releaseLock();
|
|
36
|
-
}
|
|
37
|
-
return { done: true, value: undefined };
|
|
38
|
-
},
|
|
39
|
-
[Symbol.asyncIterator]() {
|
|
40
|
-
return this;
|
|
41
|
-
},
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
exports.readableStreamToAsyncIterable = readableStreamToAsyncIterable;
|
|
45
8
|
class IterableReadableStream extends ReadableStream {
|
|
46
9
|
constructor() {
|
|
47
10
|
super(...arguments);
|
|
@@ -75,7 +38,7 @@ class IterableReadableStream extends ReadableStream {
|
|
|
75
38
|
const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
|
|
76
39
|
this.reader.releaseLock(); // release lock first
|
|
77
40
|
await cancelPromise; // now await it
|
|
78
|
-
return { done: true, value: undefined };
|
|
41
|
+
return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore chunk value anyway
|
|
79
42
|
}
|
|
80
43
|
[Symbol.asyncIterator]() {
|
|
81
44
|
return this;
|
|
@@ -86,7 +49,6 @@ class IterableReadableStream extends ReadableStream {
|
|
|
86
49
|
return new IterableReadableStream({
|
|
87
50
|
start(controller) {
|
|
88
51
|
return pump();
|
|
89
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
90
52
|
function pump() {
|
|
91
53
|
return reader.read().then(({ done, value }) => {
|
|
92
54
|
// When no more data needs to be consumed, close the stream
|
package/dist/util/stream.d.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
export declare function readableStreamToAsyncIterable(stream: any, preventCancel?: boolean): any;
|
|
2
1
|
export declare class IterableReadableStream<T> extends ReadableStream<T> {
|
|
3
2
|
reader: ReadableStreamDefaultReader<T>;
|
|
4
3
|
ensureReader(): void;
|
|
5
4
|
next(): Promise<ReadableStreamReadResult<T>>;
|
|
6
5
|
return(): Promise<{
|
|
7
6
|
done: boolean;
|
|
8
|
-
value:
|
|
7
|
+
value: T;
|
|
9
8
|
}>;
|
|
10
9
|
[Symbol.asyncIterator](): this;
|
|
11
10
|
static fromReadableStream<T>(stream: ReadableStream<T>): IterableReadableStream<T>;
|
package/dist/util/stream.js
CHANGED
|
@@ -2,42 +2,6 @@
|
|
|
2
2
|
* Support async iterator syntax for ReadableStreams in all environments.
|
|
3
3
|
* Source: https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490
|
|
4
4
|
*/
|
|
5
|
-
export function readableStreamToAsyncIterable(
|
|
6
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
7
|
-
stream, preventCancel = false) {
|
|
8
|
-
if (stream[Symbol.asyncIterator]) {
|
|
9
|
-
return stream[Symbol.asyncIterator]();
|
|
10
|
-
}
|
|
11
|
-
const reader = stream.getReader();
|
|
12
|
-
return {
|
|
13
|
-
async next() {
|
|
14
|
-
try {
|
|
15
|
-
const result = await reader.read();
|
|
16
|
-
if (result.done)
|
|
17
|
-
reader.releaseLock(); // release lock when stream becomes closed
|
|
18
|
-
return result;
|
|
19
|
-
}
|
|
20
|
-
catch (e) {
|
|
21
|
-
reader.releaseLock(); // release lock when stream becomes errored
|
|
22
|
-
throw e;
|
|
23
|
-
}
|
|
24
|
-
},
|
|
25
|
-
async return() {
|
|
26
|
-
if (!preventCancel) {
|
|
27
|
-
const cancelPromise = reader.cancel(); // cancel first, but don't await yet
|
|
28
|
-
reader.releaseLock(); // release lock first
|
|
29
|
-
await cancelPromise; // now await it
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
reader.releaseLock();
|
|
33
|
-
}
|
|
34
|
-
return { done: true, value: undefined };
|
|
35
|
-
},
|
|
36
|
-
[Symbol.asyncIterator]() {
|
|
37
|
-
return this;
|
|
38
|
-
},
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
5
|
export class IterableReadableStream extends ReadableStream {
|
|
42
6
|
constructor() {
|
|
43
7
|
super(...arguments);
|
|
@@ -71,7 +35,7 @@ export class IterableReadableStream extends ReadableStream {
|
|
|
71
35
|
const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
|
|
72
36
|
this.reader.releaseLock(); // release lock first
|
|
73
37
|
await cancelPromise; // now await it
|
|
74
|
-
return { done: true, value: undefined };
|
|
38
|
+
return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore chunk value anyway
|
|
75
39
|
}
|
|
76
40
|
[Symbol.asyncIterator]() {
|
|
77
41
|
return this;
|
|
@@ -82,7 +46,6 @@ export class IterableReadableStream extends ReadableStream {
|
|
|
82
46
|
return new IterableReadableStream({
|
|
83
47
|
start(controller) {
|
|
84
48
|
return pump();
|
|
85
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
86
49
|
function pump() {
|
|
87
50
|
return reader.read().then(({ done, value }) => {
|
|
88
51
|
// When no more data needs to be consumed, close the stream
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.CloudflareVectorizeStore = void 0;
|
|
27
|
+
const uuid = __importStar(require("uuid"));
|
|
28
|
+
const base_js_1 = require("./base.cjs");
|
|
29
|
+
const document_js_1 = require("../document.cjs");
|
|
30
|
+
const chunk_js_1 = require("../util/chunk.cjs");
|
|
31
|
+
const async_caller_js_1 = require("../util/async_caller.cjs");
|
|
32
|
+
/**
|
|
33
|
+
* Class that extends the VectorStore class and provides methods to
|
|
34
|
+
* interact with the Cloudflare Vectorize vector database.
|
|
35
|
+
*/
|
|
36
|
+
class CloudflareVectorizeStore extends base_js_1.VectorStore {
|
|
37
|
+
_vectorstoreType() {
|
|
38
|
+
return "cloudflare_vectorize";
|
|
39
|
+
}
|
|
40
|
+
constructor(embeddings, args) {
|
|
41
|
+
super(embeddings, args);
|
|
42
|
+
Object.defineProperty(this, "textKey", {
|
|
43
|
+
enumerable: true,
|
|
44
|
+
configurable: true,
|
|
45
|
+
writable: true,
|
|
46
|
+
value: void 0
|
|
47
|
+
});
|
|
48
|
+
Object.defineProperty(this, "namespace", {
|
|
49
|
+
enumerable: true,
|
|
50
|
+
configurable: true,
|
|
51
|
+
writable: true,
|
|
52
|
+
value: void 0
|
|
53
|
+
});
|
|
54
|
+
Object.defineProperty(this, "index", {
|
|
55
|
+
enumerable: true,
|
|
56
|
+
configurable: true,
|
|
57
|
+
writable: true,
|
|
58
|
+
value: void 0
|
|
59
|
+
});
|
|
60
|
+
Object.defineProperty(this, "caller", {
|
|
61
|
+
enumerable: true,
|
|
62
|
+
configurable: true,
|
|
63
|
+
writable: true,
|
|
64
|
+
value: void 0
|
|
65
|
+
});
|
|
66
|
+
this.embeddings = embeddings;
|
|
67
|
+
const { index, textKey, ...asyncCallerArgs } = args;
|
|
68
|
+
if (!index) {
|
|
69
|
+
throw new Error("Must supply a Vectorize index binding, eg { index: env.VECTORIZE }");
|
|
70
|
+
}
|
|
71
|
+
this.index = index;
|
|
72
|
+
this.textKey = textKey ?? "text";
|
|
73
|
+
this.caller = new async_caller_js_1.AsyncCaller({
|
|
74
|
+
maxConcurrency: 6,
|
|
75
|
+
maxRetries: 0,
|
|
76
|
+
...asyncCallerArgs,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Method that adds documents to the Vectorize database.
|
|
81
|
+
* @param documents Array of documents to add.
|
|
82
|
+
* @param options Optional ids for the documents.
|
|
83
|
+
* @returns Promise that resolves with the ids of the added documents.
|
|
84
|
+
*/
|
|
85
|
+
async addDocuments(documents, options) {
|
|
86
|
+
const texts = documents.map(({ pageContent }) => pageContent);
|
|
87
|
+
return this.addVectors(await this.embeddings.embedDocuments(texts), documents, options);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Method that adds vectors to the Vectorize database.
|
|
91
|
+
* @param vectors Array of vectors to add.
|
|
92
|
+
* @param documents Array of documents associated with the vectors.
|
|
93
|
+
* @param options Optional ids for the vectors.
|
|
94
|
+
* @returns Promise that resolves with the ids of the added vectors.
|
|
95
|
+
*/
|
|
96
|
+
async addVectors(vectors, documents, options) {
|
|
97
|
+
const ids = Array.isArray(options) ? options : options?.ids;
|
|
98
|
+
const documentIds = ids == null ? documents.map(() => uuid.v4()) : ids;
|
|
99
|
+
const vectorizeVectors = vectors.map((values, idx) => {
|
|
100
|
+
const metadata = {
|
|
101
|
+
...documents[idx].metadata,
|
|
102
|
+
[this.textKey]: documents[idx].pageContent,
|
|
103
|
+
};
|
|
104
|
+
return {
|
|
105
|
+
id: documentIds[idx],
|
|
106
|
+
metadata,
|
|
107
|
+
values,
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
// Stick to a limit of 500 vectors per upsert request
|
|
111
|
+
const chunkSize = 500;
|
|
112
|
+
const chunkedVectors = (0, chunk_js_1.chunkArray)(vectorizeVectors, chunkSize);
|
|
113
|
+
const batchRequests = chunkedVectors.map((chunk) => this.caller.call(async () => this.index.upsert(chunk)));
|
|
114
|
+
await Promise.all(batchRequests);
|
|
115
|
+
return documentIds;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Method that deletes vectors from the Vectorize database.
|
|
119
|
+
* @param params Parameters for the delete operation.
|
|
120
|
+
* @returns Promise that resolves when the delete operation is complete.
|
|
121
|
+
*/
|
|
122
|
+
async delete(params) {
|
|
123
|
+
const batchSize = 1000;
|
|
124
|
+
const batchedIds = (0, chunk_js_1.chunkArray)(params.ids, batchSize);
|
|
125
|
+
const batchRequests = batchedIds.map((batchIds) => this.caller.call(async () => this.index.deleteByIds(batchIds)));
|
|
126
|
+
await Promise.all(batchRequests);
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Method that performs a similarity search in the Vectorize database and
|
|
130
|
+
* returns the results along with their scores.
|
|
131
|
+
* @param query Query vector for the similarity search.
|
|
132
|
+
* @param k Number of top results to return.
|
|
133
|
+
* @returns Promise that resolves with an array of documents and their scores.
|
|
134
|
+
*/
|
|
135
|
+
async similaritySearchVectorWithScore(query, k) {
|
|
136
|
+
const results = await this.index.query(query, {
|
|
137
|
+
returnVectors: true,
|
|
138
|
+
topK: k,
|
|
139
|
+
});
|
|
140
|
+
const result = [];
|
|
141
|
+
if (results.matches) {
|
|
142
|
+
for (const res of results.matches) {
|
|
143
|
+
const { [this.textKey]: pageContent, ...metadata } = res.vector?.metadata ?? {};
|
|
144
|
+
result.push([
|
|
145
|
+
new document_js_1.Document({ metadata, pageContent: pageContent }),
|
|
146
|
+
res.score,
|
|
147
|
+
]);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return result;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
154
|
+
* from texts.
|
|
155
|
+
* @param texts Array of texts to add to the Vectorize database.
|
|
156
|
+
* @param metadatas Metadata associated with the texts.
|
|
157
|
+
* @param embeddings Embeddings to use for the texts.
|
|
158
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
159
|
+
* @param options Optional ids for the vectors.
|
|
160
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
161
|
+
*/
|
|
162
|
+
static async fromTexts(texts, metadatas, embeddings, dbConfig) {
|
|
163
|
+
const docs = [];
|
|
164
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
165
|
+
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
|
|
166
|
+
const newDoc = new document_js_1.Document({
|
|
167
|
+
pageContent: texts[i],
|
|
168
|
+
metadata,
|
|
169
|
+
});
|
|
170
|
+
docs.push(newDoc);
|
|
171
|
+
}
|
|
172
|
+
return CloudflareVectorizeStore.fromDocuments(docs, embeddings, dbConfig);
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
176
|
+
* from documents.
|
|
177
|
+
* @param docs Array of documents to add to the Vectorize database.
|
|
178
|
+
* @param embeddings Embeddings to use for the documents.
|
|
179
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
180
|
+
* @param options Optional ids for the vectors.
|
|
181
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
182
|
+
*/
|
|
183
|
+
static async fromDocuments(docs, embeddings, dbConfig) {
|
|
184
|
+
const instance = new this(embeddings, dbConfig);
|
|
185
|
+
await instance.addDocuments(docs);
|
|
186
|
+
return instance;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
190
|
+
* from an existing index.
|
|
191
|
+
* @param embeddings Embeddings to use for the documents.
|
|
192
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
193
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
194
|
+
*/
|
|
195
|
+
static async fromExistingIndex(embeddings, dbConfig) {
|
|
196
|
+
const instance = new this(embeddings, dbConfig);
|
|
197
|
+
return instance;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
exports.CloudflareVectorizeStore = CloudflareVectorizeStore;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { VectorizeIndex, VectorizeVectorMetadata } from "@cloudflare/workers-types";
|
|
2
|
+
import { VectorStore } from "./base.js";
|
|
3
|
+
import { Embeddings } from "../embeddings/base.js";
|
|
4
|
+
import { Document } from "../document.js";
|
|
5
|
+
import { AsyncCaller, type AsyncCallerParams } from "../util/async_caller.js";
|
|
6
|
+
export interface VectorizeLibArgs extends AsyncCallerParams {
|
|
7
|
+
index: VectorizeIndex;
|
|
8
|
+
textKey?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Type that defines the parameters for the delete operation in the
|
|
12
|
+
* CloudflareVectorizeStore class. It includes ids, deleteAll flag, and namespace.
|
|
13
|
+
*/
|
|
14
|
+
export type VectorizeDeleteParams = {
|
|
15
|
+
ids: string[];
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Class that extends the VectorStore class and provides methods to
|
|
19
|
+
* interact with the Cloudflare Vectorize vector database.
|
|
20
|
+
*/
|
|
21
|
+
export declare class CloudflareVectorizeStore extends VectorStore {
|
|
22
|
+
textKey: string;
|
|
23
|
+
namespace?: string;
|
|
24
|
+
index: VectorizeIndex;
|
|
25
|
+
caller: AsyncCaller;
|
|
26
|
+
_vectorstoreType(): string;
|
|
27
|
+
constructor(embeddings: Embeddings, args: VectorizeLibArgs);
|
|
28
|
+
/**
|
|
29
|
+
* Method that adds documents to the Vectorize database.
|
|
30
|
+
* @param documents Array of documents to add.
|
|
31
|
+
* @param options Optional ids for the documents.
|
|
32
|
+
* @returns Promise that resolves with the ids of the added documents.
|
|
33
|
+
*/
|
|
34
|
+
addDocuments(documents: Document[], options?: {
|
|
35
|
+
ids?: string[];
|
|
36
|
+
} | string[]): Promise<string[]>;
|
|
37
|
+
/**
|
|
38
|
+
* Method that adds vectors to the Vectorize database.
|
|
39
|
+
* @param vectors Array of vectors to add.
|
|
40
|
+
* @param documents Array of documents associated with the vectors.
|
|
41
|
+
* @param options Optional ids for the vectors.
|
|
42
|
+
* @returns Promise that resolves with the ids of the added vectors.
|
|
43
|
+
*/
|
|
44
|
+
addVectors(vectors: number[][], documents: Document[], options?: {
|
|
45
|
+
ids?: string[];
|
|
46
|
+
} | string[]): Promise<string[]>;
|
|
47
|
+
/**
|
|
48
|
+
* Method that deletes vectors from the Vectorize database.
|
|
49
|
+
* @param params Parameters for the delete operation.
|
|
50
|
+
* @returns Promise that resolves when the delete operation is complete.
|
|
51
|
+
*/
|
|
52
|
+
delete(params: VectorizeDeleteParams): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Method that performs a similarity search in the Vectorize database and
|
|
55
|
+
* returns the results along with their scores.
|
|
56
|
+
* @param query Query vector for the similarity search.
|
|
57
|
+
* @param k Number of top results to return.
|
|
58
|
+
* @returns Promise that resolves with an array of documents and their scores.
|
|
59
|
+
*/
|
|
60
|
+
similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
|
|
61
|
+
/**
|
|
62
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
63
|
+
* from texts.
|
|
64
|
+
* @param texts Array of texts to add to the Vectorize database.
|
|
65
|
+
* @param metadatas Metadata associated with the texts.
|
|
66
|
+
* @param embeddings Embeddings to use for the texts.
|
|
67
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
68
|
+
* @param options Optional ids for the vectors.
|
|
69
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
70
|
+
*/
|
|
71
|
+
static fromTexts(texts: string[], metadatas: Record<string, VectorizeVectorMetadata>[] | Record<string, VectorizeVectorMetadata>, embeddings: Embeddings, dbConfig: VectorizeLibArgs): Promise<CloudflareVectorizeStore>;
|
|
72
|
+
/**
|
|
73
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
74
|
+
* from documents.
|
|
75
|
+
* @param docs Array of documents to add to the Vectorize database.
|
|
76
|
+
* @param embeddings Embeddings to use for the documents.
|
|
77
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
78
|
+
* @param options Optional ids for the vectors.
|
|
79
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
80
|
+
*/
|
|
81
|
+
static fromDocuments(docs: Document[], embeddings: Embeddings, dbConfig: VectorizeLibArgs): Promise<CloudflareVectorizeStore>;
|
|
82
|
+
/**
|
|
83
|
+
* Static method that creates a new instance of the CloudflareVectorizeStore class
|
|
84
|
+
* from an existing index.
|
|
85
|
+
* @param embeddings Embeddings to use for the documents.
|
|
86
|
+
* @param dbConfig Configuration for the Vectorize database.
|
|
87
|
+
* @returns Promise that resolves with a new instance of the CloudflareVectorizeStore class.
|
|
88
|
+
*/
|
|
89
|
+
static fromExistingIndex(embeddings: Embeddings, dbConfig: VectorizeLibArgs): Promise<CloudflareVectorizeStore>;
|
|
90
|
+
}
|