langchain 0.0.169 → 0.0.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/agents/format_scratchpad.cjs +1 -0
- package/agents/format_scratchpad.d.ts +1 -0
- package/agents/format_scratchpad.js +1 -0
- package/chat_models/llama_cpp.cjs +1 -0
- package/chat_models/llama_cpp.d.ts +1 -0
- package/chat_models/llama_cpp.js +1 -0
- package/dist/agents/agent.cjs +49 -1
- package/dist/agents/agent.d.ts +19 -1
- package/dist/agents/agent.js +47 -0
- package/dist/agents/executor.cjs +10 -1
- package/dist/agents/executor.d.ts +22 -8
- package/dist/agents/executor.js +11 -2
- package/dist/agents/format_scratchpad.cjs +25 -0
- package/dist/agents/format_scratchpad.d.ts +10 -0
- package/dist/agents/format_scratchpad.js +21 -0
- package/dist/agents/toolkits/aws_sfn.d.ts +4 -1
- package/dist/agents/toolkits/conversational_retrieval/openai_functions.d.ts +1 -1
- package/dist/agents/toolkits/json/json.d.ts +4 -1
- package/dist/agents/toolkits/openapi/openapi.cjs +8 -0
- package/dist/agents/toolkits/openapi/openapi.d.ts +12 -1
- package/dist/agents/toolkits/openapi/openapi.js +8 -0
- package/dist/agents/toolkits/sql/sql.d.ts +4 -1
- package/dist/agents/toolkits/vectorstore/vectorstore.d.ts +8 -2
- package/dist/agents/types.d.ts +13 -1
- package/dist/chains/sql_db/sql_db_chain.cjs +11 -0
- package/dist/chains/sql_db/sql_db_chain.d.ts +11 -0
- package/dist/chains/sql_db/sql_db_chain.js +11 -0
- package/dist/chat_models/baiduwenxin.cjs +12 -1
- package/dist/chat_models/baiduwenxin.d.ts +3 -1
- package/dist/chat_models/baiduwenxin.js +12 -1
- package/dist/chat_models/llama_cpp.cjs +243 -0
- package/dist/chat_models/llama_cpp.d.ts +94 -0
- package/dist/chat_models/llama_cpp.js +239 -0
- package/dist/document_loaders/web/assemblyai.cjs +63 -114
- package/dist/document_loaders/web/assemblyai.d.ts +38 -57
- package/dist/document_loaders/web/assemblyai.js +63 -100
- package/dist/document_loaders/web/pdf.cjs +23 -5
- package/dist/document_loaders/web/pdf.d.ts +9 -1
- package/dist/document_loaders/web/pdf.js +20 -2
- package/dist/graphs/neo4j_graph.cjs +14 -0
- package/dist/graphs/neo4j_graph.d.ts +14 -0
- package/dist/graphs/neo4j_graph.js +14 -0
- package/dist/llms/googlepalm.cjs +3 -0
- package/dist/llms/googlepalm.js +3 -0
- package/dist/load/import_constants.cjs +2 -0
- package/dist/load/import_constants.js +2 -0
- package/dist/load/import_map.cjs +3 -2
- package/dist/load/import_map.d.ts +1 -0
- package/dist/load/import_map.js +1 -0
- package/dist/memory/index.d.ts +1 -1
- package/dist/memory/index.js +1 -1
- package/dist/retrievers/parent_document.cjs +22 -2
- package/dist/retrievers/parent_document.d.ts +8 -1
- package/dist/retrievers/parent_document.js +22 -2
- package/dist/retrievers/time_weighted.cjs +1 -1
- package/dist/retrievers/time_weighted.d.ts +1 -1
- package/dist/retrievers/time_weighted.js +1 -1
- package/dist/schema/runnable/base.cjs +4 -1
- package/dist/schema/runnable/base.d.ts +1 -0
- package/dist/schema/runnable/base.js +4 -1
- package/dist/schema/runnable/passthrough.cjs +35 -1
- package/dist/schema/runnable/passthrough.d.ts +11 -1
- package/dist/schema/runnable/passthrough.js +34 -1
- package/dist/sql_db.cjs +14 -0
- package/dist/sql_db.d.ts +14 -0
- package/dist/sql_db.js +14 -0
- package/dist/storage/ioredis.cjs +2 -1
- package/dist/storage/ioredis.js +2 -1
- package/dist/storage/upstash_redis.cjs +155 -0
- package/dist/storage/upstash_redis.d.ts +59 -0
- package/dist/storage/upstash_redis.js +151 -0
- package/dist/storage/vercel_kv.cjs +2 -1
- package/dist/storage/vercel_kv.js +2 -1
- package/dist/types/assemblyai-types.cjs +0 -150
- package/dist/types/assemblyai-types.d.ts +4 -670
- package/dist/types/assemblyai-types.js +1 -149
- package/dist/util/stream.cjs +3 -0
- package/dist/util/stream.js +3 -0
- package/dist/vectorstores/cassandra.cjs +25 -4
- package/dist/vectorstores/cassandra.d.ts +11 -1
- package/dist/vectorstores/cassandra.js +25 -4
- package/dist/vectorstores/momento_vector_index.cjs +3 -15
- package/dist/vectorstores/momento_vector_index.d.ts +0 -8
- package/dist/vectorstores/momento_vector_index.js +3 -15
- package/dist/vectorstores/neo4j_vector.cjs +14 -0
- package/dist/vectorstores/neo4j_vector.d.ts +14 -0
- package/dist/vectorstores/neo4j_vector.js +14 -0
- package/dist/vectorstores/pgvector.cjs +1 -1
- package/dist/vectorstores/pgvector.js +1 -1
- package/package.json +37 -8
- package/storage/upstash_redis.cjs +1 -0
- package/storage/upstash_redis.d.ts +1 -0
- package/storage/upstash_redis.js +1 -0
- package/dist/util/assemblyai-client.cjs +0 -173
- package/dist/util/assemblyai-client.d.ts +0 -63
- package/dist/util/assemblyai-client.js +0 -170
|
@@ -1,4 +1,32 @@
|
|
|
1
|
-
import { Runnable } from "./base.js";
|
|
1
|
+
import { Runnable, RunnableMap } from "./base.js";
|
|
2
|
+
/**
|
|
3
|
+
* A runnable that assigns key-value pairs to inputs of type `Record<string, unknown>`.
|
|
4
|
+
*/
|
|
5
|
+
export class RunnableAssign extends Runnable {
|
|
6
|
+
constructor(mapper) {
|
|
7
|
+
super();
|
|
8
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
9
|
+
enumerable: true,
|
|
10
|
+
configurable: true,
|
|
11
|
+
writable: true,
|
|
12
|
+
value: ["langchain", "schema", "runnable"]
|
|
13
|
+
});
|
|
14
|
+
Object.defineProperty(this, "mapper", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: void 0
|
|
19
|
+
});
|
|
20
|
+
this.mapper = mapper;
|
|
21
|
+
}
|
|
22
|
+
async invoke(input, options) {
|
|
23
|
+
const mapperResult = await this.mapper.invoke(input, options);
|
|
24
|
+
return {
|
|
25
|
+
...input,
|
|
26
|
+
...mapperResult,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
}
|
|
2
30
|
/**
|
|
3
31
|
* A runnable that passes through the input.
|
|
4
32
|
*/
|
|
@@ -24,4 +52,9 @@ export class RunnablePassthrough extends Runnable {
|
|
|
24
52
|
async invoke(input, options) {
|
|
25
53
|
return this._callWithConfig((input) => Promise.resolve(input), input, options);
|
|
26
54
|
}
|
|
55
|
+
static assign(
|
|
56
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
57
|
+
mapping) {
|
|
58
|
+
return new RunnableAssign(new RunnableMap({ steps: mapping }));
|
|
59
|
+
}
|
|
27
60
|
}
|
package/dist/sql_db.cjs
CHANGED
|
@@ -3,6 +3,20 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.SqlDatabase = void 0;
|
|
4
4
|
const sql_utils_js_1 = require("./util/sql_utils.cjs");
|
|
5
5
|
const serializable_js_1 = require("./load/serializable.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* Class that represents a SQL database in the LangChain framework.
|
|
8
|
+
*
|
|
9
|
+
* @security **Security Notice**
|
|
10
|
+
* This class generates SQL queries for the given database.
|
|
11
|
+
* The SQLDatabase class provides a getTableInfo method that can be used
|
|
12
|
+
* to get column information as well as sample data from the table.
|
|
13
|
+
* To mitigate risk of leaking sensitive data, limit permissions
|
|
14
|
+
* to read and scope to the tables that are needed.
|
|
15
|
+
* Optionally, use the includesTables or ignoreTables class parameters
|
|
16
|
+
* to limit which tables can/cannot be accessed.
|
|
17
|
+
*
|
|
18
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
19
|
+
*/
|
|
6
20
|
class SqlDatabase extends serializable_js_1.Serializable {
|
|
7
21
|
toJSON() {
|
|
8
22
|
return this.toJSONNotImplemented();
|
package/dist/sql_db.d.ts
CHANGED
|
@@ -2,6 +2,20 @@ import type { DataSource as DataSourceT, DataSourceOptions } from "typeorm";
|
|
|
2
2
|
import { SerializedSqlDatabase, SqlDatabaseDataSourceParams, SqlDatabaseOptionsParams, SqlTable } from "./util/sql_utils.js";
|
|
3
3
|
import { Serializable } from "./load/serializable.js";
|
|
4
4
|
export type { SqlDatabaseDataSourceParams, SqlDatabaseOptionsParams };
|
|
5
|
+
/**
|
|
6
|
+
* Class that represents a SQL database in the LangChain framework.
|
|
7
|
+
*
|
|
8
|
+
* @security **Security Notice**
|
|
9
|
+
* This class generates SQL queries for the given database.
|
|
10
|
+
* The SQLDatabase class provides a getTableInfo method that can be used
|
|
11
|
+
* to get column information as well as sample data from the table.
|
|
12
|
+
* To mitigate risk of leaking sensitive data, limit permissions
|
|
13
|
+
* to read and scope to the tables that are needed.
|
|
14
|
+
* Optionally, use the includesTables or ignoreTables class parameters
|
|
15
|
+
* to limit which tables can/cannot be accessed.
|
|
16
|
+
*
|
|
17
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
18
|
+
*/
|
|
5
19
|
export declare class SqlDatabase extends Serializable implements SqlDatabaseOptionsParams, SqlDatabaseDataSourceParams {
|
|
6
20
|
lc_namespace: string[];
|
|
7
21
|
toJSON(): import("./load/serializable.js").SerializedNotImplemented;
|
package/dist/sql_db.js
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { generateTableInfoFromTables, getTableAndColumnsName, verifyIgnoreTablesExistInDatabase, verifyIncludeTablesExistInDatabase, verifyListTablesExistInDatabase, } from "./util/sql_utils.js";
|
|
2
2
|
import { Serializable } from "./load/serializable.js";
|
|
3
|
+
/**
|
|
4
|
+
* Class that represents a SQL database in the LangChain framework.
|
|
5
|
+
*
|
|
6
|
+
* @security **Security Notice**
|
|
7
|
+
* This class generates SQL queries for the given database.
|
|
8
|
+
* The SQLDatabase class provides a getTableInfo method that can be used
|
|
9
|
+
* to get column information as well as sample data from the table.
|
|
10
|
+
* To mitigate risk of leaking sensitive data, limit permissions
|
|
11
|
+
* to read and scope to the tables that are needed.
|
|
12
|
+
* Optionally, use the includesTables or ignoreTables class parameters
|
|
13
|
+
* to limit which tables can/cannot be accessed.
|
|
14
|
+
*
|
|
15
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
16
|
+
*/
|
|
3
17
|
export class SqlDatabase extends Serializable {
|
|
4
18
|
toJSON() {
|
|
5
19
|
return this.toJSONNotImplemented();
|
package/dist/storage/ioredis.cjs
CHANGED
|
@@ -115,7 +115,8 @@ class RedisByteStore extends storage_js_1.BaseStore {
|
|
|
115
115
|
async *yieldKeys(prefix) {
|
|
116
116
|
let pattern;
|
|
117
117
|
if (prefix) {
|
|
118
|
-
|
|
118
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
119
|
+
pattern = this._getPrefixedKey(wildcardPrefix);
|
|
119
120
|
}
|
|
120
121
|
else {
|
|
121
122
|
pattern = this._getPrefixedKey("*");
|
package/dist/storage/ioredis.js
CHANGED
|
@@ -112,7 +112,8 @@ export class RedisByteStore extends BaseStore {
|
|
|
112
112
|
async *yieldKeys(prefix) {
|
|
113
113
|
let pattern;
|
|
114
114
|
if (prefix) {
|
|
115
|
-
|
|
115
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
116
|
+
pattern = this._getPrefixedKey(wildcardPrefix);
|
|
116
117
|
}
|
|
117
118
|
else {
|
|
118
119
|
pattern = this._getPrefixedKey("*");
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.UpstashRedisStore = void 0;
|
|
4
|
+
const redis_1 = require("@upstash/redis");
|
|
5
|
+
const storage_js_1 = require("../schema/storage.cjs");
|
|
6
|
+
/**
|
|
7
|
+
* Class that extends the BaseStore class to interact with an Upstash Redis
|
|
8
|
+
* database. It provides methods for getting, setting, and deleting data,
|
|
9
|
+
* as well as yielding keys from the database.
|
|
10
|
+
*/
|
|
11
|
+
class UpstashRedisStore extends storage_js_1.BaseStore {
|
|
12
|
+
constructor(fields) {
|
|
13
|
+
super(fields);
|
|
14
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: ["langchain", "storage"]
|
|
19
|
+
});
|
|
20
|
+
Object.defineProperty(this, "client", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
Object.defineProperty(this, "namespace", {
|
|
27
|
+
enumerable: true,
|
|
28
|
+
configurable: true,
|
|
29
|
+
writable: true,
|
|
30
|
+
value: void 0
|
|
31
|
+
});
|
|
32
|
+
Object.defineProperty(this, "yieldKeysScanBatchSize", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
writable: true,
|
|
36
|
+
value: 1000
|
|
37
|
+
});
|
|
38
|
+
Object.defineProperty(this, "sessionTTL", {
|
|
39
|
+
enumerable: true,
|
|
40
|
+
configurable: true,
|
|
41
|
+
writable: true,
|
|
42
|
+
value: void 0
|
|
43
|
+
});
|
|
44
|
+
if (fields.client) {
|
|
45
|
+
this.client = fields.client;
|
|
46
|
+
}
|
|
47
|
+
else if (fields.config) {
|
|
48
|
+
this.client = new redis_1.Redis(fields.config);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
throw new Error(`Upstash Redis store requires either a config object or a pre-configured client.`);
|
|
52
|
+
}
|
|
53
|
+
this.sessionTTL = fields.sessionTTL;
|
|
54
|
+
this.yieldKeysScanBatchSize =
|
|
55
|
+
fields.yieldKeysScanBatchSize ?? this.yieldKeysScanBatchSize;
|
|
56
|
+
this.namespace = fields.namespace;
|
|
57
|
+
}
|
|
58
|
+
_getPrefixedKey(key) {
|
|
59
|
+
if (this.namespace) {
|
|
60
|
+
const delimiter = "/";
|
|
61
|
+
return `${this.namespace}${delimiter}${key}`;
|
|
62
|
+
}
|
|
63
|
+
return key;
|
|
64
|
+
}
|
|
65
|
+
_getDeprefixedKey(key) {
|
|
66
|
+
if (this.namespace) {
|
|
67
|
+
const delimiter = "/";
|
|
68
|
+
return key.slice(this.namespace.length + delimiter.length);
|
|
69
|
+
}
|
|
70
|
+
return key;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Gets multiple keys from the Upstash Redis database.
|
|
74
|
+
* @param keys Array of keys to be retrieved.
|
|
75
|
+
* @returns An array of retrieved values.
|
|
76
|
+
*/
|
|
77
|
+
async mget(keys) {
|
|
78
|
+
const encoder = new TextEncoder();
|
|
79
|
+
const prefixedKeys = keys.map(this._getPrefixedKey.bind(this));
|
|
80
|
+
const retrievedValues = await this.client.mget(...prefixedKeys);
|
|
81
|
+
return retrievedValues.map((value) => {
|
|
82
|
+
if (!value) {
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
else if (typeof value === "object") {
|
|
86
|
+
return encoder.encode(JSON.stringify(value));
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
return encoder.encode(value);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Sets multiple keys in the Upstash Redis database.
|
|
95
|
+
* @param keyValuePairs Array of key-value pairs to be set.
|
|
96
|
+
* @returns Promise that resolves when all keys have been set.
|
|
97
|
+
*/
|
|
98
|
+
async mset(keyValuePairs) {
|
|
99
|
+
const decoder = new TextDecoder();
|
|
100
|
+
const encodedKeyValuePairs = keyValuePairs.map(([key, value]) => [
|
|
101
|
+
this._getPrefixedKey(key),
|
|
102
|
+
decoder.decode(value),
|
|
103
|
+
]);
|
|
104
|
+
const pipeline = this.client.pipeline();
|
|
105
|
+
for (const [key, value] of encodedKeyValuePairs) {
|
|
106
|
+
if (this.sessionTTL) {
|
|
107
|
+
pipeline.setex(key, this.sessionTTL, value);
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
pipeline.set(key, value);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
await pipeline.exec();
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Deletes multiple keys from the Upstash Redis database.
|
|
117
|
+
* @param keys Array of keys to be deleted.
|
|
118
|
+
* @returns Promise that resolves when all keys have been deleted.
|
|
119
|
+
*/
|
|
120
|
+
async mdelete(keys) {
|
|
121
|
+
await this.client.del(...keys.map(this._getPrefixedKey.bind(this)));
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Yields keys from the Upstash Redis database.
|
|
125
|
+
* @param prefix Optional prefix to filter the keys. A wildcard (*) is always appended to the end.
|
|
126
|
+
* @returns An AsyncGenerator that yields keys from the Upstash Redis database.
|
|
127
|
+
*/
|
|
128
|
+
async *yieldKeys(prefix) {
|
|
129
|
+
let pattern;
|
|
130
|
+
if (prefix) {
|
|
131
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
132
|
+
pattern = `${this._getPrefixedKey(wildcardPrefix)}*`;
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
pattern = this._getPrefixedKey("*");
|
|
136
|
+
}
|
|
137
|
+
let [cursor, batch] = await this.client.scan(0, {
|
|
138
|
+
match: pattern,
|
|
139
|
+
count: this.yieldKeysScanBatchSize,
|
|
140
|
+
});
|
|
141
|
+
for (const key of batch) {
|
|
142
|
+
yield this._getDeprefixedKey(key);
|
|
143
|
+
}
|
|
144
|
+
while (cursor !== 0) {
|
|
145
|
+
[cursor, batch] = await this.client.scan(cursor, {
|
|
146
|
+
match: pattern,
|
|
147
|
+
count: this.yieldKeysScanBatchSize,
|
|
148
|
+
});
|
|
149
|
+
for (const key of batch) {
|
|
150
|
+
yield this._getDeprefixedKey(key);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
exports.UpstashRedisStore = UpstashRedisStore;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Redis as UpstashRedis, type RedisConfigNodejs } from "@upstash/redis";
|
|
2
|
+
import { BaseStore } from "../schema/storage.js";
|
|
3
|
+
/**
|
|
4
|
+
* Type definition for the input parameters required to initialize an
|
|
5
|
+
* instance of the UpstashStoreInput class.
|
|
6
|
+
*/
|
|
7
|
+
export interface UpstashRedisStoreInput {
|
|
8
|
+
sessionTTL?: number;
|
|
9
|
+
config?: RedisConfigNodejs;
|
|
10
|
+
client?: UpstashRedis;
|
|
11
|
+
/**
|
|
12
|
+
* The amount of keys to retrieve per batch when yielding keys.
|
|
13
|
+
* @default 1000
|
|
14
|
+
*/
|
|
15
|
+
yieldKeysScanBatchSize?: number;
|
|
16
|
+
/**
|
|
17
|
+
* The namespace to use for the keys in the database.
|
|
18
|
+
*/
|
|
19
|
+
namespace?: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Class that extends the BaseStore class to interact with an Upstash Redis
|
|
23
|
+
* database. It provides methods for getting, setting, and deleting data,
|
|
24
|
+
* as well as yielding keys from the database.
|
|
25
|
+
*/
|
|
26
|
+
export declare class UpstashRedisStore extends BaseStore<string, Uint8Array> {
|
|
27
|
+
lc_namespace: string[];
|
|
28
|
+
protected client: UpstashRedis;
|
|
29
|
+
protected namespace?: string;
|
|
30
|
+
protected yieldKeysScanBatchSize: number;
|
|
31
|
+
private sessionTTL?;
|
|
32
|
+
constructor(fields: UpstashRedisStoreInput);
|
|
33
|
+
_getPrefixedKey(key: string): string;
|
|
34
|
+
_getDeprefixedKey(key: string): string;
|
|
35
|
+
/**
|
|
36
|
+
* Gets multiple keys from the Upstash Redis database.
|
|
37
|
+
* @param keys Array of keys to be retrieved.
|
|
38
|
+
* @returns An array of retrieved values.
|
|
39
|
+
*/
|
|
40
|
+
mget(keys: string[]): Promise<(Uint8Array | undefined)[]>;
|
|
41
|
+
/**
|
|
42
|
+
* Sets multiple keys in the Upstash Redis database.
|
|
43
|
+
* @param keyValuePairs Array of key-value pairs to be set.
|
|
44
|
+
* @returns Promise that resolves when all keys have been set.
|
|
45
|
+
*/
|
|
46
|
+
mset(keyValuePairs: [string, Uint8Array][]): Promise<void>;
|
|
47
|
+
/**
|
|
48
|
+
* Deletes multiple keys from the Upstash Redis database.
|
|
49
|
+
* @param keys Array of keys to be deleted.
|
|
50
|
+
* @returns Promise that resolves when all keys have been deleted.
|
|
51
|
+
*/
|
|
52
|
+
mdelete(keys: string[]): Promise<void>;
|
|
53
|
+
/**
|
|
54
|
+
* Yields keys from the Upstash Redis database.
|
|
55
|
+
* @param prefix Optional prefix to filter the keys. A wildcard (*) is always appended to the end.
|
|
56
|
+
* @returns An AsyncGenerator that yields keys from the Upstash Redis database.
|
|
57
|
+
*/
|
|
58
|
+
yieldKeys(prefix?: string): AsyncGenerator<string>;
|
|
59
|
+
}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { Redis as UpstashRedis } from "@upstash/redis";
|
|
2
|
+
import { BaseStore } from "../schema/storage.js";
|
|
3
|
+
/**
|
|
4
|
+
* Class that extends the BaseStore class to interact with an Upstash Redis
|
|
5
|
+
* database. It provides methods for getting, setting, and deleting data,
|
|
6
|
+
* as well as yielding keys from the database.
|
|
7
|
+
*/
|
|
8
|
+
export class UpstashRedisStore extends BaseStore {
|
|
9
|
+
constructor(fields) {
|
|
10
|
+
super(fields);
|
|
11
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
12
|
+
enumerable: true,
|
|
13
|
+
configurable: true,
|
|
14
|
+
writable: true,
|
|
15
|
+
value: ["langchain", "storage"]
|
|
16
|
+
});
|
|
17
|
+
Object.defineProperty(this, "client", {
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true,
|
|
21
|
+
value: void 0
|
|
22
|
+
});
|
|
23
|
+
Object.defineProperty(this, "namespace", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(this, "yieldKeysScanBatchSize", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: 1000
|
|
34
|
+
});
|
|
35
|
+
Object.defineProperty(this, "sessionTTL", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: void 0
|
|
40
|
+
});
|
|
41
|
+
if (fields.client) {
|
|
42
|
+
this.client = fields.client;
|
|
43
|
+
}
|
|
44
|
+
else if (fields.config) {
|
|
45
|
+
this.client = new UpstashRedis(fields.config);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
throw new Error(`Upstash Redis store requires either a config object or a pre-configured client.`);
|
|
49
|
+
}
|
|
50
|
+
this.sessionTTL = fields.sessionTTL;
|
|
51
|
+
this.yieldKeysScanBatchSize =
|
|
52
|
+
fields.yieldKeysScanBatchSize ?? this.yieldKeysScanBatchSize;
|
|
53
|
+
this.namespace = fields.namespace;
|
|
54
|
+
}
|
|
55
|
+
_getPrefixedKey(key) {
|
|
56
|
+
if (this.namespace) {
|
|
57
|
+
const delimiter = "/";
|
|
58
|
+
return `${this.namespace}${delimiter}${key}`;
|
|
59
|
+
}
|
|
60
|
+
return key;
|
|
61
|
+
}
|
|
62
|
+
_getDeprefixedKey(key) {
|
|
63
|
+
if (this.namespace) {
|
|
64
|
+
const delimiter = "/";
|
|
65
|
+
return key.slice(this.namespace.length + delimiter.length);
|
|
66
|
+
}
|
|
67
|
+
return key;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Gets multiple keys from the Upstash Redis database.
|
|
71
|
+
* @param keys Array of keys to be retrieved.
|
|
72
|
+
* @returns An array of retrieved values.
|
|
73
|
+
*/
|
|
74
|
+
async mget(keys) {
|
|
75
|
+
const encoder = new TextEncoder();
|
|
76
|
+
const prefixedKeys = keys.map(this._getPrefixedKey.bind(this));
|
|
77
|
+
const retrievedValues = await this.client.mget(...prefixedKeys);
|
|
78
|
+
return retrievedValues.map((value) => {
|
|
79
|
+
if (!value) {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
else if (typeof value === "object") {
|
|
83
|
+
return encoder.encode(JSON.stringify(value));
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
return encoder.encode(value);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Sets multiple keys in the Upstash Redis database.
|
|
92
|
+
* @param keyValuePairs Array of key-value pairs to be set.
|
|
93
|
+
* @returns Promise that resolves when all keys have been set.
|
|
94
|
+
*/
|
|
95
|
+
async mset(keyValuePairs) {
|
|
96
|
+
const decoder = new TextDecoder();
|
|
97
|
+
const encodedKeyValuePairs = keyValuePairs.map(([key, value]) => [
|
|
98
|
+
this._getPrefixedKey(key),
|
|
99
|
+
decoder.decode(value),
|
|
100
|
+
]);
|
|
101
|
+
const pipeline = this.client.pipeline();
|
|
102
|
+
for (const [key, value] of encodedKeyValuePairs) {
|
|
103
|
+
if (this.sessionTTL) {
|
|
104
|
+
pipeline.setex(key, this.sessionTTL, value);
|
|
105
|
+
}
|
|
106
|
+
else {
|
|
107
|
+
pipeline.set(key, value);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
await pipeline.exec();
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Deletes multiple keys from the Upstash Redis database.
|
|
114
|
+
* @param keys Array of keys to be deleted.
|
|
115
|
+
* @returns Promise that resolves when all keys have been deleted.
|
|
116
|
+
*/
|
|
117
|
+
async mdelete(keys) {
|
|
118
|
+
await this.client.del(...keys.map(this._getPrefixedKey.bind(this)));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Yields keys from the Upstash Redis database.
|
|
122
|
+
* @param prefix Optional prefix to filter the keys. A wildcard (*) is always appended to the end.
|
|
123
|
+
* @returns An AsyncGenerator that yields keys from the Upstash Redis database.
|
|
124
|
+
*/
|
|
125
|
+
async *yieldKeys(prefix) {
|
|
126
|
+
let pattern;
|
|
127
|
+
if (prefix) {
|
|
128
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
129
|
+
pattern = `${this._getPrefixedKey(wildcardPrefix)}*`;
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
pattern = this._getPrefixedKey("*");
|
|
133
|
+
}
|
|
134
|
+
let [cursor, batch] = await this.client.scan(0, {
|
|
135
|
+
match: pattern,
|
|
136
|
+
count: this.yieldKeysScanBatchSize,
|
|
137
|
+
});
|
|
138
|
+
for (const key of batch) {
|
|
139
|
+
yield this._getDeprefixedKey(key);
|
|
140
|
+
}
|
|
141
|
+
while (cursor !== 0) {
|
|
142
|
+
[cursor, batch] = await this.client.scan(cursor, {
|
|
143
|
+
match: pattern,
|
|
144
|
+
count: this.yieldKeysScanBatchSize,
|
|
145
|
+
});
|
|
146
|
+
for (const key of batch) {
|
|
147
|
+
yield this._getDeprefixedKey(key);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
@@ -120,7 +120,8 @@ class VercelKVStore extends storage_js_1.BaseStore {
|
|
|
120
120
|
async *yieldKeys(prefix) {
|
|
121
121
|
let pattern;
|
|
122
122
|
if (prefix) {
|
|
123
|
-
|
|
123
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
124
|
+
pattern = this._getPrefixedKey(wildcardPrefix);
|
|
124
125
|
}
|
|
125
126
|
else {
|
|
126
127
|
pattern = this._getPrefixedKey("*");
|
|
@@ -117,7 +117,8 @@ export class VercelKVStore extends BaseStore {
|
|
|
117
117
|
async *yieldKeys(prefix) {
|
|
118
118
|
let pattern;
|
|
119
119
|
if (prefix) {
|
|
120
|
-
|
|
120
|
+
const wildcardPrefix = prefix.endsWith("*") ? prefix : `${prefix}*`;
|
|
121
|
+
pattern = this._getPrefixedKey(wildcardPrefix);
|
|
121
122
|
}
|
|
122
123
|
else {
|
|
123
124
|
pattern = this._getPrefixedKey("*");
|
|
@@ -1,152 +1,2 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SubtitleFormat = exports.EntityType = exports.PiiPolicy = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* The list of PII Redaction policies to enable.
|
|
6
|
-
* See [PII redaction]{@link https://www.assemblyai.com/docs/Models/pii_redaction} for more details.
|
|
7
|
-
*/
|
|
8
|
-
exports.PiiPolicy = {
|
|
9
|
-
/**
|
|
10
|
-
* Medical process, including treatments, procedures, and tests (e.g., heart surgery, CT scan)
|
|
11
|
-
*/
|
|
12
|
-
MedicalProcess: "medical_process",
|
|
13
|
-
/**
|
|
14
|
-
* Name of a medical condition, disease, syndrome, deficit, or disorder (e.g., chronic fatigue syndrome, arrhythmia, depression)
|
|
15
|
-
*/
|
|
16
|
-
MedicalCondition: "medical_condition",
|
|
17
|
-
/**
|
|
18
|
-
* Blood type (e.g., O-, AB positive)
|
|
19
|
-
*/
|
|
20
|
-
BloodType: "blood_type",
|
|
21
|
-
/**
|
|
22
|
-
* Medications, vitamins, or supplements (e.g., Advil, Acetaminophen, Panadol)
|
|
23
|
-
*/
|
|
24
|
-
Drug: "drug",
|
|
25
|
-
/**
|
|
26
|
-
* Bodily injury (e.g., I broke my arm, I have a sprained wrist)
|
|
27
|
-
*/
|
|
28
|
-
Injury: "injury",
|
|
29
|
-
/**
|
|
30
|
-
* A "lazy" rule that will redact any sequence of numbers equal to or greater than 2
|
|
31
|
-
*/
|
|
32
|
-
NumberSequence: "number_sequence",
|
|
33
|
-
/**
|
|
34
|
-
* Email address (e.g., support@assemblyai.com)
|
|
35
|
-
*/
|
|
36
|
-
EmailAddress: "email_address",
|
|
37
|
-
/**
|
|
38
|
-
* Date of Birth (e.g., Date of Birth: March 7,1961)
|
|
39
|
-
*/
|
|
40
|
-
DateOfBirth: "date_of_birth",
|
|
41
|
-
/**
|
|
42
|
-
* Telephone or fax number
|
|
43
|
-
*/
|
|
44
|
-
PhoneNumber: "phone_number",
|
|
45
|
-
/**
|
|
46
|
-
* Social Security Number or equivalent
|
|
47
|
-
*/
|
|
48
|
-
UsSocialSecurityNumber: "us_social_security_number",
|
|
49
|
-
/**
|
|
50
|
-
* Credit card number
|
|
51
|
-
*/
|
|
52
|
-
CreditCardNumber: "credit_card_number",
|
|
53
|
-
/**
|
|
54
|
-
* Expiration date of a credit card
|
|
55
|
-
*/
|
|
56
|
-
CreditCardExpiration: "credit_card_expiration",
|
|
57
|
-
/**
|
|
58
|
-
* Credit card verification code (e.g., CVV: 080)
|
|
59
|
-
*/
|
|
60
|
-
CreditCardCvv: "credit_card_cvv",
|
|
61
|
-
/**
|
|
62
|
-
* Specific calendar date (e.g., December 18)
|
|
63
|
-
*/
|
|
64
|
-
Date: "date",
|
|
65
|
-
/**
|
|
66
|
-
* Terms indicating nationality, ethnicity, or race (e.g., American, Asian, Caucasian)
|
|
67
|
-
*/
|
|
68
|
-
Nationality: "nationality",
|
|
69
|
-
/**
|
|
70
|
-
* Name of an event or holiday (e.g., Olympics, Yom Kippur)
|
|
71
|
-
*/
|
|
72
|
-
Event: "event",
|
|
73
|
-
/**
|
|
74
|
-
* Name of a natural language (e.g., Spanish, French)
|
|
75
|
-
*/
|
|
76
|
-
Language: "language",
|
|
77
|
-
/**
|
|
78
|
-
* Any Location reference including mailing address, postal code, city, state, province, or country
|
|
79
|
-
*/
|
|
80
|
-
Location: "location",
|
|
81
|
-
/**
|
|
82
|
-
* Name and/or amount of currency (e.g., 15 pesos, $94.50)
|
|
83
|
-
*/
|
|
84
|
-
MoneyAmount: "money_amount",
|
|
85
|
-
/**
|
|
86
|
-
* Name of a person (e.g., Bob, Doug Jones)
|
|
87
|
-
*/
|
|
88
|
-
PersonName: "person_name",
|
|
89
|
-
/**
|
|
90
|
-
* Number associated with an age (e.g., 27, 75)
|
|
91
|
-
*/
|
|
92
|
-
PersonAge: "person_age",
|
|
93
|
-
/**
|
|
94
|
-
* Name of an organization (e.g., CNN, McDonalds, University of Alaska)
|
|
95
|
-
*/
|
|
96
|
-
Organization: "organization",
|
|
97
|
-
/**
|
|
98
|
-
* Terms referring to a political party, movement, or ideology (e.g., Republican, Liberal)
|
|
99
|
-
*/
|
|
100
|
-
PoliticalAffiliation: "political_affiliation",
|
|
101
|
-
/**
|
|
102
|
-
* Job title or profession (e.g., professor, actors, engineer, CPA)
|
|
103
|
-
*/
|
|
104
|
-
Occupation: "occupation",
|
|
105
|
-
/**
|
|
106
|
-
* Terms indicating religious affiliation (e.g., Hindu, Catholic)
|
|
107
|
-
*/
|
|
108
|
-
Religion: "religion",
|
|
109
|
-
/**
|
|
110
|
-
* Driver’s license number (e.g., DL# 356933-540)
|
|
111
|
-
*/
|
|
112
|
-
DriversLicense: "drivers_license",
|
|
113
|
-
/**
|
|
114
|
-
* Banking information, including account and routing numbers
|
|
115
|
-
*/
|
|
116
|
-
BankingInformation: "banking_information",
|
|
117
|
-
};
|
|
118
|
-
exports.EntityType = {
|
|
119
|
-
BloodType: "Blood type (e.g., O-, AB positive)",
|
|
120
|
-
CreditCardCvv: "Credit card verification code (e.g., CVV: 080)",
|
|
121
|
-
CreditCardExpiration: "Expiration date of a credit card",
|
|
122
|
-
CreditCardNumber: "Credit card number",
|
|
123
|
-
Date: "Specific calendar date (e.g., December 18)",
|
|
124
|
-
DateOfBirth: "Date of Birth (e.g., Date of Birth: March 7, 1961)",
|
|
125
|
-
Drug: "Medications, vitamins, or supplements (e.g., Advil, Acetaminophen, Panadol)",
|
|
126
|
-
Event: "Name of an event or holiday (e.g., Olympics, Yom Kippur)",
|
|
127
|
-
EmailAddress: "Email address (e.g., support@assemblyai.com)",
|
|
128
|
-
Injury: "Bodily injury (e.g., I broke my arm, I have a sprained wrist)",
|
|
129
|
-
Language: "Name of a natural language (e.g., Spanish, French)",
|
|
130
|
-
Location: "Any location reference including mailing address, postal code, city, state, province, or country",
|
|
131
|
-
MedicalCondition: "Name of a medical condition, disease, syndrome, deficit, or disorder (e.g., chronic fatigue syndrome, arrhythmia, depression)",
|
|
132
|
-
MedicalProcess: "Medical process, including treatments, procedures, and tests (e.g., heart surgery, CT scan)",
|
|
133
|
-
MoneyAmount: "Name and/or amount of currency (e.g., 15 pesos, $94.50)",
|
|
134
|
-
Nationality: "Terms indicating nationality, ethnicity, or race (e.g., American, Asian, Caucasian)",
|
|
135
|
-
Occupation: "Job title or profession (e.g., professor, actors, engineer, CPA)",
|
|
136
|
-
Organization: "Name of an organization (e.g., CNN, McDonalds, University of Alaska)",
|
|
137
|
-
PersonAge: "Number associated with an age (e.g., 27, 75)",
|
|
138
|
-
PersonName: "Name of a person (e.g., Bob, Doug Jones)",
|
|
139
|
-
PhoneNumber: "Telephone or fax number",
|
|
140
|
-
PoliticalAffiliation: "Terms referring to a political party, movement, or ideology (e.g., Republican, Liberal)",
|
|
141
|
-
Religion: "Terms indicating religious affiliation (e.g., Hindu, Catholic)",
|
|
142
|
-
UsSocialSecurityNumber: "Social Security Number or equivalent",
|
|
143
|
-
DriversLicense: "Driver's license number (e.g., DL #356933-540)",
|
|
144
|
-
BankingInformation: "Banking information, including account and routing numbers",
|
|
145
|
-
};
|
|
146
|
-
/**
|
|
147
|
-
* The format of the subtitles.
|
|
148
|
-
*/
|
|
149
|
-
exports.SubtitleFormat = {
|
|
150
|
-
Srt: "srt",
|
|
151
|
-
Vtt: "vtt",
|
|
152
|
-
};
|