langchain 0.0.201 → 0.0.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat_models/baiduwenxin.d.ts +1 -1
- package/dist/document_loaders/fs/pdf.cjs +1 -1
- package/dist/document_loaders/fs/pdf.js +1 -1
- package/dist/document_loaders/web/pdf.cjs +1 -1
- package/dist/document_loaders/web/pdf.js +1 -1
- package/dist/experimental/masking/index.cjs +9 -0
- package/dist/experimental/masking/index.d.ts +4 -0
- package/dist/experimental/masking/index.js +3 -0
- package/dist/experimental/masking/parser.cjs +134 -0
- package/dist/experimental/masking/parser.d.ts +44 -0
- package/dist/experimental/masking/parser.js +130 -0
- package/dist/experimental/masking/regex_masking_transformer.cjs +145 -0
- package/dist/experimental/masking/regex_masking_transformer.d.ts +48 -0
- package/dist/experimental/masking/regex_masking_transformer.js +141 -0
- package/dist/experimental/masking/transformer.cjs +9 -0
- package/dist/experimental/masking/transformer.d.ts +7 -0
- package/dist/experimental/masking/transformer.js +5 -0
- package/dist/experimental/masking/types.cjs +2 -0
- package/dist/experimental/masking/types.d.ts +25 -0
- package/dist/experimental/masking/types.js +1 -0
- package/dist/load/import_map.cjs +2 -1
- package/dist/load/import_map.d.ts +1 -0
- package/dist/load/import_map.js +1 -0
- package/dist/util/stream.cjs +15 -85
- package/dist/util/stream.d.ts +1 -15
- package/dist/util/stream.js +1 -83
- package/dist/vectorstores/prisma.cjs +1 -1
- package/dist/vectorstores/prisma.js +1 -2
- package/dist/vectorstores/xata.cjs +1 -1
- package/dist/vectorstores/xata.js +1 -1
- package/experimental/masking.cjs +1 -0
- package/experimental/masking.d.ts +1 -0
- package/experimental/masking.js +1 -0
- package/package.json +13 -5
|
@@ -131,8 +131,8 @@ export declare class ChatBaiduWenxin extends BaseChatModel implements BaiduWenxi
|
|
|
131
131
|
* Get the identifying parameters for the model
|
|
132
132
|
*/
|
|
133
133
|
identifyingParams(): {
|
|
134
|
-
stream?: boolean | undefined;
|
|
135
134
|
system?: string | undefined;
|
|
135
|
+
stream?: boolean | undefined;
|
|
136
136
|
temperature?: number | undefined;
|
|
137
137
|
top_p?: number | undefined;
|
|
138
138
|
user_id?: string | undefined;
|
|
@@ -15,7 +15,7 @@ const document_js_2 = require("../../util/document.cjs");
|
|
|
15
15
|
* ```
|
|
16
16
|
*/
|
|
17
17
|
class PDFLoader extends buffer_js_1.BufferLoader {
|
|
18
|
-
constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "
|
|
18
|
+
constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
|
|
19
19
|
super(filePathOrBlob);
|
|
20
20
|
Object.defineProperty(this, "splitPages", {
|
|
21
21
|
enumerable: true,
|
|
@@ -12,7 +12,7 @@ import { formatDocumentsAsString } from "../../util/document.js";
|
|
|
12
12
|
* ```
|
|
13
13
|
*/
|
|
14
14
|
export class PDFLoader extends BufferLoader {
|
|
15
|
-
constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "
|
|
15
|
+
constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
|
|
16
16
|
super(filePathOrBlob);
|
|
17
17
|
Object.defineProperty(this, "splitPages", {
|
|
18
18
|
enumerable: true,
|
|
@@ -14,7 +14,7 @@ const document_js_2 = require("../../util/document.cjs");
|
|
|
14
14
|
* ```
|
|
15
15
|
*/
|
|
16
16
|
class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
17
|
-
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "
|
|
17
|
+
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
|
|
18
18
|
super();
|
|
19
19
|
Object.defineProperty(this, "blob", {
|
|
20
20
|
enumerable: true,
|
|
@@ -11,7 +11,7 @@ import { formatDocumentsAsString } from "../../util/document.js";
|
|
|
11
11
|
* ```
|
|
12
12
|
*/
|
|
13
13
|
export class WebPDFLoader extends BaseDocumentLoader {
|
|
14
|
-
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "
|
|
14
|
+
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
|
|
15
15
|
super();
|
|
16
16
|
Object.defineProperty(this, "blob", {
|
|
17
17
|
enumerable: true,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MaskingTransformer = exports.RegexMaskingTransformer = exports.MaskingParser = void 0;
|
|
4
|
+
var parser_js_1 = require("./parser.cjs");
|
|
5
|
+
Object.defineProperty(exports, "MaskingParser", { enumerable: true, get: function () { return parser_js_1.MaskingParser; } });
|
|
6
|
+
var regex_masking_transformer_js_1 = require("./regex_masking_transformer.cjs");
|
|
7
|
+
Object.defineProperty(exports, "RegexMaskingTransformer", { enumerable: true, get: function () { return regex_masking_transformer_js_1.RegexMaskingTransformer; } });
|
|
8
|
+
var transformer_js_1 = require("./transformer.cjs");
|
|
9
|
+
Object.defineProperty(exports, "MaskingTransformer", { enumerable: true, get: function () { return transformer_js_1.MaskingTransformer; } });
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MaskingParser = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* MaskingParser class for handling the masking and rehydrating of messages.
|
|
6
|
+
*/
|
|
7
|
+
class MaskingParser {
|
|
8
|
+
constructor(config = {}) {
|
|
9
|
+
Object.defineProperty(this, "transformers", {
|
|
10
|
+
enumerable: true,
|
|
11
|
+
configurable: true,
|
|
12
|
+
writable: true,
|
|
13
|
+
value: void 0
|
|
14
|
+
});
|
|
15
|
+
Object.defineProperty(this, "state", {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true,
|
|
19
|
+
value: void 0
|
|
20
|
+
});
|
|
21
|
+
Object.defineProperty(this, "config", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
27
|
+
this.transformers = config.transformers ?? [];
|
|
28
|
+
this.state = new Map();
|
|
29
|
+
this.config = config;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Adds a transformer to the parser.
|
|
33
|
+
* @param transformer - An instance of a class extending MaskingTransformer.
|
|
34
|
+
*/
|
|
35
|
+
addTransformer(transformer) {
|
|
36
|
+
this.transformers.push(transformer);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Getter method for retrieving the current state.
|
|
40
|
+
* @returns The current state map.
|
|
41
|
+
*/
|
|
42
|
+
getState() {
|
|
43
|
+
return this.state;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Masks the provided message using the added transformers.
|
|
47
|
+
* This method sequentially applies each transformer's masking logic to the message.
|
|
48
|
+
* It utilizes a state map to track original values corresponding to their masked versions.
|
|
49
|
+
*
|
|
50
|
+
* @param message - The message to be masked.
|
|
51
|
+
* @returns A masked version of the message.
|
|
52
|
+
* @throws {TypeError} If the message is not a string.
|
|
53
|
+
* @throws {Error} If no transformers are added.
|
|
54
|
+
*/
|
|
55
|
+
async mask(message) {
|
|
56
|
+
// If onMaskingStart is a function, handle it accordingly
|
|
57
|
+
if (this.config.onMaskingStart) {
|
|
58
|
+
await this.config.onMaskingStart(message);
|
|
59
|
+
}
|
|
60
|
+
// Check if there are any transformers added to the parser. If not, throw an error
|
|
61
|
+
// as masking requires at least one transformer to apply its logic.
|
|
62
|
+
if (this.transformers.length === 0) {
|
|
63
|
+
throw new Error("MaskingParser.mask Error: No transformers have been added. Please add at least one transformer before parsing.");
|
|
64
|
+
}
|
|
65
|
+
if (typeof message !== "string") {
|
|
66
|
+
throw new TypeError("MaskingParser.mask Error: The 'message' argument must be a string.");
|
|
67
|
+
}
|
|
68
|
+
// Initialize the variable to hold the progressively masked message.
|
|
69
|
+
// It starts as the original message and gets transformed by each transformer.
|
|
70
|
+
let processedMessage = message;
|
|
71
|
+
// Iterate through each transformer and apply their transform method.
|
|
72
|
+
for (const transformer of this.transformers) {
|
|
73
|
+
// Transform the message and get the transformer's state changes, ensuring no direct mutation of the shared state.
|
|
74
|
+
const [transformedMessage, transformerState] = await transformer.transform(processedMessage, new Map(this.state));
|
|
75
|
+
// Update the processed message for subsequent transformers.
|
|
76
|
+
processedMessage = transformedMessage;
|
|
77
|
+
// Merge state changes from the transformer into the parser's state.
|
|
78
|
+
// This accumulates all transformations' effects on the state.
|
|
79
|
+
transformerState.forEach((value, key) => this.state.set(key, value));
|
|
80
|
+
}
|
|
81
|
+
// Handle onMaskingEnd callback
|
|
82
|
+
if (this.config.onMaskingEnd) {
|
|
83
|
+
await this.config.onMaskingEnd(processedMessage);
|
|
84
|
+
}
|
|
85
|
+
// Return the fully masked message after all transformers have been applied.
|
|
86
|
+
return processedMessage;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Rehydrates a masked message back to its original form.
|
|
90
|
+
* This method sequentially applies the rehydration logic of each added transformer in reverse order.
|
|
91
|
+
* It relies on the state map to correctly map the masked values back to their original values.
|
|
92
|
+
*
|
|
93
|
+
* The rehydration process is essential for restoring the original content of a message
|
|
94
|
+
* that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
|
|
95
|
+
*
|
|
96
|
+
* @param message - The masked message to be rehydrated.
|
|
97
|
+
* @returns The original (rehydrated) version of the message.
|
|
98
|
+
*/
|
|
99
|
+
async rehydrate(message, state) {
|
|
100
|
+
// Handle onRehydratingStart callback
|
|
101
|
+
if (this.config.onRehydratingStart) {
|
|
102
|
+
await this.config.onRehydratingStart(message);
|
|
103
|
+
}
|
|
104
|
+
if (typeof message !== "string") {
|
|
105
|
+
throw new TypeError("MaskingParser.rehydrate Error: The 'message' argument must be a string.");
|
|
106
|
+
}
|
|
107
|
+
// Check if any transformers have been added to the parser.
|
|
108
|
+
// If no transformers are present, throw an error as rehydration requires at least one transformer.
|
|
109
|
+
if (this.transformers.length === 0) {
|
|
110
|
+
throw new Error("MaskingParser.rehydrate Error: No transformers have been added. Please add at least one transformer before rehydrating.");
|
|
111
|
+
}
|
|
112
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
113
|
+
if (state && !(state instanceof Map)) {
|
|
114
|
+
throw new TypeError("MaskingParser.rehydrate Error: The 'state' argument, if provided, must be an instance of Map.");
|
|
115
|
+
}
|
|
116
|
+
const rehydrationState = state || this.state; // Use provided state or fallback to internal state
|
|
117
|
+
// Initialize the rehydratedMessage with the input masked message.
|
|
118
|
+
// This variable will undergo rehydration by each transformer in reverse order.
|
|
119
|
+
let rehydratedMessage = message;
|
|
120
|
+
// Use a reverse for...of loop to accommodate asynchronous rehydrate methods
|
|
121
|
+
const reversedTransformers = this.transformers.slice().reverse();
|
|
122
|
+
for (const transformer of reversedTransformers) {
|
|
123
|
+
// Check if the result is a Promise and use await, otherwise use it directly
|
|
124
|
+
rehydratedMessage = await transformer.rehydrate(rehydratedMessage, rehydrationState);
|
|
125
|
+
}
|
|
126
|
+
// Handle onRehydratingEnd callback
|
|
127
|
+
if (this.config.onRehydratingEnd) {
|
|
128
|
+
await this.config.onRehydratingEnd(rehydratedMessage);
|
|
129
|
+
}
|
|
130
|
+
// Return the fully rehydrated message after all transformers have been applied.
|
|
131
|
+
return rehydratedMessage;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.MaskingParser = MaskingParser;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { MaskingTransformer } from "./transformer.js";
|
|
2
|
+
import type { MaskingParserConfig } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* MaskingParser class for handling the masking and rehydrating of messages.
|
|
5
|
+
*/
|
|
6
|
+
export declare class MaskingParser {
|
|
7
|
+
private transformers;
|
|
8
|
+
private state;
|
|
9
|
+
private config;
|
|
10
|
+
constructor(config?: MaskingParserConfig);
|
|
11
|
+
/**
|
|
12
|
+
* Adds a transformer to the parser.
|
|
13
|
+
* @param transformer - An instance of a class extending MaskingTransformer.
|
|
14
|
+
*/
|
|
15
|
+
addTransformer(transformer: MaskingTransformer): void;
|
|
16
|
+
/**
|
|
17
|
+
* Getter method for retrieving the current state.
|
|
18
|
+
* @returns The current state map.
|
|
19
|
+
*/
|
|
20
|
+
getState(): Map<string, string>;
|
|
21
|
+
/**
|
|
22
|
+
* Masks the provided message using the added transformers.
|
|
23
|
+
* This method sequentially applies each transformer's masking logic to the message.
|
|
24
|
+
* It utilizes a state map to track original values corresponding to their masked versions.
|
|
25
|
+
*
|
|
26
|
+
* @param message - The message to be masked.
|
|
27
|
+
* @returns A masked version of the message.
|
|
28
|
+
* @throws {TypeError} If the message is not a string.
|
|
29
|
+
* @throws {Error} If no transformers are added.
|
|
30
|
+
*/
|
|
31
|
+
mask(message: string): Promise<string>;
|
|
32
|
+
/**
|
|
33
|
+
* Rehydrates a masked message back to its original form.
|
|
34
|
+
* This method sequentially applies the rehydration logic of each added transformer in reverse order.
|
|
35
|
+
* It relies on the state map to correctly map the masked values back to their original values.
|
|
36
|
+
*
|
|
37
|
+
* The rehydration process is essential for restoring the original content of a message
|
|
38
|
+
* that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
|
|
39
|
+
*
|
|
40
|
+
* @param message - The masked message to be rehydrated.
|
|
41
|
+
* @returns The original (rehydrated) version of the message.
|
|
42
|
+
*/
|
|
43
|
+
rehydrate(message: string, state?: Map<string, string>): Promise<string>;
|
|
44
|
+
}
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MaskingParser class for handling the masking and rehydrating of messages.
|
|
3
|
+
*/
|
|
4
|
+
export class MaskingParser {
|
|
5
|
+
constructor(config = {}) {
|
|
6
|
+
Object.defineProperty(this, "transformers", {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
configurable: true,
|
|
9
|
+
writable: true,
|
|
10
|
+
value: void 0
|
|
11
|
+
});
|
|
12
|
+
Object.defineProperty(this, "state", {
|
|
13
|
+
enumerable: true,
|
|
14
|
+
configurable: true,
|
|
15
|
+
writable: true,
|
|
16
|
+
value: void 0
|
|
17
|
+
});
|
|
18
|
+
Object.defineProperty(this, "config", {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true,
|
|
22
|
+
value: void 0
|
|
23
|
+
});
|
|
24
|
+
this.transformers = config.transformers ?? [];
|
|
25
|
+
this.state = new Map();
|
|
26
|
+
this.config = config;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Adds a transformer to the parser.
|
|
30
|
+
* @param transformer - An instance of a class extending MaskingTransformer.
|
|
31
|
+
*/
|
|
32
|
+
addTransformer(transformer) {
|
|
33
|
+
this.transformers.push(transformer);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Getter method for retrieving the current state.
|
|
37
|
+
* @returns The current state map.
|
|
38
|
+
*/
|
|
39
|
+
getState() {
|
|
40
|
+
return this.state;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Masks the provided message using the added transformers.
|
|
44
|
+
* This method sequentially applies each transformer's masking logic to the message.
|
|
45
|
+
* It utilizes a state map to track original values corresponding to their masked versions.
|
|
46
|
+
*
|
|
47
|
+
* @param message - The message to be masked.
|
|
48
|
+
* @returns A masked version of the message.
|
|
49
|
+
* @throws {TypeError} If the message is not a string.
|
|
50
|
+
* @throws {Error} If no transformers are added.
|
|
51
|
+
*/
|
|
52
|
+
async mask(message) {
|
|
53
|
+
// If onMaskingStart is a function, handle it accordingly
|
|
54
|
+
if (this.config.onMaskingStart) {
|
|
55
|
+
await this.config.onMaskingStart(message);
|
|
56
|
+
}
|
|
57
|
+
// Check if there are any transformers added to the parser. If not, throw an error
|
|
58
|
+
// as masking requires at least one transformer to apply its logic.
|
|
59
|
+
if (this.transformers.length === 0) {
|
|
60
|
+
throw new Error("MaskingParser.mask Error: No transformers have been added. Please add at least one transformer before parsing.");
|
|
61
|
+
}
|
|
62
|
+
if (typeof message !== "string") {
|
|
63
|
+
throw new TypeError("MaskingParser.mask Error: The 'message' argument must be a string.");
|
|
64
|
+
}
|
|
65
|
+
// Initialize the variable to hold the progressively masked message.
|
|
66
|
+
// It starts as the original message and gets transformed by each transformer.
|
|
67
|
+
let processedMessage = message;
|
|
68
|
+
// Iterate through each transformer and apply their transform method.
|
|
69
|
+
for (const transformer of this.transformers) {
|
|
70
|
+
// Transform the message and get the transformer's state changes, ensuring no direct mutation of the shared state.
|
|
71
|
+
const [transformedMessage, transformerState] = await transformer.transform(processedMessage, new Map(this.state));
|
|
72
|
+
// Update the processed message for subsequent transformers.
|
|
73
|
+
processedMessage = transformedMessage;
|
|
74
|
+
// Merge state changes from the transformer into the parser's state.
|
|
75
|
+
// This accumulates all transformations' effects on the state.
|
|
76
|
+
transformerState.forEach((value, key) => this.state.set(key, value));
|
|
77
|
+
}
|
|
78
|
+
// Handle onMaskingEnd callback
|
|
79
|
+
if (this.config.onMaskingEnd) {
|
|
80
|
+
await this.config.onMaskingEnd(processedMessage);
|
|
81
|
+
}
|
|
82
|
+
// Return the fully masked message after all transformers have been applied.
|
|
83
|
+
return processedMessage;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Rehydrates a masked message back to its original form.
|
|
87
|
+
* This method sequentially applies the rehydration logic of each added transformer in reverse order.
|
|
88
|
+
* It relies on the state map to correctly map the masked values back to their original values.
|
|
89
|
+
*
|
|
90
|
+
* The rehydration process is essential for restoring the original content of a message
|
|
91
|
+
* that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
|
|
92
|
+
*
|
|
93
|
+
* @param message - The masked message to be rehydrated.
|
|
94
|
+
* @returns The original (rehydrated) version of the message.
|
|
95
|
+
*/
|
|
96
|
+
async rehydrate(message, state) {
|
|
97
|
+
// Handle onRehydratingStart callback
|
|
98
|
+
if (this.config.onRehydratingStart) {
|
|
99
|
+
await this.config.onRehydratingStart(message);
|
|
100
|
+
}
|
|
101
|
+
if (typeof message !== "string") {
|
|
102
|
+
throw new TypeError("MaskingParser.rehydrate Error: The 'message' argument must be a string.");
|
|
103
|
+
}
|
|
104
|
+
// Check if any transformers have been added to the parser.
|
|
105
|
+
// If no transformers are present, throw an error as rehydration requires at least one transformer.
|
|
106
|
+
if (this.transformers.length === 0) {
|
|
107
|
+
throw new Error("MaskingParser.rehydrate Error: No transformers have been added. Please add at least one transformer before rehydrating.");
|
|
108
|
+
}
|
|
109
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
110
|
+
if (state && !(state instanceof Map)) {
|
|
111
|
+
throw new TypeError("MaskingParser.rehydrate Error: The 'state' argument, if provided, must be an instance of Map.");
|
|
112
|
+
}
|
|
113
|
+
const rehydrationState = state || this.state; // Use provided state or fallback to internal state
|
|
114
|
+
// Initialize the rehydratedMessage with the input masked message.
|
|
115
|
+
// This variable will undergo rehydration by each transformer in reverse order.
|
|
116
|
+
let rehydratedMessage = message;
|
|
117
|
+
// Use a reverse for...of loop to accommodate asynchronous rehydrate methods
|
|
118
|
+
const reversedTransformers = this.transformers.slice().reverse();
|
|
119
|
+
for (const transformer of reversedTransformers) {
|
|
120
|
+
// Check if the result is a Promise and use await, otherwise use it directly
|
|
121
|
+
rehydratedMessage = await transformer.rehydrate(rehydratedMessage, rehydrationState);
|
|
122
|
+
}
|
|
123
|
+
// Handle onRehydratingEnd callback
|
|
124
|
+
if (this.config.onRehydratingEnd) {
|
|
125
|
+
await this.config.onRehydratingEnd(rehydratedMessage);
|
|
126
|
+
}
|
|
127
|
+
// Return the fully rehydrated message after all transformers have been applied.
|
|
128
|
+
return rehydratedMessage;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RegexMaskingTransformer = void 0;
|
|
4
|
+
const transformer_js_1 = require("./transformer.cjs");
|
|
5
|
+
/**
|
|
6
|
+
* RegexMaskingTransformer class for masking and rehydrating messages with Regex.
|
|
7
|
+
*/
|
|
8
|
+
class RegexMaskingTransformer extends transformer_js_1.MaskingTransformer {
|
|
9
|
+
/**
|
|
10
|
+
* Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
|
|
11
|
+
* Validates the provided patterns to ensure they conform to the expected structure.
|
|
12
|
+
*
|
|
13
|
+
* @param patterns - An object containing masking patterns. Each pattern should include
|
|
14
|
+
* a regular expression (`regex`) and optionally a `replacement` string
|
|
15
|
+
* or a `mask` function.
|
|
16
|
+
* @param hashFunction - An optional custom hash function to be used for masking.
|
|
17
|
+
*/
|
|
18
|
+
constructor(patterns, hashFunction) {
|
|
19
|
+
super();
|
|
20
|
+
Object.defineProperty(this, "patterns", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
Object.defineProperty(this, "hashFunction", {
|
|
27
|
+
enumerable: true,
|
|
28
|
+
configurable: true,
|
|
29
|
+
writable: true,
|
|
30
|
+
value: void 0
|
|
31
|
+
});
|
|
32
|
+
// Validates the provided masking patterns before initializing the transformer.
|
|
33
|
+
// This ensures that each pattern has a valid regular expression.
|
|
34
|
+
this.validatePatterns(patterns);
|
|
35
|
+
// Assigns the validated patterns and the hash function to the transformer.
|
|
36
|
+
// If no custom hash function is provided, the default hash function is used.
|
|
37
|
+
this.patterns = patterns;
|
|
38
|
+
this.hashFunction = hashFunction || this.defaultHashFunction;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Validates the given masking patterns to ensure each pattern has a valid regular expression.
|
|
42
|
+
* Throws an error if any pattern is found to be invalid.
|
|
43
|
+
*
|
|
44
|
+
* @param patterns - The patterns object to validate.
|
|
45
|
+
*/
|
|
46
|
+
validatePatterns(patterns) {
|
|
47
|
+
for (const key of Object.keys(patterns)) {
|
|
48
|
+
const pattern = patterns[key];
|
|
49
|
+
// Checks that each pattern is an object and has a regex property that is an instance of RegExp.
|
|
50
|
+
// Throws an error if these conditions are not met, indicating an invalid pattern configuration.
|
|
51
|
+
if (!pattern ||
|
|
52
|
+
typeof pattern !== "object" ||
|
|
53
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
54
|
+
!(pattern.regex instanceof RegExp)) {
|
|
55
|
+
throw new Error("Invalid pattern configuration.");
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Masks content in a message based on the defined patterns.
|
|
61
|
+
* @param message - The message to be masked.
|
|
62
|
+
* @param state - The current state containing original values.
|
|
63
|
+
* @returns A tuple of the masked message and the updated state.
|
|
64
|
+
*/
|
|
65
|
+
async transform(message, state) {
|
|
66
|
+
if (typeof message !== "string") {
|
|
67
|
+
throw new TypeError("RegexMaskingTransformer.transform Error: The 'message' argument must be a string.");
|
|
68
|
+
}
|
|
69
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
70
|
+
if (!(state instanceof Map)) {
|
|
71
|
+
throw new TypeError("RegexMaskingTransformer.transform Error: The 'state' argument must be an instance of Map.");
|
|
72
|
+
}
|
|
73
|
+
// Holds the progressively masked message
|
|
74
|
+
let processedMessage = message;
|
|
75
|
+
// Initialize original values map with the current state or a new map
|
|
76
|
+
const originalValues = state || new Map();
|
|
77
|
+
// Iterate over each pattern defined in the transformer
|
|
78
|
+
for (const key of Object.keys(this.patterns)) {
|
|
79
|
+
const pattern = this.patterns[key];
|
|
80
|
+
// Apply the current pattern's regex to the message
|
|
81
|
+
processedMessage = processedMessage.replace(pattern.regex, (match) => {
|
|
82
|
+
// Determine the masked value: use the mask function if provided, else use the replacement string,
|
|
83
|
+
// else use the hash function.
|
|
84
|
+
const maskedValue = pattern.mask
|
|
85
|
+
? pattern.mask(match)
|
|
86
|
+
: pattern.replacement ?? this.hashFunction(match);
|
|
87
|
+
// Store the mapping of the masked value to the original value (match)
|
|
88
|
+
originalValues.set(maskedValue, match);
|
|
89
|
+
// Return the masked value to replace the original value in the message
|
|
90
|
+
return maskedValue;
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
// Return the fully masked message and the state map with all original values
|
|
94
|
+
// Wrap the synchronous return values in Promise.resolve() to maintain compatibility
|
|
95
|
+
// with the MaskingParser's expectation of a Promise return type.
|
|
96
|
+
return [processedMessage, originalValues];
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Rehydrates a masked message back to its original form using the provided state.
|
|
100
|
+
* @param message - The masked message to be rehydrated.
|
|
101
|
+
* @param state - The state map containing mappings of masked values to their original values.
|
|
102
|
+
* @returns The rehydrated (original) message.
|
|
103
|
+
*/
|
|
104
|
+
async rehydrate(message, state) {
|
|
105
|
+
if (typeof message !== "string") {
|
|
106
|
+
throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'message' argument must be a string.");
|
|
107
|
+
}
|
|
108
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
109
|
+
if (!(state instanceof Map)) {
|
|
110
|
+
throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'state' argument must be an instance of Map.");
|
|
111
|
+
}
|
|
112
|
+
// Convert the state map to an array and use reduce to sequentially replace masked values with original values.
|
|
113
|
+
const rehydratedMessage = Array.from(state).reduce((msg, [masked, original]) => {
|
|
114
|
+
// Escape special characters in the masked string to ensure it can be used in a regular expression safely.
|
|
115
|
+
// This is necessary because masked values might contain characters that have special meanings in regex.
|
|
116
|
+
const escapedMasked = masked.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
117
|
+
// Replace all instances of the escaped masked value in the message with the original value.
|
|
118
|
+
// The 'g' flag in the RegExp ensures that all occurrences of the masked value are replaced.
|
|
119
|
+
return msg.replace(new RegExp(escapedMasked, "g"), original);
|
|
120
|
+
}, message);
|
|
121
|
+
return rehydratedMessage;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Default hash function for creating unique hash values.
|
|
125
|
+
* @param input - The input string to hash.
|
|
126
|
+
* @returns The resulting hash as a string.
|
|
127
|
+
*/
|
|
128
|
+
defaultHashFunction(input) {
|
|
129
|
+
let hash = 0;
|
|
130
|
+
// Iterate over each character in the input string
|
|
131
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
132
|
+
// Get ASCII value of the character
|
|
133
|
+
const char = input.charCodeAt(i);
|
|
134
|
+
// Combine the current hash with the new character and ensure it remains a 32-bit integer
|
|
135
|
+
hash = (hash << 5) - hash + char;
|
|
136
|
+
// Bitwise OR operation to convert to a 32-bit integer.
|
|
137
|
+
// This is a common technique to ensure the final hash value stays within the 32-bit limit,
|
|
138
|
+
// effectively wrapping the value when it becomes too large.
|
|
139
|
+
hash |= 0;
|
|
140
|
+
}
|
|
141
|
+
// Convert the numerical hash value to a string and return
|
|
142
|
+
return hash.toString();
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
exports.RegexMaskingTransformer = RegexMaskingTransformer;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { MaskingTransformer } from "./transformer.js";
|
|
2
|
+
import type { HashFunction, MaskingPattern } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* RegexMaskingTransformer class for masking and rehydrating messages with Regex.
|
|
5
|
+
*/
|
|
6
|
+
export declare class RegexMaskingTransformer extends MaskingTransformer {
|
|
7
|
+
private patterns;
|
|
8
|
+
private hashFunction;
|
|
9
|
+
/**
|
|
10
|
+
* Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
|
|
11
|
+
* Validates the provided patterns to ensure they conform to the expected structure.
|
|
12
|
+
*
|
|
13
|
+
* @param patterns - An object containing masking patterns. Each pattern should include
|
|
14
|
+
* a regular expression (`regex`) and optionally a `replacement` string
|
|
15
|
+
* or a `mask` function.
|
|
16
|
+
* @param hashFunction - An optional custom hash function to be used for masking.
|
|
17
|
+
*/
|
|
18
|
+
constructor(patterns: {
|
|
19
|
+
[key: string]: MaskingPattern;
|
|
20
|
+
}, hashFunction?: HashFunction);
|
|
21
|
+
/**
|
|
22
|
+
* Validates the given masking patterns to ensure each pattern has a valid regular expression.
|
|
23
|
+
* Throws an error if any pattern is found to be invalid.
|
|
24
|
+
*
|
|
25
|
+
* @param patterns - The patterns object to validate.
|
|
26
|
+
*/
|
|
27
|
+
private validatePatterns;
|
|
28
|
+
/**
|
|
29
|
+
* Masks content in a message based on the defined patterns.
|
|
30
|
+
* @param message - The message to be masked.
|
|
31
|
+
* @param state - The current state containing original values.
|
|
32
|
+
* @returns A tuple of the masked message and the updated state.
|
|
33
|
+
*/
|
|
34
|
+
transform(message: string, state: Map<string, string>): Promise<[string, Map<string, string>]>;
|
|
35
|
+
/**
|
|
36
|
+
* Rehydrates a masked message back to its original form using the provided state.
|
|
37
|
+
* @param message - The masked message to be rehydrated.
|
|
38
|
+
* @param state - The state map containing mappings of masked values to their original values.
|
|
39
|
+
* @returns The rehydrated (original) message.
|
|
40
|
+
*/
|
|
41
|
+
rehydrate(message: string, state: Map<string, string>): Promise<string>;
|
|
42
|
+
/**
|
|
43
|
+
* Default hash function for creating unique hash values.
|
|
44
|
+
* @param input - The input string to hash.
|
|
45
|
+
* @returns The resulting hash as a string.
|
|
46
|
+
*/
|
|
47
|
+
private defaultHashFunction;
|
|
48
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { MaskingTransformer } from "./transformer.js";
|
|
2
|
+
/**
|
|
3
|
+
* RegexMaskingTransformer class for masking and rehydrating messages with Regex.
|
|
4
|
+
*/
|
|
5
|
+
export class RegexMaskingTransformer extends MaskingTransformer {
|
|
6
|
+
/**
|
|
7
|
+
* Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
|
|
8
|
+
* Validates the provided patterns to ensure they conform to the expected structure.
|
|
9
|
+
*
|
|
10
|
+
* @param patterns - An object containing masking patterns. Each pattern should include
|
|
11
|
+
* a regular expression (`regex`) and optionally a `replacement` string
|
|
12
|
+
* or a `mask` function.
|
|
13
|
+
* @param hashFunction - An optional custom hash function to be used for masking.
|
|
14
|
+
*/
|
|
15
|
+
constructor(patterns, hashFunction) {
|
|
16
|
+
super();
|
|
17
|
+
Object.defineProperty(this, "patterns", {
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true,
|
|
21
|
+
value: void 0
|
|
22
|
+
});
|
|
23
|
+
Object.defineProperty(this, "hashFunction", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
// Validates the provided masking patterns before initializing the transformer.
|
|
30
|
+
// This ensures that each pattern has a valid regular expression.
|
|
31
|
+
this.validatePatterns(patterns);
|
|
32
|
+
// Assigns the validated patterns and the hash function to the transformer.
|
|
33
|
+
// If no custom hash function is provided, the default hash function is used.
|
|
34
|
+
this.patterns = patterns;
|
|
35
|
+
this.hashFunction = hashFunction || this.defaultHashFunction;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Validates the given masking patterns to ensure each pattern has a valid regular expression.
|
|
39
|
+
* Throws an error if any pattern is found to be invalid.
|
|
40
|
+
*
|
|
41
|
+
* @param patterns - The patterns object to validate.
|
|
42
|
+
*/
|
|
43
|
+
validatePatterns(patterns) {
|
|
44
|
+
for (const key of Object.keys(patterns)) {
|
|
45
|
+
const pattern = patterns[key];
|
|
46
|
+
// Checks that each pattern is an object and has a regex property that is an instance of RegExp.
|
|
47
|
+
// Throws an error if these conditions are not met, indicating an invalid pattern configuration.
|
|
48
|
+
if (!pattern ||
|
|
49
|
+
typeof pattern !== "object" ||
|
|
50
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
51
|
+
!(pattern.regex instanceof RegExp)) {
|
|
52
|
+
throw new Error("Invalid pattern configuration.");
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Masks content in a message based on the defined patterns.
|
|
58
|
+
* @param message - The message to be masked.
|
|
59
|
+
* @param state - The current state containing original values.
|
|
60
|
+
* @returns A tuple of the masked message and the updated state.
|
|
61
|
+
*/
|
|
62
|
+
async transform(message, state) {
|
|
63
|
+
if (typeof message !== "string") {
|
|
64
|
+
throw new TypeError("RegexMaskingTransformer.transform Error: The 'message' argument must be a string.");
|
|
65
|
+
}
|
|
66
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
67
|
+
if (!(state instanceof Map)) {
|
|
68
|
+
throw new TypeError("RegexMaskingTransformer.transform Error: The 'state' argument must be an instance of Map.");
|
|
69
|
+
}
|
|
70
|
+
// Holds the progressively masked message
|
|
71
|
+
let processedMessage = message;
|
|
72
|
+
// Initialize original values map with the current state or a new map
|
|
73
|
+
const originalValues = state || new Map();
|
|
74
|
+
// Iterate over each pattern defined in the transformer
|
|
75
|
+
for (const key of Object.keys(this.patterns)) {
|
|
76
|
+
const pattern = this.patterns[key];
|
|
77
|
+
// Apply the current pattern's regex to the message
|
|
78
|
+
processedMessage = processedMessage.replace(pattern.regex, (match) => {
|
|
79
|
+
// Determine the masked value: use the mask function if provided, else use the replacement string,
|
|
80
|
+
// else use the hash function.
|
|
81
|
+
const maskedValue = pattern.mask
|
|
82
|
+
? pattern.mask(match)
|
|
83
|
+
: pattern.replacement ?? this.hashFunction(match);
|
|
84
|
+
// Store the mapping of the masked value to the original value (match)
|
|
85
|
+
originalValues.set(maskedValue, match);
|
|
86
|
+
// Return the masked value to replace the original value in the message
|
|
87
|
+
return maskedValue;
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
// Return the fully masked message and the state map with all original values
|
|
91
|
+
// Wrap the synchronous return values in Promise.resolve() to maintain compatibility
|
|
92
|
+
// with the MaskingParser's expectation of a Promise return type.
|
|
93
|
+
return [processedMessage, originalValues];
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Rehydrates a masked message back to its original form using the provided state.
|
|
97
|
+
* @param message - The masked message to be rehydrated.
|
|
98
|
+
* @param state - The state map containing mappings of masked values to their original values.
|
|
99
|
+
* @returns The rehydrated (original) message.
|
|
100
|
+
*/
|
|
101
|
+
async rehydrate(message, state) {
|
|
102
|
+
if (typeof message !== "string") {
|
|
103
|
+
throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'message' argument must be a string.");
|
|
104
|
+
}
|
|
105
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
106
|
+
if (!(state instanceof Map)) {
|
|
107
|
+
throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'state' argument must be an instance of Map.");
|
|
108
|
+
}
|
|
109
|
+
// Convert the state map to an array and use reduce to sequentially replace masked values with original values.
|
|
110
|
+
const rehydratedMessage = Array.from(state).reduce((msg, [masked, original]) => {
|
|
111
|
+
// Escape special characters in the masked string to ensure it can be used in a regular expression safely.
|
|
112
|
+
// This is necessary because masked values might contain characters that have special meanings in regex.
|
|
113
|
+
const escapedMasked = masked.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
114
|
+
// Replace all instances of the escaped masked value in the message with the original value.
|
|
115
|
+
// The 'g' flag in the RegExp ensures that all occurrences of the masked value are replaced.
|
|
116
|
+
return msg.replace(new RegExp(escapedMasked, "g"), original);
|
|
117
|
+
}, message);
|
|
118
|
+
return rehydratedMessage;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Default hash function for creating unique hash values.
|
|
122
|
+
* @param input - The input string to hash.
|
|
123
|
+
* @returns The resulting hash as a string.
|
|
124
|
+
*/
|
|
125
|
+
defaultHashFunction(input) {
|
|
126
|
+
let hash = 0;
|
|
127
|
+
// Iterate over each character in the input string
|
|
128
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
129
|
+
// Get ASCII value of the character
|
|
130
|
+
const char = input.charCodeAt(i);
|
|
131
|
+
// Combine the current hash with the new character and ensure it remains a 32-bit integer
|
|
132
|
+
hash = (hash << 5) - hash + char;
|
|
133
|
+
// Bitwise OR operation to convert to a 32-bit integer.
|
|
134
|
+
// This is a common technique to ensure the final hash value stays within the 32-bit limit,
|
|
135
|
+
// effectively wrapping the value when it becomes too large.
|
|
136
|
+
hash |= 0;
|
|
137
|
+
}
|
|
138
|
+
// Convert the numerical hash value to a string and return
|
|
139
|
+
return hash.toString();
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.MaskingTransformer = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Abstract class representing a transformer used for masking and rehydrating messages.
|
|
6
|
+
*/
|
|
7
|
+
class MaskingTransformer {
|
|
8
|
+
}
|
|
9
|
+
exports.MaskingTransformer = MaskingTransformer;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Abstract class representing a transformer used for masking and rehydrating messages.
|
|
3
|
+
*/
|
|
4
|
+
export declare abstract class MaskingTransformer {
|
|
5
|
+
abstract transform(message: string, state?: Map<string, string>): Promise<[string, Map<string, string>]>;
|
|
6
|
+
abstract rehydrate(message: string, state: Map<string, string>): Promise<string>;
|
|
7
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { MaskingTransformer } from "./transformer.js";
|
|
2
|
+
/**
|
|
3
|
+
* Configuration type for MaskingParser.
|
|
4
|
+
*/
|
|
5
|
+
export type MaskingParserConfig = {
|
|
6
|
+
transformers?: MaskingTransformer[];
|
|
7
|
+
defaultHashFunction?: HashFunction;
|
|
8
|
+
onMaskingStart?: HookFunction;
|
|
9
|
+
onMaskingEnd?: HookFunction;
|
|
10
|
+
onRehydratingStart?: HookFunction;
|
|
11
|
+
onRehydratingEnd?: HookFunction;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Regex Masking Pattern used for masking in PIIMaskingTransformer.
|
|
15
|
+
*/
|
|
16
|
+
export type MaskingPattern = {
|
|
17
|
+
regex: RegExp;
|
|
18
|
+
replacement?: string;
|
|
19
|
+
mask?: (match: string) => string;
|
|
20
|
+
};
|
|
21
|
+
export type HookFunction = ((message: string) => Promise<void>) | ((message: string) => void);
|
|
22
|
+
/**
|
|
23
|
+
* Represents a function that can hash a string input.
|
|
24
|
+
*/
|
|
25
|
+
export type HashFunction = (input: string) => string;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/load/import_map.cjs
CHANGED
|
@@ -26,7 +26,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
26
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
27
|
exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fake = exports.llms__yandex = exports.llms__fireworks = exports.llms__ollama = exports.llms__cloudflare_workersai = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__voyage = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains__combine_documents__reduce = exports.chains = exports.tools__google_places = exports.tools__render = exports.tools__connery = exports.tools = exports.base_language = exports.agents__openai__output_parser = exports.agents__xml__output_parser = exports.agents__react__output_parser = exports.agents__format_scratchpad__log_to_message = exports.agents__format_scratchpad__xml = exports.agents__format_scratchpad__log = exports.agents__format_scratchpad__openai_tools = exports.agents__format_scratchpad = exports.agents__toolkits__connery = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
|
|
28
28
|
exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__openai_files = exports.experimental__openai_assistant = exports.experimental__autogpt = exports.util__time = exports.util__math = exports.util__document = exports.storage__in_memory = exports.storage__encoder_backed = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__embeddings_filter = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__tavily_search_api = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = exports.retrievers__chaindesk = exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__prompt_template = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__fake = exports.chat_models__yandex = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__cloudflare_workersai = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = void 0;
|
|
29
|
-
exports.runnables__remote = exports.runnables = exports.evaluation = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__ollama_functions = exports.experimental__chat_models__bittensor = void 0;
|
|
29
|
+
exports.runnables__remote = exports.runnables = exports.evaluation = exports.experimental__masking = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__ollama_functions = exports.experimental__chat_models__bittensor = void 0;
|
|
30
30
|
exports.load__serializable = __importStar(require("../load/serializable.cjs"));
|
|
31
31
|
exports.agents = __importStar(require("../agents/index.cjs"));
|
|
32
32
|
exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
|
|
@@ -130,6 +130,7 @@ exports.experimental__plan_and_execute = __importStar(require("../experimental/p
|
|
|
130
130
|
exports.experimental__chat_models__bittensor = __importStar(require("../experimental/chat_models/bittensor.cjs"));
|
|
131
131
|
exports.experimental__chat_models__ollama_functions = __importStar(require("../experimental/chat_models/ollama_functions.cjs"));
|
|
132
132
|
exports.experimental__chains__violation_of_expectations = __importStar(require("../experimental/chains/violation_of_expectations/index.cjs"));
|
|
133
|
+
exports.experimental__masking = __importStar(require("../experimental/masking/index.cjs"));
|
|
133
134
|
exports.evaluation = __importStar(require("../evaluation/index.cjs"));
|
|
134
135
|
exports.runnables = __importStar(require("../runnables/index.cjs"));
|
|
135
136
|
exports.runnables__remote = __importStar(require("../runnables/remote.cjs"));
|
|
@@ -101,6 +101,7 @@ export * as experimental__plan_and_execute from "../experimental/plan_and_execut
|
|
|
101
101
|
export * as experimental__chat_models__bittensor from "../experimental/chat_models/bittensor.js";
|
|
102
102
|
export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";
|
|
103
103
|
export * as experimental__chains__violation_of_expectations from "../experimental/chains/violation_of_expectations/index.js";
|
|
104
|
+
export * as experimental__masking from "../experimental/masking/index.js";
|
|
104
105
|
export * as evaluation from "../evaluation/index.js";
|
|
105
106
|
export * as runnables from "../runnables/index.js";
|
|
106
107
|
export * as runnables__remote from "../runnables/remote.js";
|
package/dist/load/import_map.js
CHANGED
|
@@ -102,6 +102,7 @@ export * as experimental__plan_and_execute from "../experimental/plan_and_execut
|
|
|
102
102
|
export * as experimental__chat_models__bittensor from "../experimental/chat_models/bittensor.js";
|
|
103
103
|
export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";
|
|
104
104
|
export * as experimental__chains__violation_of_expectations from "../experimental/chains/violation_of_expectations/index.js";
|
|
105
|
+
export * as experimental__masking from "../experimental/masking/index.js";
|
|
105
106
|
export * as evaluation from "../evaluation/index.js";
|
|
106
107
|
export * as runnables from "../runnables/index.js";
|
|
107
108
|
export * as runnables__remote from "../runnables/remote.js";
|
package/dist/util/stream.cjs
CHANGED
|
@@ -1,87 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
Object.
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
*/
|
|
8
|
-
class IterableReadableStream extends ReadableStream {
|
|
9
|
-
constructor() {
|
|
10
|
-
super(...arguments);
|
|
11
|
-
Object.defineProperty(this, "reader", {
|
|
12
|
-
enumerable: true,
|
|
13
|
-
configurable: true,
|
|
14
|
-
writable: true,
|
|
15
|
-
value: void 0
|
|
16
|
-
});
|
|
17
|
-
}
|
|
18
|
-
ensureReader() {
|
|
19
|
-
if (!this.reader) {
|
|
20
|
-
this.reader = this.getReader();
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
async next() {
|
|
24
|
-
this.ensureReader();
|
|
25
|
-
try {
|
|
26
|
-
const result = await this.reader.read();
|
|
27
|
-
if (result.done)
|
|
28
|
-
this.reader.releaseLock(); // release lock when stream becomes closed
|
|
29
|
-
return {
|
|
30
|
-
done: result.done,
|
|
31
|
-
value: result.value, // Cloudflare Workers typing fix
|
|
32
|
-
};
|
|
33
|
-
}
|
|
34
|
-
catch (e) {
|
|
35
|
-
this.reader.releaseLock(); // release lock when stream becomes errored
|
|
36
|
-
throw e;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
async return() {
|
|
40
|
-
this.ensureReader();
|
|
41
|
-
const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
|
|
42
|
-
this.reader.releaseLock(); // release lock first
|
|
43
|
-
await cancelPromise; // now await it
|
|
44
|
-
return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore final chunk value anyway
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
45
7
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
return reader.read().then(({ done, value }) => {
|
|
57
|
-
// When no more data needs to be consumed, close the stream
|
|
58
|
-
if (done) {
|
|
59
|
-
controller.close();
|
|
60
|
-
return;
|
|
61
|
-
}
|
|
62
|
-
// Enqueue the next data chunk into our target stream
|
|
63
|
-
controller.enqueue(value);
|
|
64
|
-
return pump();
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
},
|
|
68
|
-
cancel() {
|
|
69
|
-
reader.releaseLock();
|
|
70
|
-
},
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
static fromAsyncGenerator(generator) {
|
|
74
|
-
return new IterableReadableStream({
|
|
75
|
-
async pull(controller) {
|
|
76
|
-
const { value, done } = await generator.next();
|
|
77
|
-
// When no more data needs to be consumed, close the stream
|
|
78
|
-
if (done) {
|
|
79
|
-
controller.close();
|
|
80
|
-
}
|
|
81
|
-
// Fix: `else if (value)` will hang the streaming when nullish value (e.g. empty string) is pulled
|
|
82
|
-
controller.enqueue(value);
|
|
83
|
-
},
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
exports.IterableReadableStream = IterableReadableStream;
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("@langchain/core/utils/stream"), exports);
|
package/dist/util/stream.d.ts
CHANGED
|
@@ -1,15 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
reader: ReadableStreamDefaultReader<T>;
|
|
3
|
-
ensureReader(): void;
|
|
4
|
-
next(): Promise<{
|
|
5
|
-
done: boolean;
|
|
6
|
-
value: T;
|
|
7
|
-
}>;
|
|
8
|
-
return(): Promise<{
|
|
9
|
-
done: boolean;
|
|
10
|
-
value: T;
|
|
11
|
-
}>;
|
|
12
|
-
[Symbol.asyncIterator](): this;
|
|
13
|
-
static fromReadableStream<T>(stream: ReadableStream<T>): IterableReadableStream<T>;
|
|
14
|
-
static fromAsyncGenerator<T>(generator: AsyncGenerator<T>): IterableReadableStream<T>;
|
|
15
|
-
}
|
|
1
|
+
export * from "@langchain/core/utils/stream";
|
package/dist/util/stream.js
CHANGED
|
@@ -1,83 +1 @@
|
|
|
1
|
-
|
|
2
|
-
* Support async iterator syntax for ReadableStreams in all environments.
|
|
3
|
-
* Source: https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490
|
|
4
|
-
*/
|
|
5
|
-
export class IterableReadableStream extends ReadableStream {
|
|
6
|
-
constructor() {
|
|
7
|
-
super(...arguments);
|
|
8
|
-
Object.defineProperty(this, "reader", {
|
|
9
|
-
enumerable: true,
|
|
10
|
-
configurable: true,
|
|
11
|
-
writable: true,
|
|
12
|
-
value: void 0
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
ensureReader() {
|
|
16
|
-
if (!this.reader) {
|
|
17
|
-
this.reader = this.getReader();
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
async next() {
|
|
21
|
-
this.ensureReader();
|
|
22
|
-
try {
|
|
23
|
-
const result = await this.reader.read();
|
|
24
|
-
if (result.done)
|
|
25
|
-
this.reader.releaseLock(); // release lock when stream becomes closed
|
|
26
|
-
return {
|
|
27
|
-
done: result.done,
|
|
28
|
-
value: result.value, // Cloudflare Workers typing fix
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
catch (e) {
|
|
32
|
-
this.reader.releaseLock(); // release lock when stream becomes errored
|
|
33
|
-
throw e;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
async return() {
|
|
37
|
-
this.ensureReader();
|
|
38
|
-
const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
|
|
39
|
-
this.reader.releaseLock(); // release lock first
|
|
40
|
-
await cancelPromise; // now await it
|
|
41
|
-
return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore final chunk value anyway
|
|
42
|
-
}
|
|
43
|
-
[Symbol.asyncIterator]() {
|
|
44
|
-
return this;
|
|
45
|
-
}
|
|
46
|
-
static fromReadableStream(stream) {
|
|
47
|
-
// From https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream
|
|
48
|
-
const reader = stream.getReader();
|
|
49
|
-
return new IterableReadableStream({
|
|
50
|
-
start(controller) {
|
|
51
|
-
return pump();
|
|
52
|
-
function pump() {
|
|
53
|
-
return reader.read().then(({ done, value }) => {
|
|
54
|
-
// When no more data needs to be consumed, close the stream
|
|
55
|
-
if (done) {
|
|
56
|
-
controller.close();
|
|
57
|
-
return;
|
|
58
|
-
}
|
|
59
|
-
// Enqueue the next data chunk into our target stream
|
|
60
|
-
controller.enqueue(value);
|
|
61
|
-
return pump();
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
},
|
|
65
|
-
cancel() {
|
|
66
|
-
reader.releaseLock();
|
|
67
|
-
},
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
static fromAsyncGenerator(generator) {
|
|
71
|
-
return new IterableReadableStream({
|
|
72
|
-
async pull(controller) {
|
|
73
|
-
const { value, done } = await generator.next();
|
|
74
|
-
// When no more data needs to be consumed, close the stream
|
|
75
|
-
if (done) {
|
|
76
|
-
controller.close();
|
|
77
|
-
}
|
|
78
|
-
// Fix: `else if (value)` will hang the streaming when nullish value (e.g. empty string) is pulled
|
|
79
|
-
controller.enqueue(value);
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
}
|
|
83
|
-
}
|
|
1
|
+
export * from "@langchain/core/utils/stream";
|
|
@@ -292,6 +292,7 @@ class PrismaVectorStore extends base_js_1.VectorStore {
|
|
|
292
292
|
return instance;
|
|
293
293
|
}
|
|
294
294
|
}
|
|
295
|
+
exports.PrismaVectorStore = PrismaVectorStore;
|
|
295
296
|
Object.defineProperty(PrismaVectorStore, "IdColumn", {
|
|
296
297
|
enumerable: true,
|
|
297
298
|
configurable: true,
|
|
@@ -304,4 +305,3 @@ Object.defineProperty(PrismaVectorStore, "ContentColumn", {
|
|
|
304
305
|
writable: true,
|
|
305
306
|
value: ContentColumnSymbol
|
|
306
307
|
});
|
|
307
|
-
exports.PrismaVectorStore = PrismaVectorStore;
|
|
@@ -19,7 +19,7 @@ const OpMap = {
|
|
|
19
19
|
* work with Prisma. It provides methods for adding models, documents, and
|
|
20
20
|
* vectors, as well as for performing similarity searches.
|
|
21
21
|
*/
|
|
22
|
-
class PrismaVectorStore extends VectorStore {
|
|
22
|
+
export class PrismaVectorStore extends VectorStore {
|
|
23
23
|
_vectorstoreType() {
|
|
24
24
|
return "prisma";
|
|
25
25
|
}
|
|
@@ -301,4 +301,3 @@ Object.defineProperty(PrismaVectorStore, "ContentColumn", {
|
|
|
301
301
|
writable: true,
|
|
302
302
|
value: ContentColumnSymbol
|
|
303
303
|
});
|
|
304
|
-
export { PrismaVectorStore };
|
|
@@ -89,7 +89,7 @@ class XataVectorSearch extends base_js_1.VectorStore {
|
|
|
89
89
|
* @returns Promise resolving to an array of tuples, each containing a Document and its score.
|
|
90
90
|
*/
|
|
91
91
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
92
|
-
const records = await this.client.db[this.table].vectorSearch("embedding", query, {
|
|
92
|
+
const { records } = await this.client.db[this.table].vectorSearch("embedding", query, {
|
|
93
93
|
size: k,
|
|
94
94
|
filter,
|
|
95
95
|
});
|
|
@@ -86,7 +86,7 @@ export class XataVectorSearch extends VectorStore {
|
|
|
86
86
|
* @returns Promise resolving to an array of tuples, each containing a Document and its score.
|
|
87
87
|
*/
|
|
88
88
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
89
|
-
const records = await this.client.db[this.table].vectorSearch("embedding", query, {
|
|
89
|
+
const { records } = await this.client.db[this.table].vectorSearch("embedding", query, {
|
|
90
90
|
size: k,
|
|
91
91
|
filter,
|
|
92
92
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/experimental/masking/index.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/experimental/masking/index.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/experimental/masking/index.js'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "langchain",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.203",
|
|
4
4
|
"description": "Typescript bindings for langchain",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -835,6 +835,9 @@
|
|
|
835
835
|
"experimental/chains/violation_of_expectations.cjs",
|
|
836
836
|
"experimental/chains/violation_of_expectations.js",
|
|
837
837
|
"experimental/chains/violation_of_expectations.d.ts",
|
|
838
|
+
"experimental/masking.cjs",
|
|
839
|
+
"experimental/masking.js",
|
|
840
|
+
"experimental/masking.d.ts",
|
|
838
841
|
"experimental/tools/pyinterpreter.cjs",
|
|
839
842
|
"experimental/tools/pyinterpreter.js",
|
|
840
843
|
"experimental/tools/pyinterpreter.d.ts",
|
|
@@ -944,7 +947,7 @@
|
|
|
944
947
|
"@vercel/kv": "^0.2.3",
|
|
945
948
|
"@vercel/postgres": "^0.5.0",
|
|
946
949
|
"@writerai/writer-sdk": "^0.40.2",
|
|
947
|
-
"@xata.io/client": "^0.
|
|
950
|
+
"@xata.io/client": "^0.28.0",
|
|
948
951
|
"@xenova/transformers": "^2.5.4",
|
|
949
952
|
"@zilliz/milvus2-sdk-node": ">=2.2.11",
|
|
950
953
|
"apify-client": "^2.7.1",
|
|
@@ -1012,7 +1015,7 @@
|
|
|
1012
1015
|
"srt-parser-2": "^1.2.2",
|
|
1013
1016
|
"ts-jest": "^29.1.0",
|
|
1014
1017
|
"typeorm": "^0.3.12",
|
|
1015
|
-
"typescript": "
|
|
1018
|
+
"typescript": "~5.1.6",
|
|
1016
1019
|
"typesense": "^1.5.3",
|
|
1017
1020
|
"usearch": "^1.1.1",
|
|
1018
1021
|
"vectordb": "^0.1.4",
|
|
@@ -1066,7 +1069,7 @@
|
|
|
1066
1069
|
"@vercel/kv": "^0.2.3",
|
|
1067
1070
|
"@vercel/postgres": "^0.5.0",
|
|
1068
1071
|
"@writerai/writer-sdk": "^0.40.2",
|
|
1069
|
-
"@xata.io/client": "^0.
|
|
1072
|
+
"@xata.io/client": "^0.28.0",
|
|
1070
1073
|
"@xenova/transformers": "^2.5.4",
|
|
1071
1074
|
"@zilliz/milvus2-sdk-node": ">=2.2.7",
|
|
1072
1075
|
"apify-client": "^2.7.1",
|
|
@@ -1432,7 +1435,7 @@
|
|
|
1432
1435
|
},
|
|
1433
1436
|
"dependencies": {
|
|
1434
1437
|
"@anthropic-ai/sdk": "^0.9.1",
|
|
1435
|
-
"@langchain/core": "~0.0.
|
|
1438
|
+
"@langchain/core": "~0.0.10",
|
|
1436
1439
|
"binary-extensions": "^2.2.0",
|
|
1437
1440
|
"expr-eval": "^2.0.2",
|
|
1438
1441
|
"flat": "^5.0.2",
|
|
@@ -2843,6 +2846,11 @@
|
|
|
2843
2846
|
"import": "./experimental/chains/violation_of_expectations.js",
|
|
2844
2847
|
"require": "./experimental/chains/violation_of_expectations.cjs"
|
|
2845
2848
|
},
|
|
2849
|
+
"./experimental/masking": {
|
|
2850
|
+
"types": "./experimental/masking.d.ts",
|
|
2851
|
+
"import": "./experimental/masking.js",
|
|
2852
|
+
"require": "./experimental/masking.cjs"
|
|
2853
|
+
},
|
|
2846
2854
|
"./experimental/tools/pyinterpreter": {
|
|
2847
2855
|
"types": "./experimental/tools/pyinterpreter.d.ts",
|
|
2848
2856
|
"import": "./experimental/tools/pyinterpreter.js",
|