langchain 0.0.201 → 0.0.203

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/chat_models/baiduwenxin.d.ts +1 -1
  2. package/dist/document_loaders/fs/pdf.cjs +1 -1
  3. package/dist/document_loaders/fs/pdf.js +1 -1
  4. package/dist/document_loaders/web/pdf.cjs +1 -1
  5. package/dist/document_loaders/web/pdf.js +1 -1
  6. package/dist/experimental/masking/index.cjs +9 -0
  7. package/dist/experimental/masking/index.d.ts +4 -0
  8. package/dist/experimental/masking/index.js +3 -0
  9. package/dist/experimental/masking/parser.cjs +134 -0
  10. package/dist/experimental/masking/parser.d.ts +44 -0
  11. package/dist/experimental/masking/parser.js +130 -0
  12. package/dist/experimental/masking/regex_masking_transformer.cjs +145 -0
  13. package/dist/experimental/masking/regex_masking_transformer.d.ts +48 -0
  14. package/dist/experimental/masking/regex_masking_transformer.js +141 -0
  15. package/dist/experimental/masking/transformer.cjs +9 -0
  16. package/dist/experimental/masking/transformer.d.ts +7 -0
  17. package/dist/experimental/masking/transformer.js +5 -0
  18. package/dist/experimental/masking/types.cjs +2 -0
  19. package/dist/experimental/masking/types.d.ts +25 -0
  20. package/dist/experimental/masking/types.js +1 -0
  21. package/dist/load/import_map.cjs +2 -1
  22. package/dist/load/import_map.d.ts +1 -0
  23. package/dist/load/import_map.js +1 -0
  24. package/dist/util/stream.cjs +15 -85
  25. package/dist/util/stream.d.ts +1 -15
  26. package/dist/util/stream.js +1 -83
  27. package/dist/vectorstores/prisma.cjs +1 -1
  28. package/dist/vectorstores/prisma.js +1 -2
  29. package/dist/vectorstores/xata.cjs +1 -1
  30. package/dist/vectorstores/xata.js +1 -1
  31. package/experimental/masking.cjs +1 -0
  32. package/experimental/masking.d.ts +1 -0
  33. package/experimental/masking.js +1 -0
  34. package/package.json +13 -5
@@ -131,8 +131,8 @@ export declare class ChatBaiduWenxin extends BaseChatModel implements BaiduWenxi
131
131
  * Get the identifying parameters for the model
132
132
  */
133
133
  identifyingParams(): {
134
- stream?: boolean | undefined;
135
134
  system?: string | undefined;
135
+ stream?: boolean | undefined;
136
136
  temperature?: number | undefined;
137
137
  top_p?: number | undefined;
138
138
  user_id?: string | undefined;
@@ -15,7 +15,7 @@ const document_js_2 = require("../../util/document.cjs");
15
15
  * ```
16
16
  */
17
17
  class PDFLoader extends buffer_js_1.BufferLoader {
18
- constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = " ", } = {}) {
18
+ constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
19
19
  super(filePathOrBlob);
20
20
  Object.defineProperty(this, "splitPages", {
21
21
  enumerable: true,
@@ -12,7 +12,7 @@ import { formatDocumentsAsString } from "../../util/document.js";
12
12
  * ```
13
13
  */
14
14
  export class PDFLoader extends BufferLoader {
15
- constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = " ", } = {}) {
15
+ constructor(filePathOrBlob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
16
16
  super(filePathOrBlob);
17
17
  Object.defineProperty(this, "splitPages", {
18
18
  enumerable: true,
@@ -14,7 +14,7 @@ const document_js_2 = require("../../util/document.cjs");
14
14
  * ```
15
15
  */
16
16
  class WebPDFLoader extends base_js_1.BaseDocumentLoader {
17
- constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = " ", } = {}) {
17
+ constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
18
18
  super();
19
19
  Object.defineProperty(this, "blob", {
20
20
  enumerable: true,
@@ -11,7 +11,7 @@ import { formatDocumentsAsString } from "../../util/document.js";
11
11
  * ```
12
12
  */
13
13
  export class WebPDFLoader extends BaseDocumentLoader {
14
- constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = " ", } = {}) {
14
+ constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
15
15
  super();
16
16
  Object.defineProperty(this, "blob", {
17
17
  enumerable: true,
@@ -0,0 +1,9 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MaskingTransformer = exports.RegexMaskingTransformer = exports.MaskingParser = void 0;
4
+ var parser_js_1 = require("./parser.cjs");
5
+ Object.defineProperty(exports, "MaskingParser", { enumerable: true, get: function () { return parser_js_1.MaskingParser; } });
6
+ var regex_masking_transformer_js_1 = require("./regex_masking_transformer.cjs");
7
+ Object.defineProperty(exports, "RegexMaskingTransformer", { enumerable: true, get: function () { return regex_masking_transformer_js_1.RegexMaskingTransformer; } });
8
+ var transformer_js_1 = require("./transformer.cjs");
9
+ Object.defineProperty(exports, "MaskingTransformer", { enumerable: true, get: function () { return transformer_js_1.MaskingTransformer; } });
@@ -0,0 +1,4 @@
1
+ export { MaskingParser } from "./parser.js";
2
+ export { RegexMaskingTransformer } from "./regex_masking_transformer.js";
3
+ export { MaskingTransformer } from "./transformer.js";
4
+ export { type MaskingParserConfig, type HashFunction, type HookFunction, } from "./types.js";
@@ -0,0 +1,3 @@
1
+ export { MaskingParser } from "./parser.js";
2
+ export { RegexMaskingTransformer } from "./regex_masking_transformer.js";
3
+ export { MaskingTransformer } from "./transformer.js";
@@ -0,0 +1,134 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MaskingParser = void 0;
4
+ /**
5
+ * MaskingParser class for handling the masking and rehydrating of messages.
6
+ */
7
+ class MaskingParser {
8
+ constructor(config = {}) {
9
+ Object.defineProperty(this, "transformers", {
10
+ enumerable: true,
11
+ configurable: true,
12
+ writable: true,
13
+ value: void 0
14
+ });
15
+ Object.defineProperty(this, "state", {
16
+ enumerable: true,
17
+ configurable: true,
18
+ writable: true,
19
+ value: void 0
20
+ });
21
+ Object.defineProperty(this, "config", {
22
+ enumerable: true,
23
+ configurable: true,
24
+ writable: true,
25
+ value: void 0
26
+ });
27
+ this.transformers = config.transformers ?? [];
28
+ this.state = new Map();
29
+ this.config = config;
30
+ }
31
+ /**
32
+ * Adds a transformer to the parser.
33
+ * @param transformer - An instance of a class extending MaskingTransformer.
34
+ */
35
+ addTransformer(transformer) {
36
+ this.transformers.push(transformer);
37
+ }
38
+ /**
39
+ * Getter method for retrieving the current state.
40
+ * @returns The current state map.
41
+ */
42
+ getState() {
43
+ return this.state;
44
+ }
45
+ /**
46
+ * Masks the provided message using the added transformers.
47
+ * This method sequentially applies each transformer's masking logic to the message.
48
+ * It utilizes a state map to track original values corresponding to their masked versions.
49
+ *
50
+ * @param message - The message to be masked.
51
+ * @returns A masked version of the message.
52
+ * @throws {TypeError} If the message is not a string.
53
+ * @throws {Error} If no transformers are added.
54
+ */
55
+ async mask(message) {
56
+ // If onMaskingStart is a function, handle it accordingly
57
+ if (this.config.onMaskingStart) {
58
+ await this.config.onMaskingStart(message);
59
+ }
60
+ // Check if there are any transformers added to the parser. If not, throw an error
61
+ // as masking requires at least one transformer to apply its logic.
62
+ if (this.transformers.length === 0) {
63
+ throw new Error("MaskingParser.mask Error: No transformers have been added. Please add at least one transformer before parsing.");
64
+ }
65
+ if (typeof message !== "string") {
66
+ throw new TypeError("MaskingParser.mask Error: The 'message' argument must be a string.");
67
+ }
68
+ // Initialize the variable to hold the progressively masked message.
69
+ // It starts as the original message and gets transformed by each transformer.
70
+ let processedMessage = message;
71
+ // Iterate through each transformer and apply their transform method.
72
+ for (const transformer of this.transformers) {
73
+ // Transform the message and get the transformer's state changes, ensuring no direct mutation of the shared state.
74
+ const [transformedMessage, transformerState] = await transformer.transform(processedMessage, new Map(this.state));
75
+ // Update the processed message for subsequent transformers.
76
+ processedMessage = transformedMessage;
77
+ // Merge state changes from the transformer into the parser's state.
78
+ // This accumulates all transformations' effects on the state.
79
+ transformerState.forEach((value, key) => this.state.set(key, value));
80
+ }
81
+ // Handle onMaskingEnd callback
82
+ if (this.config.onMaskingEnd) {
83
+ await this.config.onMaskingEnd(processedMessage);
84
+ }
85
+ // Return the fully masked message after all transformers have been applied.
86
+ return processedMessage;
87
+ }
88
+ /**
89
+ * Rehydrates a masked message back to its original form.
90
+ * This method sequentially applies the rehydration logic of each added transformer in reverse order.
91
+ * It relies on the state map to correctly map the masked values back to their original values.
92
+ *
93
+ * The rehydration process is essential for restoring the original content of a message
94
+ * that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
95
+ *
96
+ * @param message - The masked message to be rehydrated.
97
+ * @returns The original (rehydrated) version of the message.
98
+ */
99
+ async rehydrate(message, state) {
100
+ // Handle onRehydratingStart callback
101
+ if (this.config.onRehydratingStart) {
102
+ await this.config.onRehydratingStart(message);
103
+ }
104
+ if (typeof message !== "string") {
105
+ throw new TypeError("MaskingParser.rehydrate Error: The 'message' argument must be a string.");
106
+ }
107
+ // Check if any transformers have been added to the parser.
108
+ // If no transformers are present, throw an error as rehydration requires at least one transformer.
109
+ if (this.transformers.length === 0) {
110
+ throw new Error("MaskingParser.rehydrate Error: No transformers have been added. Please add at least one transformer before rehydrating.");
111
+ }
112
+ // eslint-disable-next-line no-instanceof/no-instanceof
113
+ if (state && !(state instanceof Map)) {
114
+ throw new TypeError("MaskingParser.rehydrate Error: The 'state' argument, if provided, must be an instance of Map.");
115
+ }
116
+ const rehydrationState = state || this.state; // Use provided state or fallback to internal state
117
+ // Initialize the rehydratedMessage with the input masked message.
118
+ // This variable will undergo rehydration by each transformer in reverse order.
119
+ let rehydratedMessage = message;
120
+ // Use a reverse for...of loop to accommodate asynchronous rehydrate methods
121
+ const reversedTransformers = this.transformers.slice().reverse();
122
+ for (const transformer of reversedTransformers) {
123
+ // Check if the result is a Promise and use await, otherwise use it directly
124
+ rehydratedMessage = await transformer.rehydrate(rehydratedMessage, rehydrationState);
125
+ }
126
+ // Handle onRehydratingEnd callback
127
+ if (this.config.onRehydratingEnd) {
128
+ await this.config.onRehydratingEnd(rehydratedMessage);
129
+ }
130
+ // Return the fully rehydrated message after all transformers have been applied.
131
+ return rehydratedMessage;
132
+ }
133
+ }
134
+ exports.MaskingParser = MaskingParser;
@@ -0,0 +1,44 @@
1
+ import { MaskingTransformer } from "./transformer.js";
2
+ import type { MaskingParserConfig } from "./types.js";
3
+ /**
4
+ * MaskingParser class for handling the masking and rehydrating of messages.
5
+ */
6
+ export declare class MaskingParser {
7
+ private transformers;
8
+ private state;
9
+ private config;
10
+ constructor(config?: MaskingParserConfig);
11
+ /**
12
+ * Adds a transformer to the parser.
13
+ * @param transformer - An instance of a class extending MaskingTransformer.
14
+ */
15
+ addTransformer(transformer: MaskingTransformer): void;
16
+ /**
17
+ * Getter method for retrieving the current state.
18
+ * @returns The current state map.
19
+ */
20
+ getState(): Map<string, string>;
21
+ /**
22
+ * Masks the provided message using the added transformers.
23
+ * This method sequentially applies each transformer's masking logic to the message.
24
+ * It utilizes a state map to track original values corresponding to their masked versions.
25
+ *
26
+ * @param message - The message to be masked.
27
+ * @returns A masked version of the message.
28
+ * @throws {TypeError} If the message is not a string.
29
+ * @throws {Error} If no transformers are added.
30
+ */
31
+ mask(message: string): Promise<string>;
32
+ /**
33
+ * Rehydrates a masked message back to its original form.
34
+ * This method sequentially applies the rehydration logic of each added transformer in reverse order.
35
+ * It relies on the state map to correctly map the masked values back to their original values.
36
+ *
37
+ * The rehydration process is essential for restoring the original content of a message
38
+ * that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
39
+ *
40
+ * @param message - The masked message to be rehydrated.
41
+ * @returns The original (rehydrated) version of the message.
42
+ */
43
+ rehydrate(message: string, state?: Map<string, string>): Promise<string>;
44
+ }
@@ -0,0 +1,130 @@
1
+ /**
2
+ * MaskingParser class for handling the masking and rehydrating of messages.
3
+ */
4
+ export class MaskingParser {
5
+ constructor(config = {}) {
6
+ Object.defineProperty(this, "transformers", {
7
+ enumerable: true,
8
+ configurable: true,
9
+ writable: true,
10
+ value: void 0
11
+ });
12
+ Object.defineProperty(this, "state", {
13
+ enumerable: true,
14
+ configurable: true,
15
+ writable: true,
16
+ value: void 0
17
+ });
18
+ Object.defineProperty(this, "config", {
19
+ enumerable: true,
20
+ configurable: true,
21
+ writable: true,
22
+ value: void 0
23
+ });
24
+ this.transformers = config.transformers ?? [];
25
+ this.state = new Map();
26
+ this.config = config;
27
+ }
28
+ /**
29
+ * Adds a transformer to the parser.
30
+ * @param transformer - An instance of a class extending MaskingTransformer.
31
+ */
32
+ addTransformer(transformer) {
33
+ this.transformers.push(transformer);
34
+ }
35
+ /**
36
+ * Getter method for retrieving the current state.
37
+ * @returns The current state map.
38
+ */
39
+ getState() {
40
+ return this.state;
41
+ }
42
+ /**
43
+ * Masks the provided message using the added transformers.
44
+ * This method sequentially applies each transformer's masking logic to the message.
45
+ * It utilizes a state map to track original values corresponding to their masked versions.
46
+ *
47
+ * @param message - The message to be masked.
48
+ * @returns A masked version of the message.
49
+ * @throws {TypeError} If the message is not a string.
50
+ * @throws {Error} If no transformers are added.
51
+ */
52
+ async mask(message) {
53
+ // If onMaskingStart is a function, handle it accordingly
54
+ if (this.config.onMaskingStart) {
55
+ await this.config.onMaskingStart(message);
56
+ }
57
+ // Check if there are any transformers added to the parser. If not, throw an error
58
+ // as masking requires at least one transformer to apply its logic.
59
+ if (this.transformers.length === 0) {
60
+ throw new Error("MaskingParser.mask Error: No transformers have been added. Please add at least one transformer before parsing.");
61
+ }
62
+ if (typeof message !== "string") {
63
+ throw new TypeError("MaskingParser.mask Error: The 'message' argument must be a string.");
64
+ }
65
+ // Initialize the variable to hold the progressively masked message.
66
+ // It starts as the original message and gets transformed by each transformer.
67
+ let processedMessage = message;
68
+ // Iterate through each transformer and apply their transform method.
69
+ for (const transformer of this.transformers) {
70
+ // Transform the message and get the transformer's state changes, ensuring no direct mutation of the shared state.
71
+ const [transformedMessage, transformerState] = await transformer.transform(processedMessage, new Map(this.state));
72
+ // Update the processed message for subsequent transformers.
73
+ processedMessage = transformedMessage;
74
+ // Merge state changes from the transformer into the parser's state.
75
+ // This accumulates all transformations' effects on the state.
76
+ transformerState.forEach((value, key) => this.state.set(key, value));
77
+ }
78
+ // Handle onMaskingEnd callback
79
+ if (this.config.onMaskingEnd) {
80
+ await this.config.onMaskingEnd(processedMessage);
81
+ }
82
+ // Return the fully masked message after all transformers have been applied.
83
+ return processedMessage;
84
+ }
85
+ /**
86
+ * Rehydrates a masked message back to its original form.
87
+ * This method sequentially applies the rehydration logic of each added transformer in reverse order.
88
+ * It relies on the state map to correctly map the masked values back to their original values.
89
+ *
90
+ * The rehydration process is essential for restoring the original content of a message
91
+ * that has been transformed (masked) by the transformers. This process is the inverse of the masking process.
92
+ *
93
+ * @param message - The masked message to be rehydrated.
94
+ * @returns The original (rehydrated) version of the message.
95
+ */
96
+ async rehydrate(message, state) {
97
+ // Handle onRehydratingStart callback
98
+ if (this.config.onRehydratingStart) {
99
+ await this.config.onRehydratingStart(message);
100
+ }
101
+ if (typeof message !== "string") {
102
+ throw new TypeError("MaskingParser.rehydrate Error: The 'message' argument must be a string.");
103
+ }
104
+ // Check if any transformers have been added to the parser.
105
+ // If no transformers are present, throw an error as rehydration requires at least one transformer.
106
+ if (this.transformers.length === 0) {
107
+ throw new Error("MaskingParser.rehydrate Error: No transformers have been added. Please add at least one transformer before rehydrating.");
108
+ }
109
+ // eslint-disable-next-line no-instanceof/no-instanceof
110
+ if (state && !(state instanceof Map)) {
111
+ throw new TypeError("MaskingParser.rehydrate Error: The 'state' argument, if provided, must be an instance of Map.");
112
+ }
113
+ const rehydrationState = state || this.state; // Use provided state or fallback to internal state
114
+ // Initialize the rehydratedMessage with the input masked message.
115
+ // This variable will undergo rehydration by each transformer in reverse order.
116
+ let rehydratedMessage = message;
117
+ // Use a reverse for...of loop to accommodate asynchronous rehydrate methods
118
+ const reversedTransformers = this.transformers.slice().reverse();
119
+ for (const transformer of reversedTransformers) {
120
+ // Check if the result is a Promise and use await, otherwise use it directly
121
+ rehydratedMessage = await transformer.rehydrate(rehydratedMessage, rehydrationState);
122
+ }
123
+ // Handle onRehydratingEnd callback
124
+ if (this.config.onRehydratingEnd) {
125
+ await this.config.onRehydratingEnd(rehydratedMessage);
126
+ }
127
+ // Return the fully rehydrated message after all transformers have been applied.
128
+ return rehydratedMessage;
129
+ }
130
+ }
@@ -0,0 +1,145 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RegexMaskingTransformer = void 0;
4
+ const transformer_js_1 = require("./transformer.cjs");
5
+ /**
6
+ * RegexMaskingTransformer class for masking and rehydrating messages with Regex.
7
+ */
8
+ class RegexMaskingTransformer extends transformer_js_1.MaskingTransformer {
9
+ /**
10
+ * Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
11
+ * Validates the provided patterns to ensure they conform to the expected structure.
12
+ *
13
+ * @param patterns - An object containing masking patterns. Each pattern should include
14
+ * a regular expression (`regex`) and optionally a `replacement` string
15
+ * or a `mask` function.
16
+ * @param hashFunction - An optional custom hash function to be used for masking.
17
+ */
18
+ constructor(patterns, hashFunction) {
19
+ super();
20
+ Object.defineProperty(this, "patterns", {
21
+ enumerable: true,
22
+ configurable: true,
23
+ writable: true,
24
+ value: void 0
25
+ });
26
+ Object.defineProperty(this, "hashFunction", {
27
+ enumerable: true,
28
+ configurable: true,
29
+ writable: true,
30
+ value: void 0
31
+ });
32
+ // Validates the provided masking patterns before initializing the transformer.
33
+ // This ensures that each pattern has a valid regular expression.
34
+ this.validatePatterns(patterns);
35
+ // Assigns the validated patterns and the hash function to the transformer.
36
+ // If no custom hash function is provided, the default hash function is used.
37
+ this.patterns = patterns;
38
+ this.hashFunction = hashFunction || this.defaultHashFunction;
39
+ }
40
+ /**
41
+ * Validates the given masking patterns to ensure each pattern has a valid regular expression.
42
+ * Throws an error if any pattern is found to be invalid.
43
+ *
44
+ * @param patterns - The patterns object to validate.
45
+ */
46
+ validatePatterns(patterns) {
47
+ for (const key of Object.keys(patterns)) {
48
+ const pattern = patterns[key];
49
+ // Checks that each pattern is an object and has a regex property that is an instance of RegExp.
50
+ // Throws an error if these conditions are not met, indicating an invalid pattern configuration.
51
+ if (!pattern ||
52
+ typeof pattern !== "object" ||
53
+ // eslint-disable-next-line no-instanceof/no-instanceof
54
+ !(pattern.regex instanceof RegExp)) {
55
+ throw new Error("Invalid pattern configuration.");
56
+ }
57
+ }
58
+ }
59
+ /**
60
+ * Masks content in a message based on the defined patterns.
61
+ * @param message - The message to be masked.
62
+ * @param state - The current state containing original values.
63
+ * @returns A tuple of the masked message and the updated state.
64
+ */
65
+ async transform(message, state) {
66
+ if (typeof message !== "string") {
67
+ throw new TypeError("RegexMaskingTransformer.transform Error: The 'message' argument must be a string.");
68
+ }
69
+ // eslint-disable-next-line no-instanceof/no-instanceof
70
+ if (!(state instanceof Map)) {
71
+ throw new TypeError("RegexMaskingTransformer.transform Error: The 'state' argument must be an instance of Map.");
72
+ }
73
+ // Holds the progressively masked message
74
+ let processedMessage = message;
75
+ // Initialize original values map with the current state or a new map
76
+ const originalValues = state || new Map();
77
+ // Iterate over each pattern defined in the transformer
78
+ for (const key of Object.keys(this.patterns)) {
79
+ const pattern = this.patterns[key];
80
+ // Apply the current pattern's regex to the message
81
+ processedMessage = processedMessage.replace(pattern.regex, (match) => {
82
+ // Determine the masked value: use the mask function if provided, else use the replacement string,
83
+ // else use the hash function.
84
+ const maskedValue = pattern.mask
85
+ ? pattern.mask(match)
86
+ : pattern.replacement ?? this.hashFunction(match);
87
+ // Store the mapping of the masked value to the original value (match)
88
+ originalValues.set(maskedValue, match);
89
+ // Return the masked value to replace the original value in the message
90
+ return maskedValue;
91
+ });
92
+ }
93
+ // Return the fully masked message and the state map with all original values
94
+ // Wrap the synchronous return values in Promise.resolve() to maintain compatibility
95
+ // with the MaskingParser's expectation of a Promise return type.
96
+ return [processedMessage, originalValues];
97
+ }
98
+ /**
99
+ * Rehydrates a masked message back to its original form using the provided state.
100
+ * @param message - The masked message to be rehydrated.
101
+ * @param state - The state map containing mappings of masked values to their original values.
102
+ * @returns The rehydrated (original) message.
103
+ */
104
+ async rehydrate(message, state) {
105
+ if (typeof message !== "string") {
106
+ throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'message' argument must be a string.");
107
+ }
108
+ // eslint-disable-next-line no-instanceof/no-instanceof
109
+ if (!(state instanceof Map)) {
110
+ throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'state' argument must be an instance of Map.");
111
+ }
112
+ // Convert the state map to an array and use reduce to sequentially replace masked values with original values.
113
+ const rehydratedMessage = Array.from(state).reduce((msg, [masked, original]) => {
114
+ // Escape special characters in the masked string to ensure it can be used in a regular expression safely.
115
+ // This is necessary because masked values might contain characters that have special meanings in regex.
116
+ const escapedMasked = masked.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
117
+ // Replace all instances of the escaped masked value in the message with the original value.
118
+ // The 'g' flag in the RegExp ensures that all occurrences of the masked value are replaced.
119
+ return msg.replace(new RegExp(escapedMasked, "g"), original);
120
+ }, message);
121
+ return rehydratedMessage;
122
+ }
123
+ /**
124
+ * Default hash function for creating unique hash values.
125
+ * @param input - The input string to hash.
126
+ * @returns The resulting hash as a string.
127
+ */
128
+ defaultHashFunction(input) {
129
+ let hash = 0;
130
+ // Iterate over each character in the input string
131
+ for (let i = 0; i < input.length; i += 1) {
132
+ // Get ASCII value of the character
133
+ const char = input.charCodeAt(i);
134
+ // Combine the current hash with the new character and ensure it remains a 32-bit integer
135
+ hash = (hash << 5) - hash + char;
136
+ // Bitwise OR operation to convert to a 32-bit integer.
137
+ // This is a common technique to ensure the final hash value stays within the 32-bit limit,
138
+ // effectively wrapping the value when it becomes too large.
139
+ hash |= 0;
140
+ }
141
+ // Convert the numerical hash value to a string and return
142
+ return hash.toString();
143
+ }
144
+ }
145
+ exports.RegexMaskingTransformer = RegexMaskingTransformer;
@@ -0,0 +1,48 @@
1
+ import { MaskingTransformer } from "./transformer.js";
2
+ import type { HashFunction, MaskingPattern } from "./types.js";
3
+ /**
4
+ * RegexMaskingTransformer class for masking and rehydrating messages with Regex.
5
+ */
6
+ export declare class RegexMaskingTransformer extends MaskingTransformer {
7
+ private patterns;
8
+ private hashFunction;
9
+ /**
10
+ * Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
11
+ * Validates the provided patterns to ensure they conform to the expected structure.
12
+ *
13
+ * @param patterns - An object containing masking patterns. Each pattern should include
14
+ * a regular expression (`regex`) and optionally a `replacement` string
15
+ * or a `mask` function.
16
+ * @param hashFunction - An optional custom hash function to be used for masking.
17
+ */
18
+ constructor(patterns: {
19
+ [key: string]: MaskingPattern;
20
+ }, hashFunction?: HashFunction);
21
+ /**
22
+ * Validates the given masking patterns to ensure each pattern has a valid regular expression.
23
+ * Throws an error if any pattern is found to be invalid.
24
+ *
25
+ * @param patterns - The patterns object to validate.
26
+ */
27
+ private validatePatterns;
28
+ /**
29
+ * Masks content in a message based on the defined patterns.
30
+ * @param message - The message to be masked.
31
+ * @param state - The current state containing original values.
32
+ * @returns A tuple of the masked message and the updated state.
33
+ */
34
+ transform(message: string, state: Map<string, string>): Promise<[string, Map<string, string>]>;
35
+ /**
36
+ * Rehydrates a masked message back to its original form using the provided state.
37
+ * @param message - The masked message to be rehydrated.
38
+ * @param state - The state map containing mappings of masked values to their original values.
39
+ * @returns The rehydrated (original) message.
40
+ */
41
+ rehydrate(message: string, state: Map<string, string>): Promise<string>;
42
+ /**
43
+ * Default hash function for creating unique hash values.
44
+ * @param input - The input string to hash.
45
+ * @returns The resulting hash as a string.
46
+ */
47
+ private defaultHashFunction;
48
+ }
@@ -0,0 +1,141 @@
1
+ import { MaskingTransformer } from "./transformer.js";
2
+ /**
3
+ * RegexMaskingTransformer class for masking and rehydrating messages with Regex.
4
+ */
5
+ export class RegexMaskingTransformer extends MaskingTransformer {
6
+ /**
7
+ * Constructs a RegexMaskingTransformer with given patterns and an optional hash function.
8
+ * Validates the provided patterns to ensure they conform to the expected structure.
9
+ *
10
+ * @param patterns - An object containing masking patterns. Each pattern should include
11
+ * a regular expression (`regex`) and optionally a `replacement` string
12
+ * or a `mask` function.
13
+ * @param hashFunction - An optional custom hash function to be used for masking.
14
+ */
15
+ constructor(patterns, hashFunction) {
16
+ super();
17
+ Object.defineProperty(this, "patterns", {
18
+ enumerable: true,
19
+ configurable: true,
20
+ writable: true,
21
+ value: void 0
22
+ });
23
+ Object.defineProperty(this, "hashFunction", {
24
+ enumerable: true,
25
+ configurable: true,
26
+ writable: true,
27
+ value: void 0
28
+ });
29
+ // Validates the provided masking patterns before initializing the transformer.
30
+ // This ensures that each pattern has a valid regular expression.
31
+ this.validatePatterns(patterns);
32
+ // Assigns the validated patterns and the hash function to the transformer.
33
+ // If no custom hash function is provided, the default hash function is used.
34
+ this.patterns = patterns;
35
+ this.hashFunction = hashFunction || this.defaultHashFunction;
36
+ }
37
+ /**
38
+ * Validates the given masking patterns to ensure each pattern has a valid regular expression.
39
+ * Throws an error if any pattern is found to be invalid.
40
+ *
41
+ * @param patterns - The patterns object to validate.
42
+ */
43
+ validatePatterns(patterns) {
44
+ for (const key of Object.keys(patterns)) {
45
+ const pattern = patterns[key];
46
+ // Checks that each pattern is an object and has a regex property that is an instance of RegExp.
47
+ // Throws an error if these conditions are not met, indicating an invalid pattern configuration.
48
+ if (!pattern ||
49
+ typeof pattern !== "object" ||
50
+ // eslint-disable-next-line no-instanceof/no-instanceof
51
+ !(pattern.regex instanceof RegExp)) {
52
+ throw new Error("Invalid pattern configuration.");
53
+ }
54
+ }
55
+ }
56
+ /**
57
+ * Masks content in a message based on the defined patterns.
58
+ * @param message - The message to be masked.
59
+ * @param state - The current state containing original values.
60
+ * @returns A tuple of the masked message and the updated state.
61
+ */
62
+ async transform(message, state) {
63
+ if (typeof message !== "string") {
64
+ throw new TypeError("RegexMaskingTransformer.transform Error: The 'message' argument must be a string.");
65
+ }
66
+ // eslint-disable-next-line no-instanceof/no-instanceof
67
+ if (!(state instanceof Map)) {
68
+ throw new TypeError("RegexMaskingTransformer.transform Error: The 'state' argument must be an instance of Map.");
69
+ }
70
+ // Holds the progressively masked message
71
+ let processedMessage = message;
72
+ // Initialize original values map with the current state or a new map
73
+ const originalValues = state || new Map();
74
+ // Iterate over each pattern defined in the transformer
75
+ for (const key of Object.keys(this.patterns)) {
76
+ const pattern = this.patterns[key];
77
+ // Apply the current pattern's regex to the message
78
+ processedMessage = processedMessage.replace(pattern.regex, (match) => {
79
+ // Determine the masked value: use the mask function if provided, else use the replacement string,
80
+ // else use the hash function.
81
+ const maskedValue = pattern.mask
82
+ ? pattern.mask(match)
83
+ : pattern.replacement ?? this.hashFunction(match);
84
+ // Store the mapping of the masked value to the original value (match)
85
+ originalValues.set(maskedValue, match);
86
+ // Return the masked value to replace the original value in the message
87
+ return maskedValue;
88
+ });
89
+ }
90
+ // Return the fully masked message and the state map with all original values
91
+ // Wrap the synchronous return values in Promise.resolve() to maintain compatibility
92
+ // with the MaskingParser's expectation of a Promise return type.
93
+ return [processedMessage, originalValues];
94
+ }
95
+ /**
96
+ * Rehydrates a masked message back to its original form using the provided state.
97
+ * @param message - The masked message to be rehydrated.
98
+ * @param state - The state map containing mappings of masked values to their original values.
99
+ * @returns The rehydrated (original) message.
100
+ */
101
+ async rehydrate(message, state) {
102
+ if (typeof message !== "string") {
103
+ throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'message' argument must be a string.");
104
+ }
105
+ // eslint-disable-next-line no-instanceof/no-instanceof
106
+ if (!(state instanceof Map)) {
107
+ throw new TypeError("RegexMaskingTransformer.rehydrate Error: The 'state' argument must be an instance of Map.");
108
+ }
109
+ // Convert the state map to an array and use reduce to sequentially replace masked values with original values.
110
+ const rehydratedMessage = Array.from(state).reduce((msg, [masked, original]) => {
111
+ // Escape special characters in the masked string to ensure it can be used in a regular expression safely.
112
+ // This is necessary because masked values might contain characters that have special meanings in regex.
113
+ const escapedMasked = masked.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
114
+ // Replace all instances of the escaped masked value in the message with the original value.
115
+ // The 'g' flag in the RegExp ensures that all occurrences of the masked value are replaced.
116
+ return msg.replace(new RegExp(escapedMasked, "g"), original);
117
+ }, message);
118
+ return rehydratedMessage;
119
+ }
120
+ /**
121
+ * Default hash function for creating unique hash values.
122
+ * @param input - The input string to hash.
123
+ * @returns The resulting hash as a string.
124
+ */
125
+ defaultHashFunction(input) {
126
+ let hash = 0;
127
+ // Iterate over each character in the input string
128
+ for (let i = 0; i < input.length; i += 1) {
129
+ // Get ASCII value of the character
130
+ const char = input.charCodeAt(i);
131
+ // Combine the current hash with the new character and ensure it remains a 32-bit integer
132
+ hash = (hash << 5) - hash + char;
133
+ // Bitwise OR operation to convert to a 32-bit integer.
134
+ // This is a common technique to ensure the final hash value stays within the 32-bit limit,
135
+ // effectively wrapping the value when it becomes too large.
136
+ hash |= 0;
137
+ }
138
+ // Convert the numerical hash value to a string and return
139
+ return hash.toString();
140
+ }
141
+ }
@@ -0,0 +1,9 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MaskingTransformer = void 0;
4
+ /**
5
+ * Abstract class representing a transformer used for masking and rehydrating messages.
6
+ */
7
+ class MaskingTransformer {
8
+ }
9
+ exports.MaskingTransformer = MaskingTransformer;
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Abstract class representing a transformer used for masking and rehydrating messages.
3
+ */
4
+ export declare abstract class MaskingTransformer {
5
+ abstract transform(message: string, state?: Map<string, string>): Promise<[string, Map<string, string>]>;
6
+ abstract rehydrate(message: string, state: Map<string, string>): Promise<string>;
7
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Abstract class representing a transformer used for masking and rehydrating messages.
3
+ */
4
+ export class MaskingTransformer {
5
+ }
@@ -0,0 +1,2 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
@@ -0,0 +1,25 @@
1
+ import { MaskingTransformer } from "./transformer.js";
2
+ /**
3
+ * Configuration type for MaskingParser.
4
+ */
5
+ export type MaskingParserConfig = {
6
+ transformers?: MaskingTransformer[];
7
+ defaultHashFunction?: HashFunction;
8
+ onMaskingStart?: HookFunction;
9
+ onMaskingEnd?: HookFunction;
10
+ onRehydratingStart?: HookFunction;
11
+ onRehydratingEnd?: HookFunction;
12
+ };
13
+ /**
14
+ * Regex Masking Pattern used for masking in PIIMaskingTransformer.
15
+ */
16
+ export type MaskingPattern = {
17
+ regex: RegExp;
18
+ replacement?: string;
19
+ mask?: (match: string) => string;
20
+ };
21
+ export type HookFunction = ((message: string) => Promise<void>) | ((message: string) => void);
22
+ /**
23
+ * Represents a function that can hash a string input.
24
+ */
25
+ export type HashFunction = (input: string) => string;
@@ -0,0 +1 @@
1
+ export {};
@@ -26,7 +26,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
26
26
  Object.defineProperty(exports, "__esModule", { value: true });
27
27
  exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fake = exports.llms__yandex = exports.llms__fireworks = exports.llms__ollama = exports.llms__cloudflare_workersai = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__voyage = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains__combine_documents__reduce = exports.chains = exports.tools__google_places = exports.tools__render = exports.tools__connery = exports.tools = exports.base_language = exports.agents__openai__output_parser = exports.agents__xml__output_parser = exports.agents__react__output_parser = exports.agents__format_scratchpad__log_to_message = exports.agents__format_scratchpad__xml = exports.agents__format_scratchpad__log = exports.agents__format_scratchpad__openai_tools = exports.agents__format_scratchpad = exports.agents__toolkits__connery = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
28
28
  exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__openai_files = exports.experimental__openai_assistant = exports.experimental__autogpt = exports.util__time = exports.util__math = exports.util__document = exports.storage__in_memory = exports.storage__encoder_backed = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__embeddings_filter = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__tavily_search_api = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = exports.retrievers__chaindesk = exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__prompt_template = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__fake = exports.chat_models__yandex = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__cloudflare_workersai = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = void 0;
29
- exports.runnables__remote = exports.runnables = exports.evaluation = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__ollama_functions = exports.experimental__chat_models__bittensor = void 0;
29
+ exports.runnables__remote = exports.runnables = exports.evaluation = exports.experimental__masking = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__ollama_functions = exports.experimental__chat_models__bittensor = void 0;
30
30
  exports.load__serializable = __importStar(require("../load/serializable.cjs"));
31
31
  exports.agents = __importStar(require("../agents/index.cjs"));
32
32
  exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
@@ -130,6 +130,7 @@ exports.experimental__plan_and_execute = __importStar(require("../experimental/p
130
130
  exports.experimental__chat_models__bittensor = __importStar(require("../experimental/chat_models/bittensor.cjs"));
131
131
  exports.experimental__chat_models__ollama_functions = __importStar(require("../experimental/chat_models/ollama_functions.cjs"));
132
132
  exports.experimental__chains__violation_of_expectations = __importStar(require("../experimental/chains/violation_of_expectations/index.cjs"));
133
+ exports.experimental__masking = __importStar(require("../experimental/masking/index.cjs"));
133
134
  exports.evaluation = __importStar(require("../evaluation/index.cjs"));
134
135
  exports.runnables = __importStar(require("../runnables/index.cjs"));
135
136
  exports.runnables__remote = __importStar(require("../runnables/remote.cjs"));
@@ -101,6 +101,7 @@ export * as experimental__plan_and_execute from "../experimental/plan_and_execut
101
101
  export * as experimental__chat_models__bittensor from "../experimental/chat_models/bittensor.js";
102
102
  export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";
103
103
  export * as experimental__chains__violation_of_expectations from "../experimental/chains/violation_of_expectations/index.js";
104
+ export * as experimental__masking from "../experimental/masking/index.js";
104
105
  export * as evaluation from "../evaluation/index.js";
105
106
  export * as runnables from "../runnables/index.js";
106
107
  export * as runnables__remote from "../runnables/remote.js";
@@ -102,6 +102,7 @@ export * as experimental__plan_and_execute from "../experimental/plan_and_execut
102
102
  export * as experimental__chat_models__bittensor from "../experimental/chat_models/bittensor.js";
103
103
  export * as experimental__chat_models__ollama_functions from "../experimental/chat_models/ollama_functions.js";
104
104
  export * as experimental__chains__violation_of_expectations from "../experimental/chains/violation_of_expectations/index.js";
105
+ export * as experimental__masking from "../experimental/masking/index.js";
105
106
  export * as evaluation from "../evaluation/index.js";
106
107
  export * as runnables from "../runnables/index.js";
107
108
  export * as runnables__remote from "../runnables/remote.js";
@@ -1,87 +1,17 @@
1
1
  "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.IterableReadableStream = void 0;
4
- /*
5
- * Support async iterator syntax for ReadableStreams in all environments.
6
- * Source: https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490
7
- */
8
- class IterableReadableStream extends ReadableStream {
9
- constructor() {
10
- super(...arguments);
11
- Object.defineProperty(this, "reader", {
12
- enumerable: true,
13
- configurable: true,
14
- writable: true,
15
- value: void 0
16
- });
17
- }
18
- ensureReader() {
19
- if (!this.reader) {
20
- this.reader = this.getReader();
21
- }
22
- }
23
- async next() {
24
- this.ensureReader();
25
- try {
26
- const result = await this.reader.read();
27
- if (result.done)
28
- this.reader.releaseLock(); // release lock when stream becomes closed
29
- return {
30
- done: result.done,
31
- value: result.value, // Cloudflare Workers typing fix
32
- };
33
- }
34
- catch (e) {
35
- this.reader.releaseLock(); // release lock when stream becomes errored
36
- throw e;
37
- }
38
- }
39
- async return() {
40
- this.ensureReader();
41
- const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
42
- this.reader.releaseLock(); // release lock first
43
- await cancelPromise; // now await it
44
- return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore final chunk value anyway
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
45
7
  }
46
- [Symbol.asyncIterator]() {
47
- return this;
48
- }
49
- static fromReadableStream(stream) {
50
- // From https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream
51
- const reader = stream.getReader();
52
- return new IterableReadableStream({
53
- start(controller) {
54
- return pump();
55
- function pump() {
56
- return reader.read().then(({ done, value }) => {
57
- // When no more data needs to be consumed, close the stream
58
- if (done) {
59
- controller.close();
60
- return;
61
- }
62
- // Enqueue the next data chunk into our target stream
63
- controller.enqueue(value);
64
- return pump();
65
- });
66
- }
67
- },
68
- cancel() {
69
- reader.releaseLock();
70
- },
71
- });
72
- }
73
- static fromAsyncGenerator(generator) {
74
- return new IterableReadableStream({
75
- async pull(controller) {
76
- const { value, done } = await generator.next();
77
- // When no more data needs to be consumed, close the stream
78
- if (done) {
79
- controller.close();
80
- }
81
- // Fix: `else if (value)` will hang the streaming when nullish value (e.g. empty string) is pulled
82
- controller.enqueue(value);
83
- },
84
- });
85
- }
86
- }
87
- exports.IterableReadableStream = IterableReadableStream;
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("@langchain/core/utils/stream"), exports);
@@ -1,15 +1 @@
1
- export declare class IterableReadableStream<T> extends ReadableStream<T> {
2
- reader: ReadableStreamDefaultReader<T>;
3
- ensureReader(): void;
4
- next(): Promise<{
5
- done: boolean;
6
- value: T;
7
- }>;
8
- return(): Promise<{
9
- done: boolean;
10
- value: T;
11
- }>;
12
- [Symbol.asyncIterator](): this;
13
- static fromReadableStream<T>(stream: ReadableStream<T>): IterableReadableStream<T>;
14
- static fromAsyncGenerator<T>(generator: AsyncGenerator<T>): IterableReadableStream<T>;
15
- }
1
+ export * from "@langchain/core/utils/stream";
@@ -1,83 +1 @@
1
- /*
2
- * Support async iterator syntax for ReadableStreams in all environments.
3
- * Source: https://github.com/MattiasBuelens/web-streams-polyfill/pull/122#issuecomment-1627354490
4
- */
5
- export class IterableReadableStream extends ReadableStream {
6
- constructor() {
7
- super(...arguments);
8
- Object.defineProperty(this, "reader", {
9
- enumerable: true,
10
- configurable: true,
11
- writable: true,
12
- value: void 0
13
- });
14
- }
15
- ensureReader() {
16
- if (!this.reader) {
17
- this.reader = this.getReader();
18
- }
19
- }
20
- async next() {
21
- this.ensureReader();
22
- try {
23
- const result = await this.reader.read();
24
- if (result.done)
25
- this.reader.releaseLock(); // release lock when stream becomes closed
26
- return {
27
- done: result.done,
28
- value: result.value, // Cloudflare Workers typing fix
29
- };
30
- }
31
- catch (e) {
32
- this.reader.releaseLock(); // release lock when stream becomes errored
33
- throw e;
34
- }
35
- }
36
- async return() {
37
- this.ensureReader();
38
- const cancelPromise = this.reader.cancel(); // cancel first, but don't await yet
39
- this.reader.releaseLock(); // release lock first
40
- await cancelPromise; // now await it
41
- return { done: true, value: undefined }; // This cast fixes TS typing, and convention is to ignore final chunk value anyway
42
- }
43
- [Symbol.asyncIterator]() {
44
- return this;
45
- }
46
- static fromReadableStream(stream) {
47
- // From https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#reading_the_stream
48
- const reader = stream.getReader();
49
- return new IterableReadableStream({
50
- start(controller) {
51
- return pump();
52
- function pump() {
53
- return reader.read().then(({ done, value }) => {
54
- // When no more data needs to be consumed, close the stream
55
- if (done) {
56
- controller.close();
57
- return;
58
- }
59
- // Enqueue the next data chunk into our target stream
60
- controller.enqueue(value);
61
- return pump();
62
- });
63
- }
64
- },
65
- cancel() {
66
- reader.releaseLock();
67
- },
68
- });
69
- }
70
- static fromAsyncGenerator(generator) {
71
- return new IterableReadableStream({
72
- async pull(controller) {
73
- const { value, done } = await generator.next();
74
- // When no more data needs to be consumed, close the stream
75
- if (done) {
76
- controller.close();
77
- }
78
- // Fix: `else if (value)` will hang the streaming when nullish value (e.g. empty string) is pulled
79
- controller.enqueue(value);
80
- },
81
- });
82
- }
83
- }
1
+ export * from "@langchain/core/utils/stream";
@@ -292,6 +292,7 @@ class PrismaVectorStore extends base_js_1.VectorStore {
292
292
  return instance;
293
293
  }
294
294
  }
295
+ exports.PrismaVectorStore = PrismaVectorStore;
295
296
  Object.defineProperty(PrismaVectorStore, "IdColumn", {
296
297
  enumerable: true,
297
298
  configurable: true,
@@ -304,4 +305,3 @@ Object.defineProperty(PrismaVectorStore, "ContentColumn", {
304
305
  writable: true,
305
306
  value: ContentColumnSymbol
306
307
  });
307
- exports.PrismaVectorStore = PrismaVectorStore;
@@ -19,7 +19,7 @@ const OpMap = {
19
19
  * work with Prisma. It provides methods for adding models, documents, and
20
20
  * vectors, as well as for performing similarity searches.
21
21
  */
22
- class PrismaVectorStore extends VectorStore {
22
+ export class PrismaVectorStore extends VectorStore {
23
23
  _vectorstoreType() {
24
24
  return "prisma";
25
25
  }
@@ -301,4 +301,3 @@ Object.defineProperty(PrismaVectorStore, "ContentColumn", {
301
301
  writable: true,
302
302
  value: ContentColumnSymbol
303
303
  });
304
- export { PrismaVectorStore };
@@ -89,7 +89,7 @@ class XataVectorSearch extends base_js_1.VectorStore {
89
89
  * @returns Promise resolving to an array of tuples, each containing a Document and its score.
90
90
  */
91
91
  async similaritySearchVectorWithScore(query, k, filter) {
92
- const records = await this.client.db[this.table].vectorSearch("embedding", query, {
92
+ const { records } = await this.client.db[this.table].vectorSearch("embedding", query, {
93
93
  size: k,
94
94
  filter,
95
95
  });
@@ -86,7 +86,7 @@ export class XataVectorSearch extends VectorStore {
86
86
  * @returns Promise resolving to an array of tuples, each containing a Document and its score.
87
87
  */
88
88
  async similaritySearchVectorWithScore(query, k, filter) {
89
- const records = await this.client.db[this.table].vectorSearch("embedding", query, {
89
+ const { records } = await this.client.db[this.table].vectorSearch("embedding", query, {
90
90
  size: k,
91
91
  filter,
92
92
  });
@@ -0,0 +1 @@
1
+ module.exports = require('../dist/experimental/masking/index.cjs');
@@ -0,0 +1 @@
1
+ export * from '../dist/experimental/masking/index.js'
@@ -0,0 +1 @@
1
+ export * from '../dist/experimental/masking/index.js'
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "langchain",
3
- "version": "0.0.201",
3
+ "version": "0.0.203",
4
4
  "description": "Typescript bindings for langchain",
5
5
  "type": "module",
6
6
  "engines": {
@@ -835,6 +835,9 @@
835
835
  "experimental/chains/violation_of_expectations.cjs",
836
836
  "experimental/chains/violation_of_expectations.js",
837
837
  "experimental/chains/violation_of_expectations.d.ts",
838
+ "experimental/masking.cjs",
839
+ "experimental/masking.js",
840
+ "experimental/masking.d.ts",
838
841
  "experimental/tools/pyinterpreter.cjs",
839
842
  "experimental/tools/pyinterpreter.js",
840
843
  "experimental/tools/pyinterpreter.d.ts",
@@ -944,7 +947,7 @@
944
947
  "@vercel/kv": "^0.2.3",
945
948
  "@vercel/postgres": "^0.5.0",
946
949
  "@writerai/writer-sdk": "^0.40.2",
947
- "@xata.io/client": "^0.25.1",
950
+ "@xata.io/client": "^0.28.0",
948
951
  "@xenova/transformers": "^2.5.4",
949
952
  "@zilliz/milvus2-sdk-node": ">=2.2.11",
950
953
  "apify-client": "^2.7.1",
@@ -1012,7 +1015,7 @@
1012
1015
  "srt-parser-2": "^1.2.2",
1013
1016
  "ts-jest": "^29.1.0",
1014
1017
  "typeorm": "^0.3.12",
1015
- "typescript": "^5.0.0",
1018
+ "typescript": "~5.1.6",
1016
1019
  "typesense": "^1.5.3",
1017
1020
  "usearch": "^1.1.1",
1018
1021
  "vectordb": "^0.1.4",
@@ -1066,7 +1069,7 @@
1066
1069
  "@vercel/kv": "^0.2.3",
1067
1070
  "@vercel/postgres": "^0.5.0",
1068
1071
  "@writerai/writer-sdk": "^0.40.2",
1069
- "@xata.io/client": "^0.25.1",
1072
+ "@xata.io/client": "^0.28.0",
1070
1073
  "@xenova/transformers": "^2.5.4",
1071
1074
  "@zilliz/milvus2-sdk-node": ">=2.2.7",
1072
1075
  "apify-client": "^2.7.1",
@@ -1432,7 +1435,7 @@
1432
1435
  },
1433
1436
  "dependencies": {
1434
1437
  "@anthropic-ai/sdk": "^0.9.1",
1435
- "@langchain/core": "~0.0.8",
1438
+ "@langchain/core": "~0.0.10",
1436
1439
  "binary-extensions": "^2.2.0",
1437
1440
  "expr-eval": "^2.0.2",
1438
1441
  "flat": "^5.0.2",
@@ -2843,6 +2846,11 @@
2843
2846
  "import": "./experimental/chains/violation_of_expectations.js",
2844
2847
  "require": "./experimental/chains/violation_of_expectations.cjs"
2845
2848
  },
2849
+ "./experimental/masking": {
2850
+ "types": "./experimental/masking.d.ts",
2851
+ "import": "./experimental/masking.js",
2852
+ "require": "./experimental/masking.cjs"
2853
+ },
2846
2854
  "./experimental/tools/pyinterpreter": {
2847
2855
  "types": "./experimental/tools/pyinterpreter.d.ts",
2848
2856
  "import": "./experimental/tools/pyinterpreter.js",