mask-privacy 1.0.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +61 -15
- package/dist/index.d.mts +106 -62
- package/dist/index.d.ts +106 -62
- package/dist/index.js +4801 -5073
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +4787 -5053
- package/dist/index.mjs.map +1 -1
- package/examples/test_local_nlp.ts +21 -0
- package/jest.config.js +1 -0
- package/package.json +27 -12
- package/src/client.ts +70 -39
- package/src/core/crypto.ts +156 -38
- package/src/core/exceptions.ts +3 -0
- package/src/core/fpe.ts +35 -97
- package/src/core/fpe_utils.ts +60 -0
- package/src/core/key_provider.ts +128 -45
- package/src/core/nlp_worker.js +45 -0
- package/src/core/remote_scanner.ts +86 -0
- package/src/core/scanner.ts +26 -78
- package/src/core/search.ts +52 -0
- package/src/core/timeout.ts +32 -0
- package/src/core/transformers_scanner.ts +215 -0
- package/src/core/utils.ts +85 -44
- package/src/core/vault.ts +272 -138
- package/src/index.ts +8 -2
- package/src/integrations/adk_hooks.ts +12 -7
- package/src/integrations/langchain_hooks.ts +20 -11
- package/src/integrations/llamaindex_hooks.ts +14 -5
- package/src/telemetry/audit_logger.ts +131 -75
- package/tests/async.test.ts +2 -1
- package/tests/audit_logger.test.ts +7 -11
- package/tests/exceptions.test.ts +8 -1
- package/tests/fail_strategy.test.ts +30 -25
- package/tests/fpe.test.ts +34 -27
- package/tests/hooks.test.ts +1 -1
- package/tests/jest.setup.ts +30 -0
- package/tests/key_provider.test.ts +36 -18
- package/tests/langchain.test.ts +15 -4
- package/tests/llamaindex.test.ts +11 -1
- package/tests/ner_masking.test.ts +59 -0
- package/tests/remote_scanner.test.ts +73 -0
- package/tests/search.test.ts +59 -0
- package/tests/security_hardening.test.ts +52 -0
- package/tests/test_cross.ts +14 -0
- package/tests/vault.test.ts +11 -11
- package/tests/vault_backends.test.ts +23 -15
- package/.test_audit.db +0 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright 2026 Mask AI Solutions
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -27,9 +27,9 @@ Instead of trusting the LLM to safeguard plain-text data, the system strictly en
|
|
|
27
27
|
|
|
28
28
|
This guarantees that the LLM can orchestrate workflows involving sensitive data without ever actually exposing the raw data to the model or its remote provider logs.
|
|
29
29
|
|
|
30
|
-
Additionally,
|
|
31
|
-
1. **
|
|
32
|
-
2. **
|
|
30
|
+
Additionally, the SDK addresses two technical considerations for production use:
|
|
31
|
+
1. **Distributed State Management**: Traditional "vaults" may lose state in multi-node environments. Pluggable backends (Redis, DynamoDB, Memcached) ensure detokenization state is shared across all pods.
|
|
32
|
+
2. **Schema Compatibility**: Downstream tools frequently require specific formats. Format-Preserving Tokenization (Emails, US Phones, SSNs, etc.) generates tokens that retain the format of original data.
|
|
33
33
|
|
|
34
34
|
### How We Handle Data (Local-First by Default)
|
|
35
35
|
|
|
@@ -49,7 +49,7 @@ The Data Plane is the open-source, transparent, auditable runtime execution laye
|
|
|
49
49
|
* **JIT Cryptography Engine:** The core pre-tool decryption and post-tool encryption hooks that intercept and mutate data in-flight.
|
|
50
50
|
* **Format-Preserving Tokenization Router:** Ensures downstream databases and strict schemas don't break when handed a token. Tokens look like real data; the real values are stored encrypted and retrieved via the vault.
|
|
51
51
|
* **Pluggable Distributed Vaults:** Support for enterprise-native caching layers (Redis, DynamoDB, Memcached) to ensure horizontally-scaled edge agents have synchronized access to detokenization mapping.
|
|
52
|
-
* **Local Audit Logger:** An asynchronous AuditLogger that buffers privacy events
|
|
52
|
+
* **Local Audit Logger:** An asynchronous AuditLogger that buffers privacy events in memory and emits structured JSON logs to stdout for SIEM ingestion.
|
|
53
53
|
|
|
54
54
|
---
|
|
55
55
|
|
|
@@ -79,21 +79,50 @@ const client = new MaskClient({
|
|
|
79
79
|
const safeToken = await client.encode("user@tenant.com");
|
|
80
80
|
```
|
|
81
81
|
|
|
82
|
-
### 3.
|
|
83
|
-
|
|
84
|
-
* SSN tokens always begin with `000` (The Social Security Administration
|
|
82
|
+
### 3. Collision Avoidance
|
|
83
|
+
Mask prevents the misidentification of real data as tokens by using universally invalid prefixes for token generation:
|
|
84
|
+
* SSN tokens always begin with `000` (The Social Security Administration does not issue Area Numbers of 000).
|
|
85
85
|
* Routing tokens always begin with `0000` (The Federal Reserve valid range starts at 01).
|
|
86
86
|
* Credit Card tokens use the `4000-0000-0000` Visa reserved test BIN.
|
|
87
|
-
|
|
87
|
+
|
|
88
|
+
This prefix-based approach ensures that the SDK does not inadvertently process valid PII as an existing token.
|
|
88
89
|
|
|
89
90
|
### 4. Enterprise Async Support
|
|
90
91
|
Mask is built from the ground up for high-concurrency Node.js environments. All core operations are asynchronous and promised-based. Calling `encode()`, `decode()`, or `scanAndTokenize()` allows your event loop to remain unblocked while handling PII tokenization tasks.
|
|
91
92
|
|
|
92
|
-
### 5. Pluggable Key
|
|
93
|
-
For zero-trust environments, `MASK_ENCRYPTION_KEY`
|
|
93
|
+
### 5. Pluggable Key Management (Enterprise KMS)
|
|
94
|
+
For zero-trust environments, `MASK_ENCRYPTION_KEY` can be managed outside of static environment variables. Mask supports a pluggable `BaseKeyProvider` architecture that allows you to fetch secrets dynamically from dedicated Key Management Services.
|
|
95
|
+
|
|
96
|
+
#### Supported Providers
|
|
97
|
+
* **EnvKeyProvider (Default)**: Reads from `MASK_ENCRYPTION_KEY` and `MASK_MASTER_KEY`.
|
|
98
|
+
* **AwsKmsKeyProvider (Stub)**: Placeholder for AWS KMS. Requires implementation of `@aws-sdk/client-kms`.
|
|
99
|
+
* **AzureKeyVaultProvider (Stub)**: Placeholder for Azure Key Vault. Requires implementation of `@azure/keyvault-keys`.
|
|
100
|
+
* **HashiCorpVaultProvider (Stub)**: Placeholder for HashiCorp Vault.
|
|
101
|
+
|
|
102
|
+
> [!NOTE]
|
|
103
|
+
> All KMS stub providers are designed for **Fail-Shut** operation. If you attempt to use a stub provider that is not yet implemented, the SDK will throw an `Error` rather than fall back to insecure defaults.
|
|
104
|
+
|
|
105
|
+
#### Example: Implementing a Custom Provider
|
|
106
|
+
If you need to support a specific KMS today, you can easily implement the `BaseKeyProvider` interface:
|
|
94
107
|
|
|
95
|
-
|
|
96
|
-
|
|
108
|
+
```typescript
|
|
109
|
+
import { BaseKeyProvider, setKeyProvider } from 'mask-privacy/core/key_provider';
|
|
110
|
+
|
|
111
|
+
class MyCustomKmsProvider extends BaseKeyProvider {
|
|
112
|
+
async getEncryptionKey(): Promise<string> {
|
|
113
|
+
// Logic to fetch from your KMS
|
|
114
|
+
return "your-secret-key";
|
|
115
|
+
}
|
|
116
|
+
async getMasterKey(): Promise<string> {
|
|
117
|
+
return "your-master-key";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
setKeyProvider(new MyCustomKmsProvider());
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### 6. Local NLP Scanning (Default)
|
|
125
|
+
Performance-sensitive deployments utilize the built-in `LocalTransformersScanner` by default. This uses HuggingFace Transformers to run PII detection locally within your Node.js process, eliminating the need for external NLP services.
|
|
97
126
|
|
|
98
127
|
### 7. Sub-string Detokenization
|
|
99
128
|
Mask includes the ability to detokenize PII embedded within larger text blocks (like email bodies or chat messages). `detokenizeText()` uses high-performance regex to find and restore all tokens within a paragraph before they hit your tools.
|
|
@@ -110,6 +139,7 @@ Add peer dependencies depending on your infrastructure:
|
|
|
110
139
|
npm install ioredis # For Redis vaults
|
|
111
140
|
npm install @aws-sdk/client-dynamodb @aws-sdk/lib-dynamodb # For DynamoDB vaults
|
|
112
141
|
npm install memjs # For Memcached vaults
|
|
142
|
+
npm install @huggingface/transformers # Required for local NLP scanning
|
|
113
143
|
```
|
|
114
144
|
|
|
115
145
|
### Framework Support
|
|
@@ -150,6 +180,19 @@ export MASK_DYNAMODB_REGION=us-east-1
|
|
|
150
180
|
# For Memcached:
|
|
151
181
|
export MASK_MEMCACHED_HOST=localhost
|
|
152
182
|
export MASK_MEMCACHED_PORT=11211
|
|
183
|
+
|
|
184
|
+
#### 4. Security Enforcement
|
|
185
|
+
# Enable strict mode to refuse startup without MASK_ENCRYPTION_KEY
|
|
186
|
+
export MASK_STRICT_PROD=true
|
|
187
|
+
|
|
188
|
+
# Configure Blind Index Salt (Optional)
|
|
189
|
+
export MASK_BLIND_INDEX_SALT="custom-salt-here"
|
|
190
|
+
|
|
191
|
+
> [!IMPORTANT]
|
|
192
|
+
> **Security Warning:** In production, you **must** change the default `MASK_BLIND_INDEX_SALT`. Using the default salt makes your blind indices vulnerable to pre-computed hash (rainbow table) attacks across different SDK installations.
|
|
193
|
+
|
|
194
|
+
# Configure MemoryVault cleanup aggressiveness (default: 0.01)
|
|
195
|
+
export MASK_VAULT_CLEANUP_FREQUENCY=0.05
|
|
153
196
|
```
|
|
154
197
|
|
|
155
198
|
---
|
|
@@ -236,11 +279,14 @@ The SDK includes a thread-safe, asynchronous AuditLogger built-in.
|
|
|
236
279
|
|
|
237
280
|
As your agents encrypt and decrypt data, the logger buffers these privacy events (Action, Agent, TTL). **Raw PII is never logged.**
|
|
238
281
|
|
|
239
|
-
Audit events are
|
|
282
|
+
Audit events are buffered in memory and flushed periodically to stdout as structured JSON. Pipe these logs into your existing Datadog or Splunk agents for SOC2/HIPAA compliance reporting.
|
|
283
|
+
|
|
284
|
+
The `AuditLogger` includes graceful shutdown hooks (`SIGTERM`, `SIGINT`) to ensure all buffered events are flushed before the process exits.
|
|
285
|
+
|
|
286
|
+
To prevent memory issues in high-volume environments, the buffer size can be capped:
|
|
240
287
|
|
|
241
|
-
To disable the local SQLite buffer:
|
|
242
288
|
```bash
|
|
243
|
-
export
|
|
289
|
+
export MASK_AUDIT_MAX_BUFFER_SIZE=5000
|
|
244
290
|
```
|
|
245
291
|
|
|
246
292
|
## License
|
package/dist/index.d.mts
CHANGED
|
@@ -1,21 +1,24 @@
|
|
|
1
1
|
/** Interface every vault backend must implement. */
|
|
2
2
|
declare abstract class BaseVault {
|
|
3
3
|
/** Persist a token → plaintext mapping with a TTL. Optionally save a reverse lookup hash. */
|
|
4
|
-
abstract store(token: string, plaintext: string, ttlSeconds: number, ptHash?: string | null): Promise<void
|
|
4
|
+
abstract store(token: string, plaintext: string, ttlSeconds: number, ptHash?: string | null): Promise<void>;
|
|
5
5
|
/** Return the existing unexpired token for a given plaintext hash, or null. */
|
|
6
|
-
abstract getTokenByPlaintextHash(ptHash: string): Promise<string | null
|
|
6
|
+
abstract getTokenByPlaintextHash(ptHash: string): Promise<string | null>;
|
|
7
7
|
/** Return the plaintext for token, or null if missing/expired. */
|
|
8
|
-
abstract retrieve(token: string): Promise<string | null
|
|
8
|
+
abstract retrieve(token: string): Promise<string | null>;
|
|
9
9
|
/** Delete a token and its reverse mapping. */
|
|
10
|
-
abstract delete(token: string): Promise<void
|
|
10
|
+
abstract delete(token: string): Promise<void>;
|
|
11
11
|
}
|
|
12
12
|
declare function getVault(): BaseVault;
|
|
13
|
+
type EncodeOptions = {
|
|
14
|
+
ttl?: number;
|
|
15
|
+
searchBuckets?: ('year' | 'month' | 'day' | 'numeric')[];
|
|
16
|
+
searchBucketSize?: number;
|
|
17
|
+
};
|
|
13
18
|
/**
|
|
14
19
|
* Tokenise rawText, encrypt it, store in vault, return the FPE token.
|
|
15
20
|
*/
|
|
16
|
-
declare function encode(rawText: string, options?:
|
|
17
|
-
ttl?: number;
|
|
18
|
-
}): Promise<string>;
|
|
21
|
+
declare function encode(rawText: string, options?: EncodeOptions): Promise<string>;
|
|
19
22
|
/** Async wrapper for encode (parity with Python aencode). */
|
|
20
23
|
declare const aencode: typeof encode;
|
|
21
24
|
/**
|
|
@@ -31,42 +34,26 @@ declare function detokenizeText(text: string): Promise<string>;
|
|
|
31
34
|
/** Async wrapper for detokenizeText (parity with Python adetokenize_text). */
|
|
32
35
|
declare const adetokenizeText: typeof detokenizeText;
|
|
33
36
|
|
|
37
|
+
/**
|
|
38
|
+
* Heuristic: return true if value appears to be a Mask token.
|
|
39
|
+
*/
|
|
40
|
+
declare function looksLikeToken(value: string | any): boolean;
|
|
41
|
+
|
|
34
42
|
/**
|
|
35
43
|
* Format-Preserving Encryption (FPE) token generation.
|
|
36
44
|
*
|
|
37
45
|
* Generates structurally valid, **deterministic** tokens that preserve the
|
|
38
46
|
* format of the original data type so downstream tools, schemas, and
|
|
39
47
|
* validators continue to work without modification.
|
|
40
|
-
*
|
|
41
|
-
* Determinism is achieved via HMAC-SHA256 keyed with a master key, ensuring
|
|
42
|
-
* the same plaintext always produces the same token. This preserves entity
|
|
43
|
-
* relationships for LLMs (e.g. "John" is always [TKN-abc]) without leaking
|
|
44
|
-
* the identity.
|
|
45
|
-
*
|
|
46
|
-
* Supported formats:
|
|
47
|
-
* - Email → tkn-<hex>@email.com
|
|
48
|
-
* - Phone → +1-555-<7 digits>
|
|
49
|
-
* - SSN → 000-00-<4 digits>
|
|
50
|
-
* - CC → 4000-0000-0000-<4 digits>
|
|
51
|
-
* - Routing→ 000000<3 digits>
|
|
52
|
-
* - Default→ [TKN-<hex>]
|
|
53
48
|
*/
|
|
54
49
|
/** Clear the cached master key. Useful in tests. */
|
|
55
50
|
declare function resetMasterKey(): void;
|
|
56
51
|
/**
|
|
57
52
|
* Return a **deterministic**, format-preserving token for rawText.
|
|
58
|
-
*
|
|
59
|
-
* The token is structurally compatible with the original data type
|
|
60
|
-
* so that downstream schema validators, regex checks, and database
|
|
61
|
-
* constraints continue to pass.
|
|
62
53
|
*/
|
|
63
|
-
declare function generateFPEToken(rawText: string): string
|
|
64
|
-
/**
|
|
65
|
-
* Heuristic: return true if value appears to be a Mask token.
|
|
66
|
-
*/
|
|
67
|
-
declare function looksLikeToken(value: string): boolean;
|
|
54
|
+
declare function generateFPEToken(rawText: string): Promise<string>;
|
|
68
55
|
|
|
69
|
-
declare class
|
|
56
|
+
declare class BaseScanner {
|
|
70
57
|
protected _supportedEntities: string[];
|
|
71
58
|
constructor();
|
|
72
59
|
setSupportedEntities(entities: string[]): void;
|
|
@@ -92,15 +79,45 @@ declare class PresidioScanner {
|
|
|
92
79
|
aggressive?: boolean;
|
|
93
80
|
}): Promise<any[]>;
|
|
94
81
|
}
|
|
82
|
+
/** @deprecated Use BaseScanner instead. Kept for backwards compatibility. */
|
|
83
|
+
declare const PresidioScanner: typeof BaseScanner;
|
|
84
|
+
declare function getScanner(): BaseScanner;
|
|
85
|
+
|
|
95
86
|
/**
|
|
96
|
-
*
|
|
87
|
+
* Local Transformers-based NER Scanner for Mask SDK.
|
|
88
|
+
*
|
|
89
|
+
* Uses @huggingface/transformers via a Piscina worker pool to provide
|
|
90
|
+
* state-of-the-art PII detection (Names, Locations, Organizations) entirely
|
|
91
|
+
* on the local machine via ONNX Runtime — without blocking the Node.js event loop.
|
|
97
92
|
*/
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
93
|
+
|
|
94
|
+
declare class LocalTransformersScanner extends BaseScanner {
|
|
95
|
+
private _pool;
|
|
96
|
+
private _modelName;
|
|
97
|
+
private _warmupPromise;
|
|
98
|
+
constructor(modelName?: string);
|
|
99
|
+
/**
|
|
100
|
+
* Initialize the Piscina worker pool and pre-warm the ONNX model.
|
|
101
|
+
* The model is loaded eagerly at construction time to avoid P99 latency
|
|
102
|
+
* spikes on the first user request.
|
|
103
|
+
*/
|
|
104
|
+
private _initPool;
|
|
105
|
+
/**
|
|
106
|
+
* Map Transformer entity types to Mask internal entity types.
|
|
107
|
+
*/
|
|
108
|
+
private _mapEntityType;
|
|
101
109
|
protected _tier2Nlp(text: string, encodeFn: (val: string) => Promise<string>, boostEntities: Set<string>, aggressive: boolean, confidenceThreshold: number): Promise<[string, any[]]>;
|
|
110
|
+
/**
|
|
111
|
+
* Merges sub-tokens and entities of the same type while precisely tracking
|
|
112
|
+
* offsets in the original text.
|
|
113
|
+
*/
|
|
114
|
+
private _mergeResultsWithOffsets;
|
|
115
|
+
/**
|
|
116
|
+
* Gracefully shut down the worker pool.
|
|
117
|
+
* Call this during application shutdown to prevent the Node.js process from hanging.
|
|
118
|
+
*/
|
|
119
|
+
close(): Promise<void>;
|
|
102
120
|
}
|
|
103
|
-
declare function getScanner(): PresidioScanner;
|
|
104
121
|
|
|
105
122
|
/**
|
|
106
123
|
* Custom exception hierarchy for the Mask SDK.
|
|
@@ -120,6 +137,9 @@ declare class MaskDecryptionError extends MaskError {
|
|
|
120
137
|
/** Raised when spaCy / Presidio analysis exceeds the time budget. */
|
|
121
138
|
declare class MaskNLPTimeout extends MaskError {
|
|
122
139
|
}
|
|
140
|
+
/** Raised when mandatory security keys (MASK_MASTER_KEY, etc.) are missing. */
|
|
141
|
+
declare class MaskSecurityError extends MaskError {
|
|
142
|
+
}
|
|
123
143
|
|
|
124
144
|
/**
|
|
125
145
|
* Core cryptography engine for Mask SDK.
|
|
@@ -128,46 +148,62 @@ declare class MaskNLPTimeout extends MaskError {
|
|
|
128
148
|
* ensuring that plaintext PII is encrypted locally before being
|
|
129
149
|
* transmitted and stored in distributed vaults (Redis/Memcached/DynamoDB).
|
|
130
150
|
*
|
|
151
|
+
* Uses AES-256-GCM (authenticated encryption) via native Node.js crypto.
|
|
152
|
+
* Includes a compatibility layer to decrypt legacy Fernet-format tokens.
|
|
153
|
+
*
|
|
131
154
|
* Requires MASK_ENCRYPTION_KEY to be set in the environment.
|
|
132
155
|
*/
|
|
133
156
|
declare class CryptoEngine {
|
|
134
157
|
private static _instance;
|
|
135
|
-
private
|
|
158
|
+
private _aesKey;
|
|
159
|
+
private _indexSecret;
|
|
136
160
|
private constructor();
|
|
161
|
+
/**
|
|
162
|
+
* Return the singleton instance, initialising it if necessary.
|
|
163
|
+
* This is asynchronous because key providers (KMS, etc.) might be async.
|
|
164
|
+
*/
|
|
165
|
+
static getInstanceAsync(): Promise<CryptoEngine>;
|
|
166
|
+
/** Legacy synchronous accessor — will throw if not already initialised. */
|
|
137
167
|
static getInstance(): CryptoEngine;
|
|
138
168
|
/** Clear the singleton instance to force re-initialization (useful for key rotation). */
|
|
139
169
|
static reset(): void;
|
|
140
170
|
private _init;
|
|
171
|
+
/** Return the secret used for HMAC-based blind indexing. */
|
|
172
|
+
getIndexSecret(): Promise<Buffer>;
|
|
141
173
|
encrypt(plaintext: string): string;
|
|
142
174
|
decrypt(ciphertext: string): string;
|
|
175
|
+
/** Decrypt an AES-256-GCM token (base64 encoded). */
|
|
176
|
+
private _decryptAesGcm;
|
|
177
|
+
/**
|
|
178
|
+
* Attempt to decrypt a legacy Fernet-format token.
|
|
179
|
+
*
|
|
180
|
+
* Fernet format: Version (1) || Timestamp (8) || IV (16) || Ciphertext (var) || HMAC (32)
|
|
181
|
+
* All base64url-encoded.
|
|
182
|
+
*
|
|
183
|
+
* We try to use the `fernet` npm package if available, otherwise throw.
|
|
184
|
+
*/
|
|
185
|
+
private _decryptLegacyFernet;
|
|
143
186
|
}
|
|
144
187
|
|
|
145
|
-
/**
|
|
146
|
-
* Asynchronous audit logger for Mask Privacy SDK.
|
|
147
|
-
*
|
|
148
|
-
* Logs every tokenisation / detokenisation event *without* recording the
|
|
149
|
-
* plaintext PII. Events are batched and flushed to:
|
|
150
|
-
* - stdout / Console (default)
|
|
151
|
-
* - Customer SIEM via structured JSON log lines
|
|
152
|
-
*
|
|
153
|
-
* NOTE: This SDK is LOCAL-FIRST. Audits are stored in a local
|
|
154
|
-
* SQLite file (.mask_audit.db) and are NOT sent anywhere externally.
|
|
155
|
-
*/
|
|
156
188
|
declare class AuditLogger {
|
|
157
189
|
private static _instance;
|
|
158
|
-
private _dbPath;
|
|
159
190
|
private _flushInterval;
|
|
160
191
|
private _running;
|
|
161
192
|
private _timer;
|
|
193
|
+
private _isFlushing;
|
|
162
194
|
private _buffer;
|
|
163
|
-
private
|
|
164
|
-
private
|
|
195
|
+
private _maxBufferSize;
|
|
196
|
+
private _strictMode;
|
|
197
|
+
private _bufferFullWarned;
|
|
198
|
+
private _shutdownRegistered;
|
|
165
199
|
private constructor();
|
|
166
200
|
static getInstance(): AuditLogger;
|
|
167
201
|
log(action: string, token: string, dataType?: string, agent?: string, tool?: string, extra?: Record<string, any>): void;
|
|
168
202
|
start(): void;
|
|
169
|
-
stop(): void
|
|
203
|
+
stop(): Promise<void>;
|
|
170
204
|
private _flush;
|
|
205
|
+
/** Synchronous flush for use in signal handlers where async is unreliable. */
|
|
206
|
+
private _flushSync;
|
|
171
207
|
}
|
|
172
208
|
|
|
173
209
|
/**
|
|
@@ -180,29 +216,33 @@ declare class AuditLogger {
|
|
|
180
216
|
declare class MaskClient {
|
|
181
217
|
vault: BaseVault;
|
|
182
218
|
crypto: CryptoEngine;
|
|
183
|
-
scanner:
|
|
219
|
+
scanner: BaseScanner;
|
|
184
220
|
auditLogger: AuditLogger;
|
|
185
221
|
/** backward compat alias */
|
|
186
222
|
logger: AuditLogger;
|
|
187
223
|
ttl: number;
|
|
188
224
|
/**
|
|
189
225
|
* Initialise the client with specific component instances.
|
|
190
|
-
*
|
|
191
|
-
* If an instance is not provided, the client will fall back to
|
|
192
|
-
* the standard environment-configured singleton for that component.
|
|
193
226
|
*/
|
|
194
227
|
constructor(options?: {
|
|
195
228
|
vault?: BaseVault;
|
|
196
229
|
crypto?: CryptoEngine;
|
|
197
|
-
scanner?:
|
|
230
|
+
scanner?: BaseScanner;
|
|
198
231
|
auditLogger?: AuditLogger;
|
|
199
232
|
ttl?: number;
|
|
200
233
|
});
|
|
234
|
+
/**
|
|
235
|
+
* Static factory to create an initialized MaskClient.
|
|
236
|
+
*/
|
|
237
|
+
static create(options?: {
|
|
238
|
+
vault?: BaseVault;
|
|
239
|
+
crypto?: CryptoEngine;
|
|
240
|
+
scanner?: BaseScanner;
|
|
241
|
+
auditLogger?: AuditLogger;
|
|
242
|
+
ttl?: number;
|
|
243
|
+
}): Promise<MaskClient>;
|
|
201
244
|
/**
|
|
202
245
|
* Tokenise rawText, encrypt it, and store it in the vault.
|
|
203
|
-
*
|
|
204
|
-
* Includes deduplication: if the same plaintext has been encoded
|
|
205
|
-
* before and the token is still active, the existing token is returned.
|
|
206
246
|
*/
|
|
207
247
|
encode(rawText: string): Promise<string>;
|
|
208
248
|
/** Retrieve token from vault and decrypt it. */
|
|
@@ -215,10 +255,14 @@ declare class MaskClient {
|
|
|
215
255
|
adecode(token: string): Promise<string>;
|
|
216
256
|
/** Async wrapper for scanAndTokenize (parity with Python ascan_and_tokenize). */
|
|
217
257
|
ascanAndTokenize(text: string): Promise<string>;
|
|
218
|
-
/** Find and replace all tokens within text with their plaintext. */
|
|
219
258
|
detokenizeText(text: string): Promise<string>;
|
|
220
259
|
/** Async wrapper for detokenizeText (parity with Python adetokenize_text). */
|
|
221
260
|
adetokenizeText(text: string): Promise<string>;
|
|
261
|
+
/**
|
|
262
|
+
* Gracefully shut down all SDK resources (worker pools, vault connections, audit logger).
|
|
263
|
+
* Call this during application shutdown to prevent the Node.js process from hanging.
|
|
264
|
+
*/
|
|
265
|
+
close(): Promise<void>;
|
|
222
266
|
}
|
|
223
267
|
|
|
224
268
|
/**
|
|
@@ -228,7 +272,7 @@ declare class MaskClient {
|
|
|
228
272
|
* Provides format-preserving encryption, local/distributed vaulting,
|
|
229
273
|
* and framework-agnostic tool interception hooks.
|
|
230
274
|
*/
|
|
231
|
-
declare const VERSION = "
|
|
275
|
+
declare const VERSION = "2.0.0";
|
|
232
276
|
|
|
233
277
|
/**
|
|
234
278
|
* Detect PII entities in text and return a list of objects with metadata.
|
|
@@ -254,4 +298,4 @@ declare function ascanAndTokenize(text: string, options?: {
|
|
|
254
298
|
*/
|
|
255
299
|
declare function secureTool(...args: any[]): any;
|
|
256
300
|
|
|
257
|
-
export { MaskClient, MaskDecryptionError, MaskError, MaskNLPTimeout, MaskVaultConnectionError,
|
|
301
|
+
export { BaseScanner, LocalTransformersScanner, MaskClient, MaskDecryptionError, MaskError, MaskNLPTimeout, MaskSecurityError, MaskVaultConnectionError, PresidioScanner, VERSION, adecode, adetokenizeText, aencode, ascanAndTokenize, decode, detectEntitiesWithConfidence, detokenizeText, encode, generateFPEToken, getScanner, getVault, looksLikeToken, resetMasterKey, secureTool };
|