n8n-nodes-redactor 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -1
- package/dist/credentials/RedactorVaultApi.credentials.js +23 -0
- package/dist/credentials/redactorVault.dark.svg +1 -0
- package/dist/credentials/redactorVault.svg +1 -0
- package/dist/nodes/Redactor/Redactor.node.js +42 -11
- package/dist/nodes/Redactor/engine.js +199 -181
- package/dist/nodes/Redactor/injection.js +1 -1
- package/dist/nodes/Redactor/profiles.js +7 -6
- package/dist/nodes/Redactor/regexSafety.js +111 -0
- package/dist/nodes/Redactor/runtime.js +75 -0
- package/dist/nodes/Redactor/vault.js +115 -70
- package/package.json +7 -4
- package/dist/nodes/Redactor/Redactor.node.d.ts +0 -5
- package/dist/nodes/Redactor/audit.d.ts +0 -21
- package/dist/nodes/Redactor/classification.d.ts +0 -13
- package/dist/nodes/Redactor/context.d.ts +0 -7
- package/dist/nodes/Redactor/encryption.d.ts +0 -4
- package/dist/nodes/Redactor/engine.d.ts +0 -6
- package/dist/nodes/Redactor/injection.d.ts +0 -20
- package/dist/nodes/Redactor/names.d.ts +0 -8
- package/dist/nodes/Redactor/patterns.d.ts +0 -8
- package/dist/nodes/Redactor/presidio.d.ts +0 -36
- package/dist/nodes/Redactor/profiles.d.ts +0 -20
- package/dist/nodes/Redactor/pseudonymize.d.ts +0 -1
- package/dist/nodes/Redactor/ropa.d.ts +0 -51
- package/dist/nodes/Redactor/types.d.ts +0 -73
- package/dist/nodes/Redactor/vault.d.ts +0 -57
package/README.md
CHANGED
|
@@ -110,7 +110,7 @@ Data source > Detect > Redact > LLM > Verify > Restore > Output > Purge
|
|
|
110
110
|
| Deduplication | The same value gets the same token everywhere, so the model keeps relationships between fields |
|
|
111
111
|
| Field targeting | Allowlist or denylist paths such as `user.email`, `*.phone` or `metadata.*` |
|
|
112
112
|
| Vault | Workflow Storage by default, so Restore works in later production runs of the same workflow, or In-Memory. Sessions expire after 60 minutes by default |
|
|
113
|
-
| Vault encryption | AES-256-GCM with scrypt key derivation, one passphrase per tenant |
|
|
113
|
+
| Vault encryption | AES-256-GCM with scrypt key derivation, one passphrase per tenant, kept in a Redactor Vault credential. See [Vault encryption](#vault-encryption) |
|
|
114
114
|
| Confidence threshold | Each match carries a 0.0 to 1.0 score. Checksum validated patterns score 0.95 |
|
|
115
115
|
| Allow and deny lists | Values to never redact, such as your support address, and values to always redact, such as project codenames |
|
|
116
116
|
| Prompt injection check | 25+ patterns, including instruction overrides, delimiter escapes, encoded payloads and zero width characters. Returns BLOCK, FLAG or CLEAN |
|
|
@@ -122,6 +122,20 @@ Data source > Detect > Redact > LLM > Verify > Restore > Output > Purge
|
|
|
122
122
|
|
|
123
123
|
This node reduces what reaches a model. It is not legal advice and does not on its own make a workflow GDPR or HIPAA compliant.
|
|
124
124
|
|
|
125
|
+
## Vault encryption
|
|
126
|
+
|
|
127
|
+
Workflow Storage keeps sessions with the workflow. To encrypt them, create a **Redactor Vault** credential with a passphrase of at least 12 characters and attach it to both the Redact and the Restore node. n8n stores credentials encrypted and apart from the workflow, so the passphrase never sits next to the data it protects. Give each tenant its own credential.
|
|
128
|
+
|
|
129
|
+
The older Vault Encryption Passphrase field still works so workflows saved with it keep restoring, but it is saved in plain text inside the workflow. When both are set, the credential wins.
|
|
130
|
+
|
|
131
|
+
Restore needs the same passphrase that Redact used. If Redact meets a session ID that a different passphrase wrote, it stops with an error instead of overwriting it.
|
|
132
|
+
|
|
133
|
+
### Concurrent runs
|
|
134
|
+
|
|
135
|
+
n8n saves workflow storage as one block at the end of each production run and does not merge it. If two runs of the same workflow overlap, the one that finishes last wins. A session created by the other run can be lost, and a session purged by the other run can come back until its TTL expires. The node cannot prevent this from inside n8n. To stay safe, keep the Session TTL short, avoid overlapping runs of a workflow that uses Purge (for example by limiting production concurrency on your instance), or use In-Memory storage when Redact and Restore run in the same execution.
|
|
136
|
+
|
|
137
|
+
In-Memory sessions are kept per workflow and are never visible to another workflow.
|
|
138
|
+
|
|
125
139
|
## For developers
|
|
126
140
|
|
|
127
141
|
| Area | Detail |
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RedactorVaultApi = void 0;
|
|
4
|
+
class RedactorVaultApi {
|
|
5
|
+
constructor() {
|
|
6
|
+
this.name = 'redactorVaultApi';
|
|
7
|
+
this.displayName = 'Redactor Vault API';
|
|
8
|
+
this.icon = { light: 'file:redactorVault.svg', dark: 'file:redactorVault.dark.svg' };
|
|
9
|
+
this.documentationUrl = 'https://github.com/mjmirza/n8n-nodes-redactor#vault-encryption';
|
|
10
|
+
this.properties = [
|
|
11
|
+
{
|
|
12
|
+
displayName: 'Vault Passphrase',
|
|
13
|
+
name: 'passphrase',
|
|
14
|
+
type: 'string',
|
|
15
|
+
typeOptions: { password: true },
|
|
16
|
+
default: '',
|
|
17
|
+
required: true,
|
|
18
|
+
description: 'Encrypts stored sessions with AES-256-GCM. Use at least 12 characters, and a different passphrase for each tenant. Sessions written with this passphrase cannot be read without it.',
|
|
19
|
+
},
|
|
20
|
+
];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
exports.RedactorVaultApi = RedactorVaultApi;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="60" height="60" viewBox="16 15 370 370"><g transform="translate(0,400) scale(0.1,-0.1)"><path fill="#FF6B6B" d="M1880 3520 c-200 -204 -414 -321 -714 -390 l-99 -23 6 -171 c6 -183 24 -288 71 -431 55 -163 163 -360 258 -469 34 -40 34 -40 68 18 145 254 391 434 638 467 416 56 762 -273 883 -841 23 -108 34 -413 17 -485 -17 -70 105 200 146 325 116 350 145 599 146 1221 l0 346 -162 6 c-420 16 -743 151 -1016 424 -68 68 -124 123 -125 122 -1 0 -54 -54 -117 -119z"/><path fill="#FFA42E" d="M703 2628 c6 -546 24 -707 113 -1015 177 -610 573 -1066 1070 -1232 113 -38 115 -38 224 -1 193 64 391 184 472 285 339 421 307 1184 -66 1532 -157 147 -364 176 -528 72 -51 -33 -118 -109 -82 -94 101 42 155 43 241 5 254 -112 199 -491 -85 -601 -265 -103 -614 59 -864 401 -206 281 -312 593 -325 958 l-6 152 -84 0 -85 0 5 -462z"/></g></svg>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="60" height="60" viewBox="16 15 370 370"><g transform="translate(0,400) scale(0.1,-0.1)"><path fill="#FF5757" d="M1880 3520 c-200 -204 -414 -321 -714 -390 l-99 -23 6 -171 c6 -183 24 -288 71 -431 55 -163 163 -360 258 -469 34 -40 34 -40 68 18 145 254 391 434 638 467 416 56 762 -273 883 -841 23 -108 34 -413 17 -485 -17 -70 105 200 146 325 116 350 145 599 146 1221 l0 346 -162 6 c-420 16 -743 151 -1016 424 -68 68 -124 123 -125 122 -1 0 -54 -54 -117 -119z"/><path fill="#F79410" d="M703 2628 c6 -546 24 -707 113 -1015 177 -610 573 -1066 1070 -1232 113 -38 115 -38 224 -1 193 64 391 184 472 285 339 421 307 1184 -66 1532 -157 147 -364 176 -528 72 -51 -33 -118 -109 -82 -94 101 42 155 43 241 5 254 -112 199 -491 -85 -601 -265 -103 -614 59 -864 401 -206 281 -312 593 -325 958 l-6 152 -84 0 -85 0 5 -462z"/></g></svg>
|
|
@@ -11,6 +11,7 @@ const audit_1 = require("./audit");
|
|
|
11
11
|
const presidio_1 = require("./presidio");
|
|
12
12
|
const profiles_1 = require("./profiles");
|
|
13
13
|
const ropa_1 = require("./ropa");
|
|
14
|
+
const runtime_1 = require("./runtime");
|
|
14
15
|
const RECOMMENDED_PATTERNS = [
|
|
15
16
|
'email', 'phone', 'phoneDE', 'phoneUK', 'phoneAT', 'phoneCH', 'phoneFR',
|
|
16
17
|
'phoneNL', 'phoneES', 'phoneIT', 'phoneAU', 'phoneIN', 'phoneBR',
|
|
@@ -57,6 +58,19 @@ class Redactor {
|
|
|
57
58
|
name: 'Redactor',
|
|
58
59
|
},
|
|
59
60
|
inputs: [n8n_workflow_1.NodeConnectionTypes.Main],
|
|
61
|
+
credentials: [
|
|
62
|
+
{
|
|
63
|
+
name: 'redactorVaultApi',
|
|
64
|
+
required: false,
|
|
65
|
+
testedBy: 'redactorVaultPassphrase',
|
|
66
|
+
displayOptions: {
|
|
67
|
+
show: {
|
|
68
|
+
operation: ['redact', 'restore', 'stats', 'purge'],
|
|
69
|
+
vaultStorage: ['workflow', 'file'],
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
],
|
|
60
74
|
outputs: `={{((parameters) => (parameters["includeReport"] || parameters["auditLog"]) ? [{ type: "main", displayName: "Data" }, { type: "main", displayName: "Report" }] : [{ type: "main", displayName: "Data" }])($parameter)}}`,
|
|
61
75
|
properties: [
|
|
62
76
|
{
|
|
@@ -601,7 +615,7 @@ class Redactor {
|
|
|
601
615
|
},
|
|
602
616
|
},
|
|
603
617
|
default: '',
|
|
604
|
-
description: '
|
|
618
|
+
description: 'Deprecated. This field is saved in plain text inside the workflow. Attach a Redactor Vault credential instead, which n8n stores encrypted and which takes priority. Kept so workflows saved with it still restore.',
|
|
605
619
|
},
|
|
606
620
|
{
|
|
607
621
|
displayName: 'Session TTL (Minutes)',
|
|
@@ -739,6 +753,18 @@ class Redactor {
|
|
|
739
753
|
},
|
|
740
754
|
],
|
|
741
755
|
};
|
|
756
|
+
this.methods = {
|
|
757
|
+
credentialTest: {
|
|
758
|
+
async redactorVaultPassphrase(credential) {
|
|
759
|
+
var _a, _b;
|
|
760
|
+
const passphrase = String((_b = (_a = credential.data) === null || _a === void 0 ? void 0 : _a.passphrase) !== null && _b !== void 0 ? _b : '');
|
|
761
|
+
if (passphrase.length < 12) {
|
|
762
|
+
return { status: 'Error', message: 'Use a passphrase of at least 12 characters.' };
|
|
763
|
+
}
|
|
764
|
+
return { status: 'OK', message: 'Passphrase accepted.' };
|
|
765
|
+
},
|
|
766
|
+
},
|
|
767
|
+
};
|
|
742
768
|
}
|
|
743
769
|
async execute() {
|
|
744
770
|
var _a, _b, _c, _d, _e, _f, _g;
|
|
@@ -761,6 +787,7 @@ class Redactor {
|
|
|
761
787
|
fieldRules: [],
|
|
762
788
|
fieldMode: 'all',
|
|
763
789
|
skipSemantic: operation === 'verify',
|
|
790
|
+
scanNumbers: true,
|
|
764
791
|
};
|
|
765
792
|
const allHits = [];
|
|
766
793
|
for (let i = 0; i < items.length; i++) {
|
|
@@ -859,14 +886,9 @@ class Redactor {
|
|
|
859
886
|
}]];
|
|
860
887
|
}
|
|
861
888
|
const vaultStorage = this.getNodeParameter('vaultStorage', 0, 'workflow');
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
vaultPassphrase = this.getNodeParameter('vaultPassphrase', 0, '');
|
|
865
|
-
}
|
|
866
|
-
catch { }
|
|
867
|
-
const vault = (0, vault_1.createVault)(vaultStorage, vaultStorage === 'memory' ? undefined : this.getWorkflowStaticData('global'), vaultPassphrase || undefined);
|
|
889
|
+
const vaultPassphrase = vaultStorage === 'memory' ? '' : await (0, runtime_1.resolvePassphrase)(this);
|
|
890
|
+
const vault = (0, vault_1.createVault)(vaultStorage, vaultStorage === 'memory' ? undefined : this.getWorkflowStaticData('global'), vaultPassphrase || undefined, (0, runtime_1.workflowNamespace)(this));
|
|
868
891
|
if (operation === 'stats') {
|
|
869
|
-
vault.cleanup();
|
|
870
892
|
const sessions = vault.listSessions();
|
|
871
893
|
const totalEntries = sessions.reduce((sum, s) => sum + s.entryCount, 0);
|
|
872
894
|
return [[
|
|
@@ -991,7 +1013,16 @@ class Redactor {
|
|
|
991
1013
|
if (profile.auditLog !== undefined)
|
|
992
1014
|
auditLog = profile.auditLog;
|
|
993
1015
|
}
|
|
994
|
-
|
|
1016
|
+
(0, runtime_1.assertUserRegexes)(this, customPatterns, [
|
|
1017
|
+
{ name: 'Allow list', entries: allowList },
|
|
1018
|
+
{ name: 'Deny list', entries: denyList },
|
|
1019
|
+
]);
|
|
1020
|
+
try {
|
|
1021
|
+
vault.getOrCreateSession(sessionId, sessionTtl * 60 * 1000);
|
|
1022
|
+
}
|
|
1023
|
+
catch (error) {
|
|
1024
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), error.message);
|
|
1025
|
+
}
|
|
995
1026
|
const ctx = {
|
|
996
1027
|
enabledPatterns,
|
|
997
1028
|
customPatterns,
|
|
@@ -1030,7 +1061,7 @@ class Redactor {
|
|
|
1030
1061
|
for (let i = 0; i < items.length; i++) {
|
|
1031
1062
|
const result = (0, injection_1.scanForInjection)(items[i].json);
|
|
1032
1063
|
if (result.detected && (!injectionResult || result.score > injectionResult.score)) {
|
|
1033
|
-
injectionResult = result;
|
|
1064
|
+
injectionResult = (0, runtime_1.sanitizeInjection)(result);
|
|
1034
1065
|
}
|
|
1035
1066
|
}
|
|
1036
1067
|
}
|
|
@@ -1065,7 +1096,7 @@ class Redactor {
|
|
|
1065
1096
|
const deleteAfterRestore = this.getNodeParameter('deleteAfterRestore', 0, true);
|
|
1066
1097
|
const session = vault.getSession(sessionId);
|
|
1067
1098
|
if (!session) {
|
|
1068
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `No vault found for session "${sessionId}". Ensure a Redact node ran first with the same Session ID
|
|
1099
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `No vault found for session "${sessionId}" that this passphrase can open. Ensure a Redact node ran first with the same Session ID, vault storage type and passphrase.`, {
|
|
1069
1100
|
description: 'Testing step by step? Each "Execute step" gets a new execution ID, and n8n saves workflow storage only after production runs of an active workflow. Run the whole workflow, or set the same fixed Session ID (for example "test-1") on both nodes. Restore only works in the workflow that ran Redact.',
|
|
1070
1101
|
});
|
|
1071
1102
|
}
|
|
@@ -39,6 +39,7 @@ exports.enhanceWithPresidio = enhanceWithPresidio;
|
|
|
39
39
|
exports.buildReport = buildReport;
|
|
40
40
|
const crypto = __importStar(require("crypto"));
|
|
41
41
|
const patterns_1 = require("./patterns");
|
|
42
|
+
const regexSafety_1 = require("./regexSafety");
|
|
42
43
|
const context_1 = require("./context");
|
|
43
44
|
const names_1 = require("./names");
|
|
44
45
|
const pseudonymize_1 = require("./pseudonymize");
|
|
@@ -95,55 +96,53 @@ function maskValue(value, label) {
|
|
|
95
96
|
function hashValue(value) {
|
|
96
97
|
return crypto.createHash('sha256').update(value).digest('hex').slice(0, 12);
|
|
97
98
|
}
|
|
99
|
+
function storeReversible(match, pattern, vault, sessionId, dedup, makeToken) {
|
|
100
|
+
if (dedup) {
|
|
101
|
+
const existing = vault.findByOriginal(sessionId, match);
|
|
102
|
+
if (existing)
|
|
103
|
+
return existing.token;
|
|
104
|
+
}
|
|
105
|
+
const token = makeToken();
|
|
106
|
+
vault.addEntry(sessionId, {
|
|
107
|
+
token,
|
|
108
|
+
original: match,
|
|
109
|
+
patternLabel: pattern.label,
|
|
110
|
+
category: ('category' in pattern ? pattern.category : 'identity'),
|
|
111
|
+
createdAt: new Date().toISOString(),
|
|
112
|
+
});
|
|
113
|
+
return token;
|
|
114
|
+
}
|
|
115
|
+
const IRREVERSIBLE = {
|
|
116
|
+
mask: (match, label) => maskValue(match, label),
|
|
117
|
+
hash: (match, label) => `[${label}:${hashValue(match)}]`,
|
|
118
|
+
redact: () => '[REDACTED]',
|
|
119
|
+
blackout: (match) => '\u2588'.repeat(match.length),
|
|
120
|
+
remove: () => '',
|
|
121
|
+
};
|
|
98
122
|
function applyRedaction(match, pattern, mode, vault, sessionId, dedup) {
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (dedup) {
|
|
102
|
-
const existing = vault.findByOriginal(sessionId, match);
|
|
103
|
-
if (existing)
|
|
104
|
-
return existing.token;
|
|
105
|
-
}
|
|
123
|
+
if (mode === 'token') {
|
|
124
|
+
return storeReversible(match, pattern, vault, sessionId, dedup, () => {
|
|
106
125
|
const session = vault.getSession(sessionId);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
vault.addEntry(sessionId, {
|
|
110
|
-
token,
|
|
111
|
-
original: match,
|
|
112
|
-
patternLabel: pattern.label,
|
|
113
|
-
category: ('category' in pattern ? pattern.category : 'identity'),
|
|
114
|
-
createdAt: new Date().toISOString(),
|
|
115
|
-
});
|
|
116
|
-
return token;
|
|
117
|
-
}
|
|
118
|
-
case 'mask':
|
|
119
|
-
return maskValue(match, pattern.label);
|
|
120
|
-
case 'hash':
|
|
121
|
-
return `[${pattern.label}:${hashValue(match)}]`;
|
|
122
|
-
case 'redact':
|
|
123
|
-
return `[REDACTED]`;
|
|
124
|
-
case 'blackout':
|
|
125
|
-
return '\u2588'.repeat(match.length);
|
|
126
|
-
case 'remove':
|
|
127
|
-
return '';
|
|
128
|
-
case 'pseudonymize': {
|
|
129
|
-
if (dedup) {
|
|
130
|
-
const existing = vault.findByOriginal(sessionId, match);
|
|
131
|
-
if (existing)
|
|
132
|
-
return existing.token;
|
|
133
|
-
}
|
|
134
|
-
const pseudonym = (0, pseudonymize_1.generatePseudonym)(sessionId, match, pattern.label);
|
|
135
|
-
vault.addEntry(sessionId, {
|
|
136
|
-
token: pseudonym,
|
|
137
|
-
original: match,
|
|
138
|
-
patternLabel: pattern.label,
|
|
139
|
-
category: ('category' in pattern ? pattern.category : 'identity'),
|
|
140
|
-
createdAt: new Date().toISOString(),
|
|
141
|
-
});
|
|
142
|
-
return pseudonym;
|
|
143
|
-
}
|
|
144
|
-
default:
|
|
145
|
-
return match;
|
|
126
|
+
return generateToken(pattern.label, session ? Object.keys(session.entries).length : 0);
|
|
127
|
+
});
|
|
146
128
|
}
|
|
129
|
+
if (mode === 'pseudonymize') {
|
|
130
|
+
return storeReversible(match, pattern, vault, sessionId, dedup, () => uniquePseudonym(vault, sessionId, match, pattern.label));
|
|
131
|
+
}
|
|
132
|
+
const irreversible = IRREVERSIBLE[mode];
|
|
133
|
+
return irreversible ? irreversible(match, pattern.label) : match;
|
|
134
|
+
}
|
|
135
|
+
function uniquePseudonym(vault, sessionId, original, label) {
|
|
136
|
+
var _a, _b;
|
|
137
|
+
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
138
|
+
for (let attempt = 0; attempt < 32; attempt++) {
|
|
139
|
+
const seed = attempt === 0 ? original : `${original}\u0000${attempt}`;
|
|
140
|
+
const candidate = (0, pseudonymize_1.generatePseudonym)(sessionId, seed, label);
|
|
141
|
+
const taken = entries[candidate];
|
|
142
|
+
if (!taken || taken.original === original)
|
|
143
|
+
return candidate;
|
|
144
|
+
}
|
|
145
|
+
return generateToken(label, Object.keys(entries).length);
|
|
147
146
|
}
|
|
148
147
|
function shouldProcessField(fieldPath, fieldMode, fieldRules) {
|
|
149
148
|
if (fieldMode === 'all')
|
|
@@ -169,17 +168,7 @@ function fieldPathMatches(actual, pattern) {
|
|
|
169
168
|
}
|
|
170
169
|
}
|
|
171
170
|
function isSafeRegex(regexStr) {
|
|
172
|
-
|
|
173
|
-
return false;
|
|
174
|
-
if (regexStr.length > 500)
|
|
175
|
-
return false;
|
|
176
|
-
try {
|
|
177
|
-
new RegExp(regexStr);
|
|
178
|
-
return true;
|
|
179
|
-
}
|
|
180
|
-
catch {
|
|
181
|
-
return false;
|
|
182
|
-
}
|
|
171
|
+
return (0, regexSafety_1.checkRegex)(regexStr) === null;
|
|
183
172
|
}
|
|
184
173
|
const SEMANTIC_FIELD_MAP = [
|
|
185
174
|
{
|
|
@@ -386,70 +375,87 @@ function getSemanticLabel(fieldKey, siblingKeys) {
|
|
|
386
375
|
}
|
|
387
376
|
return null;
|
|
388
377
|
}
|
|
389
|
-
function
|
|
390
|
-
var _a
|
|
378
|
+
function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
379
|
+
var _a;
|
|
380
|
+
const semantic = getSemanticLabel(path, siblingKeys);
|
|
381
|
+
if (!semantic || !semantic.label || raw.trim().length === 0)
|
|
382
|
+
return null;
|
|
383
|
+
if (isAllowListed(raw, ctx.allowList))
|
|
384
|
+
return null;
|
|
385
|
+
const semConf = semantic.label === 'IDENTIFIER' ? context_1.AMBIGUOUS_FIELD_CONFIDENCE : context_1.SEMANTIC_CONFIDENCE;
|
|
386
|
+
if (semConf < ((_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0))
|
|
387
|
+
return null;
|
|
388
|
+
const replacement = applyRedaction(raw, { label: semantic.label, category: semantic.category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
389
|
+
hits.push({
|
|
390
|
+
token: replacement, original: '***',
|
|
391
|
+
patternName: `semantic_${semantic.label}`,
|
|
392
|
+
patternLabel: semantic.label,
|
|
393
|
+
category: semantic.category,
|
|
394
|
+
field: display, itemIndex,
|
|
395
|
+
confidence: semConf,
|
|
396
|
+
});
|
|
397
|
+
return replacement;
|
|
398
|
+
}
|
|
399
|
+
function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
400
|
+
const text = String(value);
|
|
401
|
+
if (!ctx.skipSemantic) {
|
|
402
|
+
const semantic = redactSemantic(text, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys);
|
|
403
|
+
if (semantic !== null)
|
|
404
|
+
return semantic;
|
|
405
|
+
}
|
|
406
|
+
if (!ctx.scanNumbers)
|
|
407
|
+
return value;
|
|
408
|
+
const scanned = redactText(text, ctx, vault, sessionId, hits, itemIndex, display);
|
|
409
|
+
return scanned === text ? value : scanned;
|
|
410
|
+
}
|
|
411
|
+
function redactKey(key, taken, ctx, vault, sessionId, itemIndex, keyHits) {
|
|
412
|
+
let out = redactText(key, ctx, vault, sessionId, keyHits, itemIndex, '');
|
|
413
|
+
if (out === key)
|
|
414
|
+
return key;
|
|
415
|
+
const base = out;
|
|
416
|
+
for (let n = 1; out === '' || out in taken; n++)
|
|
417
|
+
out = `${base}_${n}`;
|
|
418
|
+
return out;
|
|
419
|
+
}
|
|
420
|
+
function redactObject(value, ctx, vault, sessionId, hits, itemIndex, path, display) {
|
|
421
|
+
const entries = Object.entries(value);
|
|
422
|
+
const siblingKeys = entries.map(([k]) => k);
|
|
423
|
+
const obj = {};
|
|
424
|
+
for (const [k, v] of entries) {
|
|
425
|
+
const fieldPath = path ? `${path}.${k}` : k;
|
|
426
|
+
const keyHits = [];
|
|
427
|
+
const outKey = shouldProcessField(fieldPath, ctx.fieldMode, ctx.fieldRules)
|
|
428
|
+
? redactKey(k, obj, ctx, vault, sessionId, itemIndex, keyHits)
|
|
429
|
+
: k;
|
|
430
|
+
const childDisplay = display ? `${display}.${outKey}` : outKey;
|
|
431
|
+
for (const h of keyHits)
|
|
432
|
+
hits.push({ ...h, field: childDisplay });
|
|
433
|
+
obj[outKey] = redactValue(v, ctx, vault, sessionId, hits, itemIndex, fieldPath, siblingKeys, childDisplay);
|
|
434
|
+
}
|
|
435
|
+
return obj;
|
|
436
|
+
}
|
|
437
|
+
function redactValue(value, ctx, vault, sessionId, hits, itemIndex, currentPath = '', siblingKeys, displayPath = currentPath) {
|
|
438
|
+
const processed = shouldProcessField(currentPath, ctx.fieldMode, ctx.fieldRules);
|
|
391
439
|
if (typeof value === 'string') {
|
|
392
|
-
if (!
|
|
440
|
+
if (!processed)
|
|
393
441
|
return value;
|
|
394
|
-
}
|
|
395
442
|
if (!ctx.skipSemantic) {
|
|
396
|
-
const semantic =
|
|
397
|
-
if (semantic
|
|
398
|
-
|
|
399
|
-
}
|
|
400
|
-
else {
|
|
401
|
-
const semConf = semantic.label === 'IDENTIFIER' ? context_1.AMBIGUOUS_FIELD_CONFIDENCE : context_1.SEMANTIC_CONFIDENCE;
|
|
402
|
-
if (semConf >= ((_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0)) {
|
|
403
|
-
const replacement = applyRedaction(value, { label: semantic.label, category: semantic.category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
404
|
-
hits.push({
|
|
405
|
-
token: replacement, original: '***',
|
|
406
|
-
patternName: `semantic_${semantic.label}`,
|
|
407
|
-
patternLabel: semantic.label,
|
|
408
|
-
category: semantic.category,
|
|
409
|
-
field: currentPath, itemIndex,
|
|
410
|
-
confidence: semConf,
|
|
411
|
-
});
|
|
412
|
-
return replacement;
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
}
|
|
443
|
+
const semantic = redactSemantic(value, ctx, vault, sessionId, hits, itemIndex, currentPath, displayPath, siblingKeys);
|
|
444
|
+
if (semantic !== null)
|
|
445
|
+
return semantic;
|
|
416
446
|
}
|
|
417
|
-
return redactText(value, ctx, vault, sessionId, hits, itemIndex,
|
|
447
|
+
return redactText(value, ctx, vault, sessionId, hits, itemIndex, displayPath);
|
|
418
448
|
}
|
|
419
|
-
if (typeof value === 'number'
|
|
420
|
-
if (!
|
|
449
|
+
if (typeof value === 'number') {
|
|
450
|
+
if (!processed)
|
|
421
451
|
return value;
|
|
422
|
-
|
|
423
|
-
const semantic = getSemanticLabel(currentPath, siblingKeys);
|
|
424
|
-
if (semantic && semantic.label) {
|
|
425
|
-
const semConf = semantic.label === 'IDENTIFIER' ? context_1.AMBIGUOUS_FIELD_CONFIDENCE : context_1.SEMANTIC_CONFIDENCE;
|
|
426
|
-
if (semConf >= ((_b = ctx.confidenceThreshold) !== null && _b !== void 0 ? _b : 0)) {
|
|
427
|
-
const strValue = String(value);
|
|
428
|
-
const replacement = applyRedaction(strValue, { label: semantic.label, category: semantic.category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
429
|
-
hits.push({
|
|
430
|
-
token: replacement, original: '***',
|
|
431
|
-
patternName: `semantic_${semantic.label}`,
|
|
432
|
-
patternLabel: semantic.label,
|
|
433
|
-
category: semantic.category,
|
|
434
|
-
field: currentPath, itemIndex,
|
|
435
|
-
confidence: semConf,
|
|
436
|
-
});
|
|
437
|
-
return replacement;
|
|
438
|
-
}
|
|
439
|
-
}
|
|
452
|
+
return redactNumber(value, ctx, vault, sessionId, hits, itemIndex, currentPath, displayPath, siblingKeys);
|
|
440
453
|
}
|
|
441
454
|
if (Array.isArray(value)) {
|
|
442
|
-
return value.map((v, i) => redactValue(v, ctx, vault, sessionId, hits, itemIndex, `${currentPath}[${i}]`));
|
|
455
|
+
return value.map((v, i) => redactValue(v, ctx, vault, sessionId, hits, itemIndex, `${currentPath}[${i}]`, undefined, `${displayPath}[${i}]`));
|
|
443
456
|
}
|
|
444
457
|
if (value !== null && typeof value === 'object') {
|
|
445
|
-
|
|
446
|
-
const siblingKeys = entries.map(([k]) => k);
|
|
447
|
-
const obj = {};
|
|
448
|
-
for (const [k, v] of entries) {
|
|
449
|
-
const fieldPath = currentPath ? `${currentPath}.${k}` : k;
|
|
450
|
-
obj[k] = redactValue(v, ctx, vault, sessionId, hits, itemIndex, fieldPath, siblingKeys);
|
|
451
|
-
}
|
|
452
|
-
return obj;
|
|
458
|
+
return redactObject(value, ctx, vault, sessionId, hits, itemIndex, currentPath, displayPath);
|
|
453
459
|
}
|
|
454
460
|
return value;
|
|
455
461
|
}
|
|
@@ -571,94 +577,106 @@ function redactText(text, ctx, vault, sessionId, hits, itemIndex, fieldPath) {
|
|
|
571
577
|
}
|
|
572
578
|
return result;
|
|
573
579
|
}
|
|
580
|
+
function makeRestorer(entries) {
|
|
581
|
+
const tokens = Object.keys(entries).filter((t) => t.length > 0).sort((a, b) => b.length - a.length);
|
|
582
|
+
if (tokens.length === 0)
|
|
583
|
+
return (text) => text;
|
|
584
|
+
const re = new RegExp(tokens.map((t) => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'), 'g');
|
|
585
|
+
return (text) => text.replace(re, (token) => entries[token].original);
|
|
586
|
+
}
|
|
587
|
+
function restoreWith(value, restore) {
|
|
588
|
+
if (typeof value === 'string')
|
|
589
|
+
return restore(value);
|
|
590
|
+
if (Array.isArray(value))
|
|
591
|
+
return value.map((v) => restoreWith(v, restore));
|
|
592
|
+
if (value !== null && typeof value === 'object') {
|
|
593
|
+
const obj = {};
|
|
594
|
+
for (const [k, v] of Object.entries(value)) {
|
|
595
|
+
obj[restore(k)] = restoreWith(v, restore);
|
|
596
|
+
}
|
|
597
|
+
return obj;
|
|
598
|
+
}
|
|
599
|
+
return value;
|
|
600
|
+
}
|
|
574
601
|
function restoreValue(value, vault, sessionId) {
|
|
575
602
|
const session = vault.getSession(sessionId);
|
|
576
603
|
if (!session)
|
|
577
604
|
return value;
|
|
605
|
+
return restoreWith(value, makeRestorer(session.entries));
|
|
606
|
+
}
|
|
607
|
+
function toUtf16Spans(text, entities) {
|
|
608
|
+
const offsets = [0];
|
|
609
|
+
for (const ch of text)
|
|
610
|
+
offsets.push(offsets[offsets.length - 1] + ch.length);
|
|
611
|
+
return entities
|
|
612
|
+
.filter((e) => Number.isInteger(e.start) && Number.isInteger(e.end) && e.start >= 0 && e.end > e.start && e.end < offsets.length)
|
|
613
|
+
.map((e) => ({ start: offsets[e.start], end: offsets[e.end], score: e.score, entityType: e.entity_type }));
|
|
614
|
+
}
|
|
615
|
+
function resolveOverlaps(spans) {
|
|
616
|
+
const ranked = [...spans].sort((a, b) => b.score - a.score || (b.end - b.start) - (a.end - a.start) || a.start - b.start);
|
|
617
|
+
const kept = [];
|
|
618
|
+
for (const span of ranked) {
|
|
619
|
+
if (!kept.some((k) => span.start < k.end && k.start < span.end))
|
|
620
|
+
kept.push(span);
|
|
621
|
+
}
|
|
622
|
+
return kept.sort((a, b) => b.start - a.start);
|
|
623
|
+
}
|
|
624
|
+
async function presidioRedactString(value, ctx, vault, sessionId, hits, itemIndex, path) {
|
|
625
|
+
var _a;
|
|
626
|
+
if (value.trim().length === 0 || TOKEN_RE.test(value))
|
|
627
|
+
return value;
|
|
628
|
+
const entities = await (0, presidio_1.analyzeWithPresidio)(ctx.presidioUrl, value, ctx.presidioLanguage || 'en', 0.4, 10000, ctx.presidioRequest);
|
|
629
|
+
const threshold = (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0;
|
|
630
|
+
const spans = resolveOverlaps(toUtf16Spans(value, entities).filter((s) => s.score >= threshold));
|
|
631
|
+
let text = value;
|
|
632
|
+
for (const span of spans) {
|
|
633
|
+
const matchedText = value.slice(span.start, span.end);
|
|
634
|
+
if (TOKEN_RE.test(matchedText) || (matchedText.startsWith('[') && matchedText.endsWith(']')))
|
|
635
|
+
continue;
|
|
636
|
+
if (isAllowListed(matchedText, ctx.allowList))
|
|
637
|
+
continue;
|
|
638
|
+
const label = (0, presidio_1.getRedactorLabel)(span.entityType);
|
|
639
|
+
const category = (0, presidio_1.getRedactorCategory)(span.entityType);
|
|
640
|
+
const replacement = applyRedaction(matchedText, { label, category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
641
|
+
text = text.slice(0, span.start) + replacement + text.slice(span.end);
|
|
642
|
+
hits.push({
|
|
643
|
+
token: replacement, original: '***',
|
|
644
|
+
patternName: `presidio_${span.entityType}`, patternLabel: label,
|
|
645
|
+
category, field: path, itemIndex, confidence: span.score,
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
return text;
|
|
649
|
+
}
|
|
650
|
+
async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path) {
|
|
578
651
|
if (typeof value === 'string') {
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
652
|
+
if (!shouldProcessField(path, ctx.fieldMode, ctx.fieldRules))
|
|
653
|
+
return value;
|
|
654
|
+
try {
|
|
655
|
+
return await presidioRedactString(value, ctx, vault, sessionId, hits, itemIndex, path);
|
|
656
|
+
}
|
|
657
|
+
catch {
|
|
658
|
+
return value;
|
|
582
659
|
}
|
|
583
|
-
return result;
|
|
584
660
|
}
|
|
585
661
|
if (Array.isArray(value)) {
|
|
586
|
-
|
|
662
|
+
const out = [];
|
|
663
|
+
for (let i = 0; i < value.length; i++)
|
|
664
|
+
out.push(await presidioWalk(value[i], ctx, vault, sessionId, hits, itemIndex, `${path}[${i}]`));
|
|
665
|
+
return out;
|
|
587
666
|
}
|
|
588
667
|
if (value !== null && typeof value === 'object') {
|
|
589
|
-
const
|
|
668
|
+
const out = {};
|
|
590
669
|
for (const [k, v] of Object.entries(value)) {
|
|
591
|
-
|
|
670
|
+
out[k] = await presidioWalk(v, ctx, vault, sessionId, hits, itemIndex, path ? `${path}.${k}` : k);
|
|
592
671
|
}
|
|
593
|
-
return
|
|
672
|
+
return out;
|
|
594
673
|
}
|
|
595
674
|
return value;
|
|
596
675
|
}
|
|
597
676
|
async function enhanceWithPresidio(data, ctx, vault, sessionId, hits, itemIndex) {
|
|
598
|
-
var _a;
|
|
599
677
|
if (!ctx.presidioUrl)
|
|
600
678
|
return data;
|
|
601
|
-
|
|
602
|
-
for (const [key, value] of Object.entries(result)) {
|
|
603
|
-
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
|
604
|
-
result[key] = await enhanceWithPresidio(value, ctx, vault, sessionId, hits, itemIndex);
|
|
605
|
-
continue;
|
|
606
|
-
}
|
|
607
|
-
if (Array.isArray(value)) {
|
|
608
|
-
const arr = [];
|
|
609
|
-
for (const item of value) {
|
|
610
|
-
if (item !== null && typeof item === 'object') {
|
|
611
|
-
arr.push(await enhanceWithPresidio(item, ctx, vault, sessionId, hits, itemIndex));
|
|
612
|
-
}
|
|
613
|
-
else {
|
|
614
|
-
arr.push(item);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
result[key] = arr;
|
|
618
|
-
continue;
|
|
619
|
-
}
|
|
620
|
-
if (typeof value !== 'string')
|
|
621
|
-
continue;
|
|
622
|
-
if (value.trim().length === 0)
|
|
623
|
-
continue;
|
|
624
|
-
if (TOKEN_RE.test(value))
|
|
625
|
-
continue;
|
|
626
|
-
try {
|
|
627
|
-
const entities = await (0, presidio_1.analyzeWithPresidio)(ctx.presidioUrl, value, ctx.presidioLanguage || 'en', 0.4, 10000, ctx.presidioRequest);
|
|
628
|
-
if (entities.length === 0)
|
|
629
|
-
continue;
|
|
630
|
-
const sorted = entities.sort((a, b) => b.start - a.start);
|
|
631
|
-
let text = value;
|
|
632
|
-
for (const entity of sorted) {
|
|
633
|
-
const matchedText = text.slice(entity.start, entity.end);
|
|
634
|
-
if (TOKEN_RE.test(matchedText) || (matchedText.startsWith('[') && matchedText.endsWith(']')))
|
|
635
|
-
continue;
|
|
636
|
-
if (isAllowListed(matchedText, ctx.allowList))
|
|
637
|
-
continue;
|
|
638
|
-
if (entity.score < ((_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0))
|
|
639
|
-
continue;
|
|
640
|
-
const label = (0, presidio_1.getRedactorLabel)(entity.entity_type);
|
|
641
|
-
const category = (0, presidio_1.getRedactorCategory)(entity.entity_type);
|
|
642
|
-
const replacement = applyRedaction(matchedText, { label, category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
643
|
-
text = text.slice(0, entity.start) + replacement + text.slice(entity.end);
|
|
644
|
-
hits.push({
|
|
645
|
-
token: replacement,
|
|
646
|
-
original: '***',
|
|
647
|
-
patternName: `presidio_${entity.entity_type}`,
|
|
648
|
-
patternLabel: label,
|
|
649
|
-
category,
|
|
650
|
-
field: key,
|
|
651
|
-
itemIndex,
|
|
652
|
-
confidence: entity.score,
|
|
653
|
-
});
|
|
654
|
-
}
|
|
655
|
-
result[key] = text;
|
|
656
|
-
}
|
|
657
|
-
catch {
|
|
658
|
-
continue;
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
return result;
|
|
679
|
+
return (await presidioWalk(data, ctx, vault, sessionId, hits, itemIndex, ''));
|
|
662
680
|
}
|
|
663
681
|
function buildReport(sessionId, hits) {
|
|
664
682
|
const hitsByCategory = {};
|
|
@@ -258,7 +258,7 @@ function detectPromptInjection(text) {
|
|
|
258
258
|
matches.push({
|
|
259
259
|
category: 'encoded_payload',
|
|
260
260
|
pattern: 'Base64-encoded injection payload detected',
|
|
261
|
-
matchedText: `
|
|
261
|
+
matchedText: `(decoded payload of ${decoded.length} characters withheld)`,
|
|
262
262
|
severity: 'HIGH',
|
|
263
263
|
score: 0.35,
|
|
264
264
|
});
|