n8n-nodes-redactor 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/nodes/Redactor/Redactor.node.js +18 -36
- package/dist/nodes/Redactor/engine.js +201 -140
- package/dist/nodes/Redactor/injection.js +6 -0
- package/dist/nodes/Redactor/regexSafety.js +7 -4
- package/dist/nodes/Redactor/runtime.js +28 -23
- package/dist/nodes/Redactor/vault.js +30 -0
- package/package.json +1 -1
|
@@ -1036,6 +1036,9 @@ class Redactor {
|
|
|
1036
1036
|
presidioUrl: enablePresidio ? presidioUrl : undefined,
|
|
1037
1037
|
presidioLanguage: enablePresidio ? presidioLanguage : undefined,
|
|
1038
1038
|
presidioRequest: enablePresidio ? presidioRequester : undefined,
|
|
1039
|
+
scanNumbers: true,
|
|
1040
|
+
reservedTokens: (0, engine_1.collectTokenLike)(items.map((it) => it.json)),
|
|
1041
|
+
keyOrigins: new WeakMap(),
|
|
1039
1042
|
};
|
|
1040
1043
|
const allHits = [];
|
|
1041
1044
|
const returnData = [];
|
|
@@ -1118,48 +1121,27 @@ class Redactor {
|
|
|
1118
1121
|
if (!purgeSessionId) {
|
|
1119
1122
|
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Session ID is required for purging a specific session.');
|
|
1120
1123
|
}
|
|
1121
|
-
const
|
|
1122
|
-
if (session) {
|
|
1123
|
-
const entryCount = Object.keys(session.entries).length;
|
|
1124
|
-
vault.deleteSession(purgeSessionId);
|
|
1125
|
-
return [[{
|
|
1126
|
-
json: {
|
|
1127
|
-
purged: true,
|
|
1128
|
-
sessionId: purgeSessionId,
|
|
1129
|
-
entriesDeleted: entryCount,
|
|
1130
|
-
message: `Session purged successfully.`,
|
|
1131
|
-
},
|
|
1132
|
-
pairedItem: allPaired,
|
|
1133
|
-
}]];
|
|
1134
|
-
}
|
|
1135
|
-
else {
|
|
1136
|
-
return [[{
|
|
1137
|
-
json: {
|
|
1138
|
-
purged: false,
|
|
1139
|
-
sessionId: purgeSessionId,
|
|
1140
|
-
message: `No session found with this ID.`,
|
|
1141
|
-
},
|
|
1142
|
-
pairedItem: allPaired,
|
|
1143
|
-
}]];
|
|
1144
|
-
}
|
|
1145
|
-
}
|
|
1146
|
-
else {
|
|
1147
|
-
const sessions = vault.listSessions();
|
|
1148
|
-
let totalEntries = 0;
|
|
1149
|
-
for (const s of sessions) {
|
|
1150
|
-
totalEntries += s.entryCount;
|
|
1151
|
-
vault.deleteSession(s.sessionId);
|
|
1152
|
-
}
|
|
1124
|
+
const purged = vault.purgeSession(purgeSessionId);
|
|
1153
1125
|
return [[{
|
|
1154
1126
|
json: {
|
|
1155
|
-
purged:
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
message:
|
|
1127
|
+
purged: purged.found,
|
|
1128
|
+
sessionId: purgeSessionId,
|
|
1129
|
+
...(purged.found ? { entriesDeleted: purged.entries } : {}),
|
|
1130
|
+
message: purged.found ? 'Session purged successfully.' : 'No session found with this ID.',
|
|
1159
1131
|
},
|
|
1160
1132
|
pairedItem: allPaired,
|
|
1161
1133
|
}]];
|
|
1162
1134
|
}
|
|
1135
|
+
const all = vault.purgeAll();
|
|
1136
|
+
return [[{
|
|
1137
|
+
json: {
|
|
1138
|
+
purged: true,
|
|
1139
|
+
sessionsPurged: all.sessions,
|
|
1140
|
+
totalEntriesDeleted: all.entries,
|
|
1141
|
+
message: `All ${all.sessions} session(s) purged successfully.`,
|
|
1142
|
+
},
|
|
1143
|
+
pairedItem: allPaired,
|
|
1144
|
+
}]];
|
|
1163
1145
|
}
|
|
1164
1146
|
return [[]];
|
|
1165
1147
|
}
|
|
@@ -33,6 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.collectTokenLike = collectTokenLike;
|
|
36
37
|
exports.redactValue = redactValue;
|
|
37
38
|
exports.restoreValue = restoreValue;
|
|
38
39
|
exports.enhanceWithPresidio = enhanceWithPresidio;
|
|
@@ -99,7 +100,7 @@ function hashValue(value) {
|
|
|
99
100
|
function storeReversible(match, pattern, vault, sessionId, dedup, makeToken) {
|
|
100
101
|
if (dedup) {
|
|
101
102
|
const existing = vault.findByOriginal(sessionId, match);
|
|
102
|
-
if (existing)
|
|
103
|
+
if (existing && !existing.numeric)
|
|
103
104
|
return existing.token;
|
|
104
105
|
}
|
|
105
106
|
const token = makeToken();
|
|
@@ -119,12 +120,9 @@ const IRREVERSIBLE = {
|
|
|
119
120
|
blackout: (match) => '\u2588'.repeat(match.length),
|
|
120
121
|
remove: () => '',
|
|
121
122
|
};
|
|
122
|
-
function applyRedaction(match, pattern, mode, vault, sessionId, dedup) {
|
|
123
|
+
function applyRedaction(match, pattern, mode, vault, sessionId, dedup, reserved) {
|
|
123
124
|
if (mode === 'token') {
|
|
124
|
-
return storeReversible(match, pattern, vault, sessionId, dedup, () =>
|
|
125
|
-
const session = vault.getSession(sessionId);
|
|
126
|
-
return generateToken(pattern.label, session ? Object.keys(session.entries).length : 0);
|
|
127
|
-
});
|
|
125
|
+
return storeReversible(match, pattern, vault, sessionId, dedup, () => freshToken(pattern.label, vault, sessionId, reserved));
|
|
128
126
|
}
|
|
129
127
|
if (mode === 'pseudonymize') {
|
|
130
128
|
return storeReversible(match, pattern, vault, sessionId, dedup, () => uniquePseudonym(vault, sessionId, match, pattern.label));
|
|
@@ -132,6 +130,33 @@ function applyRedaction(match, pattern, mode, vault, sessionId, dedup) {
|
|
|
132
130
|
const irreversible = IRREVERSIBLE[mode];
|
|
133
131
|
return irreversible ? irreversible(match, pattern.label) : match;
|
|
134
132
|
}
|
|
133
|
+
function freshToken(label, vault, sessionId, reserved) {
|
|
134
|
+
var _a, _b;
|
|
135
|
+
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
136
|
+
let index = Object.keys(entries).length;
|
|
137
|
+
let token = generateToken(label, index);
|
|
138
|
+
while (entries[token] || (reserved === null || reserved === void 0 ? void 0 : reserved.has(token)))
|
|
139
|
+
token = generateToken(label, ++index);
|
|
140
|
+
return token;
|
|
141
|
+
}
|
|
142
|
+
const TOKEN_LIKE_RE = /\[[A-Z][A-Z0-9_]*_\d+\]/g;
|
|
143
|
+
function collectTokenLike(value, into = new Set()) {
|
|
144
|
+
const scan = (text) => {
|
|
145
|
+
for (const m of text.matchAll(TOKEN_LIKE_RE))
|
|
146
|
+
into.add(m[0]);
|
|
147
|
+
};
|
|
148
|
+
if (typeof value === 'string')
|
|
149
|
+
scan(value);
|
|
150
|
+
else if (Array.isArray(value))
|
|
151
|
+
value.forEach((v) => collectTokenLike(v, into));
|
|
152
|
+
else if (value !== null && typeof value === 'object') {
|
|
153
|
+
for (const [k, v] of Object.entries(value)) {
|
|
154
|
+
scan(k);
|
|
155
|
+
collectTokenLike(v, into);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return into;
|
|
159
|
+
}
|
|
135
160
|
function uniquePseudonym(vault, sessionId, original, label) {
|
|
136
161
|
var _a, _b;
|
|
137
162
|
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
@@ -139,6 +164,8 @@ function uniquePseudonym(vault, sessionId, original, label) {
|
|
|
139
164
|
const seed = attempt === 0 ? original : `${original}\u0000${attempt}`;
|
|
140
165
|
const candidate = (0, pseudonymize_1.generatePseudonym)(sessionId, seed, label);
|
|
141
166
|
const taken = entries[candidate];
|
|
167
|
+
if (candidate === original)
|
|
168
|
+
continue;
|
|
142
169
|
if (!taken || taken.original === original)
|
|
143
170
|
return candidate;
|
|
144
171
|
}
|
|
@@ -385,7 +412,7 @@ function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
385
412
|
const semConf = semantic.label === 'IDENTIFIER' ? context_1.AMBIGUOUS_FIELD_CONFIDENCE : context_1.SEMANTIC_CONFIDENCE;
|
|
386
413
|
if (semConf < ((_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0))
|
|
387
414
|
return null;
|
|
388
|
-
const replacement = applyRedaction(raw, { label: semantic.label, category: semantic.category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
415
|
+
const replacement = applyRedaction(raw, { label: semantic.label, category: semantic.category }, ctx.mode, vault, sessionId, ctx.dedup, ctx.reservedTokens);
|
|
389
416
|
hits.push({
|
|
390
417
|
token: replacement, original: '***',
|
|
391
418
|
patternName: `semantic_${semantic.label}`,
|
|
@@ -397,41 +424,59 @@ function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
397
424
|
return replacement;
|
|
398
425
|
}
|
|
399
426
|
function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
427
|
+
var _a;
|
|
400
428
|
const text = String(value);
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
429
|
+
const numCtx = { ...ctx, dedup: false };
|
|
430
|
+
let out = null;
|
|
431
|
+
if (!ctx.skipSemantic)
|
|
432
|
+
out = redactSemantic(text, numCtx, vault, sessionId, hits, itemIndex, path, display, siblingKeys);
|
|
433
|
+
if (out === null && ctx.scanNumbers)
|
|
434
|
+
out = redactText(text, numCtx, vault, sessionId, hits, itemIndex, display);
|
|
435
|
+
if (out === null || out === text)
|
|
407
436
|
return value;
|
|
408
|
-
const
|
|
409
|
-
|
|
437
|
+
const entry = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries[out];
|
|
438
|
+
if (entry && entry.original === text)
|
|
439
|
+
entry.numeric = true;
|
|
440
|
+
return out;
|
|
410
441
|
}
|
|
411
|
-
function redactKey(key, taken, ctx, vault, sessionId, itemIndex, keyHits) {
|
|
412
|
-
|
|
442
|
+
function redactKey(key, taken, sourceKeys, ctx, vault, sessionId, itemIndex, keyHits) {
|
|
443
|
+
var _a;
|
|
444
|
+
const out = redactText(key, ctx, vault, sessionId, keyHits, itemIndex, '');
|
|
413
445
|
if (out === key)
|
|
414
446
|
return key;
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
out
|
|
418
|
-
|
|
447
|
+
const clashes = (k) => k === '' || k in taken || sourceKeys.has(k);
|
|
448
|
+
if (!clashes(out))
|
|
449
|
+
return out;
|
|
450
|
+
if (ctx.mode === 'token' || ctx.mode === 'pseudonymize') {
|
|
451
|
+
const token = freshToken('KEY', vault, sessionId, new Set([...((_a = ctx.reservedTokens) !== null && _a !== void 0 ? _a : []), ...sourceKeys, ...Object.keys(taken)]));
|
|
452
|
+
vault.addEntry(sessionId, { token, original: key, patternLabel: 'KEY', category: 'other', createdAt: new Date().toISOString() });
|
|
453
|
+
return token;
|
|
454
|
+
}
|
|
455
|
+
let suffixed = out;
|
|
456
|
+
for (let n = 1; clashes(suffixed); n++)
|
|
457
|
+
suffixed = `${out}_${n}`;
|
|
458
|
+
return suffixed;
|
|
419
459
|
}
|
|
420
460
|
function redactObject(value, ctx, vault, sessionId, hits, itemIndex, path, display) {
|
|
461
|
+
var _a;
|
|
421
462
|
const entries = Object.entries(value);
|
|
422
463
|
const siblingKeys = entries.map(([k]) => k);
|
|
464
|
+
const sourceKeys = new Set(siblingKeys);
|
|
465
|
+
const origins = new Map();
|
|
423
466
|
const obj = {};
|
|
424
467
|
for (const [k, v] of entries) {
|
|
425
468
|
const fieldPath = path ? `${path}.${k}` : k;
|
|
426
469
|
const keyHits = [];
|
|
427
470
|
const outKey = shouldProcessField(fieldPath, ctx.fieldMode, ctx.fieldRules)
|
|
428
|
-
? redactKey(k, obj, ctx, vault, sessionId, itemIndex, keyHits)
|
|
471
|
+
? redactKey(k, obj, sourceKeys, ctx, vault, sessionId, itemIndex, keyHits)
|
|
429
472
|
: k;
|
|
473
|
+
origins.set(outKey, k);
|
|
430
474
|
const childDisplay = display ? `${display}.${outKey}` : outKey;
|
|
431
475
|
for (const h of keyHits)
|
|
432
476
|
hits.push({ ...h, field: childDisplay });
|
|
433
477
|
obj[outKey] = redactValue(v, ctx, vault, sessionId, hits, itemIndex, fieldPath, siblingKeys, childDisplay);
|
|
434
478
|
}
|
|
479
|
+
(_a = ctx.keyOrigins) === null || _a === void 0 ? void 0 : _a.set(obj, origins);
|
|
435
480
|
return obj;
|
|
436
481
|
}
|
|
437
482
|
function redactValue(value, ctx, vault, sessionId, hits, itemIndex, currentPath = '', siblingKeys, displayPath = currentPath) {
|
|
@@ -459,123 +504,129 @@ function redactValue(value, ctx, vault, sessionId, hits, itemIndex, currentPath
|
|
|
459
504
|
}
|
|
460
505
|
return value;
|
|
461
506
|
}
|
|
462
|
-
|
|
507
|
+
const escapeRe = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
508
|
+
function segments(text, produced) {
|
|
463
509
|
var _a;
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
break;
|
|
476
|
-
case 'contains':
|
|
477
|
-
re = new RegExp(entry.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
|
|
478
|
-
break;
|
|
479
|
-
default:
|
|
480
|
-
re = new RegExp(entry.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'gi');
|
|
481
|
-
}
|
|
482
|
-
result = result.replace(re, (match) => {
|
|
483
|
-
if (TOKEN_RE.test(match))
|
|
484
|
-
return match;
|
|
485
|
-
const replacement = applyRedaction(match, { label: 'DENY_LIST', category: 'other' }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
486
|
-
hits.push({
|
|
487
|
-
token: replacement, original: '***',
|
|
488
|
-
patternName: 'denyList', patternLabel: 'DENY_LIST',
|
|
489
|
-
category: 'other', field: fieldPath, itemIndex,
|
|
490
|
-
confidence: context_1.DENY_LIST_CONFIDENCE,
|
|
491
|
-
});
|
|
492
|
-
return replacement;
|
|
493
|
-
});
|
|
494
|
-
}
|
|
495
|
-
catch { }
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
const enabled = (0, patterns_1.getPatternsByNames)(ctx.enabledPatterns);
|
|
499
|
-
const activePatterns = [...enabled.filter((p) => p.validate), ...enabled.filter((p) => !p.validate)];
|
|
500
|
-
for (const pattern of activePatterns) {
|
|
501
|
-
const re = new RegExp(pattern.regex.source, pattern.regex.flags);
|
|
502
|
-
result = result.replace(re, (match, ...args) => {
|
|
503
|
-
if (TOKEN_RE.test(match) || (match.startsWith('[') && match.endsWith(']')))
|
|
504
|
-
return match;
|
|
505
|
-
const validatorPassed = pattern.validate ? pattern.validate(match) : true;
|
|
506
|
-
if (pattern.validate && !validatorPassed)
|
|
507
|
-
return match;
|
|
508
|
-
if (isAllowListed(match, ctx.allowList))
|
|
509
|
-
return match;
|
|
510
|
-
const matchOffset = typeof args[args.length - 2] === 'number' ? args[args.length - 2] : 0;
|
|
511
|
-
const confidence = (0, context_1.calculateConfidence)(pattern.name, !!pattern.validate, validatorPassed, text, matchOffset, matchOffset + match.length);
|
|
512
|
-
if (confidence < threshold)
|
|
513
|
-
return match;
|
|
514
|
-
const replacement = applyRedaction(match, pattern, ctx.mode, vault, sessionId, ctx.dedup);
|
|
515
|
-
hits.push({
|
|
516
|
-
token: replacement, original: '***',
|
|
517
|
-
patternName: pattern.name, patternLabel: pattern.label,
|
|
518
|
-
category: pattern.category, field: fieldPath, itemIndex,
|
|
519
|
-
confidence,
|
|
520
|
-
});
|
|
521
|
-
return replacement;
|
|
522
|
-
});
|
|
510
|
+
if (produced.size === 0)
|
|
511
|
+
return [{ text, locked: false }];
|
|
512
|
+
const guard = new RegExp([...produced].sort((a, b) => b.length - a.length).map(escapeRe).join('|'), 'g');
|
|
513
|
+
const out = [];
|
|
514
|
+
let last = 0;
|
|
515
|
+
for (const m of text.matchAll(guard)) {
|
|
516
|
+
const at = (_a = m.index) !== null && _a !== void 0 ? _a : 0;
|
|
517
|
+
if (at > last)
|
|
518
|
+
out.push({ text: text.slice(last, at), locked: false });
|
|
519
|
+
out.push({ text: m[0], locked: true });
|
|
520
|
+
last = at + m[0].length;
|
|
523
521
|
}
|
|
524
|
-
if (
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
522
|
+
if (last < text.length)
|
|
523
|
+
out.push({ text: text.slice(last), locked: false });
|
|
524
|
+
return out;
|
|
525
|
+
}
|
|
526
|
+
function replaceOutside(text, produced, re, fn) {
|
|
527
|
+
return segments(text, produced)
|
|
528
|
+
.map((seg) => (seg.locked ? seg.text : seg.text.replace(new RegExp(re.source, re.flags), fn)))
|
|
529
|
+
.join('');
|
|
530
|
+
}
|
|
531
|
+
function occursOutside(text, produced, needle) {
|
|
532
|
+
return segments(text, produced).some((seg) => !seg.locked && seg.text.includes(needle));
|
|
533
|
+
}
|
|
534
|
+
function redactMatch(pass, match, pattern, detail) {
|
|
535
|
+
const { ctx, vault, sessionId } = pass;
|
|
536
|
+
const replacement = applyRedaction(match, pattern, ctx.mode, vault, sessionId, ctx.dedup, ctx.reservedTokens);
|
|
537
|
+
pass.hits.push({ token: replacement, original: '***', field: pass.fieldPath, itemIndex: pass.itemIndex, ...detail });
|
|
538
|
+
if (replacement)
|
|
539
|
+
pass.produced.add(replacement);
|
|
540
|
+
return replacement;
|
|
541
|
+
}
|
|
542
|
+
function denyListRegex(entry) {
|
|
543
|
+
if (entry.type === 'regex')
|
|
544
|
+
return isSafeRegex(entry.value) ? new RegExp(entry.value, 'gi') : null;
|
|
545
|
+
return new RegExp(escapeRe(entry.value), 'gi');
|
|
546
|
+
}
|
|
547
|
+
function denyListStage(pass) {
|
|
548
|
+
var _a;
|
|
549
|
+
for (const entry of (_a = pass.ctx.denyList) !== null && _a !== void 0 ? _a : []) {
|
|
550
|
+
try {
|
|
551
|
+
const re = denyListRegex(entry);
|
|
552
|
+
if (!re)
|
|
536
553
|
continue;
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
category: 'identity', field: fieldPath, itemIndex,
|
|
543
|
-
confidence: nameConfidence,
|
|
544
|
-
});
|
|
554
|
+
pass.result = replaceOutside(pass.result, pass.produced, re, (match) => TOKEN_RE.test(match)
|
|
555
|
+
? match
|
|
556
|
+
: redactMatch(pass, match, { label: 'DENY_LIST', category: 'other' }, {
|
|
557
|
+
patternName: 'denyList', patternLabel: 'DENY_LIST', category: 'other', confidence: context_1.DENY_LIST_CONFIDENCE,
|
|
558
|
+
}));
|
|
545
559
|
}
|
|
560
|
+
catch { }
|
|
546
561
|
}
|
|
547
|
-
|
|
548
|
-
|
|
562
|
+
}
|
|
563
|
+
function builtInMatch(pass, pattern, match, args) {
|
|
564
|
+
if (TOKEN_RE.test(match) || (match.startsWith('[') && match.endsWith(']')))
|
|
565
|
+
return match;
|
|
566
|
+
const validatorPassed = pattern.validate ? pattern.validate(match) : true;
|
|
567
|
+
if (!validatorPassed || isAllowListed(match, pass.ctx.allowList))
|
|
568
|
+
return match;
|
|
569
|
+
const offsetArg = args[args.length - 2];
|
|
570
|
+
const offset = typeof offsetArg === 'number' ? offsetArg : 0;
|
|
571
|
+
const confidence = (0, context_1.calculateConfidence)(pattern.name, !!pattern.validate, validatorPassed, pass.source, offset, offset + match.length);
|
|
572
|
+
if (confidence < pass.threshold)
|
|
573
|
+
return match;
|
|
574
|
+
return redactMatch(pass, match, pattern, {
|
|
575
|
+
patternName: pattern.name, patternLabel: pattern.label, category: pattern.category, confidence,
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
function builtInStage(pass) {
|
|
579
|
+
const enabled = (0, patterns_1.getPatternsByNames)(pass.ctx.enabledPatterns);
|
|
580
|
+
for (const pattern of [...enabled.filter((p) => p.validate), ...enabled.filter((p) => !p.validate)]) {
|
|
581
|
+
const re = new RegExp(pattern.regex.source, pattern.regex.flags);
|
|
582
|
+
pass.result = replaceOutside(pass.result, pass.produced, re, (match, ...args) => builtInMatch(pass, pattern, match, args));
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
function nameStage(pass) {
|
|
586
|
+
if (pass.ctx.skipSemantic)
|
|
587
|
+
return;
|
|
588
|
+
for (const nm of (0, names_1.detectNamesInText)(pass.result)) {
|
|
589
|
+
if ((nm.name.includes('[') && nm.name.includes(']')) || isAllowListed(nm.name, pass.ctx.allowList))
|
|
549
590
|
continue;
|
|
550
|
-
|
|
591
|
+
const confidence = (0, context_1.hasContextWords)(pass.source, nm.start, nm.end, names_1.NAME_CONTEXT_WORDS) ? 0.8 : 0.65;
|
|
592
|
+
if (confidence < pass.threshold || !occursOutside(pass.result, pass.produced, nm.name))
|
|
551
593
|
continue;
|
|
594
|
+
const before = new Set(pass.produced);
|
|
595
|
+
const replacement = redactMatch(pass, nm.name, { label: 'PERSON_NAME', category: 'identity' }, {
|
|
596
|
+
patternName: 'personNameDict', patternLabel: 'PERSON_NAME', category: 'identity', confidence,
|
|
597
|
+
});
|
|
598
|
+
pass.result = replaceOutside(pass.result, before, new RegExp(escapeRe(nm.name), 'g'), () => replacement);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
function customStage(pass) {
|
|
602
|
+
if (context_1.CUSTOM_PATTERN_CONFIDENCE < pass.threshold)
|
|
603
|
+
return;
|
|
604
|
+
for (const cp of pass.ctx.customPatterns) {
|
|
605
|
+
if (!cp.label || !cp.regex || !isSafeRegex(cp.regex))
|
|
606
|
+
continue;
|
|
607
|
+
const pattern = { label: cp.label.toUpperCase(), category: cp.category || 'identity' };
|
|
552
608
|
try {
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
if (isAllowListed(match, ctx.allowList))
|
|
558
|
-
return match;
|
|
559
|
-
if (context_1.CUSTOM_PATTERN_CONFIDENCE < threshold)
|
|
560
|
-
return match;
|
|
561
|
-
const pseudoPattern = {
|
|
562
|
-
label: cp.label.toUpperCase(),
|
|
563
|
-
category: cp.category || 'identity',
|
|
564
|
-
};
|
|
565
|
-
const replacement = applyRedaction(match, pseudoPattern, ctx.mode, vault, sessionId, ctx.dedup);
|
|
566
|
-
hits.push({
|
|
567
|
-
token: replacement, original: '***',
|
|
568
|
-
patternName: `custom_${cp.label}`, patternLabel: pseudoPattern.label,
|
|
569
|
-
category: pseudoPattern.category, field: fieldPath, itemIndex,
|
|
609
|
+
pass.result = replaceOutside(pass.result, pass.produced, new RegExp(cp.regex, 'g'), (match) => TOKEN_RE.test(match) || isAllowListed(match, pass.ctx.allowList)
|
|
610
|
+
? match
|
|
611
|
+
: redactMatch(pass, match, pattern, {
|
|
612
|
+
patternName: `custom_${cp.label}`, patternLabel: pattern.label, category: pattern.category,
|
|
570
613
|
confidence: context_1.CUSTOM_PATTERN_CONFIDENCE,
|
|
571
|
-
});
|
|
572
|
-
return replacement;
|
|
573
|
-
});
|
|
574
|
-
}
|
|
575
|
-
catch {
|
|
614
|
+
}));
|
|
576
615
|
}
|
|
616
|
+
catch { }
|
|
577
617
|
}
|
|
578
|
-
|
|
618
|
+
}
|
|
619
|
+
function redactText(text, ctx, vault, sessionId, hits, itemIndex, fieldPath) {
|
|
620
|
+
var _a;
|
|
621
|
+
const pass = {
|
|
622
|
+
source: text, result: text, produced: new Set(), threshold: (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0,
|
|
623
|
+
ctx, vault, sessionId, hits, itemIndex, fieldPath,
|
|
624
|
+
};
|
|
625
|
+
denyListStage(pass);
|
|
626
|
+
builtInStage(pass);
|
|
627
|
+
nameStage(pass);
|
|
628
|
+
customStage(pass);
|
|
629
|
+
return pass.result;
|
|
579
630
|
}
|
|
580
631
|
function makeRestorer(entries) {
|
|
581
632
|
const tokens = Object.keys(entries).filter((t) => t.length > 0).sort((a, b) => b.length - a.length);
|
|
@@ -584,15 +635,19 @@ function makeRestorer(entries) {
|
|
|
584
635
|
const re = new RegExp(tokens.map((t) => t.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'), 'g');
|
|
585
636
|
return (text) => text.replace(re, (token) => entries[token].original);
|
|
586
637
|
}
|
|
587
|
-
function restoreWith(value, restore) {
|
|
588
|
-
if (typeof value === 'string')
|
|
638
|
+
function restoreWith(value, restore, entries) {
|
|
639
|
+
if (typeof value === 'string') {
|
|
640
|
+
const exact = entries[value];
|
|
641
|
+
if ((exact === null || exact === void 0 ? void 0 : exact.numeric) && Number.isFinite(Number(exact.original)))
|
|
642
|
+
return Number(exact.original);
|
|
589
643
|
return restore(value);
|
|
644
|
+
}
|
|
590
645
|
if (Array.isArray(value))
|
|
591
|
-
return value.map((v) => restoreWith(v, restore));
|
|
646
|
+
return value.map((v) => restoreWith(v, restore, entries));
|
|
592
647
|
if (value !== null && typeof value === 'object') {
|
|
593
648
|
const obj = {};
|
|
594
649
|
for (const [k, v] of Object.entries(value)) {
|
|
595
|
-
obj[restore(k)] = restoreWith(v, restore);
|
|
650
|
+
obj[restore(k)] = restoreWith(v, restore, entries);
|
|
596
651
|
}
|
|
597
652
|
return obj;
|
|
598
653
|
}
|
|
@@ -602,7 +657,7 @@ function restoreValue(value, vault, sessionId) {
|
|
|
602
657
|
const session = vault.getSession(sessionId);
|
|
603
658
|
if (!session)
|
|
604
659
|
return value;
|
|
605
|
-
return restoreWith(value, makeRestorer(session.entries));
|
|
660
|
+
return restoreWith(value, makeRestorer(session.entries), session.entries);
|
|
606
661
|
}
|
|
607
662
|
function toUtf16Spans(text, entities) {
|
|
608
663
|
const offsets = [0];
|
|
@@ -637,7 +692,7 @@ async function presidioRedactString(value, ctx, vault, sessionId, hits, itemInde
|
|
|
637
692
|
continue;
|
|
638
693
|
const label = (0, presidio_1.getRedactorLabel)(span.entityType);
|
|
639
694
|
const category = (0, presidio_1.getRedactorCategory)(span.entityType);
|
|
640
|
-
const replacement = applyRedaction(matchedText, { label, category }, ctx.mode, vault, sessionId, ctx.dedup);
|
|
695
|
+
const replacement = applyRedaction(matchedText, { label, category }, ctx.mode, vault, sessionId, ctx.dedup, ctx.reservedTokens);
|
|
641
696
|
text = text.slice(0, span.start) + replacement + text.slice(span.end);
|
|
642
697
|
hits.push({
|
|
643
698
|
token: replacement, original: '***',
|
|
@@ -647,9 +702,10 @@ async function presidioRedactString(value, ctx, vault, sessionId, hits, itemInde
|
|
|
647
702
|
}
|
|
648
703
|
return text;
|
|
649
704
|
}
|
|
650
|
-
async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path) {
|
|
705
|
+
async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path, policyPath) {
|
|
706
|
+
var _a, _b;
|
|
651
707
|
if (typeof value === 'string') {
|
|
652
|
-
if (!shouldProcessField(
|
|
708
|
+
if (!shouldProcessField(policyPath, ctx.fieldMode, ctx.fieldRules))
|
|
653
709
|
return value;
|
|
654
710
|
try {
|
|
655
711
|
return await presidioRedactString(value, ctx, vault, sessionId, hits, itemIndex, path);
|
|
@@ -660,14 +716,19 @@ async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path)
|
|
|
660
716
|
}
|
|
661
717
|
if (Array.isArray(value)) {
|
|
662
718
|
const out = [];
|
|
663
|
-
for (let i = 0; i < value.length; i++)
|
|
664
|
-
out.push(await presidioWalk(value[i], ctx, vault, sessionId, hits, itemIndex, `${path}[${i}]`));
|
|
719
|
+
for (let i = 0; i < value.length; i++) {
|
|
720
|
+
out.push(await presidioWalk(value[i], ctx, vault, sessionId, hits, itemIndex, `${path}[${i}]`, `${policyPath}[${i}]`));
|
|
721
|
+
}
|
|
665
722
|
return out;
|
|
666
723
|
}
|
|
667
724
|
if (value !== null && typeof value === 'object') {
|
|
725
|
+
const origins = (_a = ctx.keyOrigins) === null || _a === void 0 ? void 0 : _a.get(value);
|
|
668
726
|
const out = {};
|
|
669
727
|
for (const [k, v] of Object.entries(value)) {
|
|
670
|
-
|
|
728
|
+
const original = (_b = origins === null || origins === void 0 ? void 0 : origins.get(k)) !== null && _b !== void 0 ? _b : k;
|
|
729
|
+
const child = path ? `${path}.${k}` : k;
|
|
730
|
+
const childPolicy = policyPath ? `${policyPath}.${original}` : original;
|
|
731
|
+
out[k] = await presidioWalk(v, ctx, vault, sessionId, hits, itemIndex, child, childPolicy);
|
|
671
732
|
}
|
|
672
733
|
return out;
|
|
673
734
|
}
|
|
@@ -676,7 +737,7 @@ async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path)
|
|
|
676
737
|
async function enhanceWithPresidio(data, ctx, vault, sessionId, hits, itemIndex) {
|
|
677
738
|
if (!ctx.presidioUrl)
|
|
678
739
|
return data;
|
|
679
|
-
return (await presidioWalk(data, ctx, vault, sessionId, hits, itemIndex, ''));
|
|
740
|
+
return (await presidioWalk(data, ctx, vault, sessionId, hits, itemIndex, '', ''));
|
|
680
741
|
}
|
|
681
742
|
function buildReport(sessionId, hits) {
|
|
682
743
|
const hitsByCategory = {};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.INJECTION_DESCRIPTIONS = void 0;
|
|
3
4
|
exports.detectPromptInjection = detectPromptInjection;
|
|
4
5
|
exports.scanForInjection = scanForInjection;
|
|
5
6
|
const INJECTION_PATTERNS = [
|
|
@@ -200,6 +201,11 @@ const INJECTION_PATTERNS = [
|
|
|
200
201
|
description: 'Fake end-of-prompt marker',
|
|
201
202
|
},
|
|
202
203
|
];
|
|
204
|
+
exports.INJECTION_DESCRIPTIONS = [
|
|
205
|
+
...INJECTION_PATTERNS.map((p) => p.description),
|
|
206
|
+
'Invisible Unicode characters detected',
|
|
207
|
+
'Base64-encoded injection payload detected',
|
|
208
|
+
];
|
|
203
209
|
const ZERO_WIDTH_RE = /(?:\u200B|\u200C|\u200D|\uFEFF|\u00AD|\u2060|\u2061|\u2062|\u2063|\u2064|\u2066|\u2067|\u2068|\u2069|\u206A|\u206B|\u206C|\u206D|\u206E|\u206F)/g;
|
|
204
210
|
const BASE64_SUSPICIOUS_RE = /[A-Za-z0-9+/]{40,}={0,2}/g;
|
|
205
211
|
const INJECTION_SEEDS = [
|
|
@@ -69,12 +69,15 @@ const NESTED = 'a repeated group contains a quantifier or alternation (catastrop
|
|
|
69
69
|
function isRisky(frame) {
|
|
70
70
|
return !!frame && (frame.hasVariableQuantifier || frame.hasAlternation);
|
|
71
71
|
}
|
|
72
|
+
const NO_GROUP = { hasVariableQuantifier: false, hasAlternation: false };
|
|
73
|
+
const repeats = (q) => q !== null && q.max > 1;
|
|
74
|
+
const isVariable = (q) => q !== null && q.min !== q.max;
|
|
72
75
|
function applyQuantifier(q, closed, top) {
|
|
73
|
-
|
|
74
|
-
if (repeats && isRisky(closed))
|
|
76
|
+
if (repeats(q) && isRisky(closed))
|
|
75
77
|
return NESTED;
|
|
76
|
-
const
|
|
77
|
-
top.hasVariableQuantifier
|
|
78
|
+
const inner = closed !== null && closed !== void 0 ? closed : NO_GROUP;
|
|
79
|
+
top.hasVariableQuantifier || (top.hasVariableQuantifier = isVariable(q) || inner.hasVariableQuantifier);
|
|
80
|
+
top.hasAlternation || (top.hasAlternation = inner.hasAlternation);
|
|
78
81
|
return null;
|
|
79
82
|
}
|
|
80
83
|
function structuralProblem(src) {
|
|
@@ -5,18 +5,27 @@ exports.workflowNamespace = workflowNamespace;
|
|
|
5
5
|
exports.assertUserRegexes = assertUserRegexes;
|
|
6
6
|
exports.sanitizeInjection = sanitizeInjection;
|
|
7
7
|
const n8n_workflow_1 = require("n8n-workflow");
|
|
8
|
-
const patterns_1 = require("./patterns");
|
|
9
|
-
const vault_1 = require("./vault");
|
|
10
|
-
const engine_1 = require("./engine");
|
|
11
8
|
const regexSafety_1 = require("./regexSafety");
|
|
9
|
+
const injection_1 = require("./injection");
|
|
12
10
|
const VAULT_CREDENTIAL = 'redactorVaultApi';
|
|
13
11
|
async function resolvePassphrase(fns) {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
12
|
+
var _a;
|
|
13
|
+
if ((_a = fns.getNode().credentials) === null || _a === void 0 ? void 0 : _a[VAULT_CREDENTIAL]) {
|
|
14
|
+
let cred;
|
|
15
|
+
try {
|
|
16
|
+
cred = (await fns.getCredentials(VAULT_CREDENTIAL));
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), 'The Redactor Vault credential could not be read, so the vault stays locked.', {
|
|
20
|
+
description: `Nothing was stored. Re-enter the passphrase in the credential. Cause: ${error.message}`,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
if (typeof cred.passphrase !== 'string' || cred.passphrase === '') {
|
|
24
|
+
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), 'The Redactor Vault credential has an empty passphrase.', {
|
|
25
|
+
description: 'Nothing was stored. Set a passphrase in the credential, or remove the credential from the node.',
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
return cred.passphrase;
|
|
20
29
|
}
|
|
21
30
|
try {
|
|
22
31
|
return fns.getNodeParameter('vaultPassphrase', 0, '') || '';
|
|
@@ -57,19 +66,15 @@ function assertUserRegexes(fns, customPatterns, lists) {
|
|
|
57
66
|
}
|
|
58
67
|
}
|
|
59
68
|
function sanitizeInjection(result) {
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
};
|
|
71
|
-
const scrub = (text) => (0, engine_1.redactValue)(text, ctx, vault, sessionId, [], 0);
|
|
72
|
-
const matchedPatterns = result.matchedPatterns.map((m) => ({ ...m, pattern: scrub(m.pattern), matchedText: scrub(m.matchedText) }));
|
|
73
|
-
vault.deleteSession(sessionId);
|
|
69
|
+
const matchedPatterns = result.matchedPatterns.map((m) => {
|
|
70
|
+
var _a;
|
|
71
|
+
return ({
|
|
72
|
+
category: m.category,
|
|
73
|
+
pattern: (_a = injection_1.INJECTION_DESCRIPTIONS.find((d) => m.pattern === d || m.pattern.endsWith(` ${d}`))) !== null && _a !== void 0 ? _a : 'Prompt injection detector',
|
|
74
|
+
matchedText: '(withheld)',
|
|
75
|
+
severity: m.severity,
|
|
76
|
+
score: m.score,
|
|
77
|
+
});
|
|
78
|
+
});
|
|
74
79
|
return { ...result, matchedPatterns };
|
|
75
80
|
}
|
|
@@ -124,6 +124,17 @@ class MemoryVault {
|
|
|
124
124
|
deleteSession(sessionId) {
|
|
125
125
|
this.drop(this.key(sessionId));
|
|
126
126
|
}
|
|
127
|
+
purgeAll() {
|
|
128
|
+
const keys = this.ownKeys();
|
|
129
|
+
const entries = keys.reduce((n, k) => { var _a, _b; return n + Object.keys((_b = (_a = memoryStore.get(k)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {}).length; }, 0);
|
|
130
|
+
keys.forEach((k) => this.drop(k));
|
|
131
|
+
return { sessions: keys.length, entries };
|
|
132
|
+
}
|
|
133
|
+
purgeSession(sessionId) {
|
|
134
|
+
const session = memoryStore.get(this.key(sessionId));
|
|
135
|
+
this.deleteSession(sessionId);
|
|
136
|
+
return { found: !!session, entries: session ? Object.keys(session.entries).length : null };
|
|
137
|
+
}
|
|
127
138
|
listSessions() {
|
|
128
139
|
return this.ownKeys()
|
|
129
140
|
.map((k) => memoryStore.get(k))
|
|
@@ -242,6 +253,25 @@ class WorkflowVault {
|
|
|
242
253
|
if (k in this.sessions)
|
|
243
254
|
delete this.writableSessions()[k];
|
|
244
255
|
}
|
|
256
|
+
purgeAll() {
|
|
257
|
+
const stored = Object.values(this.sessions);
|
|
258
|
+
const entries = stored.reduce((n, rec) => {
|
|
259
|
+
const session = rec && typeof rec === 'object' ? this.decode(rec) : null;
|
|
260
|
+
return n + (session ? Object.keys(session.entries).length : 0);
|
|
261
|
+
}, 0);
|
|
262
|
+
this.cache.clear();
|
|
263
|
+
if (stored.length)
|
|
264
|
+
this.store[STORE_KEY] = {};
|
|
265
|
+
return { sessions: stored.length, entries };
|
|
266
|
+
}
|
|
267
|
+
purgeSession(sessionId) {
|
|
268
|
+
var _a;
|
|
269
|
+
const stored = this.sessions[this.key(sessionId)];
|
|
270
|
+
const live = (_a = this.cache.get(sessionId)) !== null && _a !== void 0 ? _a : (stored && typeof stored === 'object' ? this.decode(stored) : null);
|
|
271
|
+
const found = !!stored || this.cache.has(sessionId);
|
|
272
|
+
this.deleteSession(sessionId);
|
|
273
|
+
return { found, entries: live && live.sessionId === sessionId ? Object.keys(live.entries).length : null };
|
|
274
|
+
}
|
|
245
275
|
listSessions() {
|
|
246
276
|
const out = [];
|
|
247
277
|
for (const stored of Object.values(this.sessions)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-redactor",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.2.0",
|
|
4
4
|
"description": "Redact, mask and anonymize PII before it reaches an LLM, then restore it. 216 patterns for emails, phones, addresses, IBAN, cards, names, SSN, passport and tax IDs. Reversible encrypted vault, GDPR Art.30 ROPA record, prompt-injection check. Runs on n8n Cloud and self-hosted n8n 2.x.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"n8n-community-node-package",
|