n8n-nodes-redactor 4.2.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1039,6 +1039,8 @@ class Redactor {
|
|
|
1039
1039
|
scanNumbers: true,
|
|
1040
1040
|
reservedTokens: (0, engine_1.collectTokenLike)(items.map((it) => it.json)),
|
|
1041
1041
|
keyOrigins: new WeakMap(),
|
|
1042
|
+
inputText: (0, engine_1.collectInputText)(items.map((it) => it.json)),
|
|
1043
|
+
lockedSpans: new Map(),
|
|
1042
1044
|
};
|
|
1043
1045
|
const allHits = [];
|
|
1044
1046
|
const returnData = [];
|
|
@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.collectTokenLike = collectTokenLike;
|
|
37
|
+
exports.collectInputText = collectInputText;
|
|
37
38
|
exports.redactValue = redactValue;
|
|
38
39
|
exports.restoreValue = restoreValue;
|
|
39
40
|
exports.enhanceWithPresidio = enhanceWithPresidio;
|
|
@@ -97,8 +98,8 @@ function maskValue(value, label) {
|
|
|
97
98
|
function hashValue(value) {
|
|
98
99
|
return crypto.createHash('sha256').update(value).digest('hex').slice(0, 12);
|
|
99
100
|
}
|
|
100
|
-
function storeReversible(match, pattern, vault, sessionId,
|
|
101
|
-
if (dedup) {
|
|
101
|
+
function storeReversible(match, pattern, ctx, vault, sessionId, makeToken) {
|
|
102
|
+
if (ctx.dedup) {
|
|
102
103
|
const existing = vault.findByOriginal(sessionId, match);
|
|
103
104
|
if (existing && !existing.numeric)
|
|
104
105
|
return existing.token;
|
|
@@ -120,14 +121,14 @@ const IRREVERSIBLE = {
|
|
|
120
121
|
blackout: (match) => '\u2588'.repeat(match.length),
|
|
121
122
|
remove: () => '',
|
|
122
123
|
};
|
|
123
|
-
function applyRedaction(match, pattern,
|
|
124
|
-
if (mode === 'token') {
|
|
125
|
-
return storeReversible(match, pattern, vault, sessionId,
|
|
124
|
+
function applyRedaction(match, pattern, ctx, vault, sessionId) {
|
|
125
|
+
if (ctx.mode === 'token') {
|
|
126
|
+
return storeReversible(match, pattern, ctx, vault, sessionId, () => freshToken(pattern.label, vault, sessionId, ctx.reservedTokens));
|
|
126
127
|
}
|
|
127
|
-
if (mode === 'pseudonymize') {
|
|
128
|
-
return storeReversible(match, pattern, vault, sessionId,
|
|
128
|
+
if (ctx.mode === 'pseudonymize') {
|
|
129
|
+
return storeReversible(match, pattern, ctx, vault, sessionId, () => uniquePseudonym(vault, sessionId, match, pattern.label, ctx));
|
|
129
130
|
}
|
|
130
|
-
const irreversible = IRREVERSIBLE[mode];
|
|
131
|
+
const irreversible = IRREVERSIBLE[ctx.mode];
|
|
131
132
|
return irreversible ? irreversible(match, pattern.label) : match;
|
|
132
133
|
}
|
|
133
134
|
function freshToken(label, vault, sessionId, reserved) {
|
|
@@ -140,36 +141,46 @@ function freshToken(label, vault, sessionId, reserved) {
|
|
|
140
141
|
return token;
|
|
141
142
|
}
|
|
142
143
|
const TOKEN_LIKE_RE = /\[[A-Z][A-Z0-9_]*_\d+\]/g;
|
|
143
|
-
function
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
into.add(m[0]);
|
|
147
|
-
};
|
|
148
|
-
if (typeof value === 'string')
|
|
149
|
-
scan(value);
|
|
144
|
+
function walkText(value, visit) {
|
|
145
|
+
if (typeof value === 'string' || typeof value === 'number')
|
|
146
|
+
visit(String(value));
|
|
150
147
|
else if (Array.isArray(value))
|
|
151
|
-
value.forEach((v) =>
|
|
148
|
+
value.forEach((v) => walkText(v, visit));
|
|
152
149
|
else if (value !== null && typeof value === 'object') {
|
|
153
150
|
for (const [k, v] of Object.entries(value)) {
|
|
154
|
-
|
|
155
|
-
|
|
151
|
+
visit(k);
|
|
152
|
+
walkText(v, visit);
|
|
156
153
|
}
|
|
157
154
|
}
|
|
155
|
+
}
|
|
156
|
+
function collectTokenLike(value, into = new Set()) {
|
|
157
|
+
walkText(value, (text) => {
|
|
158
|
+
for (const m of text.matchAll(TOKEN_LIKE_RE))
|
|
159
|
+
into.add(m[0]);
|
|
160
|
+
});
|
|
158
161
|
return into;
|
|
159
162
|
}
|
|
160
|
-
function
|
|
161
|
-
|
|
163
|
+
function collectInputText(value) {
|
|
164
|
+
const parts = [];
|
|
165
|
+
walkText(value, (text) => parts.push(text));
|
|
166
|
+
return parts.join('\u0000');
|
|
167
|
+
}
|
|
168
|
+
function reusable(taken, original, ctx) {
|
|
169
|
+
return !!taken && taken.original === original && !!taken.numeric === !!ctx.numeric;
|
|
170
|
+
}
|
|
171
|
+
function uniquePseudonym(vault, sessionId, original, label, ctx) {
|
|
172
|
+
var _a, _b, _c;
|
|
162
173
|
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
163
174
|
for (let attempt = 0; attempt < 32; attempt++) {
|
|
164
175
|
const seed = attempt === 0 ? original : `${original}\u0000${attempt}`;
|
|
165
176
|
const candidate = (0, pseudonymize_1.generatePseudonym)(sessionId, seed, label);
|
|
166
|
-
|
|
167
|
-
if (candidate === original)
|
|
177
|
+
if (candidate === original || ((_c = ctx.inputText) === null || _c === void 0 ? void 0 : _c.includes(candidate)))
|
|
168
178
|
continue;
|
|
169
|
-
|
|
179
|
+
const taken = entries[candidate];
|
|
180
|
+
if (!taken || reusable(taken, original, ctx))
|
|
170
181
|
return candidate;
|
|
171
182
|
}
|
|
172
|
-
return
|
|
183
|
+
return freshToken(label, vault, sessionId, ctx.reservedTokens);
|
|
173
184
|
}
|
|
174
185
|
function shouldProcessField(fieldPath, fieldMode, fieldRules) {
|
|
175
186
|
if (fieldMode === 'all')
|
|
@@ -403,7 +414,7 @@ function getSemanticLabel(fieldKey, siblingKeys) {
|
|
|
403
414
|
return null;
|
|
404
415
|
}
|
|
405
416
|
function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
406
|
-
var _a;
|
|
417
|
+
var _a, _b;
|
|
407
418
|
const semantic = getSemanticLabel(path, siblingKeys);
|
|
408
419
|
if (!semantic || !semantic.label || raw.trim().length === 0)
|
|
409
420
|
return null;
|
|
@@ -412,7 +423,9 @@ function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
412
423
|
const semConf = semantic.label === 'IDENTIFIER' ? context_1.AMBIGUOUS_FIELD_CONFIDENCE : context_1.SEMANTIC_CONFIDENCE;
|
|
413
424
|
if (semConf < ((_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0))
|
|
414
425
|
return null;
|
|
415
|
-
const replacement = applyRedaction(raw, { label: semantic.label, category: semantic.category }, ctx
|
|
426
|
+
const replacement = applyRedaction(raw, { label: semantic.label, category: semantic.category }, ctx, vault, sessionId);
|
|
427
|
+
if (replacement)
|
|
428
|
+
(_b = ctx.lockedSpans) === null || _b === void 0 ? void 0 : _b.set(lockKey(itemIndex, display), [[0, replacement.length]]);
|
|
416
429
|
hits.push({
|
|
417
430
|
token: replacement, original: '***',
|
|
418
431
|
patternName: `semantic_${semantic.label}`,
|
|
@@ -423,10 +436,24 @@ function redactSemantic(raw, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
423
436
|
});
|
|
424
437
|
return replacement;
|
|
425
438
|
}
|
|
439
|
+
function numericMapping(out, text, ctx, vault, sessionId) {
|
|
440
|
+
var _a, _b;
|
|
441
|
+
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
442
|
+
const existing = entries[out];
|
|
443
|
+
if ((existing === null || existing === void 0 ? void 0 : existing.original) === text) {
|
|
444
|
+
existing.numeric = true;
|
|
445
|
+
return out;
|
|
446
|
+
}
|
|
447
|
+
const token = existing ? freshToken('NUMBER', vault, sessionId, ctx.reservedTokens) : out;
|
|
448
|
+
vault.addEntry(sessionId, {
|
|
449
|
+
token, original: text, patternLabel: 'NUMBER', category: 'other', createdAt: new Date().toISOString(), numeric: true,
|
|
450
|
+
});
|
|
451
|
+
return token;
|
|
452
|
+
}
|
|
426
453
|
function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
427
454
|
var _a;
|
|
428
455
|
const text = String(value);
|
|
429
|
-
const numCtx = { ...ctx, dedup: false };
|
|
456
|
+
const numCtx = { ...ctx, dedup: false, numeric: true };
|
|
430
457
|
let out = null;
|
|
431
458
|
if (!ctx.skipSemantic)
|
|
432
459
|
out = redactSemantic(text, numCtx, vault, sessionId, hits, itemIndex, path, display, siblingKeys);
|
|
@@ -434,10 +461,11 @@ function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
434
461
|
out = redactText(text, numCtx, vault, sessionId, hits, itemIndex, display);
|
|
435
462
|
if (out === null || out === text)
|
|
436
463
|
return value;
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
464
|
+
if (ctx.mode !== 'token' && ctx.mode !== 'pseudonymize')
|
|
465
|
+
return out;
|
|
466
|
+
const mapped = numericMapping(out, text, ctx, vault, sessionId);
|
|
467
|
+
(_a = ctx.lockedSpans) === null || _a === void 0 ? void 0 : _a.set(lockKey(itemIndex, display), [[0, mapped.length]]);
|
|
468
|
+
return mapped;
|
|
441
469
|
}
|
|
442
470
|
function redactKey(key, taken, sourceKeys, ctx, vault, sessionId, itemIndex, keyHits) {
|
|
443
471
|
var _a;
|
|
@@ -505,38 +533,53 @@ function redactValue(value, ctx, vault, sessionId, hits, itemIndex, currentPath
|
|
|
505
533
|
return value;
|
|
506
534
|
}
|
|
507
535
|
const escapeRe = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
if (produced.size === 0)
|
|
511
|
-
return [{ text, locked: false }];
|
|
512
|
-
const guard = new RegExp([...produced].sort((a, b) => b.length - a.length).map(escapeRe).join('|'), 'g');
|
|
536
|
+
const USER_REGEX_WINDOW = 2048;
|
|
537
|
+
function windows(seg, size) {
|
|
513
538
|
const out = [];
|
|
539
|
+
let at = 0;
|
|
540
|
+
while (seg.text.length - at > size) {
|
|
541
|
+
const cut = seg.text.lastIndexOf(' ', at + size);
|
|
542
|
+
const end = cut > at + size / 2 ? cut + 1 : at + size;
|
|
543
|
+
out.push({ text: seg.text.slice(at, end), locked: false, src: seg.src + at });
|
|
544
|
+
at = end;
|
|
545
|
+
}
|
|
546
|
+
out.push({ text: seg.text.slice(at), locked: false, src: seg.src + at });
|
|
547
|
+
return out;
|
|
548
|
+
}
|
|
549
|
+
function replaceInFree(seg, re, fn, into) {
|
|
550
|
+
var _a;
|
|
514
551
|
let last = 0;
|
|
515
|
-
for (const m of text.matchAll(
|
|
552
|
+
for (const m of seg.text.matchAll(re)) {
|
|
516
553
|
const at = (_a = m.index) !== null && _a !== void 0 ? _a : 0;
|
|
554
|
+
if (m[0] === '')
|
|
555
|
+
continue;
|
|
556
|
+
const replacement = fn(m[0], seg.src + at);
|
|
557
|
+
if (replacement === null)
|
|
558
|
+
continue;
|
|
517
559
|
if (at > last)
|
|
518
|
-
|
|
519
|
-
|
|
560
|
+
into.push({ text: seg.text.slice(last, at), locked: false, src: seg.src + last });
|
|
561
|
+
into.push({ text: replacement, locked: true, src: -1 });
|
|
520
562
|
last = at + m[0].length;
|
|
521
563
|
}
|
|
522
|
-
if (last < text.length)
|
|
523
|
-
|
|
524
|
-
return out;
|
|
525
|
-
}
|
|
526
|
-
function replaceOutside(text, produced, re, fn) {
|
|
527
|
-
return segments(text, produced)
|
|
528
|
-
.map((seg) => (seg.locked ? seg.text : seg.text.replace(new RegExp(re.source, re.flags), fn)))
|
|
529
|
-
.join('');
|
|
564
|
+
if (last < seg.text.length)
|
|
565
|
+
into.push({ text: seg.text.slice(last), locked: false, src: seg.src + last });
|
|
530
566
|
}
|
|
531
|
-
function
|
|
532
|
-
|
|
567
|
+
function replaceFree(pass, re, fn, window) {
|
|
568
|
+
const flags = re.flags.includes('g') ? re.flags : `${re.flags}g`;
|
|
569
|
+
const global = new RegExp(re.source, flags);
|
|
570
|
+
const next = [];
|
|
571
|
+
for (const seg of pass.segs) {
|
|
572
|
+
if (seg.locked)
|
|
573
|
+
next.push(seg);
|
|
574
|
+
else
|
|
575
|
+
for (const piece of window ? windows(seg, window) : [seg])
|
|
576
|
+
replaceInFree(piece, global, fn, next);
|
|
577
|
+
}
|
|
578
|
+
pass.segs = next;
|
|
533
579
|
}
|
|
534
580
|
function redactMatch(pass, match, pattern, detail) {
|
|
535
|
-
const
|
|
536
|
-
const replacement = applyRedaction(match, pattern, ctx.mode, vault, sessionId, ctx.dedup, ctx.reservedTokens);
|
|
581
|
+
const replacement = applyRedaction(match, pattern, pass.ctx, pass.vault, pass.sessionId);
|
|
537
582
|
pass.hits.push({ token: replacement, original: '***', field: pass.fieldPath, itemIndex: pass.itemIndex, ...detail });
|
|
538
|
-
if (replacement)
|
|
539
|
-
pass.produced.add(replacement);
|
|
540
583
|
return replacement;
|
|
541
584
|
}
|
|
542
585
|
function denyListRegex(entry) {
|
|
@@ -551,26 +594,25 @@ function denyListStage(pass) {
|
|
|
551
594
|
const re = denyListRegex(entry);
|
|
552
595
|
if (!re)
|
|
553
596
|
continue;
|
|
554
|
-
|
|
555
|
-
|
|
597
|
+
const window = entry.type === 'regex' ? USER_REGEX_WINDOW : undefined;
|
|
598
|
+
replaceFree(pass, re, (match) => TOKEN_RE.test(match)
|
|
599
|
+
? null
|
|
556
600
|
: redactMatch(pass, match, { label: 'DENY_LIST', category: 'other' }, {
|
|
557
601
|
patternName: 'denyList', patternLabel: 'DENY_LIST', category: 'other', confidence: context_1.DENY_LIST_CONFIDENCE,
|
|
558
|
-
}));
|
|
602
|
+
}), window);
|
|
559
603
|
}
|
|
560
604
|
catch { }
|
|
561
605
|
}
|
|
562
606
|
}
|
|
563
|
-
function builtInMatch(pass, pattern, match,
|
|
607
|
+
function builtInMatch(pass, pattern, match, offset) {
|
|
564
608
|
if (TOKEN_RE.test(match) || (match.startsWith('[') && match.endsWith(']')))
|
|
565
|
-
return
|
|
609
|
+
return null;
|
|
566
610
|
const validatorPassed = pattern.validate ? pattern.validate(match) : true;
|
|
567
611
|
if (!validatorPassed || isAllowListed(match, pass.ctx.allowList))
|
|
568
|
-
return
|
|
569
|
-
const offsetArg = args[args.length - 2];
|
|
570
|
-
const offset = typeof offsetArg === 'number' ? offsetArg : 0;
|
|
612
|
+
return null;
|
|
571
613
|
const confidence = (0, context_1.calculateConfidence)(pattern.name, !!pattern.validate, validatorPassed, pass.source, offset, offset + match.length);
|
|
572
614
|
if (confidence < pass.threshold)
|
|
573
|
-
return
|
|
615
|
+
return null;
|
|
574
616
|
return redactMatch(pass, match, pattern, {
|
|
575
617
|
patternName: pattern.name, patternLabel: pattern.label, category: pattern.category, confidence,
|
|
576
618
|
});
|
|
@@ -578,24 +620,36 @@ function builtInMatch(pass, pattern, match, args) {
|
|
|
578
620
|
function builtInStage(pass) {
|
|
579
621
|
const enabled = (0, patterns_1.getPatternsByNames)(pass.ctx.enabledPatterns);
|
|
580
622
|
for (const pattern of [...enabled.filter((p) => p.validate), ...enabled.filter((p) => !p.validate)]) {
|
|
581
|
-
|
|
582
|
-
|
|
623
|
+
replaceFree(pass, pattern.regex, (match, offset) => builtInMatch(pass, pattern, match, offset));
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
function freeNames(pass) {
|
|
627
|
+
const names = new Map();
|
|
628
|
+
for (const seg of pass.segs) {
|
|
629
|
+
if (seg.locked)
|
|
630
|
+
continue;
|
|
631
|
+
for (const nm of (0, names_1.detectNamesInText)(seg.text))
|
|
632
|
+
if (!names.has(nm.name))
|
|
633
|
+
names.set(nm.name, seg.src + nm.start);
|
|
583
634
|
}
|
|
635
|
+
return names;
|
|
584
636
|
}
|
|
585
637
|
function nameStage(pass) {
|
|
586
638
|
if (pass.ctx.skipSemantic)
|
|
587
639
|
return;
|
|
588
|
-
for (const
|
|
589
|
-
if ((
|
|
640
|
+
for (const [name, start] of freeNames(pass)) {
|
|
641
|
+
if ((name.includes('[') && name.includes(']')) || isAllowListed(name, pass.ctx.allowList))
|
|
590
642
|
continue;
|
|
591
|
-
const confidence = (0, context_1.hasContextWords)(pass.source,
|
|
592
|
-
if (confidence < pass.threshold
|
|
643
|
+
const confidence = (0, context_1.hasContextWords)(pass.source, start, start + name.length, names_1.NAME_CONTEXT_WORDS) ? 0.8 : 0.65;
|
|
644
|
+
if (confidence < pass.threshold)
|
|
593
645
|
continue;
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
646
|
+
let replacement;
|
|
647
|
+
replaceFree(pass, new RegExp(escapeRe(name), 'g'), (match) => {
|
|
648
|
+
replacement !== null && replacement !== void 0 ? replacement : (replacement = redactMatch(pass, match, { label: 'PERSON_NAME', category: 'identity' }, {
|
|
649
|
+
patternName: 'personNameDict', patternLabel: 'PERSON_NAME', category: 'identity', confidence,
|
|
650
|
+
}));
|
|
651
|
+
return replacement;
|
|
597
652
|
});
|
|
598
|
-
pass.result = replaceOutside(pass.result, before, new RegExp(escapeRe(nm.name), 'g'), () => replacement);
|
|
599
653
|
}
|
|
600
654
|
}
|
|
601
655
|
function customStage(pass) {
|
|
@@ -606,27 +660,42 @@ function customStage(pass) {
|
|
|
606
660
|
continue;
|
|
607
661
|
const pattern = { label: cp.label.toUpperCase(), category: cp.category || 'identity' };
|
|
608
662
|
try {
|
|
609
|
-
|
|
610
|
-
?
|
|
663
|
+
replaceFree(pass, new RegExp(cp.regex, 'g'), (match) => TOKEN_RE.test(match) || isAllowListed(match, pass.ctx.allowList)
|
|
664
|
+
? null
|
|
611
665
|
: redactMatch(pass, match, pattern, {
|
|
612
666
|
patternName: `custom_${cp.label}`, patternLabel: pattern.label, category: pattern.category,
|
|
613
667
|
confidence: context_1.CUSTOM_PATTERN_CONFIDENCE,
|
|
614
|
-
}));
|
|
668
|
+
}), USER_REGEX_WINDOW);
|
|
615
669
|
}
|
|
616
670
|
catch { }
|
|
617
671
|
}
|
|
618
672
|
}
|
|
673
|
+
function recordLocked(ctx, itemIndex, fieldPath, segs) {
|
|
674
|
+
if (!ctx.lockedSpans || fieldPath === '')
|
|
675
|
+
return;
|
|
676
|
+
const spans = [];
|
|
677
|
+
let at = 0;
|
|
678
|
+
for (const seg of segs) {
|
|
679
|
+
if (seg.locked)
|
|
680
|
+
spans.push([at, at + seg.text.length]);
|
|
681
|
+
at += seg.text.length;
|
|
682
|
+
}
|
|
683
|
+
if (spans.length > 0)
|
|
684
|
+
ctx.lockedSpans.set(lockKey(itemIndex, fieldPath), spans);
|
|
685
|
+
}
|
|
686
|
+
const lockKey = (itemIndex, fieldPath) => `${itemIndex}\u0000${fieldPath}`;
|
|
619
687
|
function redactText(text, ctx, vault, sessionId, hits, itemIndex, fieldPath) {
|
|
620
688
|
var _a;
|
|
621
689
|
const pass = {
|
|
622
|
-
source: text,
|
|
690
|
+
source: text, segs: [{ text, locked: false, src: 0 }], threshold: (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0,
|
|
623
691
|
ctx, vault, sessionId, hits, itemIndex, fieldPath,
|
|
624
692
|
};
|
|
625
693
|
denyListStage(pass);
|
|
626
694
|
builtInStage(pass);
|
|
627
695
|
nameStage(pass);
|
|
628
696
|
customStage(pass);
|
|
629
|
-
|
|
697
|
+
recordLocked(ctx, itemIndex, fieldPath, pass.segs);
|
|
698
|
+
return pass.segs.map((s) => s.text).join('');
|
|
630
699
|
}
|
|
631
700
|
function makeRestorer(entries) {
|
|
632
701
|
const tokens = Object.keys(entries).filter((t) => t.length > 0).sort((a, b) => b.length - a.length);
|
|
@@ -676,13 +745,18 @@ function resolveOverlaps(spans) {
|
|
|
676
745
|
}
|
|
677
746
|
return kept.sort((a, b) => b.start - a.start);
|
|
678
747
|
}
|
|
748
|
+
function presidioSpans(value, entities, ctx, key) {
|
|
749
|
+
var _a, _b, _c;
|
|
750
|
+
const threshold = (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0;
|
|
751
|
+
const locked = (_c = (_b = ctx.lockedSpans) === null || _b === void 0 ? void 0 : _b.get(key)) !== null && _c !== void 0 ? _c : [];
|
|
752
|
+
const free = (s) => !locked.some(([start, end]) => s.start < end && start < s.end);
|
|
753
|
+
return resolveOverlaps(toUtf16Spans(value, entities).filter((s) => s.score >= threshold && free(s)));
|
|
754
|
+
}
|
|
679
755
|
async function presidioRedactString(value, ctx, vault, sessionId, hits, itemIndex, path) {
|
|
680
|
-
var _a;
|
|
681
756
|
if (value.trim().length === 0 || TOKEN_RE.test(value))
|
|
682
757
|
return value;
|
|
683
758
|
const entities = await (0, presidio_1.analyzeWithPresidio)(ctx.presidioUrl, value, ctx.presidioLanguage || 'en', 0.4, 10000, ctx.presidioRequest);
|
|
684
|
-
const
|
|
685
|
-
const spans = resolveOverlaps(toUtf16Spans(value, entities).filter((s) => s.score >= threshold));
|
|
759
|
+
const spans = presidioSpans(value, entities, ctx, lockKey(itemIndex, path));
|
|
686
760
|
let text = value;
|
|
687
761
|
for (const span of spans) {
|
|
688
762
|
const matchedText = value.slice(span.start, span.end);
|
|
@@ -692,7 +766,7 @@ async function presidioRedactString(value, ctx, vault, sessionId, hits, itemInde
|
|
|
692
766
|
continue;
|
|
693
767
|
const label = (0, presidio_1.getRedactorLabel)(span.entityType);
|
|
694
768
|
const category = (0, presidio_1.getRedactorCategory)(span.entityType);
|
|
695
|
-
const replacement = applyRedaction(matchedText, { label, category }, ctx
|
|
769
|
+
const replacement = applyRedaction(matchedText, { label, category }, ctx, vault, sessionId);
|
|
696
770
|
text = text.slice(0, span.start) + replacement + text.slice(span.end);
|
|
697
771
|
hits.push({
|
|
698
772
|
token: replacement, original: '***',
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.emptySet = void 0;
|
|
4
|
+
exports.union = union;
|
|
5
|
+
exports.overlaps = overlaps;
|
|
6
|
+
exports.atomCharset = atomCharset;
|
|
7
|
+
const emptySet = () => ({ ascii: new Array(128).fill(false), other: false });
|
|
8
|
+
exports.emptySet = emptySet;
|
|
9
|
+
const fullSet = () => ({ ascii: new Array(128).fill(true), other: true });
|
|
10
|
+
function addCode(set, code) {
|
|
11
|
+
if (code >= 128) {
|
|
12
|
+
set.other = true;
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
set.ascii[code] = true;
|
|
16
|
+
const ch = String.fromCharCode(code);
|
|
17
|
+
for (const folded of [ch.toLowerCase(), ch.toUpperCase()])
|
|
18
|
+
set.ascii[folded.charCodeAt(0)] = true;
|
|
19
|
+
}
|
|
20
|
+
function addRange(set, from, to) {
|
|
21
|
+
for (let c = from; c <= Math.min(to, 127); c++)
|
|
22
|
+
addCode(set, c);
|
|
23
|
+
if (to >= 128)
|
|
24
|
+
set.other = true;
|
|
25
|
+
}
|
|
26
|
+
function union(a, b) {
|
|
27
|
+
return { ascii: a.ascii.map((v, i) => v || b.ascii[i]), other: a.other || b.other };
|
|
28
|
+
}
|
|
29
|
+
function overlaps(a, b) {
|
|
30
|
+
return (a.other && b.other) || a.ascii.some((v, i) => v && b.ascii[i]);
|
|
31
|
+
}
|
|
32
|
+
function complement(set) {
|
|
33
|
+
return { ascii: set.ascii.map((v) => !v), other: true };
|
|
34
|
+
}
|
|
35
|
+
const CLASS_ESCAPES = {
|
|
36
|
+
d: (s) => addRange(s, 48, 57),
|
|
37
|
+
w: (s) => {
|
|
38
|
+
addRange(s, 48, 57);
|
|
39
|
+
addRange(s, 65, 90);
|
|
40
|
+
addCode(s, 95);
|
|
41
|
+
},
|
|
42
|
+
s: (s) => {
|
|
43
|
+
for (const c of [9, 10, 11, 12, 13, 32])
|
|
44
|
+
addCode(s, c);
|
|
45
|
+
s.other = true;
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
function escapeSet(letter) {
|
|
49
|
+
const lower = letter.toLowerCase();
|
|
50
|
+
const fill = CLASS_ESCAPES[lower];
|
|
51
|
+
if (fill) {
|
|
52
|
+
const set = (0, exports.emptySet)();
|
|
53
|
+
fill(set);
|
|
54
|
+
return letter === lower ? set : complement(set);
|
|
55
|
+
}
|
|
56
|
+
if (/[a-zA-Z0-9]/.test(letter))
|
|
57
|
+
return fullSet();
|
|
58
|
+
const set = (0, exports.emptySet)();
|
|
59
|
+
addCode(set, letter.charCodeAt(0));
|
|
60
|
+
return set;
|
|
61
|
+
}
|
|
62
|
+
function classMember(body, j) {
|
|
63
|
+
var _a, _b;
|
|
64
|
+
if (body[j] === '\\') {
|
|
65
|
+
const set = escapeSet((_a = body[j + 1]) !== null && _a !== void 0 ? _a : '\\');
|
|
66
|
+
const single = /[a-zA-Z0-9]/.test((_b = body[j + 1]) !== null && _b !== void 0 ? _b : '') ? null : body.charCodeAt(j + 1);
|
|
67
|
+
return { set, next: j + 2, code: single };
|
|
68
|
+
}
|
|
69
|
+
const set = (0, exports.emptySet)();
|
|
70
|
+
addCode(set, body.charCodeAt(j));
|
|
71
|
+
return { set, next: j + 1, code: body.charCodeAt(j) };
|
|
72
|
+
}
|
|
73
|
+
function classSet(body) {
|
|
74
|
+
const negated = body.startsWith('^');
|
|
75
|
+
let set = (0, exports.emptySet)();
|
|
76
|
+
let j = negated ? 1 : 0;
|
|
77
|
+
while (j < body.length) {
|
|
78
|
+
const first = classMember(body, j);
|
|
79
|
+
j = first.next;
|
|
80
|
+
if (body[j] === '-' && j + 1 < body.length && first.code !== null) {
|
|
81
|
+
const last = classMember(body, j + 1);
|
|
82
|
+
if (last.code !== null)
|
|
83
|
+
addRange(set, first.code, last.code);
|
|
84
|
+
else
|
|
85
|
+
set = union(union(set, first.set), last.set);
|
|
86
|
+
j = last.next;
|
|
87
|
+
}
|
|
88
|
+
else
|
|
89
|
+
set = union(set, first.set);
|
|
90
|
+
}
|
|
91
|
+
return negated ? complement(set) : set;
|
|
92
|
+
}
|
|
93
|
+
function atomCharset(src, start, end) {
|
|
94
|
+
var _a, _b;
|
|
95
|
+
const c = src[start];
|
|
96
|
+
if (c === '^' || c === '$')
|
|
97
|
+
return null;
|
|
98
|
+
if (c === '\\')
|
|
99
|
+
return /[bB]/.test((_a = src[start + 1]) !== null && _a !== void 0 ? _a : '') ? null : escapeSet((_b = src[start + 1]) !== null && _b !== void 0 ? _b : '\\');
|
|
100
|
+
if (c === '[')
|
|
101
|
+
return classSet(src.slice(start + 1, end - 1));
|
|
102
|
+
if (c === '.')
|
|
103
|
+
return fullSet();
|
|
104
|
+
const set = (0, exports.emptySet)();
|
|
105
|
+
addCode(set, src.charCodeAt(start));
|
|
106
|
+
return set;
|
|
107
|
+
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.checkRegex = checkRegex;
|
|
4
|
+
const regexCharset_1 = require("./regexCharset");
|
|
4
5
|
const MAX_REGEX_LENGTH = 500;
|
|
6
|
+
const newFrame = (zeroWidth = false) => ({
|
|
7
|
+
hasVariableQuantifier: false, hasAlternation: false, run: null, chars: (0, regexCharset_1.emptySet)(), zeroWidth,
|
|
8
|
+
});
|
|
5
9
|
function readBraces(src, i) {
|
|
6
10
|
const m = /^\{(\d+)(,(\d*))?\}/.exec(src.slice(i));
|
|
7
11
|
if (!m)
|
|
@@ -55,11 +59,12 @@ function readToken(src, i, stack) {
|
|
|
55
59
|
if (c === '[')
|
|
56
60
|
return { atomEnd: skipClass(src, i), closed: null };
|
|
57
61
|
if (c === '(') {
|
|
58
|
-
stack.push(
|
|
62
|
+
stack.push(newFrame(/^\(\?(?:=|!|<=|<!)/.test(src.slice(i))));
|
|
59
63
|
return { atomEnd: i, closed: null, next: i + groupPrefixLength(src, i) };
|
|
60
64
|
}
|
|
61
65
|
if (c === '|') {
|
|
62
66
|
stack[stack.length - 1].hasAlternation = true;
|
|
67
|
+
stack[stack.length - 1].run = null;
|
|
63
68
|
return { atomEnd: i, closed: null, next: i + 1 };
|
|
64
69
|
}
|
|
65
70
|
const closed = c === ')' && stack.length > 1 ? stack.pop() : null;
|
|
@@ -69,7 +74,7 @@ const NESTED = 'a repeated group contains a quantifier or alternation (catastrop
|
|
|
69
74
|
function isRisky(frame) {
|
|
70
75
|
return !!frame && (frame.hasVariableQuantifier || frame.hasAlternation);
|
|
71
76
|
}
|
|
72
|
-
const NO_GROUP =
|
|
77
|
+
const NO_GROUP = newFrame();
|
|
73
78
|
const repeats = (q) => q !== null && q.max > 1;
|
|
74
79
|
const isVariable = (q) => q !== null && q.min !== q.max;
|
|
75
80
|
function applyQuantifier(q, closed, top) {
|
|
@@ -80,8 +85,37 @@ function applyQuantifier(q, closed, top) {
|
|
|
80
85
|
top.hasAlternation || (top.hasAlternation = inner.hasAlternation);
|
|
81
86
|
return null;
|
|
82
87
|
}
|
|
88
|
+
const SEQUENTIAL = 'two repetitions in a row can match the same characters (catastrophic backtracking)';
|
|
89
|
+
function applySequence(chars, variable, required, top) {
|
|
90
|
+
if (chars === null)
|
|
91
|
+
return null;
|
|
92
|
+
top.chars = (0, regexCharset_1.union)(top.chars, chars);
|
|
93
|
+
const shared = top.run !== null && (0, regexCharset_1.overlaps)(top.run, chars);
|
|
94
|
+
if (variable) {
|
|
95
|
+
if (shared)
|
|
96
|
+
return SEQUENTIAL;
|
|
97
|
+
top.run = top.run ? (0, regexCharset_1.union)(top.run, chars) : chars;
|
|
98
|
+
}
|
|
99
|
+
else if (required && !shared)
|
|
100
|
+
top.run = null;
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
function stepChars(src, i, step) {
|
|
104
|
+
const closed = step.closed;
|
|
105
|
+
if (!closed)
|
|
106
|
+
return (0, regexCharset_1.atomCharset)(src, i, step.atomEnd);
|
|
107
|
+
return closed.zeroWidth ? null : closed.chars;
|
|
108
|
+
}
|
|
109
|
+
function stepProblem(src, i, step, q, top) {
|
|
110
|
+
var _a, _b;
|
|
111
|
+
const problem = applyQuantifier(q, step.closed, top);
|
|
112
|
+
if (problem || (q === null || q === void 0 ? void 0 : q.max) === 0)
|
|
113
|
+
return problem;
|
|
114
|
+
const variable = isVariable(q) || ((_a = step.closed) === null || _a === void 0 ? void 0 : _a.hasVariableQuantifier) === true;
|
|
115
|
+
return applySequence(stepChars(src, i, step), variable, ((_b = q === null || q === void 0 ? void 0 : q.min) !== null && _b !== void 0 ? _b : 1) >= 1, top);
|
|
116
|
+
}
|
|
83
117
|
function structuralProblem(src) {
|
|
84
|
-
const stack = [
|
|
118
|
+
const stack = [newFrame()];
|
|
85
119
|
let i = 0;
|
|
86
120
|
while (i < src.length) {
|
|
87
121
|
const step = readToken(src, i, stack);
|
|
@@ -92,7 +126,7 @@ function structuralProblem(src) {
|
|
|
92
126
|
continue;
|
|
93
127
|
}
|
|
94
128
|
const q = readQuantifier(src, step.atomEnd);
|
|
95
|
-
const problem =
|
|
129
|
+
const problem = stepProblem(src, i, step, q, stack[stack.length - 1]);
|
|
96
130
|
if (problem)
|
|
97
131
|
return problem;
|
|
98
132
|
i = step.atomEnd + (q ? q.length : 0);
|
|
@@ -27,12 +27,26 @@ async function resolvePassphrase(fns) {
|
|
|
27
27
|
}
|
|
28
28
|
return cred.passphrase;
|
|
29
29
|
}
|
|
30
|
+
return legacyPassphrase(fns);
|
|
31
|
+
}
|
|
32
|
+
function legacyPassphrase(fns) {
|
|
33
|
+
let raw;
|
|
30
34
|
try {
|
|
31
|
-
|
|
35
|
+
raw = fns.getNodeParameter('vaultPassphrase', 0, '');
|
|
32
36
|
}
|
|
33
|
-
catch {
|
|
37
|
+
catch (error) {
|
|
38
|
+
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), 'The legacy vault passphrase could not be evaluated, so the vault stays locked.', {
|
|
39
|
+
description: `Nothing was stored. Fix the expression, or move the passphrase into a Redactor Vault credential. Cause: ${error.message}`,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
if (raw === undefined || raw === null || raw === '')
|
|
34
43
|
return '';
|
|
44
|
+
if (typeof raw !== 'string') {
|
|
45
|
+
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), 'The legacy vault passphrase is not text, so the vault stays locked.', {
|
|
46
|
+
description: 'Nothing was stored. Set the passphrase to a text value, or use a Redactor Vault credential.',
|
|
47
|
+
});
|
|
35
48
|
}
|
|
49
|
+
return raw;
|
|
36
50
|
}
|
|
37
51
|
function workflowNamespace(fns) {
|
|
38
52
|
try {
|
|
@@ -47,7 +61,7 @@ function workflowNamespace(fns) {
|
|
|
47
61
|
function assertUserRegexes(fns, customPatterns, lists) {
|
|
48
62
|
const refuse = (what, problem) => {
|
|
49
63
|
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), `${what} regex ${problem}`, {
|
|
50
|
-
description: 'Rewrite the pattern without nested or alternated repetition, for example [ab]+ instead of (a|b)+.',
|
|
64
|
+
description: 'Rewrite the pattern without nested or alternated repetition, for example [ab]+ instead of (a|b)+, and without two repetitions in a row that can match the same characters, for example \\d+ instead of \\d+\\d+. Put a fixed separator such as - or @ between repeated parts.',
|
|
51
65
|
});
|
|
52
66
|
};
|
|
53
67
|
for (const cp of customPatterns) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-redactor",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.3.0",
|
|
4
4
|
"description": "Redact, mask and anonymize PII before it reaches an LLM, then restore it. 216 patterns for emails, phones, addresses, IBAN, cards, names, SSN, passport and tax IDs. Reversible encrypted vault, GDPR Art.30 ROPA record, prompt-injection check. Runs on n8n Cloud and self-hosted n8n 2.x.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"n8n-community-node-package",
|