n8n-nodes-redactor 5.1.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/nodes/Redactor/Redactor.node.js +24 -6
- package/dist/nodes/Redactor/classification.js +9 -6
- package/dist/nodes/Redactor/engine.js +239 -87
- package/dist/nodes/Redactor/injection.js +43 -66
- package/dist/nodes/Redactor/names.js +27 -14
- package/dist/nodes/Redactor/patterns.js +43 -9
- package/dist/nodes/Redactor/profiles.js +1 -1
- package/dist/nodes/Redactor/regexCharset.js +2 -0
- package/dist/nodes/Redactor/regexSafety.js +8 -4
- package/dist/nodes/Redactor/ropa.js +19 -3
- package/dist/nodes/Redactor/runtime.js +9 -0
- package/dist/nodes/Redactor/types.js +7 -0
- package/dist/nodes/Redactor/vault.js +17 -1
- package/package.json +1 -1
|
@@ -43,6 +43,16 @@ const RECOMMENDED_PATTERNS = [
|
|
|
43
43
|
'vin',
|
|
44
44
|
'uuid', 'imei', 'iccid',
|
|
45
45
|
];
|
|
46
|
+
function versionRopaSummary(ropa, byLevel, typeVersion) {
|
|
47
|
+
const summary = ropa.processingSummary;
|
|
48
|
+
if (typeVersion >= 2) {
|
|
49
|
+
summary.hitsBySensitivity = { ...byLevel };
|
|
50
|
+
}
|
|
51
|
+
else {
|
|
52
|
+
summary.hitsByPiiCategory = summary.hitsByCategory;
|
|
53
|
+
summary.hitsByCategory = { ...byLevel };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
46
56
|
class Redactor {
|
|
47
57
|
constructor() {
|
|
48
58
|
this.description = {
|
|
@@ -50,7 +60,8 @@ class Redactor {
|
|
|
50
60
|
name: 'redactor',
|
|
51
61
|
icon: { light: 'file:redactor.svg', dark: 'file:redactor.dark.svg' },
|
|
52
62
|
group: ['transform'],
|
|
53
|
-
version: 1,
|
|
63
|
+
version: [1, 2],
|
|
64
|
+
defaultVersion: 2,
|
|
54
65
|
subtitle: '={{$parameter["operation"]}}',
|
|
55
66
|
usableAsTool: true,
|
|
56
67
|
description: 'Redact and remove sensitive data, PII, personal information, emails, phone numbers, addresses, IBAN, credit cards, names before sending to LLM. Anonymize, mask, sanitize, scrub customer data. Restore original values after. Helps GDPR, HIPAA and CCPA workflows. Data privacy, data protection, data masking, data anonymization.',
|
|
@@ -767,7 +778,7 @@ class Redactor {
|
|
|
767
778
|
};
|
|
768
779
|
}
|
|
769
780
|
async execute() {
|
|
770
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
781
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
771
782
|
const items = this.getInputData();
|
|
772
783
|
const allPaired = items.map((_, i) => ({ item: i }));
|
|
773
784
|
const operation = this.getNodeParameter('operation', 0);
|
|
@@ -793,7 +804,8 @@ class Redactor {
|
|
|
793
804
|
for (let i = 0; i < items.length; i++) {
|
|
794
805
|
const hits = [];
|
|
795
806
|
(0, engine_1.redactValue)(items[i].json, ctx, tempVault, tempSessionId, hits, i);
|
|
796
|
-
|
|
807
|
+
for (const hit of hits)
|
|
808
|
+
allHits.push(hit);
|
|
797
809
|
}
|
|
798
810
|
tempVault.deleteSession(tempSessionId);
|
|
799
811
|
if (operation === 'detect') {
|
|
@@ -867,12 +879,14 @@ class Redactor {
|
|
|
867
879
|
dedup: true,
|
|
868
880
|
fieldRules: [],
|
|
869
881
|
fieldMode: 'all',
|
|
882
|
+
scanNumbers: true,
|
|
870
883
|
};
|
|
871
884
|
const allHits = [];
|
|
872
885
|
for (let i = 0; i < items.length; i++) {
|
|
873
886
|
const hits = [];
|
|
874
887
|
(0, engine_1.redactValue)(items[i].json, ctx, tempVault, tempSessionId, hits, i);
|
|
875
|
-
|
|
888
|
+
for (const hit of hits)
|
|
889
|
+
allHits.push(hit);
|
|
876
890
|
}
|
|
877
891
|
tempVault.deleteSession(tempSessionId);
|
|
878
892
|
const classification = (0, classification_1.classifyData)(allHits);
|
|
@@ -1041,6 +1055,8 @@ class Redactor {
|
|
|
1041
1055
|
keyOrigins: new WeakMap(),
|
|
1042
1056
|
inputText: (0, engine_1.collectInputText)(items.map((it) => it.json)),
|
|
1043
1057
|
lockedSpans: new WeakMap(),
|
|
1058
|
+
keyLocks: new WeakMap(),
|
|
1059
|
+
priorTokens: (0, engine_1.priorTokenMatcher)((_e = vault.getSession(sessionId)) === null || _e === void 0 ? void 0 : _e.entries),
|
|
1044
1060
|
};
|
|
1045
1061
|
const allHits = [];
|
|
1046
1062
|
const returnData = [];
|
|
@@ -1056,7 +1072,8 @@ class Redactor {
|
|
|
1056
1072
|
}
|
|
1057
1073
|
}
|
|
1058
1074
|
returnData.push({ json: redacted, pairedItem: { item: i } });
|
|
1059
|
-
|
|
1075
|
+
for (const hit of hits)
|
|
1076
|
+
allHits.push(hit);
|
|
1060
1077
|
}
|
|
1061
1078
|
vault.save(sessionId);
|
|
1062
1079
|
let injectionResult = null;
|
|
@@ -1091,7 +1108,8 @@ class Redactor {
|
|
|
1091
1108
|
if (includeReport && this.getNodeParameter('includeRopa', 0, false)) {
|
|
1092
1109
|
const d = this.getNodeParameter('ropa', 0, {});
|
|
1093
1110
|
const list = (v) => String(v !== null && v !== void 0 ? v : '').split(',').map((x) => x.trim()).filter(Boolean);
|
|
1094
|
-
report.ropa = (0, ropa_1.generateRopaRecord)(allHits, (0, classification_1.classifyData)(allHits), items.length, enabledPatterns.length, mode, sessionTtl, !!vaultPassphrase, auditLog, { name: String((
|
|
1111
|
+
report.ropa = (0, ropa_1.generateRopaRecord)(allHits, (0, classification_1.classifyData)(allHits), items.length, enabledPatterns.length, mode, sessionTtl, !!vaultPassphrase, auditLog, { name: String((_f = d.controllerName) !== null && _f !== void 0 ? _f : ''), contactEmail: String((_g = d.controllerContact) !== null && _g !== void 0 ? _g : ''), dpo: d.dpo ? String(d.dpo) : undefined }, list(d.purposes), String((_h = d.legalBasis) !== null && _h !== void 0 ? _h : 'legitimate_interest'), list(d.dataSubjects), list(d.recipients), []);
|
|
1112
|
+
versionRopaSummary(report.ropa, (0, classification_1.classifyData)(allHits).hitCountByLevel, this.getNode().typeVersion);
|
|
1095
1113
|
}
|
|
1096
1114
|
return [returnData, [{ json: report, pairedItem: allPaired }]];
|
|
1097
1115
|
}
|
|
@@ -22,6 +22,7 @@ const LEVEL_NUMERIC = {
|
|
|
22
22
|
RESTRICTED: 3,
|
|
23
23
|
};
|
|
24
24
|
function classifyData(hits) {
|
|
25
|
+
var _a;
|
|
25
26
|
if (hits.length === 0) {
|
|
26
27
|
return {
|
|
27
28
|
level: 'PUBLIC',
|
|
@@ -40,12 +41,14 @@ function classifyData(hits) {
|
|
|
40
41
|
CONFIDENTIAL: 0,
|
|
41
42
|
RESTRICTED: 0,
|
|
42
43
|
};
|
|
43
|
-
const categoryCounts =
|
|
44
|
+
const categoryCounts = new Map();
|
|
44
45
|
let totalConfidence = 0;
|
|
45
46
|
for (const hit of hits) {
|
|
46
|
-
const catLevel = CATEGORY_SENSITIVITY
|
|
47
|
+
const catLevel = Object.prototype.hasOwnProperty.call(CATEGORY_SENSITIVITY, hit.category)
|
|
48
|
+
? CATEGORY_SENSITIVITY[hit.category]
|
|
49
|
+
: 'INTERNAL';
|
|
47
50
|
hitCountByLevel[catLevel]++;
|
|
48
|
-
categoryCounts
|
|
51
|
+
categoryCounts.set(hit.category, ((_a = categoryCounts.get(hit.category)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
49
52
|
totalConfidence += hit.confidence || 0.65;
|
|
50
53
|
}
|
|
51
54
|
let level = 'PUBLIC';
|
|
@@ -64,10 +67,10 @@ function classifyData(hits) {
|
|
|
64
67
|
level = 'CONFIDENTIAL';
|
|
65
68
|
escalated = true;
|
|
66
69
|
}
|
|
67
|
-
const
|
|
68
|
-
|
|
70
|
+
const ranked = [...categoryCounts.entries()].sort((a, b) => b[1] - a[1]);
|
|
71
|
+
const topCategory = ranked[0][0];
|
|
69
72
|
const reasonParts = [];
|
|
70
|
-
for (const [cat, count] of
|
|
73
|
+
for (const [cat, count] of ranked) {
|
|
71
74
|
reasonParts.push(`${count} ${cat}`);
|
|
72
75
|
}
|
|
73
76
|
let reason = `Contains: ${reasonParts.join(', ')}.`;
|
|
@@ -36,12 +36,14 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.collectTokenLike = collectTokenLike;
|
|
37
37
|
exports.collectInputText = collectInputText;
|
|
38
38
|
exports.redactValue = redactValue;
|
|
39
|
+
exports.priorTokenMatcher = priorTokenMatcher;
|
|
39
40
|
exports.restoreValue = restoreValue;
|
|
40
41
|
exports.enhanceWithPresidio = enhanceWithPresidio;
|
|
41
42
|
exports.buildReport = buildReport;
|
|
42
43
|
const crypto = __importStar(require("crypto"));
|
|
43
44
|
const patterns_1 = require("./patterns");
|
|
44
45
|
const regexSafety_1 = require("./regexSafety");
|
|
46
|
+
const types_1 = require("./types");
|
|
45
47
|
const context_1 = require("./context");
|
|
46
48
|
const names_1 = require("./names");
|
|
47
49
|
const pseudonymize_1 = require("./pseudonymize");
|
|
@@ -76,7 +78,6 @@ function generateToken(label, index) {
|
|
|
76
78
|
function setOwn(obj, key, value) {
|
|
77
79
|
Object.defineProperty(obj, key, { value, enumerable: true, writable: true, configurable: true });
|
|
78
80
|
}
|
|
79
|
-
const TOKEN_RE = /^\[[A-Z][A-Z0-9_]*_\d+\]$/;
|
|
80
81
|
function maskValue(value, label) {
|
|
81
82
|
if (value.length <= 2)
|
|
82
83
|
return '*'.repeat(value.length);
|
|
@@ -135,13 +136,15 @@ function applyRedaction(match, pattern, ctx, vault, sessionId) {
|
|
|
135
136
|
const irreversible = IRREVERSIBLE[ctx.mode];
|
|
136
137
|
return irreversible ? irreversible(match, pattern.label) : match;
|
|
137
138
|
}
|
|
139
|
+
const nextIndex = new WeakMap();
|
|
138
140
|
function freshToken(label, vault, sessionId, reserved) {
|
|
139
|
-
var _a, _b;
|
|
141
|
+
var _a, _b, _c;
|
|
140
142
|
const entries = (_b = (_a = vault.getSession(sessionId)) === null || _a === void 0 ? void 0 : _a.entries) !== null && _b !== void 0 ? _b : {};
|
|
141
|
-
let index = Object.keys(entries).length;
|
|
143
|
+
let index = (_c = nextIndex.get(entries)) !== null && _c !== void 0 ? _c : Object.keys(entries).length;
|
|
142
144
|
let token = generateToken(label, index);
|
|
143
145
|
while (entries[token] || (reserved === null || reserved === void 0 ? void 0 : reserved.has(token)))
|
|
144
146
|
token = generateToken(label, ++index);
|
|
147
|
+
nextIndex.set(entries, index + 1);
|
|
145
148
|
return token;
|
|
146
149
|
}
|
|
147
150
|
const TOKEN_LIKE_RE = /\[[A-Z][A-Z0-9_]*_\d+\]/g;
|
|
@@ -454,7 +457,7 @@ function numericMapping(out, text, ctx, vault, sessionId) {
|
|
|
454
457
|
return token;
|
|
455
458
|
}
|
|
456
459
|
function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, display, siblingKeys) {
|
|
457
|
-
const text = String(value);
|
|
460
|
+
const text = Object.is(value, -0) ? '-0' : String(value);
|
|
458
461
|
const numCtx = { ...ctx, dedup: false, numeric: true };
|
|
459
462
|
let out = null;
|
|
460
463
|
if (!ctx.skipSemantic)
|
|
@@ -471,17 +474,12 @@ function redactNumber(value, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
471
474
|
ctx.lastLocked = [[0, mapped.length]];
|
|
472
475
|
return mapped;
|
|
473
476
|
}
|
|
474
|
-
function
|
|
477
|
+
function distinctKey(out, original, clashes, ctx, vault, sessionId, reserved) {
|
|
475
478
|
var _a;
|
|
476
|
-
const out = redactText(key, ctx, vault, sessionId, keyHits, itemIndex, '');
|
|
477
|
-
if (out === key)
|
|
478
|
-
return key;
|
|
479
|
-
const clashes = (k) => k === '' || Object.prototype.hasOwnProperty.call(taken, k) || sourceKeys.has(k);
|
|
480
|
-
if (!clashes(out))
|
|
481
|
-
return out;
|
|
482
479
|
if (ctx.mode === 'token' || ctx.mode === 'pseudonymize') {
|
|
483
|
-
const token = freshToken('KEY', vault, sessionId, new Set([...((_a = ctx.reservedTokens) !== null && _a !== void 0 ? _a : []), ...
|
|
484
|
-
vault.addEntry(sessionId, { token, original
|
|
480
|
+
const token = freshToken('KEY', vault, sessionId, new Set([...((_a = ctx.reservedTokens) !== null && _a !== void 0 ? _a : []), ...reserved]));
|
|
481
|
+
vault.addEntry(sessionId, { token, original, patternLabel: 'KEY', category: 'other', createdAt: new Date().toISOString() });
|
|
482
|
+
ctx.lastLocked = [[0, token.length]];
|
|
485
483
|
return token;
|
|
486
484
|
}
|
|
487
485
|
let suffixed = out;
|
|
@@ -489,6 +487,24 @@ function redactKey(key, taken, sourceKeys, ctx, vault, sessionId, itemIndex, key
|
|
|
489
487
|
suffixed = `${out}_${n}`;
|
|
490
488
|
return suffixed;
|
|
491
489
|
}
|
|
490
|
+
function redactKey(key, taken, sourceKeys, ctx, vault, sessionId, itemIndex, keyHits) {
|
|
491
|
+
const out = redactText(key, ctx, vault, sessionId, keyHits, itemIndex, '');
|
|
492
|
+
if (out === key)
|
|
493
|
+
return key;
|
|
494
|
+
const clashes = (k) => k === '' || Object.prototype.hasOwnProperty.call(taken, k) || sourceKeys.has(k);
|
|
495
|
+
if (!clashes(out))
|
|
496
|
+
return out;
|
|
497
|
+
return distinctKey(out, key, clashes, ctx, vault, sessionId, [...sourceKeys, ...Object.keys(taken)]);
|
|
498
|
+
}
|
|
499
|
+
function rememberKeyLock(ctx, container, key) {
|
|
500
|
+
var _a;
|
|
501
|
+
if (ctx.lastLocked && ctx.keyLocks) {
|
|
502
|
+
const byKey = (_a = ctx.keyLocks.get(container)) !== null && _a !== void 0 ? _a : new Map();
|
|
503
|
+
byKey.set(key, ctx.lastLocked);
|
|
504
|
+
ctx.keyLocks.set(container, byKey);
|
|
505
|
+
}
|
|
506
|
+
ctx.lastLocked = undefined;
|
|
507
|
+
}
|
|
492
508
|
function redactObject(value, ctx, vault, sessionId, hits, itemIndex, path, display) {
|
|
493
509
|
var _a;
|
|
494
510
|
const entries = Object.entries(value);
|
|
@@ -499,9 +515,11 @@ function redactObject(value, ctx, vault, sessionId, hits, itemIndex, path, displ
|
|
|
499
515
|
for (const [k, v] of entries) {
|
|
500
516
|
const fieldPath = path ? `${path}.${k}` : k;
|
|
501
517
|
const keyHits = [];
|
|
518
|
+
ctx.lastLocked = undefined;
|
|
502
519
|
const outKey = shouldProcessField(fieldPath, ctx.fieldMode, ctx.fieldRules)
|
|
503
520
|
? redactKey(k, obj, sourceKeys, ctx, vault, sessionId, itemIndex, keyHits)
|
|
504
|
-
: k;
|
|
521
|
+
: protectLiterals(k, ctx, vault, sessionId, hits, itemIndex);
|
|
522
|
+
rememberKeyLock(ctx, obj, outKey);
|
|
505
523
|
origins.set(outKey, k);
|
|
506
524
|
const childDisplay = display ? `${display}.${outKey}` : outKey;
|
|
507
525
|
for (const h of keyHits)
|
|
@@ -515,7 +533,7 @@ function redactValue(value, ctx, vault, sessionId, hits, itemIndex, currentPath
|
|
|
515
533
|
const processed = shouldProcessField(currentPath, ctx.fieldMode, ctx.fieldRules);
|
|
516
534
|
if (typeof value === 'string') {
|
|
517
535
|
if (!processed)
|
|
518
|
-
return value;
|
|
536
|
+
return protectLiterals(value, ctx, vault, sessionId, hits, itemIndex);
|
|
519
537
|
if (!ctx.skipSemantic) {
|
|
520
538
|
const semantic = redactSemantic(value, ctx, vault, sessionId, hits, itemIndex, currentPath, displayPath, siblingKeys);
|
|
521
539
|
if (semantic !== null)
|
|
@@ -548,44 +566,131 @@ function assertUserRegexInput(text, what) {
|
|
|
548
566
|
throw new Error(`${what} cannot scan a value of ${text.length} characters, the limit is ${USER_REGEX_MAX_INPUT}. ` +
|
|
549
567
|
'Exclude the field with field rules, or remove the regex.');
|
|
550
568
|
}
|
|
551
|
-
|
|
552
|
-
|
|
569
|
+
class SegmentCursor {
|
|
570
|
+
constructor(segs) {
|
|
571
|
+
this.segs = segs;
|
|
572
|
+
this.next = [];
|
|
573
|
+
this.i = 0;
|
|
574
|
+
this.cur = segs[0];
|
|
575
|
+
}
|
|
576
|
+
freeAt(start, end) {
|
|
577
|
+
while (this.cur && (this.cur.locked || this.cur.end <= start))
|
|
578
|
+
this.take();
|
|
579
|
+
const cur = this.cur;
|
|
580
|
+
return cur && cur.src <= start && end <= cur.end ? cur : undefined;
|
|
581
|
+
}
|
|
582
|
+
lock(start, end, replacement) {
|
|
583
|
+
const cur = this.cur;
|
|
584
|
+
if (start > cur.src)
|
|
585
|
+
this.next.push(freeSlice(cur, cur.src, start));
|
|
586
|
+
this.next.push({ text: replacement, locked: true, src: start, end });
|
|
587
|
+
if (end < cur.end)
|
|
588
|
+
this.cur = freeSlice(cur, end, cur.end);
|
|
589
|
+
else
|
|
590
|
+
this.cur = this.segs[++this.i];
|
|
591
|
+
}
|
|
592
|
+
finish() {
|
|
593
|
+
while (this.cur)
|
|
594
|
+
this.take();
|
|
595
|
+
return this.next;
|
|
596
|
+
}
|
|
597
|
+
take() {
|
|
598
|
+
this.next.push(this.cur);
|
|
599
|
+
this.cur = this.segs[++this.i];
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
function freeSlice(seg, start, end) {
|
|
603
|
+
return { text: seg.text.slice(start - seg.src, end - seg.src), locked: false, src: start, end };
|
|
604
|
+
}
|
|
605
|
+
function redactMatch(pass, match, decision) {
|
|
606
|
+
if (decision.store)
|
|
607
|
+
return decision.store(match);
|
|
608
|
+
const replacement = applyRedaction(match, decision.pattern, pass.ctx, pass.vault, pass.sessionId);
|
|
609
|
+
if (decision.detail) {
|
|
610
|
+
pass.hits.push({ token: replacement, original: '***', field: pass.fieldPath, itemIndex: pass.itemIndex, ...decision.detail });
|
|
611
|
+
}
|
|
612
|
+
return replacement;
|
|
553
613
|
}
|
|
554
|
-
function
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
const
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
614
|
+
function segmentAfter(segs, pos) {
|
|
615
|
+
let lo = 0;
|
|
616
|
+
let hi = segs.length;
|
|
617
|
+
while (lo < hi) {
|
|
618
|
+
const mid = (lo + hi) >> 1;
|
|
619
|
+
if (segs[mid].end <= pos)
|
|
620
|
+
lo = mid + 1;
|
|
621
|
+
else
|
|
622
|
+
hi = mid;
|
|
623
|
+
}
|
|
624
|
+
return lo;
|
|
625
|
+
}
|
|
626
|
+
function claimPending(pass, pending) {
|
|
627
|
+
for (const p of pending) {
|
|
628
|
+
let i = segmentAfter(pass.segs, p.start);
|
|
629
|
+
while (i < pass.segs.length && pass.segs[i].src < p.end) {
|
|
630
|
+
const seg = pass.segs[i];
|
|
631
|
+
if (seg.locked) {
|
|
632
|
+
i++;
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
635
|
+
const start = Math.max(seg.src, p.start);
|
|
636
|
+
const end = Math.min(seg.end, p.end);
|
|
637
|
+
const parts = [];
|
|
638
|
+
if (start > seg.src)
|
|
639
|
+
parts.push(freeSlice(seg, seg.src, start));
|
|
640
|
+
parts.push({ text: redactMatch(pass, pass.source.slice(start, end), p.decision), locked: true, src: start, end });
|
|
641
|
+
if (end < seg.end)
|
|
642
|
+
parts.push(freeSlice(seg, end, seg.end));
|
|
643
|
+
pass.segs.splice(i, 1, ...parts);
|
|
644
|
+
i += parts.length;
|
|
645
|
+
}
|
|
577
646
|
}
|
|
578
647
|
}
|
|
579
|
-
function
|
|
580
|
-
const
|
|
581
|
-
|
|
582
|
-
|
|
648
|
+
function claimOne(pass, cursor, pending, start, end, decide) {
|
|
649
|
+
const text = pass.source.slice(start, end);
|
|
650
|
+
const free = cursor.freeAt(start, end) !== undefined;
|
|
651
|
+
const decision = decide(text, start);
|
|
652
|
+
if (decision && free)
|
|
653
|
+
cursor.lock(start, end, redactMatch(pass, text, decision));
|
|
654
|
+
else if (decision)
|
|
655
|
+
pending.push({ start, end, decision });
|
|
656
|
+
return free;
|
|
657
|
+
}
|
|
658
|
+
function replaceFree(pass, re, decide) {
|
|
659
|
+
const global = new RegExp(re.source, re.flags.includes('g') ? re.flags : `${re.flags}g`);
|
|
660
|
+
const cursor = new SegmentCursor(pass.segs);
|
|
661
|
+
const pending = [];
|
|
662
|
+
let m;
|
|
663
|
+
while ((m = global.exec(pass.source)) !== null) {
|
|
664
|
+
const start = m.index;
|
|
665
|
+
if (m[0] === '' || !claimOne(pass, cursor, pending, start, start + m[0].length, decide))
|
|
666
|
+
global.lastIndex = start + 1;
|
|
667
|
+
}
|
|
668
|
+
pass.segs = cursor.finish();
|
|
669
|
+
claimPending(pass, pending);
|
|
670
|
+
}
|
|
671
|
+
function replaceSpans(pass, spans, decide) {
|
|
672
|
+
const ordered = [...spans].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
673
|
+
const cursor = new SegmentCursor(pass.segs);
|
|
674
|
+
const pending = [];
|
|
675
|
+
let reach = -1;
|
|
676
|
+
for (const span of ordered) {
|
|
677
|
+
if (span.end <= reach)
|
|
678
|
+
continue;
|
|
679
|
+
claimOne(pass, cursor, pending, span.start, span.end, decide);
|
|
680
|
+
reach = span.end;
|
|
681
|
+
}
|
|
682
|
+
pass.segs = cursor.finish();
|
|
683
|
+
claimPending(pass, pending);
|
|
583
684
|
}
|
|
584
685
|
function denyListRegex(entry) {
|
|
585
686
|
if (entry.type === 'regex')
|
|
586
687
|
return isSafeRegex(entry.value) ? new RegExp(entry.value, 'gi') : null;
|
|
587
688
|
return new RegExp(escapeRe(entry.value), 'gi');
|
|
588
689
|
}
|
|
690
|
+
const DENY_DECISION = {
|
|
691
|
+
pattern: { label: 'DENY_LIST', category: 'other' },
|
|
692
|
+
detail: { patternName: 'denyList', patternLabel: 'DENY_LIST', category: 'other', confidence: context_1.DENY_LIST_CONFIDENCE },
|
|
693
|
+
};
|
|
589
694
|
function denyListStage(pass) {
|
|
590
695
|
var _a;
|
|
591
696
|
for (const entry of (_a = pass.ctx.denyList) !== null && _a !== void 0 ? _a : []) {
|
|
@@ -593,57 +698,57 @@ function denyListStage(pass) {
|
|
|
593
698
|
assertUserRegexInput(pass.source, `Deny list regex "${entry.value}"`);
|
|
594
699
|
try {
|
|
595
700
|
const re = denyListRegex(entry);
|
|
596
|
-
if (
|
|
597
|
-
|
|
598
|
-
replaceFree(pass, re, (match) => TOKEN_RE.test(match)
|
|
599
|
-
? null
|
|
600
|
-
: redactMatch(pass, match, { label: 'DENY_LIST', category: 'other' }, {
|
|
601
|
-
patternName: 'denyList', patternLabel: 'DENY_LIST', category: 'other', confidence: context_1.DENY_LIST_CONFIDENCE,
|
|
602
|
-
}));
|
|
701
|
+
if (re)
|
|
702
|
+
replaceFree(pass, re, () => DENY_DECISION);
|
|
603
703
|
}
|
|
604
704
|
catch { }
|
|
605
705
|
}
|
|
606
706
|
}
|
|
607
|
-
function
|
|
608
|
-
if (TOKEN_RE.test(match) || (match.startsWith('[') && match.endsWith(']')))
|
|
609
|
-
return null;
|
|
707
|
+
function builtInDecision(pass, pattern, match, offset) {
|
|
610
708
|
const validatorPassed = pattern.validate ? pattern.validate(match) : true;
|
|
611
709
|
if (!validatorPassed || isAllowListed(match, pass.ctx.allowList))
|
|
612
710
|
return null;
|
|
613
711
|
const confidence = (0, context_1.calculateConfidence)(pattern.name, !!pattern.validate, validatorPassed, pass.source, offset, offset + match.length);
|
|
614
712
|
if (confidence < pass.threshold)
|
|
615
713
|
return null;
|
|
616
|
-
return
|
|
617
|
-
patternName: pattern.name, patternLabel: pattern.label, category: pattern.category, confidence,
|
|
618
|
-
});
|
|
714
|
+
return { pattern, detail: { patternName: pattern.name, patternLabel: pattern.label, category: pattern.category, confidence } };
|
|
619
715
|
}
|
|
620
716
|
function builtInStage(pass) {
|
|
621
717
|
const enabled = (0, patterns_1.getPatternsByNames)(pass.ctx.enabledPatterns);
|
|
622
718
|
for (const pattern of [...enabled.filter((p) => p.validate), ...enabled.filter((p) => !p.validate)]) {
|
|
623
|
-
|
|
719
|
+
if (pattern.anchor && !pass.source.includes(pattern.anchor))
|
|
720
|
+
continue;
|
|
721
|
+
const decide = (match, offset) => builtInDecision(pass, pattern, match, offset);
|
|
722
|
+
if (pattern.scan)
|
|
723
|
+
replaceSpans(pass, pattern.scan(pass.source), decide);
|
|
724
|
+
else
|
|
725
|
+
replaceFree(pass, pattern.regex, decide);
|
|
624
726
|
}
|
|
625
727
|
}
|
|
626
|
-
function
|
|
627
|
-
const
|
|
628
|
-
|
|
728
|
+
function nameSpans(pass) {
|
|
729
|
+
const spans = (0, names_1.detectNamesInText)(pass.source).map((nm) => ({ start: nm.start, end: nm.end }));
|
|
730
|
+
for (const seg of pass.segs) {
|
|
731
|
+
if (seg.locked)
|
|
732
|
+
continue;
|
|
733
|
+
for (const nm of (0, names_1.detectNamesInText)(seg.text))
|
|
734
|
+
spans.push({ start: seg.src + nm.start, end: seg.src + nm.end });
|
|
735
|
+
}
|
|
736
|
+
return spans;
|
|
629
737
|
}
|
|
630
738
|
function nameStage(pass) {
|
|
631
739
|
if (pass.ctx.skipSemantic)
|
|
632
740
|
return;
|
|
633
|
-
|
|
634
|
-
if (
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
return replacement;
|
|
645
|
-
});
|
|
646
|
-
}
|
|
741
|
+
replaceSpans(pass, nameSpans(pass), (match, start) => {
|
|
742
|
+
if (isAllowListed(match, pass.ctx.allowList))
|
|
743
|
+
return null;
|
|
744
|
+
const confidence = (0, context_1.hasContextWords)(pass.source, start, start + match.length, names_1.NAME_CONTEXT_WORDS) ? 0.8 : 0.65;
|
|
745
|
+
if (confidence < pass.threshold)
|
|
746
|
+
return null;
|
|
747
|
+
return {
|
|
748
|
+
pattern: { label: 'PERSON_NAME', category: 'identity' },
|
|
749
|
+
detail: { patternName: 'personNameDict', patternLabel: 'PERSON_NAME', category: 'identity', confidence },
|
|
750
|
+
};
|
|
751
|
+
});
|
|
647
752
|
}
|
|
648
753
|
function customStage(pass) {
|
|
649
754
|
if (context_1.CUSTOM_PATTERN_CONFIDENCE < pass.threshold)
|
|
@@ -651,15 +756,15 @@ function customStage(pass) {
|
|
|
651
756
|
for (const cp of pass.ctx.customPatterns) {
|
|
652
757
|
if (!cp.label || !cp.regex || !isSafeRegex(cp.regex))
|
|
653
758
|
continue;
|
|
654
|
-
const
|
|
759
|
+
const category = (0, types_1.isPiiCategory)(cp.category) ? cp.category : cp.category ? 'other' : 'identity';
|
|
760
|
+
const pattern = { label: cp.label.toUpperCase(), category };
|
|
761
|
+
const decision = {
|
|
762
|
+
pattern,
|
|
763
|
+
detail: { patternName: `custom_${cp.label}`, patternLabel: pattern.label, category: pattern.category, confidence: context_1.CUSTOM_PATTERN_CONFIDENCE },
|
|
764
|
+
};
|
|
655
765
|
assertUserRegexInput(pass.source, `Custom pattern "${cp.label}"`);
|
|
656
766
|
try {
|
|
657
|
-
replaceFree(pass, new RegExp(cp.regex, 'g'), (match) =>
|
|
658
|
-
? null
|
|
659
|
-
: redactMatch(pass, match, pattern, {
|
|
660
|
-
patternName: `custom_${cp.label}`, patternLabel: pattern.label, category: pattern.category,
|
|
661
|
-
confidence: context_1.CUSTOM_PATTERN_CONFIDENCE,
|
|
662
|
-
}));
|
|
767
|
+
replaceFree(pass, new RegExp(cp.regex, 'g'), (match) => (isAllowListed(match, pass.ctx.allowList) ? null : decision));
|
|
663
768
|
}
|
|
664
769
|
catch { }
|
|
665
770
|
}
|
|
@@ -686,12 +791,42 @@ function lockChild(ctx, container, key, compute) {
|
|
|
686
791
|
ctx.lastLocked = undefined;
|
|
687
792
|
return out;
|
|
688
793
|
}
|
|
794
|
+
function priorTokenMatcher(entries) {
|
|
795
|
+
const tokens = Object.keys(entries !== null && entries !== void 0 ? entries : {}).filter((t) => t.length > 0).sort((a, b) => b.length - a.length);
|
|
796
|
+
return tokens.length > 0 ? new RegExp(tokens.map(escapeRe).join('|'), 'g') : undefined;
|
|
797
|
+
}
|
|
798
|
+
function literalStage(pass) {
|
|
799
|
+
const { ctx, vault, sessionId } = pass;
|
|
800
|
+
if (!ctx.priorTokens || (ctx.mode !== 'token' && ctx.mode !== 'pseudonymize'))
|
|
801
|
+
return;
|
|
802
|
+
const decision = {
|
|
803
|
+
pattern: { label: 'LITERAL', category: 'other' },
|
|
804
|
+
store: (match) => {
|
|
805
|
+
const token = freshToken('LITERAL', vault, sessionId, ctx.reservedTokens);
|
|
806
|
+
vault.addEntry(sessionId, { token, original: match, patternLabel: 'LITERAL', category: 'other', createdAt: new Date().toISOString() });
|
|
807
|
+
return token;
|
|
808
|
+
},
|
|
809
|
+
};
|
|
810
|
+
replaceFree(pass, ctx.priorTokens, () => decision);
|
|
811
|
+
}
|
|
812
|
+
function protectLiterals(text, ctx, vault, sessionId, hits, itemIndex) {
|
|
813
|
+
if (!ctx.priorTokens)
|
|
814
|
+
return text;
|
|
815
|
+
const pass = {
|
|
816
|
+
source: text, segs: [{ text, locked: false, src: 0, end: text.length }], threshold: 0,
|
|
817
|
+
ctx, vault, sessionId, hits, itemIndex, fieldPath: '',
|
|
818
|
+
};
|
|
819
|
+
literalStage(pass);
|
|
820
|
+
recordLocked(ctx, pass.segs);
|
|
821
|
+
return pass.segs.map((seg) => seg.text).join('');
|
|
822
|
+
}
|
|
689
823
|
function redactText(text, ctx, vault, sessionId, hits, itemIndex, fieldPath) {
|
|
690
824
|
var _a;
|
|
691
825
|
const pass = {
|
|
692
|
-
source: text, segs: [{ text, locked: false, src: 0 }], threshold: (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0,
|
|
826
|
+
source: text, segs: [{ text, locked: false, src: 0, end: text.length }], threshold: (_a = ctx.confidenceThreshold) !== null && _a !== void 0 ? _a : 0,
|
|
693
827
|
ctx, vault, sessionId, hits, itemIndex, fieldPath,
|
|
694
828
|
};
|
|
829
|
+
literalStage(pass);
|
|
695
830
|
denyListStage(pass);
|
|
696
831
|
builtInStage(pass);
|
|
697
832
|
nameStage(pass);
|
|
@@ -754,15 +889,13 @@ function presidioSpans(value, entities, ctx, locked) {
|
|
|
754
889
|
return resolveOverlaps(toUtf16Spans(value, entities).filter((s) => s.score >= threshold && free(s)));
|
|
755
890
|
}
|
|
756
891
|
async function presidioRedactString(value, ctx, vault, sessionId, hits, itemIndex, path, locked) {
|
|
757
|
-
if (value.trim().length === 0
|
|
892
|
+
if (value.trim().length === 0)
|
|
758
893
|
return value;
|
|
759
894
|
const entities = await (0, presidio_1.analyzeWithPresidio)(ctx.presidioUrl, value, ctx.presidioLanguage || 'en', 0.4, 10000, ctx.presidioRequest);
|
|
760
895
|
const spans = presidioSpans(value, entities, ctx, locked);
|
|
761
896
|
let text = value;
|
|
762
897
|
for (const span of spans) {
|
|
763
898
|
const matchedText = value.slice(span.start, span.end);
|
|
764
|
-
if (TOKEN_RE.test(matchedText) || (matchedText.startsWith('[') && matchedText.endsWith(']')))
|
|
765
|
-
continue;
|
|
766
899
|
if (isAllowListed(matchedText, ctx.allowList))
|
|
767
900
|
continue;
|
|
768
901
|
const label = (0, presidio_1.getRedactorLabel)(span.entityType);
|
|
@@ -777,6 +910,24 @@ async function presidioRedactString(value, ctx, vault, sessionId, hits, itemInde
|
|
|
777
910
|
}
|
|
778
911
|
return text;
|
|
779
912
|
}
|
|
913
|
+
const IDENTIFIER_KEY = /^[\p{L}_$][\p{L}\p{N}_$-]*$/u;
|
|
914
|
+
async function presidioKey(key, original, taken, ctx, vault, sessionId, hits, itemIndex, path, policyPath) {
|
|
915
|
+
var _a, _b, _c;
|
|
916
|
+
if (IDENTIFIER_KEY.test(key) || !shouldProcessField(policyPath, ctx.fieldMode, ctx.fieldRules))
|
|
917
|
+
return key;
|
|
918
|
+
const locked = (_c = (_b = (_a = ctx.keyLocks) === null || _a === void 0 ? void 0 : _a.get(taken)) === null || _b === void 0 ? void 0 : _b.get(key)) !== null && _c !== void 0 ? _c : [];
|
|
919
|
+
let out = key;
|
|
920
|
+
try {
|
|
921
|
+
out = await presidioRedactString(key, ctx, vault, sessionId, hits, itemIndex, path, locked);
|
|
922
|
+
}
|
|
923
|
+
catch {
|
|
924
|
+
return key;
|
|
925
|
+
}
|
|
926
|
+
const clashes = (k) => k === '' || Object.prototype.hasOwnProperty.call(taken, k);
|
|
927
|
+
if (out === key || !clashes(out))
|
|
928
|
+
return out;
|
|
929
|
+
return distinctKey(out, original, clashes, ctx, vault, sessionId, Object.keys(taken));
|
|
930
|
+
}
|
|
780
931
|
async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path, policyPath, locked = []) {
|
|
781
932
|
var _a, _b, _c, _d, _e, _f;
|
|
782
933
|
if (typeof value === 'string') {
|
|
@@ -805,7 +956,8 @@ async function presidioWalk(value, ctx, vault, sessionId, hits, itemIndex, path,
|
|
|
805
956
|
const child = path ? `${path}.${k}` : k;
|
|
806
957
|
const childPolicy = policyPath ? `${policyPath}.${original}` : original;
|
|
807
958
|
const lock = (_f = (_e = ctx.lockedSpans) === null || _e === void 0 ? void 0 : _e.get(value)) === null || _f === void 0 ? void 0 : _f.get(k);
|
|
808
|
-
|
|
959
|
+
const key = await presidioKey(k, original, out, ctx, vault, sessionId, hits, itemIndex, child, childPolicy);
|
|
960
|
+
setOwn(out, key, await presidioWalk(v, ctx, vault, sessionId, hits, itemIndex, child, childPolicy, lock));
|
|
809
961
|
}
|
|
810
962
|
return out;
|
|
811
963
|
}
|
|
@@ -307,72 +307,49 @@ function detectPromptInjection(text) {
|
|
|
307
307
|
message,
|
|
308
308
|
};
|
|
309
309
|
}
|
|
310
|
-
function
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const fieldPath = currentPath ? `${currentPath}.${key}` : key;
|
|
322
|
-
if (typeof value === 'string' && value.trim().length > 0) {
|
|
323
|
-
const result = detectPromptInjection(value);
|
|
324
|
-
if (result.score > worstResult.score) {
|
|
325
|
-
worstResult = {
|
|
326
|
-
...result,
|
|
327
|
-
matchedPatterns: result.matchedPatterns.map((m) => ({
|
|
328
|
-
...m,
|
|
329
|
-
pattern: `[${fieldPath}] ${m.pattern}`,
|
|
330
|
-
})),
|
|
331
|
-
};
|
|
332
|
-
}
|
|
333
|
-
else if (result.detected) {
|
|
334
|
-
worstResult.matchedPatterns.push(...result.matchedPatterns.map((m) => ({
|
|
335
|
-
...m,
|
|
336
|
-
pattern: `[${fieldPath}] ${m.pattern}`,
|
|
337
|
-
})));
|
|
338
|
-
worstResult.score = Math.min(1.0, worstResult.score + result.score * 0.3);
|
|
339
|
-
worstResult.detected = true;
|
|
340
|
-
if (result.encodedPayloadsFound)
|
|
341
|
-
worstResult.encodedPayloadsFound = true;
|
|
342
|
-
if (result.zeroWidthCharsFound)
|
|
343
|
-
worstResult.zeroWidthCharsFound = true;
|
|
344
|
-
}
|
|
345
|
-
}
|
|
346
|
-
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
|
347
|
-
const nested = scanForInjection(value, fieldPath);
|
|
348
|
-
if (nested.score > worstResult.score) {
|
|
349
|
-
worstResult = nested;
|
|
350
|
-
}
|
|
351
|
-
else if (nested.detected) {
|
|
352
|
-
worstResult.matchedPatterns.push(...nested.matchedPatterns);
|
|
353
|
-
worstResult.score = Math.min(1.0, worstResult.score + nested.score * 0.3);
|
|
354
|
-
worstResult.detected = true;
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
if (Array.isArray(value)) {
|
|
358
|
-
for (let i = 0; i < value.length; i++) {
|
|
359
|
-
const item = value[i];
|
|
360
|
-
if (item !== null && typeof item === 'object') {
|
|
361
|
-
const nested = scanForInjection(item, `${fieldPath}[${i}]`);
|
|
362
|
-
if (nested.detected) {
|
|
363
|
-
worstResult.matchedPatterns.push(...nested.matchedPatterns);
|
|
364
|
-
worstResult.score = Math.min(1.0, worstResult.score + nested.score * 0.3);
|
|
365
|
-
worstResult.detected = true;
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
}
|
|
310
|
+
function collectStrings(value, path, into) {
|
|
311
|
+
if (typeof value === 'string') {
|
|
312
|
+
if (value.trim().length > 0)
|
|
313
|
+
into.push({ path, text: value });
|
|
314
|
+
}
|
|
315
|
+
else if (Array.isArray(value)) {
|
|
316
|
+
value.forEach((item, i) => collectStrings(item, `${path}[${i}]`, into));
|
|
317
|
+
}
|
|
318
|
+
else if (value !== null && typeof value === 'object') {
|
|
319
|
+
for (const [key, item] of Object.entries(value)) {
|
|
320
|
+
collectStrings(item, path ? `${path}.${key}` : key, into);
|
|
369
321
|
}
|
|
370
322
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
return
|
|
323
|
+
}
|
|
324
|
+
function actionFor(score, detected) {
|
|
325
|
+
if (score >= 0.7)
|
|
326
|
+
return 'BLOCK';
|
|
327
|
+
if (score >= 0.3)
|
|
328
|
+
return 'FLAG';
|
|
329
|
+
return detected ? 'LOG_ONLY' : 'CLEAN';
|
|
330
|
+
}
|
|
331
|
+
function scanForInjection(data, currentPath = '') {
|
|
332
|
+
const fields = [];
|
|
333
|
+
collectStrings(data, currentPath, fields);
|
|
334
|
+
const found = fields
|
|
335
|
+
.map((f) => ({ path: f.path, result: detectPromptInjection(f.text) }))
|
|
336
|
+
.filter((f) => f.result.detected)
|
|
337
|
+
.sort((a, b) => b.result.score - a.result.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
338
|
+
if (found.length === 0) {
|
|
339
|
+
return {
|
|
340
|
+
detected: false, score: 0, action: 'CLEAN', matchedPatterns: [],
|
|
341
|
+
encodedPayloadsFound: false, zeroWidthCharsFound: false, message: 'No prompt injection detected.',
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
const score = Math.min(1.0, found.slice(1).reduce((sum, f) => sum + f.result.score * 0.3, found[0].result.score));
|
|
345
|
+
const matchedPatterns = found.flatMap((f) => f.result.matchedPatterns.map((m) => ({ ...m, pattern: `[${f.path}] ${m.pattern}` })));
|
|
346
|
+
return {
|
|
347
|
+
...found[0].result,
|
|
348
|
+
detected: true,
|
|
349
|
+
score,
|
|
350
|
+
action: actionFor(score, true),
|
|
351
|
+
matchedPatterns,
|
|
352
|
+
encodedPayloadsFound: found.some((f) => f.result.encodedPayloadsFound),
|
|
353
|
+
zeroWidthCharsFound: found.some((f) => f.result.zeroWidthCharsFound),
|
|
354
|
+
};
|
|
378
355
|
}
|
|
@@ -1609,29 +1609,42 @@ const NOT_NAMES = new Set([
|
|
|
1609
1609
|
'True', 'False', 'Null', 'None', 'Default', 'Admin', 'System', 'Test', 'Debug', 'Config',
|
|
1610
1610
|
'Company', 'Corporation', 'Inc', 'Ltd', 'GmbH', 'LLC', 'Group', 'Team', 'Department', 'Division',
|
|
1611
1611
|
'Total', 'Amount', 'Price', 'Cost', 'Rate', 'Tax', 'Fee', 'Balance',
|
|
1612
|
-
'Solangi', 'Ghaffar', 'Shaikh', 'Syed', 'Qureshi', 'Mirza', 'Abbasi', 'Awan', 'Bhutto', 'Butt', 'Chaudhry', 'Durrani', 'Gilani', 'Hashmi', 'Iqbal', 'Jatoi', 'Leghari', 'Mengal', 'Niazi', 'Raisani', 'Talpur', 'Zardari',
|
|
1613
1612
|
]);
|
|
1613
|
+
for (const surname of ['Solangi', 'Ghaffar', 'Shaikh', 'Syed', 'Qureshi', 'Mirza', 'Abbasi', 'Awan', 'Bhutto', 'Butt', 'Chaudhry', 'Durrani', 'Gilani', 'Hashmi', 'Iqbal', 'Jatoi', 'Leghari', 'Mengal', 'Niazi', 'Raisani', 'Talpur', 'Zardari']) {
|
|
1614
|
+
exports.LAST_NAMES.add(surname);
|
|
1615
|
+
}
|
|
1614
1616
|
function detectNamesInText(text) {
|
|
1615
1617
|
const results = [];
|
|
1616
1618
|
const capitalWordSeq = /\b([A-ZÀ-ÿ][a-zà-ÿ]{1,20}(?:\s+[A-ZÀ-ÿ][a-zà-ÿ]{1,20}){1,3})\b/g;
|
|
1617
1619
|
let match;
|
|
1618
1620
|
while ((match = capitalWordSeq.exec(text)) !== null) {
|
|
1619
|
-
const
|
|
1620
|
-
|
|
1621
|
-
if (words.some((w) => NOT_NAMES.has(w)))
|
|
1622
|
-
continue;
|
|
1623
|
-
const hasFirstName = words.some((w) => exports.FIRST_NAMES.has(w));
|
|
1624
|
-
const hasLastName = words.some((w) => exports.LAST_NAMES.has(w));
|
|
1625
|
-
if (hasFirstName || hasLastName) {
|
|
1626
|
-
results.push({
|
|
1627
|
-
name: fullMatch,
|
|
1628
|
-
start: match.index,
|
|
1629
|
-
end: match.index + fullMatch.length,
|
|
1630
|
-
});
|
|
1631
|
-
}
|
|
1621
|
+
for (const run of nameRuns(match[1], match.index))
|
|
1622
|
+
results.push(run);
|
|
1632
1623
|
}
|
|
1633
1624
|
return results;
|
|
1634
1625
|
}
|
|
1626
|
+
function nameRuns(sequence, offset) {
|
|
1627
|
+
var _a;
|
|
1628
|
+
const runs = [];
|
|
1629
|
+
let words = [];
|
|
1630
|
+
const flush = () => {
|
|
1631
|
+
const known = words.some((w) => exports.FIRST_NAMES.has(w.word) || exports.LAST_NAMES.has(w.word));
|
|
1632
|
+
if (words.length >= 2 && known) {
|
|
1633
|
+
const start = offset + words[0].at;
|
|
1634
|
+
const last = words[words.length - 1];
|
|
1635
|
+
runs.push({ name: sequence.slice(words[0].at, last.at + last.word.length), start, end: offset + last.at + last.word.length });
|
|
1636
|
+
}
|
|
1637
|
+
words = [];
|
|
1638
|
+
};
|
|
1639
|
+
for (const m of sequence.matchAll(/\S+/g)) {
|
|
1640
|
+
if (NOT_NAMES.has(m[0]))
|
|
1641
|
+
flush();
|
|
1642
|
+
else
|
|
1643
|
+
words.push({ word: m[0], at: (_a = m.index) !== null && _a !== void 0 ? _a : 0 });
|
|
1644
|
+
}
|
|
1645
|
+
flush();
|
|
1646
|
+
return runs;
|
|
1647
|
+
}
|
|
1635
1648
|
exports.NAME_CONTEXT_WORDS = [
|
|
1636
1649
|
'spoke with', 'contact', 'customer', 'client', 'patient', 'employee',
|
|
1637
1650
|
'user', 'member', 'subscriber', 'applicant', 'candidate', 'tenant',
|
|
@@ -19,6 +19,38 @@ function luhnCheck(num) {
|
|
|
19
19
|
}
|
|
20
20
|
return sum % 10 === 0;
|
|
21
21
|
}
|
|
22
|
+
const EMAIL_LOCAL = /[A-Za-z0-9._%+'!#$&*/=?^`{|}~-]/;
|
|
23
|
+
const EMAIL_DOMAIN = /[\p{L}\p{N}.-]/u;
|
|
24
|
+
const EMAIL_DOMAIN_SHAPE = /^[\p{L}\p{N}-]+(?:[.][\p{L}\p{N}-]+)*[.]\p{L}{2,}$/u;
|
|
25
|
+
function emailLocalStart(text, at) {
|
|
26
|
+
if (text[at - 1] === '"') {
|
|
27
|
+
const open = text.lastIndexOf('"', at - 2);
|
|
28
|
+
const quoted = open >= 0 ? text.slice(open + 1, at - 1) : '';
|
|
29
|
+
return quoted.length > 0 && !/[\r\n"]/.test(quoted) ? open : -1;
|
|
30
|
+
}
|
|
31
|
+
let start = at;
|
|
32
|
+
while (start > 0 && EMAIL_LOCAL.test(text[start - 1]))
|
|
33
|
+
start--;
|
|
34
|
+
return start < at ? start : -1;
|
|
35
|
+
}
|
|
36
|
+
function emailDomainEnd(text, at) {
|
|
37
|
+
let end = at + 1;
|
|
38
|
+
while (end < text.length && EMAIL_DOMAIN.test(text[end]))
|
|
39
|
+
end++;
|
|
40
|
+
while (end > at + 1 && /[.-]/.test(text[end - 1]))
|
|
41
|
+
end--;
|
|
42
|
+
return EMAIL_DOMAIN_SHAPE.test(text.slice(at + 1, end)) ? end : -1;
|
|
43
|
+
}
|
|
44
|
+
function scanEmails(text) {
|
|
45
|
+
const spans = [];
|
|
46
|
+
for (let at = text.indexOf('@'); at >= 0; at = text.indexOf('@', at + 1)) {
|
|
47
|
+
const start = emailLocalStart(text, at);
|
|
48
|
+
const end = start < 0 ? -1 : emailDomainEnd(text, at);
|
|
49
|
+
if (end > 0)
|
|
50
|
+
spans.push({ start, end });
|
|
51
|
+
}
|
|
52
|
+
return spans;
|
|
53
|
+
}
|
|
22
54
|
function ibanCheck(iban) {
|
|
23
55
|
const cleaned = iban.replace(/\s/g, '').toUpperCase();
|
|
24
56
|
if (cleaned.length < 15 || cleaned.length > 34)
|
|
@@ -35,8 +67,10 @@ exports.PII_PATTERNS = [
|
|
|
35
67
|
name: 'email',
|
|
36
68
|
label: 'EMAIL',
|
|
37
69
|
category: 'contact',
|
|
38
|
-
regex: /[a-zA-Z0-9._
|
|
39
|
-
|
|
70
|
+
regex: /[a-zA-Z0-9._%+'-]+@[\p{L}\p{N}.-]+[.]\p{L}{2,}/gu,
|
|
71
|
+
anchor: '@',
|
|
72
|
+
scan: scanEmails,
|
|
73
|
+
description: 'Email addresses, including quoted local parts and international domains',
|
|
40
74
|
},
|
|
41
75
|
{
|
|
42
76
|
name: 'phone',
|
|
@@ -56,7 +90,7 @@ exports.PII_PATTERNS = [
|
|
|
56
90
|
name: 'phoneDE',
|
|
57
91
|
label: 'PHONE_DE',
|
|
58
92
|
category: 'contact',
|
|
59
|
-
regex: /(?:0|\+49)\s?\d{2,4}[\s/]\d{3,8}(?:[\s-]\d{1,5})
|
|
93
|
+
regex: /(?:0|\+49)\s?\d{2,4}[\s/]\d{3,8}(?:[\s-]\d{1,5})?|\+49\d{9,12}\b/g,
|
|
60
94
|
description: 'German phone numbers (089 1234 5678, +49 30 12345678)',
|
|
61
95
|
},
|
|
62
96
|
{
|
|
@@ -277,9 +311,9 @@ exports.PII_PATTERNS = [
|
|
|
277
311
|
name: 'creditCard',
|
|
278
312
|
label: 'CREDIT_CARD',
|
|
279
313
|
category: 'financial',
|
|
280
|
-
regex: /\b(?:\d
|
|
314
|
+
regex: /\b(?:\d[\s-]?){12,18}\d\b/g,
|
|
281
315
|
validate: (match) => luhnCheck(match),
|
|
282
|
-
description: 'Credit/debit card numbers (Luhn validated)',
|
|
316
|
+
description: 'Credit/debit card numbers, 13 to 19 digits (Luhn validated)',
|
|
283
317
|
},
|
|
284
318
|
{
|
|
285
319
|
name: 'iban',
|
|
@@ -825,8 +859,8 @@ exports.PII_PATTERNS = [
|
|
|
825
859
|
name: 'pemPrivateKey',
|
|
826
860
|
label: 'PRIVATE_KEY',
|
|
827
861
|
category: 'enterprise',
|
|
828
|
-
regex: /-----BEGIN (
|
|
829
|
-
description: 'PEM private
|
|
862
|
+
regex: /-----BEGIN (RSA |DSA |EC |OPENSSH |ENCRYPTED )?PRIVATE KEY-----(?:[\s\S]*?-----END \1?PRIVATE KEY-----|[A-Za-z0-9+/=\s]*)/g,
|
|
863
|
+
description: 'PEM private keys, the whole block through the END line',
|
|
830
864
|
},
|
|
831
865
|
{
|
|
832
866
|
name: 'sshPublicKey',
|
|
@@ -1448,8 +1482,8 @@ exports.PII_PATTERNS = [
|
|
|
1448
1482
|
name: 'pgpPrivateKey',
|
|
1449
1483
|
label: 'PGP_KEY',
|
|
1450
1484
|
category: 'enterprise',
|
|
1451
|
-
regex: /-----BEGIN PGP PRIVATE KEY BLOCK
|
|
1452
|
-
description: 'PGP private key block
|
|
1485
|
+
regex: /-----BEGIN PGP PRIVATE KEY BLOCK-----(?:[\s\S]*?-----END PGP PRIVATE KEY BLOCK-----|[A-Za-z0-9+/=:\s]*)/g,
|
|
1486
|
+
description: 'PGP private key blocks, the whole block through the END line',
|
|
1453
1487
|
},
|
|
1454
1488
|
{
|
|
1455
1489
|
name: 'envSecrets',
|
|
@@ -120,7 +120,7 @@ function parseTenantProfile(input, knownPatterns) {
|
|
|
120
120
|
}
|
|
121
121
|
if (!isObject(raw))
|
|
122
122
|
fail('must be a JSON object');
|
|
123
|
-
const unknown = Object.keys(raw).filter((k) => !(k
|
|
123
|
+
const unknown = Object.keys(raw).filter((k) => !Object.prototype.hasOwnProperty.call(PARSERS, k));
|
|
124
124
|
if (unknown.length)
|
|
125
125
|
fail(`unknown field(s) ${unknown.join(', ')}`);
|
|
126
126
|
const profile = {};
|
|
@@ -93,6 +93,8 @@ function escapeSet(src, i) {
|
|
|
93
93
|
function classMember(body, j) {
|
|
94
94
|
if (body[j] === '\\' && body[j + 1] === 'b')
|
|
95
95
|
return { set: codeSet(8), next: j + 2, code: 8 };
|
|
96
|
+
if (body[j] === '\\' && body[j + 1] === 'B')
|
|
97
|
+
return { set: codeSet(66), next: j + 2, code: 66 };
|
|
96
98
|
if (body[j] === '\\') {
|
|
97
99
|
const { length, code } = readEscape(body, j);
|
|
98
100
|
return { set: escapeSet(body, j), next: j + length, code };
|
|
@@ -3,9 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.checkRegex = checkRegex;
|
|
4
4
|
const regexCharset_1 = require("./regexCharset");
|
|
5
5
|
const MAX_REGEX_LENGTH = 500;
|
|
6
|
-
const newFrame = (zeroWidth = false) => ({
|
|
6
|
+
const newFrame = (zeroWidth = false, inLook = zeroWidth) => ({
|
|
7
7
|
hasVariableQuantifier: false, hasAlternation: false, run: null, chars: (0, regexCharset_1.emptySet)(), zeroWidth,
|
|
8
|
-
branchConsumes: false, allConsume: true, nullable: false,
|
|
8
|
+
branchConsumes: false, allConsume: true, nullable: false, inLook,
|
|
9
9
|
});
|
|
10
10
|
function endBranch(frame) {
|
|
11
11
|
frame.allConsume && (frame.allConsume = frame.branchConsumes);
|
|
@@ -48,7 +48,7 @@ function skipClass(src, i) {
|
|
|
48
48
|
function groupPrefixLength(src, i) {
|
|
49
49
|
if (src[i + 1] !== '?')
|
|
50
50
|
return 1;
|
|
51
|
-
const named = /^\(\?<[
|
|
51
|
+
const named = /^\(\?<[^=!>][^>]*>/.exec(src.slice(i));
|
|
52
52
|
if (named)
|
|
53
53
|
return named[0].length;
|
|
54
54
|
const look = /^\(\?(?::|=|!|<=|<!)/.exec(src.slice(i));
|
|
@@ -79,7 +79,8 @@ function readToken(src, i, stack) {
|
|
|
79
79
|
if (c === '[')
|
|
80
80
|
return { atomEnd: skipClass(src, i), closed: null };
|
|
81
81
|
if (c === '(') {
|
|
82
|
-
|
|
82
|
+
const look = /^\(\?(?:=|!|<=|<!)/.test(src.slice(i));
|
|
83
|
+
stack.push(newFrame(look, look || stack[stack.length - 1].inLook));
|
|
83
84
|
return { atomEnd: i, closed: null, next: i + groupPrefixLength(src, i) };
|
|
84
85
|
}
|
|
85
86
|
if (c === '|')
|
|
@@ -127,8 +128,11 @@ function consumes(q, chars, closed) {
|
|
|
127
128
|
return false;
|
|
128
129
|
return !(closed === null || closed === void 0 ? void 0 : closed.nullable);
|
|
129
130
|
}
|
|
131
|
+
const IN_LOOK = 'repeats a variable part inside a lookahead or lookbehind, which is not supported';
|
|
130
132
|
function stepProblem(src, i, step, q, top) {
|
|
131
133
|
var _a;
|
|
134
|
+
if (top.inLook && isVariable(q))
|
|
135
|
+
return IN_LOOK;
|
|
132
136
|
const problem = applyQuantifier(q, step.closed, top);
|
|
133
137
|
if (problem || (q === null || q === void 0 ? void 0 : q.max) === 0)
|
|
134
138
|
return problem;
|
|
@@ -1,6 +1,22 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.generateRopaRecord = generateRopaRecord;
|
|
4
|
+
function retentionDescription(vaultTtlMinutes, auditLogEnabled) {
|
|
5
|
+
const vault = vaultTtlMinutes > 0
|
|
6
|
+
? `Vault sessions expire after ${vaultTtlMinutes} minutes.`
|
|
7
|
+
: 'Vault sessions do not expire automatically. Purge them when they are no longer needed.';
|
|
8
|
+
const audit = auditLogEnabled
|
|
9
|
+
? 'Audit entries go to the Report output. Their retention depends on where the workflow stores them.'
|
|
10
|
+
: 'No audit log is written.';
|
|
11
|
+
return `${vault} ${audit}`;
|
|
12
|
+
}
|
|
13
|
+
function countByCategory(hits) {
|
|
14
|
+
var _a;
|
|
15
|
+
const counts = new Map();
|
|
16
|
+
for (const hit of hits)
|
|
17
|
+
counts.set(hit.category, ((_a = counts.get(hit.category)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
18
|
+
return Object.fromEntries(counts);
|
|
19
|
+
}
|
|
4
20
|
function generateRopaRecord(hits, classification, itemCount, patternsUsed, redactionMode, vaultTtlMinutes, encryptionEnabled, auditLogEnabled, controller, processingPurposes, legalBasis, dataSubjectCategories, recipientCategories, thirdCountryTransfers) {
|
|
5
21
|
const categoryMap = {};
|
|
6
22
|
for (const hit of hits) {
|
|
@@ -30,8 +46,8 @@ function generateRopaRecord(hits, classification, itemCount, patternsUsed, redac
|
|
|
30
46
|
personalDataCategories,
|
|
31
47
|
retentionPolicy: {
|
|
32
48
|
vaultTtlMinutes,
|
|
33
|
-
auditLogRetentionDays:
|
|
34
|
-
description:
|
|
49
|
+
auditLogRetentionDays: null,
|
|
50
|
+
description: retentionDescription(vaultTtlMinutes, auditLogEnabled),
|
|
35
51
|
},
|
|
36
52
|
securityMeasures: {
|
|
37
53
|
encryptionAtRest: encryptionEnabled,
|
|
@@ -46,7 +62,7 @@ function generateRopaRecord(hits, classification, itemCount, patternsUsed, redac
|
|
|
46
62
|
redactionMode,
|
|
47
63
|
patternsUsed,
|
|
48
64
|
classificationLevel: classification.level,
|
|
49
|
-
hitsByCategory:
|
|
65
|
+
hitsByCategory: countByCategory(hits),
|
|
50
66
|
},
|
|
51
67
|
};
|
|
52
68
|
}
|
|
@@ -6,6 +6,7 @@ exports.assertUserRegexes = assertUserRegexes;
|
|
|
6
6
|
exports.sanitizeInjection = sanitizeInjection;
|
|
7
7
|
const n8n_workflow_1 = require("n8n-workflow");
|
|
8
8
|
const regexSafety_1 = require("./regexSafety");
|
|
9
|
+
const types_1 = require("./types");
|
|
9
10
|
const injection_1 = require("./injection");
|
|
10
11
|
const VAULT_CREDENTIAL = 'redactorVaultApi';
|
|
11
12
|
async function resolvePassphrase(fns) {
|
|
@@ -58,6 +59,13 @@ function workflowNamespace(fns) {
|
|
|
58
59
|
}
|
|
59
60
|
return 'wf:unsaved';
|
|
60
61
|
}
|
|
62
|
+
function assertCategory(fns, cp) {
|
|
63
|
+
if (fns.getNode().typeVersion < 2 || !cp.category || (0, types_1.isPiiCategory)(cp.category))
|
|
64
|
+
return;
|
|
65
|
+
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), `Custom pattern "${cp.label}" has an unknown category "${cp.category}"`, {
|
|
66
|
+
description: `Use one of ${types_1.PII_CATEGORIES.join(', ')}.`,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
61
69
|
function assertUserRegexes(fns, customPatterns, lists) {
|
|
62
70
|
const refuse = (what, problem) => {
|
|
63
71
|
throw new n8n_workflow_1.NodeOperationError(fns.getNode(), `${what} regex ${problem}`, {
|
|
@@ -67,6 +75,7 @@ function assertUserRegexes(fns, customPatterns, lists) {
|
|
|
67
75
|
for (const cp of customPatterns) {
|
|
68
76
|
if (!cp.label || !cp.regex)
|
|
69
77
|
continue;
|
|
78
|
+
assertCategory(fns, cp);
|
|
70
79
|
const problem = (0, regexSafety_1.checkRegex)(cp.regex);
|
|
71
80
|
if (problem)
|
|
72
81
|
refuse(`Custom pattern "${cp.label}"`, problem);
|
|
@@ -1,2 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isPiiCategory = exports.PII_CATEGORIES = void 0;
|
|
4
|
+
exports.PII_CATEGORIES = [
|
|
5
|
+
'identity', 'financial', 'contact', 'network', 'location', 'temporal',
|
|
6
|
+
'medical', 'crypto', 'enterprise', 'vehicle', 'biometric', 'other',
|
|
7
|
+
];
|
|
8
|
+
const isPiiCategory = (value) => typeof value === 'string' && exports.PII_CATEGORIES.includes(value);
|
|
9
|
+
exports.isPiiCategory = isPiiCategory;
|
|
@@ -167,6 +167,7 @@ class WorkflowVault {
|
|
|
167
167
|
constructor(store, passphrase = '') {
|
|
168
168
|
this.store = store;
|
|
169
169
|
this.passphrase = passphrase;
|
|
170
|
+
this.originals = new WeakMap();
|
|
170
171
|
this.cache = new Map();
|
|
171
172
|
}
|
|
172
173
|
get sessions() {
|
|
@@ -240,12 +241,27 @@ class WorkflowVault {
|
|
|
240
241
|
return;
|
|
241
242
|
assertNoConflict(session, entry);
|
|
242
243
|
session.entries[entry.token] = entry;
|
|
244
|
+
const index = this.originals.get(session);
|
|
245
|
+
const first = index === null || index === void 0 ? void 0 : index.get(entry.original);
|
|
246
|
+
if (index && (!first || first.token === entry.token))
|
|
247
|
+
index.set(entry.original, entry);
|
|
243
248
|
}
|
|
244
249
|
findByOriginal(sessionId, original) {
|
|
245
250
|
const session = this.cache.get(sessionId);
|
|
246
251
|
if (!session)
|
|
247
252
|
return undefined;
|
|
248
|
-
return
|
|
253
|
+
return this.originalIndex(session).get(original);
|
|
254
|
+
}
|
|
255
|
+
originalIndex(session) {
|
|
256
|
+
let index = this.originals.get(session);
|
|
257
|
+
if (!index) {
|
|
258
|
+
index = new Map();
|
|
259
|
+
for (const entry of Object.values(session.entries))
|
|
260
|
+
if (!index.has(entry.original))
|
|
261
|
+
index.set(entry.original, entry);
|
|
262
|
+
this.originals.set(session, index);
|
|
263
|
+
}
|
|
264
|
+
return index;
|
|
249
265
|
}
|
|
250
266
|
deleteSession(sessionId) {
|
|
251
267
|
this.cache.delete(sessionId);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-redactor",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "6.0.0",
|
|
4
4
|
"description": "Redact, mask and anonymize PII before it reaches an LLM, then restore it. 216 patterns for emails, phones, addresses, IBAN, cards, names, SSN, passport and tax IDs. Reversible encrypted vault, GDPR Art.30 ROPA record, prompt-injection check. Runs on n8n Cloud and self-hosted n8n 2.x.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"n8n-community-node-package",
|