n8n-nodes-redactor 6.1.0 → 7.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -9
- package/dist/nodes/Redactor/Redactor.node.js +202 -83
- package/dist/nodes/Redactor/audit.js +18 -9
- package/dist/nodes/Redactor/engine.js +609 -101
- package/dist/nodes/Redactor/injection.js +1 -1
- package/dist/nodes/Redactor/names.js +54 -15
- package/dist/nodes/Redactor/patterns.js +454 -44
- package/dist/nodes/Redactor/presidio.js +57 -21
- package/dist/nodes/Redactor/profiles.js +5 -3
- package/dist/nodes/Redactor/pseudonymize.js +2 -2
- package/dist/nodes/Redactor/runtime.js +59 -3
- package/dist/nodes/Redactor/vault.js +3 -3
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="https://raw.githubusercontent.com/mjmirza/n8n-nodes-redactor/
|
|
2
|
+
<img src="https://raw.githubusercontent.com/mjmirza/n8n-nodes-redactor/main/.github/assets/logo.png" alt="Redactor" width="96" />
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<h1 align="center">n8n-nodes-redactor</h1>
|
|
@@ -12,16 +12,16 @@
|
|
|
12
12
|
<p align="center">
|
|
13
13
|
<a href="https://www.npmjs.com/package/n8n-nodes-redactor"><img src="https://img.shields.io/npm/v/n8n-nodes-redactor?label=npm" alt="npm version" /></a>
|
|
14
14
|
<a href="https://www.npmjs.com/package/n8n-nodes-redactor"><img src="https://img.shields.io/npm/dm/n8n-nodes-redactor?label=downloads" alt="npm downloads" /></a>
|
|
15
|
-
<a href="https://github.com/mjmirza/n8n-nodes-redactor/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/mjmirza/n8n-nodes-redactor/ci.yml?branch=
|
|
15
|
+
<a href="https://github.com/mjmirza/n8n-nodes-redactor/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/mjmirza/n8n-nodes-redactor/ci.yml?branch=main&label=CI" alt="CI status" /></a>
|
|
16
16
|
<a href="https://www.npmjs.com/package/n8n-nodes-redactor#provenance"><img src="https://img.shields.io/badge/npm-provenance%20signed-brightgreen" alt="npm provenance" /></a>
|
|
17
17
|
<a href="#license"><img src="https://img.shields.io/npm/l/n8n-nodes-redactor" alt="MIT License" /></a>
|
|
18
|
-
<img src="https://img.shields.io/badge/PII%20patterns-
|
|
18
|
+
<img src="https://img.shields.io/badge/PII%20patterns-218-blue" alt="218 PII patterns" />
|
|
19
19
|
<img src="https://img.shields.io/badge/n8n-Cloud%20%2B%20self--hosted-FF6D5A" alt="n8n Cloud and self-hosted" />
|
|
20
20
|
</p>
|
|
21
21
|
|
|
22
22
|
## What you get
|
|
23
23
|
|
|
24
|
-
- **Personal data
|
|
24
|
+
- **Personal data is kept from the model.** Emails, phone numbers, IBANs, card numbers, national IDs and 200 more kinds of data are swapped for safe tokens before the LLM call. Pattern detection cannot know every name, so add Presidio for free-text names, and keep Verify in the chain to catch what slips through.
|
|
25
25
|
- **The answer still makes sense.** After the model replies, the node puts the real values back, so your output reads naturally.
|
|
26
26
|
- **Fewer false alarms.** Card numbers are Luhn checked, IBANs checksum verified, NHS numbers mod 11 validated, so random digits are left alone.
|
|
27
27
|
- **Proof for your records.** An optional audit report, and a GDPR Article 30 processing record built from each run.
|
|
@@ -54,7 +54,7 @@ After Restore, your workflow gets the real values back.
|
|
|
54
54
|
| Self-hosted n8n 1.x and 2.x | Supported |
|
|
55
55
|
| n8n verified community nodes | Eligible from 4.0.0. Passes n8n's verification scanner with no findings |
|
|
56
56
|
|
|
57
|
-
Version 4.0.0 uses no local disk. The vault lives in workflow storage and audit records go to the Report output. Upgrading from 1.x? Read [docs/MIGRATION.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/
|
|
57
|
+
Version 4.0.0 uses no local disk. The vault lives in workflow storage and audit records go to the Report output. Upgrading from 1.x? Read [docs/MIGRATION.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/main/docs/MIGRATION.md). The 1.x line stays installable with `npm install n8n-nodes-redactor@v1-legacy`.
|
|
58
58
|
|
|
59
59
|
## Operations
|
|
60
60
|
|
|
@@ -71,9 +71,13 @@ Version 4.0.0 uses no local disk. The vault lives in workflow storage and audit
|
|
|
71
71
|
The recommended production chain looks like this.
|
|
72
72
|
|
|
73
73
|
```
|
|
74
|
-
Data source >
|
|
74
|
+
Data source > Redact > Verify > LLM > Restore > Output > Purge
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
+
Detect is for a dry run before you set up redaction. It adds a summary item to its output by default, so keep it out of this chain, or turn on Include Audit Report, which moves the summary to a second output.
|
|
78
|
+
|
|
79
|
+
Set Verify Output on the Verify node to Items, Stop on FAIL. Verify then passes the redacted items on unchanged when nothing leaks, and stops the run before the LLM when something does, so the model never receives data that failed the check. With the default Report output, Verify returns one report instead of the items, so it cannot sit in this chain. Verify also fails when an item carries a binary attachment, because attachments are not scanned. Redact removes attachments by default. Set Binary Attachments to Keep Unscanned only when the files hold no personal data.
|
|
80
|
+
|
|
77
81
|
## Redaction modes
|
|
78
82
|
|
|
79
83
|
| Mode | Output | Reversible | Best for |
|
|
@@ -88,7 +92,7 @@ Data source > Detect > Redact > LLM > Verify > Restore > Output > Purge
|
|
|
88
92
|
|
|
89
93
|
## What it detects
|
|
90
94
|
|
|
91
|
-
|
|
95
|
+
218 patterns across 11 categories.
|
|
92
96
|
|
|
93
97
|
| Category | Examples |
|
|
94
98
|
|---|---|
|
|
@@ -117,8 +121,8 @@ Data source > Detect > Redact > LLM > Verify > Restore > Output > Purge
|
|
|
117
121
|
| Audit report | A second output with hits by category and pattern |
|
|
118
122
|
| Audit log | Turn on Audit Log to get an `audit` record (counts, labels, timing, never values) on the Report output. Send it to any database, sheet or SIEM node |
|
|
119
123
|
| ROPA record | Turn on Include Audit Report, then Include GDPR Art.30 ROPA Record, and fill in the controller details |
|
|
120
|
-
| Presidio | Optional Microsoft Presidio container for names and organisations in free text. See [docs/PRESIDIO_SETUP.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/
|
|
121
|
-
| Per-tenant profiles | Turn on Use Tenant Profile and paste JSON. Validated on every run. Fields are listed in [docs/MIGRATION.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/
|
|
124
|
+
| Presidio | Optional Microsoft Presidio container for names and organisations in free text. See [docs/PRESIDIO_SETUP.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/main/docs/PRESIDIO_SETUP.md). When Presidio cannot scan a text the run stops, unless If Presidio Fails is set to continue with patterns only |
|
|
125
|
+
| Per-tenant profiles | Turn on Use Tenant Profile and paste JSON. Validated on every run. Fields are listed in [docs/MIGRATION.md](https://github.com/mjmirza/n8n-nodes-redactor/blob/main/docs/MIGRATION.md#tenant-profile-fields) |
|
|
122
126
|
|
|
123
127
|
This node reduces what reaches a model. It is not legal advice and does not on its own make a workflow GDPR or HIPAA compliant.
|
|
124
128
|
|
|
@@ -13,7 +13,7 @@ const profiles_1 = require("./profiles");
|
|
|
13
13
|
const ropa_1 = require("./ropa");
|
|
14
14
|
const runtime_1 = require("./runtime");
|
|
15
15
|
const RECOMMENDED_PATTERNS = [
|
|
16
|
-
'email', 'phone', 'phoneDE', 'phoneUK', 'phoneAT', 'phoneCH', 'phoneFR',
|
|
16
|
+
'email', 'phone', 'phoneE164', 'phoneDE', 'phoneUK', 'phoneAT', 'phoneCH', 'phoneFR',
|
|
17
17
|
'phoneNL', 'phoneES', 'phoneIT', 'phoneAU', 'phoneIN', 'phoneBR',
|
|
18
18
|
'personName', 'ssn', 'itinUS', 'sinCA', 'ninoUK', 'passportUS', 'passportEU',
|
|
19
19
|
'passportDE', 'nationalIdDE', 'nhsNumber', 'taxIdUS', 'taxIdDE',
|
|
@@ -35,7 +35,7 @@ const RECOMMENDED_PATTERNS = [
|
|
|
35
35
|
'internalHostname', 'uncPath', 'ldapDN', 'adUsername',
|
|
36
36
|
'dbConnectionString', 'connStringKV',
|
|
37
37
|
'awsAccessKey', 'gcpApiKey', 'stripeKey', 'openaiKey', 'githubToken',
|
|
38
|
-
'slackToken', 'bearerToken', 'jwtToken', 'pemPrivateKey', 'sshPublicKey',
|
|
38
|
+
'slackToken', 'bearerToken', 'httpBasicAuth', 'httpBasicCredential', 'jwtToken', 'pemPrivateKey', 'sshPublicKey',
|
|
39
39
|
'genericSecret', 'azureKey', 'slackWebhook',
|
|
40
40
|
'sendgridKey', 'twilioKey', 'anthropicKey', 'gitlabToken',
|
|
41
41
|
'x509Certificate', 'pgpPrivateKey', 'envSecrets',
|
|
@@ -43,6 +43,19 @@ const RECOMMENDED_PATTERNS = [
|
|
|
43
43
|
'vin',
|
|
44
44
|
'uuid', 'imei', 'iccid',
|
|
45
45
|
];
|
|
46
|
+
function verifyMessage(leaks, attachments) {
|
|
47
|
+
if (leaks === 0 && attachments === 0)
|
|
48
|
+
return 'No PII detected in values. Data appears properly redacted.';
|
|
49
|
+
const parts = [];
|
|
50
|
+
if (leaks > 0)
|
|
51
|
+
parts.push(`Found ${leaks} potential PII leak(s). Values contain detectable sensitive data.`);
|
|
52
|
+
if (attachments > 0)
|
|
53
|
+
parts.push(`${attachments} binary attachment(s) were not scanned and may contain PII.`);
|
|
54
|
+
return parts.join(' ');
|
|
55
|
+
}
|
|
56
|
+
function withBinary(out, source) {
|
|
57
|
+
return source.binary ? { ...out, binary: source.binary } : out;
|
|
58
|
+
}
|
|
46
59
|
function versionRopaSummary(ropa, byLevel, typeVersion) {
|
|
47
60
|
const summary = ropa.processingSummary;
|
|
48
61
|
if (typeVersion >= 2) {
|
|
@@ -529,6 +542,54 @@ class Redactor {
|
|
|
529
542
|
default: false,
|
|
530
543
|
description: 'Whether to scan input for prompt injection attempts (jailbreaks, instruction overrides, delimiter escapes, encoded payloads) before it reaches an LLM. Detected injections are flagged in the output.',
|
|
531
544
|
},
|
|
545
|
+
{
|
|
546
|
+
displayName: 'Verify Output',
|
|
547
|
+
name: 'verifyOutput',
|
|
548
|
+
type: 'options',
|
|
549
|
+
displayOptions: {
|
|
550
|
+
show: {
|
|
551
|
+
operation: ['verify'],
|
|
552
|
+
},
|
|
553
|
+
},
|
|
554
|
+
options: [
|
|
555
|
+
{
|
|
556
|
+
name: 'Report',
|
|
557
|
+
value: 'report',
|
|
558
|
+
description: 'Return one report with PASS or FAIL',
|
|
559
|
+
},
|
|
560
|
+
{
|
|
561
|
+
name: 'Items, Stop on FAIL',
|
|
562
|
+
value: 'gate',
|
|
563
|
+
description: 'Pass the checked items on unchanged when they pass, and stop the run when they fail. Place it before the LLM.',
|
|
564
|
+
},
|
|
565
|
+
],
|
|
566
|
+
default: 'report',
|
|
567
|
+
description: 'What Verify returns',
|
|
568
|
+
},
|
|
569
|
+
{
|
|
570
|
+
displayName: 'Binary Attachments',
|
|
571
|
+
name: 'binaryAttachments',
|
|
572
|
+
type: 'options',
|
|
573
|
+
displayOptions: {
|
|
574
|
+
show: {
|
|
575
|
+
operation: ['redact'],
|
|
576
|
+
},
|
|
577
|
+
},
|
|
578
|
+
options: [
|
|
579
|
+
{
|
|
580
|
+
name: 'Remove',
|
|
581
|
+
value: 'remove',
|
|
582
|
+
description: 'Drop attachments from the output, so no unscanned data reaches the next node',
|
|
583
|
+
},
|
|
584
|
+
{
|
|
585
|
+
name: 'Keep Unscanned',
|
|
586
|
+
value: 'keep',
|
|
587
|
+
description: 'Pass attachments through unchanged. Their content is not scanned or redacted.',
|
|
588
|
+
},
|
|
589
|
+
],
|
|
590
|
+
default: 'remove',
|
|
591
|
+
description: 'What to do with binary attachments. The node redacts JSON only, so a file can still hold personal data.',
|
|
592
|
+
},
|
|
532
593
|
{
|
|
533
594
|
displayName: 'Audit Log',
|
|
534
595
|
name: 'auditLog',
|
|
@@ -551,7 +612,7 @@ class Redactor {
|
|
|
551
612
|
},
|
|
552
613
|
},
|
|
553
614
|
default: false,
|
|
554
|
-
description: 'Whether to add NLP-based detection using Microsoft Presidio. Catches person names, locations, and organizations in free text that regex cannot detect. Requires a Presidio Docker container running locally. Setup: docker run -d -p 5002:3000 mcr.microsoft.com/presidio-analyzer',
|
|
615
|
+
description: 'Whether to add NLP-based detection using Microsoft Presidio. Catches person names, locations, and organizations in free text that regex cannot detect. Requires a Presidio Docker container running locally. Setup: docker run -d -p 127.0.0.1:5002:3000 mcr.microsoft.com/presidio-analyzer',
|
|
555
616
|
},
|
|
556
617
|
{
|
|
557
618
|
displayName: 'Presidio URL',
|
|
@@ -567,6 +628,31 @@ class Redactor {
|
|
|
567
628
|
placeholder: 'http://localhost:5002',
|
|
568
629
|
description: 'URL of the Presidio Analyzer service. Default: http://localhost:5002. If using docker-compose, use the service name: http://presidio:3000',
|
|
569
630
|
},
|
|
631
|
+
{
|
|
632
|
+
displayName: 'If Presidio Fails',
|
|
633
|
+
name: 'presidioOnFailure',
|
|
634
|
+
type: 'options',
|
|
635
|
+
displayOptions: {
|
|
636
|
+
show: {
|
|
637
|
+
operation: ['redact'],
|
|
638
|
+
enablePresidio: [true],
|
|
639
|
+
},
|
|
640
|
+
},
|
|
641
|
+
options: [
|
|
642
|
+
{
|
|
643
|
+
name: 'Stop the Run',
|
|
644
|
+
value: 'stop',
|
|
645
|
+
description: 'Output nothing when any text could not be scanned by Presidio',
|
|
646
|
+
},
|
|
647
|
+
{
|
|
648
|
+
name: 'Continue With Patterns Only',
|
|
649
|
+
value: 'continue',
|
|
650
|
+
description: 'Output the pattern-only result and count the unscanned texts in the report',
|
|
651
|
+
},
|
|
652
|
+
],
|
|
653
|
+
default: 'stop',
|
|
654
|
+
description: 'What to do when Presidio cannot be reached or returns an error for a text',
|
|
655
|
+
},
|
|
570
656
|
{
|
|
571
657
|
displayName: 'Presidio Language',
|
|
572
658
|
name: 'presidioLanguage',
|
|
@@ -781,9 +867,8 @@ class Redactor {
|
|
|
781
867
|
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
782
868
|
const items = this.getInputData();
|
|
783
869
|
const allPaired = items.map((_, i) => ({ item: i }));
|
|
784
|
-
const operation =
|
|
870
|
+
const operation = (0, runtime_1.runSetting)(this, 'operation');
|
|
785
871
|
const startTime = Date.now();
|
|
786
|
-
(0, presidio_1.resetCircuitBreaker)();
|
|
787
872
|
if (operation === 'detect' || operation === 'verify') {
|
|
788
873
|
const enabledPatterns = (0, patterns_1.getPatternOptions)().map((p) => p.value);
|
|
789
874
|
const customPatterns = [];
|
|
@@ -810,7 +895,7 @@ class Redactor {
|
|
|
810
895
|
}
|
|
811
896
|
tempVault.deleteSession(tempSessionId);
|
|
812
897
|
if (operation === 'detect') {
|
|
813
|
-
const returnData = items.map((item, i) => ({ json: item.json, pairedItem: { item: i } }));
|
|
898
|
+
const returnData = items.map((item, i) => withBinary({ json: item.json, pairedItem: { item: i } }, item));
|
|
814
899
|
const report = (0, engine_1.buildReport)(tempSessionId, allHits);
|
|
815
900
|
const summary = {
|
|
816
901
|
_redactorScan: true,
|
|
@@ -821,16 +906,8 @@ class Redactor {
|
|
|
821
906
|
hitsByPattern: report.hitsByPattern,
|
|
822
907
|
timestamp: report.timestamp,
|
|
823
908
|
};
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
includeReport = this.getNodeParameter('includeReport', 0, false);
|
|
827
|
-
}
|
|
828
|
-
catch { }
|
|
829
|
-
let auditLog = false;
|
|
830
|
-
try {
|
|
831
|
-
auditLog = this.getNodeParameter('auditLog', 0, false);
|
|
832
|
-
}
|
|
833
|
-
catch { }
|
|
909
|
+
const includeReport = (0, runtime_1.runSetting)(this, 'includeReport', false);
|
|
910
|
+
const auditLog = (0, runtime_1.runSetting)(this, 'auditLog', false);
|
|
834
911
|
if (includeReport || auditLog) {
|
|
835
912
|
const reportJson = includeReport
|
|
836
913
|
? { ...summary, hits: report.hits }
|
|
@@ -845,17 +922,25 @@ class Redactor {
|
|
|
845
922
|
returnData.push({ json: summary, pairedItem: allPaired });
|
|
846
923
|
return [returnData];
|
|
847
924
|
}
|
|
848
|
-
const
|
|
925
|
+
const unscannedAttachments = items.reduce((n, item) => { var _a; return n + Object.keys((_a = item.binary) !== null && _a !== void 0 ? _a : {}).length; }, 0);
|
|
926
|
+
const passed = allHits.length === 0 && unscannedAttachments === 0;
|
|
849
927
|
const verifyReport = (0, engine_1.buildReport)(tempSessionId, allHits);
|
|
928
|
+
if ((0, runtime_1.runSetting)(this, 'verifyOutput', 'report') === 'gate') {
|
|
929
|
+
if (!passed) {
|
|
930
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), verifyMessage(allHits.length, unscannedAttachments), {
|
|
931
|
+
description: 'Verify stopped the run before the data reached the next node. Fix the redaction settings and run again.',
|
|
932
|
+
});
|
|
933
|
+
}
|
|
934
|
+
return [items.map((item, i) => ({ ...item, pairedItem: { item: i } }))];
|
|
935
|
+
}
|
|
850
936
|
return [[{
|
|
851
937
|
json: {
|
|
852
938
|
verified: passed,
|
|
853
939
|
status: passed ? 'PASS' : 'FAIL',
|
|
854
940
|
leaksFound: allHits.length,
|
|
941
|
+
unscannedAttachments,
|
|
855
942
|
itemsScanned: items.length,
|
|
856
|
-
message:
|
|
857
|
-
? 'No PII detected in values. Data appears properly redacted.'
|
|
858
|
-
: `Found ${allHits.length} potential PII leak(s). Values contain detectable sensitive data.`,
|
|
943
|
+
message: verifyMessage(allHits.length, unscannedAttachments),
|
|
859
944
|
hitsByCategory: verifyReport.hitsByCategory,
|
|
860
945
|
hitsByPattern: verifyReport.hitsByPattern,
|
|
861
946
|
leaks: allHits.map((h) => ({
|
|
@@ -900,7 +985,7 @@ class Redactor {
|
|
|
900
985
|
pairedItem: allPaired,
|
|
901
986
|
}]];
|
|
902
987
|
}
|
|
903
|
-
const vaultStorage =
|
|
988
|
+
const vaultStorage = (0, runtime_1.runSetting)(this, 'vaultStorage', 'workflow');
|
|
904
989
|
const vaultPassphrase = vaultStorage === 'memory' ? '' : await (0, runtime_1.resolvePassphrase)(this);
|
|
905
990
|
const vault = (0, vault_1.createVault)(vaultStorage, vaultStorage === 'memory' ? undefined : this.getWorkflowStaticData('global'), vaultPassphrase || undefined, (0, runtime_1.workflowNamespace)(this));
|
|
906
991
|
if (operation === 'stats') {
|
|
@@ -927,31 +1012,23 @@ class Redactor {
|
|
|
927
1012
|
}
|
|
928
1013
|
let sessionId = '';
|
|
929
1014
|
if (operation === 'redact' || operation === 'restore') {
|
|
930
|
-
sessionId =
|
|
931
|
-
if (!sessionId) {
|
|
932
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Session ID is required');
|
|
933
|
-
}
|
|
1015
|
+
sessionId = (0, runtime_1.readSessionId)(this, 'sessionId');
|
|
934
1016
|
}
|
|
935
1017
|
if (operation === 'redact') {
|
|
936
|
-
let mode =
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
}
|
|
942
|
-
else if (detectionScope === 'recommended') {
|
|
943
|
-
enabledPatterns = RECOMMENDED_PATTERNS;
|
|
944
|
-
}
|
|
945
|
-
else {
|
|
946
|
-
enabledPatterns = this.getNodeParameter('piiTypes', 0);
|
|
1018
|
+
let mode = (0, runtime_1.runSetting)(this, 'redactionMode', 'token');
|
|
1019
|
+
if (!profiles_1.MODES.includes(mode)) {
|
|
1020
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unknown redaction mode "${String(mode)}"`, {
|
|
1021
|
+
description: `Use one of: ${profiles_1.MODES.join(', ')}.`,
|
|
1022
|
+
});
|
|
947
1023
|
}
|
|
948
|
-
let
|
|
949
|
-
let
|
|
950
|
-
let
|
|
951
|
-
|
|
952
|
-
const
|
|
1024
|
+
let enabledPatterns = (0, runtime_1.selectedPatterns)(this, (0, patterns_1.getPatternOptions)().map((p) => p.value), RECOMMENDED_PATTERNS);
|
|
1025
|
+
let dedup = (0, runtime_1.runSetting)(this, 'dedup', true);
|
|
1026
|
+
let fieldMode = (0, runtime_1.runSetting)(this, 'fieldMode', 'all');
|
|
1027
|
+
let sessionTtl = (0, runtime_1.runSetting)(this, 'sessionTtl', 60);
|
|
1028
|
+
const includeReport = (0, runtime_1.runSetting)(this, 'includeReport', false);
|
|
1029
|
+
const customPatternsRaw = (0, runtime_1.runSetting)(this, 'customPatterns', {});
|
|
953
1030
|
let customPatterns = (_a = customPatternsRaw.patterns) !== null && _a !== void 0 ? _a : [];
|
|
954
|
-
const fieldRulesRaw =
|
|
1031
|
+
const fieldRulesRaw = (0, runtime_1.runSetting)(this, 'fieldRules', {});
|
|
955
1032
|
let fieldRules = ((_b = fieldRulesRaw.rules) !== null && _b !== void 0 ? _b : []).map((r) => ({
|
|
956
1033
|
field: r.field,
|
|
957
1034
|
mode: fieldMode === 'allowlist' ? 'include' : 'exclude',
|
|
@@ -968,33 +1045,39 @@ class Redactor {
|
|
|
968
1045
|
});
|
|
969
1046
|
return { statusCode: response.statusCode, body: response.body };
|
|
970
1047
|
};
|
|
971
|
-
|
|
1048
|
+
const binaryAttachments = (0, runtime_1.runSetting)(this, 'binaryAttachments', 'remove');
|
|
1049
|
+
const enablePresidio = (0, runtime_1.runSetting)(this, 'enablePresidio', false);
|
|
972
1050
|
let presidioUrl = '';
|
|
973
1051
|
let presidioLanguage = 'en';
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
1052
|
+
let presidioOnFailure = 'stop';
|
|
1053
|
+
if (enablePresidio) {
|
|
1054
|
+
presidioUrl = (0, runtime_1.runSetting)(this, 'presidioUrl', 'http://localhost:5002');
|
|
1055
|
+
presidioLanguage = (0, runtime_1.runSetting)(this, 'presidioLanguage', 'en');
|
|
1056
|
+
presidioOnFailure = (0, runtime_1.runSetting)(this, 'presidioOnFailure', 'stop');
|
|
1057
|
+
if (presidioOnFailure !== 'stop' && presidioOnFailure !== 'continue') {
|
|
1058
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'If Presidio Fails must be stop or continue');
|
|
1059
|
+
}
|
|
1060
|
+
if (!/^https?:[/][/][^/\s]/i.test(String(presidioUrl).trim())) {
|
|
1061
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Presidio URL must be an http or https address', {
|
|
1062
|
+
description: 'Set Presidio URL, for example http://localhost:5002, or turn off Enhanced NLP Detection.',
|
|
1063
|
+
});
|
|
979
1064
|
}
|
|
980
1065
|
}
|
|
981
|
-
|
|
982
|
-
}
|
|
983
|
-
const allowListRaw = this.getNodeParameter('allowList', 0, {});
|
|
1066
|
+
const allowListRaw = (0, runtime_1.runSetting)(this, 'allowList', {});
|
|
984
1067
|
let allowList = ((_c = allowListRaw.entries) !== null && _c !== void 0 ? _c : []).map((e) => ({
|
|
985
1068
|
value: e.value,
|
|
986
1069
|
type: (e.type || 'contains'),
|
|
987
1070
|
}));
|
|
988
|
-
const denyListRaw =
|
|
1071
|
+
const denyListRaw = (0, runtime_1.runSetting)(this, 'denyList', {});
|
|
989
1072
|
let denyList = ((_d = denyListRaw.entries) !== null && _d !== void 0 ? _d : []).map((e) => ({
|
|
990
1073
|
value: e.value,
|
|
991
1074
|
type: (e.type || 'exact'),
|
|
992
1075
|
}));
|
|
993
|
-
let confidenceThreshold =
|
|
994
|
-
const hasReportOutput = includeReport ||
|
|
995
|
-
let auditLog =
|
|
996
|
-
const useTenantProfile =
|
|
997
|
-
const profileRaw = useTenantProfile ?
|
|
1076
|
+
let confidenceThreshold = (0, runtime_1.runSetting)(this, 'confidenceThreshold', 0);
|
|
1077
|
+
const hasReportOutput = includeReport || (0, runtime_1.runSetting)(this, 'auditLog', false);
|
|
1078
|
+
let auditLog = (0, runtime_1.runSetting)(this, 'auditLog', false);
|
|
1079
|
+
const useTenantProfile = (0, runtime_1.runSetting)(this, 'useTenantProfile', false);
|
|
1080
|
+
const profileRaw = useTenantProfile ? (0, runtime_1.runSetting)(this, 'tenantProfile', '') : '';
|
|
998
1081
|
if (profileRaw !== '' && profileRaw !== undefined && profileRaw !== null) {
|
|
999
1082
|
let profile;
|
|
1000
1083
|
try {
|
|
@@ -1027,6 +1110,20 @@ class Redactor {
|
|
|
1027
1110
|
sessionTtl = profile.retentionMinutes;
|
|
1028
1111
|
if (profile.auditLog !== undefined)
|
|
1029
1112
|
auditLog = profile.auditLog;
|
|
1113
|
+
if (profile.fieldMode && !profile.fieldRules) {
|
|
1114
|
+
fieldRules = fieldRules.map((rule) => ({
|
|
1115
|
+
field: rule.field,
|
|
1116
|
+
mode: fieldMode === 'allowlist' ? 'include' : 'exclude',
|
|
1117
|
+
}));
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
if (typeof confidenceThreshold !== 'number' || !Number.isFinite(confidenceThreshold) || confidenceThreshold < 0 || confidenceThreshold > 1) {
|
|
1121
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Confidence Threshold must be a number from 0 to 1, got ${String(confidenceThreshold)}`);
|
|
1122
|
+
}
|
|
1123
|
+
if (typeof sessionTtl !== 'number' || !Number.isFinite(sessionTtl) || sessionTtl < 0) {
|
|
1124
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Session TTL must be 0 or more minutes, got ${String(sessionTtl)}`, {
|
|
1125
|
+
description: 'Use 0 to keep the session until it is purged.',
|
|
1126
|
+
});
|
|
1030
1127
|
}
|
|
1031
1128
|
(0, runtime_1.assertUserRegexes)(this, customPatterns, [
|
|
1032
1129
|
{ name: 'Allow list', entries: allowList },
|
|
@@ -1056,44 +1153,61 @@ class Redactor {
|
|
|
1056
1153
|
reservedTokens: (0, engine_1.collectTokenLike)(items.map((it) => it.json)),
|
|
1057
1154
|
keyOrigins: new WeakMap(),
|
|
1058
1155
|
inputText: (0, engine_1.collectInputText)(items.map((it) => it.json)),
|
|
1156
|
+
inputIndex: {},
|
|
1059
1157
|
lockedSpans: new WeakMap(),
|
|
1060
1158
|
keyLocks: new WeakMap(),
|
|
1061
|
-
priorTokens: (0, engine_1.priorTokenMatcher)((_e = vault.getSession(sessionId)) === null || _e === void 0 ? void 0 : _e.entries),
|
|
1062
1159
|
};
|
|
1160
|
+
ctx.priorTokens = (0, engine_1.priorTokensInInput)(ctx, (_e = vault.getSession(sessionId)) === null || _e === void 0 ? void 0 : _e.entries);
|
|
1063
1161
|
const allHits = [];
|
|
1064
1162
|
const returnData = [];
|
|
1065
1163
|
for (let i = 0; i < items.length; i++) {
|
|
1066
1164
|
const item = items[i];
|
|
1067
1165
|
const hits = [];
|
|
1068
1166
|
let redacted = (0, engine_1.redactValue)(item.json, ctx, vault, sessionId, hits, i);
|
|
1069
|
-
if (
|
|
1167
|
+
if (enablePresidio) {
|
|
1070
1168
|
try {
|
|
1071
1169
|
redacted = await (0, engine_1.enhanceWithPresidio)(redacted, ctx, vault, sessionId, hits, i);
|
|
1072
1170
|
}
|
|
1073
|
-
catch {
|
|
1171
|
+
catch (error) {
|
|
1172
|
+
if (presidioOnFailure === 'stop') {
|
|
1173
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Presidio scan failed: ${error.message}`, { itemIndex: i });
|
|
1174
|
+
}
|
|
1175
|
+
(0, presidio_1.markUnscanned)(presidioRequester);
|
|
1074
1176
|
}
|
|
1075
1177
|
}
|
|
1076
|
-
|
|
1178
|
+
const redactedItem = { json: redacted, pairedItem: { item: i } };
|
|
1179
|
+
returnData.push(binaryAttachments === 'keep' ? withBinary(redactedItem, items[i]) : redactedItem);
|
|
1077
1180
|
for (const hit of hits)
|
|
1078
1181
|
allHits.push(hit);
|
|
1079
1182
|
}
|
|
1183
|
+
const restorable = (0, engine_1.ensureRestorable)(items.map((it) => it.json), returnData.map((r) => r.json), ctx, vault, sessionId, allHits);
|
|
1184
|
+
restorable.forEach((json, i) => {
|
|
1185
|
+
returnData[i].json = json;
|
|
1186
|
+
});
|
|
1187
|
+
const unscannedTexts = enablePresidio ? (0, presidio_1.presidioUnscanned)(presidioRequester) : 0;
|
|
1188
|
+
if (unscannedTexts > 0 && presidioOnFailure === 'stop') {
|
|
1189
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Presidio could not scan ${unscannedTexts} text(s), so nothing was output`, {
|
|
1190
|
+
description: 'Check that Presidio is running at the Presidio URL, or set If Presidio Fails to Continue With Patterns Only.',
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
const promptInjection = (0, runtime_1.runSetting)(this, 'promptInjection', false);
|
|
1194
|
+
if (!vault.getSession(sessionId)) {
|
|
1195
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Session expired during the run', {
|
|
1196
|
+
description: 'Its TTL ran out before Redact finished, so nothing was output. Run again, or raise Session TTL.',
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1080
1199
|
vault.save(sessionId);
|
|
1081
1200
|
let injectionResult = null;
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
if (result.detected && (!injectionResult || result.score > injectionResult.score)) {
|
|
1088
|
-
injectionResult = (0, runtime_1.sanitizeInjection)(result);
|
|
1089
|
-
}
|
|
1201
|
+
if (promptInjection) {
|
|
1202
|
+
for (let i = 0; i < items.length; i++) {
|
|
1203
|
+
const result = (0, injection_1.scanForInjection)(items[i].json);
|
|
1204
|
+
if (result.detected && (!injectionResult || result.score > injectionResult.score)) {
|
|
1205
|
+
injectionResult = (0, runtime_1.sanitizeInjection)(result);
|
|
1090
1206
|
}
|
|
1091
1207
|
}
|
|
1092
1208
|
}
|
|
1093
|
-
catch {
|
|
1094
|
-
}
|
|
1095
1209
|
const auditEntry = auditLog && hasReportOutput
|
|
1096
|
-
? (0, audit_1.buildAuditEntry)('REDACT', mode, allHits, items.length, Date.now() - startTime, enabledPatterns.length, !!ctx.presidioUrl, (0, classification_1.classifyData)(allHits), sessionId)
|
|
1210
|
+
? (0, audit_1.buildAuditEntry)('REDACT', mode, allHits, items.length, Date.now() - startTime, enabledPatterns.length, !!ctx.presidioUrl, (0, classification_1.classifyData)(allHits), sessionId, unscannedTexts)
|
|
1097
1211
|
: undefined;
|
|
1098
1212
|
if (injectionResult && injectionResult.detected) {
|
|
1099
1213
|
returnData.push({
|
|
@@ -1107,8 +1221,10 @@ class Redactor {
|
|
|
1107
1221
|
const report = includeReport ? (0, engine_1.buildReport)(sessionId, allHits) : {};
|
|
1108
1222
|
if (auditEntry)
|
|
1109
1223
|
report.audit = auditEntry;
|
|
1110
|
-
if (
|
|
1111
|
-
|
|
1224
|
+
if (unscannedTexts > 0)
|
|
1225
|
+
report.presidioUnscannedTexts = unscannedTexts;
|
|
1226
|
+
if (includeReport && (0, runtime_1.runSetting)(this, 'includeRopa', false)) {
|
|
1227
|
+
const d = (0, runtime_1.runSetting)(this, 'ropa', {});
|
|
1112
1228
|
const list = (v) => String(v !== null && v !== void 0 ? v : '').split(',').map((x) => x.trim()).filter(Boolean);
|
|
1113
1229
|
report.ropa = (0, ropa_1.generateRopaRecord)(allHits, (0, classification_1.classifyData)(allHits), items.length, enabledPatterns.length, mode, sessionTtl, !!vaultPassphrase, auditLog, { name: String((_f = d.controllerName) !== null && _f !== void 0 ? _f : ''), contactEmail: String((_g = d.controllerContact) !== null && _g !== void 0 ? _g : ''), dpo: d.dpo ? String(d.dpo) : undefined }, list(d.purposes), String((_h = d.legalBasis) !== null && _h !== void 0 ? _h : 'legitimate_interest'), list(d.dataSubjects), list(d.recipients), []);
|
|
1114
1230
|
versionRopaSummary(report.ropa, (0, classification_1.classifyData)(allHits).hitCountByLevel, this.getNode().typeVersion);
|
|
@@ -1118,18 +1234,24 @@ class Redactor {
|
|
|
1118
1234
|
return [returnData];
|
|
1119
1235
|
}
|
|
1120
1236
|
if (operation === 'restore') {
|
|
1121
|
-
const deleteAfterRestore =
|
|
1237
|
+
const deleteAfterRestore = (0, runtime_1.runSetting)(this, 'deleteAfterRestore', true);
|
|
1122
1238
|
const session = vault.getSession(sessionId);
|
|
1123
1239
|
if (!session) {
|
|
1124
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(),
|
|
1240
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No vault found for this session that this passphrase can open. Ensure a Redact node ran first with the same Session ID, vault storage type and passphrase.', {
|
|
1125
1241
|
description: 'Testing step by step? Each "Execute step" gets a new execution ID, and n8n saves workflow storage only after production runs of an active workflow. Run the whole workflow, or set the same fixed Session ID (for example "test-1") on both nodes. Restore only works in the workflow that ran Redact.',
|
|
1126
1242
|
});
|
|
1127
1243
|
}
|
|
1128
1244
|
const returnData = [];
|
|
1129
1245
|
for (let i = 0; i < items.length; i++) {
|
|
1130
1246
|
const item = items[i];
|
|
1131
|
-
|
|
1132
|
-
|
|
1247
|
+
let restored;
|
|
1248
|
+
try {
|
|
1249
|
+
restored = (0, engine_1.restoreFromSession)(item.json, session);
|
|
1250
|
+
}
|
|
1251
|
+
catch (error) {
|
|
1252
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), error.message, { itemIndex: i });
|
|
1253
|
+
}
|
|
1254
|
+
returnData.push(withBinary({ json: restored, pairedItem: { item: i } }, item));
|
|
1133
1255
|
}
|
|
1134
1256
|
if (deleteAfterRestore) {
|
|
1135
1257
|
vault.deleteSession(sessionId);
|
|
@@ -1137,12 +1259,9 @@ class Redactor {
|
|
|
1137
1259
|
return [returnData];
|
|
1138
1260
|
}
|
|
1139
1261
|
if (operation === 'purge') {
|
|
1140
|
-
const purgeScope =
|
|
1262
|
+
const purgeScope = (0, runtime_1.runSetting)(this, 'purgeScope', 'all');
|
|
1141
1263
|
if (purgeScope === 'specific') {
|
|
1142
|
-
const purgeSessionId =
|
|
1143
|
-
if (!purgeSessionId) {
|
|
1144
|
-
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Session ID is required for purging a specific session.');
|
|
1145
|
-
}
|
|
1264
|
+
const purgeSessionId = (0, runtime_1.readSessionId)(this, 'purgeSessionId');
|
|
1146
1265
|
const purged = vault.purgeSession(purgeSessionId);
|
|
1147
1266
|
return [[{
|
|
1148
1267
|
json: {
|
|
@@ -33,8 +33,20 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.sessionReference = sessionReference;
|
|
37
|
+
exports.countHits = countHits;
|
|
36
38
|
exports.buildAuditEntry = buildAuditEntry;
|
|
37
39
|
const crypto = __importStar(require("crypto"));
|
|
40
|
+
function sessionReference(sessionId) {
|
|
41
|
+
return `session:${crypto.createHash('sha256').update('redactor-session:').update(sessionId).digest('hex')}`;
|
|
42
|
+
}
|
|
43
|
+
function countHits(hits, field) {
|
|
44
|
+
var _a;
|
|
45
|
+
const counts = new Map();
|
|
46
|
+
for (const hit of hits)
|
|
47
|
+
counts.set(hit[field], ((_a = counts.get(hit[field])) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
48
|
+
return Object.fromEntries(counts);
|
|
49
|
+
}
|
|
38
50
|
function determineSeverity(classificationLevel, totalHits = 0) {
|
|
39
51
|
if (classificationLevel === 'RESTRICTED' || totalHits >= 50)
|
|
40
52
|
return 'CRITICAL';
|
|
@@ -44,13 +56,9 @@ function determineSeverity(classificationLevel, totalHits = 0) {
|
|
|
44
56
|
return 'MEDIUM';
|
|
45
57
|
return 'LOW';
|
|
46
58
|
}
|
|
47
|
-
function buildAuditEntry(action, mode, hits, inputItemCount, processingTimeMs, patternsUsed, presidioEnabled, classification, sessionId) {
|
|
48
|
-
const hitsByCategory =
|
|
49
|
-
const hitsByPattern =
|
|
50
|
-
for (const hit of hits) {
|
|
51
|
-
hitsByCategory[hit.category] = (hitsByCategory[hit.category] || 0) + 1;
|
|
52
|
-
hitsByPattern[hit.patternLabel] = (hitsByPattern[hit.patternLabel] || 0) + 1;
|
|
53
|
-
}
|
|
59
|
+
function buildAuditEntry(action, mode, hits, inputItemCount, processingTimeMs, patternsUsed, presidioEnabled, classification, sessionId, presidioUnscannedTexts = 0) {
|
|
60
|
+
const hitsByCategory = countHits(hits, 'category');
|
|
61
|
+
const hitsByPattern = countHits(hits, 'patternLabel');
|
|
54
62
|
return {
|
|
55
63
|
version: '2.0',
|
|
56
64
|
timestamp: new Date().toISOString(),
|
|
@@ -58,7 +66,7 @@ function buildAuditEntry(action, mode, hits, inputItemCount, processingTimeMs, p
|
|
|
58
66
|
severity: determineSeverity(classification === null || classification === void 0 ? void 0 : classification.level, hits.length),
|
|
59
67
|
action,
|
|
60
68
|
mode,
|
|
61
|
-
status: 'SUCCESS',
|
|
69
|
+
status: presidioUnscannedTexts > 0 ? 'PARTIAL' : 'SUCCESS',
|
|
62
70
|
totalHits: hits.length,
|
|
63
71
|
hitsByCategory,
|
|
64
72
|
hitsByPattern,
|
|
@@ -67,6 +75,7 @@ function buildAuditEntry(action, mode, hits, inputItemCount, processingTimeMs, p
|
|
|
67
75
|
processingTimeMs,
|
|
68
76
|
patternsUsed,
|
|
69
77
|
presidioEnabled,
|
|
70
|
-
|
|
78
|
+
...(presidioUnscannedTexts > 0 ? { presidioUnscannedTexts } : {}),
|
|
79
|
+
sessionId: sessionId === undefined ? undefined : sessionReference(sessionId),
|
|
71
80
|
};
|
|
72
81
|
}
|