@ogcio/o11y-sdk-node 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/lib/exporter/pii-exporter-decorator.js +9 -9
- package/dist/lib/internals/redaction/pii-detection.d.ts +11 -9
- package/dist/lib/internals/redaction/pii-detection.js +17 -24
- package/dist/lib/internals/redaction/redactors/ip.js +2 -2
- package/dist/package.json +1 -1
- package/lib/exporter/pii-exporter-decorator.ts +13 -10
- package/lib/internals/redaction/pii-detection.ts +27 -40
- package/lib/internals/redaction/redactors/ip.ts +2 -2
- package/package.json +1 -1
- package/test/internals/pii-detection.test.ts +142 -20
- package/test/internals/redactors/ip.test.ts +4 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.4.1](https://github.com/ogcio/o11y/compare/@ogcio/o11y-sdk-node@v0.4.0...@ogcio/o11y-sdk-node@v0.4.1) (2025-09-03)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* nested attributes redaction node sdk AB[#30826](https://github.com/ogcio/o11y/issues/30826) ([#198](https://github.com/ogcio/o11y/issues/198)) ([b926ba5](https://github.com/ogcio/o11y/commit/b926ba54097fd6028c8aa78cbe1c276ce2cc2166))
|
|
9
|
+
|
|
3
10
|
## [0.4.0](https://github.com/ogcio/o11y/compare/@ogcio/o11y-sdk-node@v0.3.1...@ogcio/o11y-sdk-node@v0.4.0) (2025-09-02)
|
|
4
11
|
|
|
5
12
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { OTLPExporterBase } from "@opentelemetry/otlp-exporter-base";
|
|
2
|
-
import {
|
|
2
|
+
import { _cleanStringPII, _recursiveObjectClean, } from "../internals/redaction/pii-detection.js";
|
|
3
3
|
import { redactors, } from "../internals/redaction/redactors/index.js";
|
|
4
4
|
export class PIIExporterDecorator extends OTLPExporterBase {
|
|
5
5
|
_exporter;
|
|
@@ -67,22 +67,22 @@ export class PIIExporterDecorator extends OTLPExporterBase {
|
|
|
67
67
|
Object.assign(span, {
|
|
68
68
|
name: _cleanStringPII(span.name, "trace", this._redactors),
|
|
69
69
|
attributes: span.attributes &&
|
|
70
|
-
|
|
70
|
+
_recursiveObjectClean(span.attributes, "trace", this._redactors),
|
|
71
71
|
resource: {
|
|
72
72
|
attributes: span?.resource?.attributes &&
|
|
73
|
-
|
|
73
|
+
_recursiveObjectClean(span.resource.attributes, "trace", this._redactors),
|
|
74
74
|
},
|
|
75
75
|
links: span?.links?.map((link) => {
|
|
76
76
|
Object.assign(link, {
|
|
77
77
|
attributes: link?.attributes &&
|
|
78
|
-
|
|
78
|
+
_recursiveObjectClean(link.attributes, "trace", this._redactors),
|
|
79
79
|
});
|
|
80
80
|
}),
|
|
81
81
|
events: span?.events?.map((event) => {
|
|
82
82
|
Object.assign(event, {
|
|
83
83
|
name: _cleanStringPII(event.name, "trace", this._redactors),
|
|
84
84
|
attributes: event?.attributes &&
|
|
85
|
-
|
|
85
|
+
_recursiveObjectClean(event.attributes, "trace", this._redactors),
|
|
86
86
|
});
|
|
87
87
|
return event;
|
|
88
88
|
}),
|
|
@@ -91,12 +91,12 @@ export class PIIExporterDecorator extends OTLPExporterBase {
|
|
|
91
91
|
_redactLogRecord(log) {
|
|
92
92
|
return {
|
|
93
93
|
...log,
|
|
94
|
-
body:
|
|
94
|
+
body: _recursiveObjectClean(log.body, "log", this._redactors),
|
|
95
95
|
attributes: log.attributes &&
|
|
96
|
-
|
|
96
|
+
_recursiveObjectClean(log.attributes, "log", this._redactors),
|
|
97
97
|
resource: log.resource && {
|
|
98
98
|
...log.resource,
|
|
99
|
-
attributes:
|
|
99
|
+
attributes: _recursiveObjectClean(log.resource.attributes, "log", this._redactors),
|
|
100
100
|
},
|
|
101
101
|
};
|
|
102
102
|
}
|
|
@@ -104,7 +104,7 @@ export class PIIExporterDecorator extends OTLPExporterBase {
|
|
|
104
104
|
Object.assign(metric, {
|
|
105
105
|
resource: {
|
|
106
106
|
attributes: metric?.resource?.attributes &&
|
|
107
|
-
|
|
107
|
+
_recursiveObjectClean(metric.resource.attributes, "metric", this._redactors),
|
|
108
108
|
},
|
|
109
109
|
});
|
|
110
110
|
}
|
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
import type { AnyValue } from "@opentelemetry/api-logs";
|
|
2
2
|
import { Redactor } from "./redactors/index.js";
|
|
3
|
-
import { AttributeValue } from "@opentelemetry/api";
|
|
4
3
|
export type PIISource = "trace" | "log" | "metric";
|
|
4
|
+
/**
|
|
5
|
+
* Checks whether a string contains URI-encoded components.
|
|
6
|
+
*
|
|
7
|
+
* @param {string} value - The string to inspect.
|
|
8
|
+
* @returns {boolean} `true` if the string is encoded, `false` otherwise.
|
|
9
|
+
*/
|
|
10
|
+
export declare function _containsEncodedComponents(value: string): boolean;
|
|
5
11
|
/**
|
|
6
12
|
* Cleans a string by redacting configured PIIs and emitting metrics for redacted values.
|
|
7
13
|
*
|
|
8
14
|
* If the string is URL-encoded, it will be decoded before redaction.
|
|
9
|
-
* Metrics are emitted for:
|
|
10
|
-
* - each domain found in redacted email addresses.
|
|
11
|
-
* - IPv4|IPv6 addresses redacted.
|
|
12
15
|
*
|
|
13
16
|
* @template T
|
|
14
17
|
*
|
|
15
|
-
* @param {
|
|
18
|
+
* @param {string} value - The input value to sanitize.
|
|
16
19
|
* @param {"trace" | "log"} source - The source context of the input, used in metrics.
|
|
17
20
|
* @param {Redactor[]} redactors - The string processors containing the redaction logic.
|
|
18
21
|
*
|
|
19
|
-
* @returns {
|
|
22
|
+
* @returns {string} The cleaned string with any configured PII replaced by `[REDACTED PII_TYPE]`.
|
|
20
23
|
*/
|
|
21
|
-
export declare function _cleanStringPII
|
|
22
|
-
export declare function
|
|
23
|
-
export declare function _cleanLogBodyPII<T extends AnyValue>(value: T, redactors: Redactor[]): T;
|
|
24
|
+
export declare function _cleanStringPII(value: string, source: PIISource, redactors: Redactor[]): string;
|
|
25
|
+
export declare function _recursiveObjectClean<T extends AnyValue>(value: T, source: PIISource, redactors: Redactor[]): T;
|
|
@@ -6,34 +6,36 @@ const encoder = new TextEncoder();
|
|
|
6
6
|
* @param {string} value - The string to inspect.
|
|
7
7
|
* @returns {boolean} `true` if the string is encoded, `false` otherwise.
|
|
8
8
|
*/
|
|
9
|
-
function _containsEncodedComponents(value) {
|
|
9
|
+
export function _containsEncodedComponents(value) {
|
|
10
10
|
try {
|
|
11
|
-
|
|
11
|
+
const decodedURIComponent = decodeURIComponent(value);
|
|
12
|
+
if (decodeURI(value) !== decodedURIComponent) {
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
if (value !== decodedURIComponent) {
|
|
16
|
+
return (encodeURIComponent(decodedURIComponent) === value ||
|
|
17
|
+
encodeURI(decodedURIComponent) === value);
|
|
18
|
+
}
|
|
12
19
|
}
|
|
13
20
|
catch {
|
|
14
21
|
return false;
|
|
15
22
|
}
|
|
23
|
+
return false;
|
|
16
24
|
}
|
|
17
25
|
/**
|
|
18
26
|
* Cleans a string by redacting configured PIIs and emitting metrics for redacted values.
|
|
19
27
|
*
|
|
20
28
|
* If the string is URL-encoded, it will be decoded before redaction.
|
|
21
|
-
* Metrics are emitted for:
|
|
22
|
-
* - each domain found in redacted email addresses.
|
|
23
|
-
* - IPv4|IPv6 addresses redacted.
|
|
24
29
|
*
|
|
25
30
|
* @template T
|
|
26
31
|
*
|
|
27
|
-
* @param {
|
|
32
|
+
* @param {string} value - The input value to sanitize.
|
|
28
33
|
* @param {"trace" | "log"} source - The source context of the input, used in metrics.
|
|
29
34
|
* @param {Redactor[]} redactors - The string processors containing the redaction logic.
|
|
30
35
|
*
|
|
31
|
-
* @returns {
|
|
36
|
+
* @returns {string} The cleaned string with any configured PII replaced by `[REDACTED PII_TYPE]`.
|
|
32
37
|
*/
|
|
33
38
|
export function _cleanStringPII(value, source, redactors) {
|
|
34
|
-
if (Array.isArray(value)) {
|
|
35
|
-
return value.map((v) => _cleanStringPII(v, source, redactors));
|
|
36
|
-
}
|
|
37
39
|
if (typeof value !== "string") {
|
|
38
40
|
return value;
|
|
39
41
|
}
|
|
@@ -45,18 +47,9 @@ export function _cleanStringPII(value, source, redactors) {
|
|
|
45
47
|
}
|
|
46
48
|
return redactors.reduce((redactedValue, currentRedactor) => currentRedactor(redactedValue, source, kind), decodedValue);
|
|
47
49
|
}
|
|
48
|
-
export function
|
|
49
|
-
if (!entry) {
|
|
50
|
-
return entry;
|
|
51
|
-
}
|
|
52
|
-
return Object.fromEntries(Object.entries(entry).map(([k, v]) => [
|
|
53
|
-
k,
|
|
54
|
-
_cleanStringPII(v, source, redactors),
|
|
55
|
-
]));
|
|
56
|
-
}
|
|
57
|
-
export function _cleanLogBodyPII(value, redactors) {
|
|
50
|
+
export function _recursiveObjectClean(value, source, redactors) {
|
|
58
51
|
if (typeof value === "string") {
|
|
59
|
-
return _cleanStringPII(value,
|
|
52
|
+
return _cleanStringPII(value, source, redactors);
|
|
60
53
|
}
|
|
61
54
|
if (typeof value === "number" ||
|
|
62
55
|
typeof value === "boolean" ||
|
|
@@ -66,7 +59,7 @@ export function _cleanLogBodyPII(value, redactors) {
|
|
|
66
59
|
if (value instanceof Uint8Array) {
|
|
67
60
|
try {
|
|
68
61
|
const decoded = decoder.decode(value);
|
|
69
|
-
const sanitized = _cleanStringPII(decoded,
|
|
62
|
+
const sanitized = _cleanStringPII(decoded, source, redactors);
|
|
70
63
|
return encoder.encode(sanitized);
|
|
71
64
|
}
|
|
72
65
|
catch {
|
|
@@ -74,12 +67,12 @@ export function _cleanLogBodyPII(value, redactors) {
|
|
|
74
67
|
}
|
|
75
68
|
}
|
|
76
69
|
if (Array.isArray(value)) {
|
|
77
|
-
return value.map((value) =>
|
|
70
|
+
return value.map((value) => _recursiveObjectClean(value, source, redactors));
|
|
78
71
|
}
|
|
79
72
|
if (typeof value === "object") {
|
|
80
73
|
const sanitized = {};
|
|
81
74
|
for (const [key, val] of Object.entries(value)) {
|
|
82
|
-
sanitized[key] =
|
|
75
|
+
sanitized[key] = _recursiveObjectClean(val, source, redactors);
|
|
83
76
|
}
|
|
84
77
|
return sanitized;
|
|
85
78
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { _getPIICounterRedactionMetric } from "../../shared-metrics.js";
|
|
2
2
|
// Generous IP address matchers (might match some invalid addresses like 192.168.01.1)
|
|
3
|
-
const IPV4_REGEX = /(?<!\d)(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(?!\d)/gi;
|
|
4
|
-
const IPV6_REGEX = /(?<![0-9a-f:])((?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|:(?::[0-9A-Fa-f]{1,4}){1,7}:?|(?:[0-9A-Fa-f]{1,4}:){1,4}:(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})){3})(?![0-9a-f:])/gi;
|
|
3
|
+
const IPV4_REGEX = /(?<!\d)(?:%[0-9A-Fa-f]{2})?(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(?:%[0-9A-Fa-f]{2})?(?!\d)/gi;
|
|
4
|
+
const IPV6_REGEX = /(?<![0-9a-f:])(?:%[0-9A-Fa-f]{2})?((?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|:(?::[0-9A-Fa-f]{1,4}){1,7}:?|(?:[0-9A-Fa-f]{1,4}:){1,4}:(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})){3})(?:%[0-9A-Fa-f]{2})?(?![0-9a-f:])/gi;
|
|
5
5
|
/**
|
|
6
6
|
* Redacts all ip addresses in the input string and collects metadata.
|
|
7
7
|
*
|
package/dist/package.json
CHANGED
|
@@ -8,9 +8,8 @@ import {
|
|
|
8
8
|
import { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base";
|
|
9
9
|
import { NodeSDKConfig } from "../index.js";
|
|
10
10
|
import {
|
|
11
|
-
_cleanLogBodyPII,
|
|
12
|
-
_cleanObjectPII,
|
|
13
11
|
_cleanStringPII,
|
|
12
|
+
_recursiveObjectClean,
|
|
14
13
|
} from "../internals/redaction/pii-detection.js";
|
|
15
14
|
import {
|
|
16
15
|
Redactor,
|
|
@@ -114,17 +113,21 @@ export class PIIExporterDecorator
|
|
|
114
113
|
name: _cleanStringPII(span.name, "trace", this._redactors),
|
|
115
114
|
attributes:
|
|
116
115
|
span.attributes &&
|
|
117
|
-
|
|
116
|
+
_recursiveObjectClean(span.attributes, "trace", this._redactors),
|
|
118
117
|
resource: {
|
|
119
118
|
attributes:
|
|
120
119
|
span?.resource?.attributes &&
|
|
121
|
-
|
|
120
|
+
_recursiveObjectClean(
|
|
121
|
+
span.resource.attributes,
|
|
122
|
+
"trace",
|
|
123
|
+
this._redactors,
|
|
124
|
+
),
|
|
122
125
|
},
|
|
123
126
|
links: span?.links?.map((link) => {
|
|
124
127
|
Object.assign(link, {
|
|
125
128
|
attributes:
|
|
126
129
|
link?.attributes &&
|
|
127
|
-
|
|
130
|
+
_recursiveObjectClean(link.attributes, "trace", this._redactors),
|
|
128
131
|
});
|
|
129
132
|
}),
|
|
130
133
|
events: span?.events?.map((event) => {
|
|
@@ -132,7 +135,7 @@ export class PIIExporterDecorator
|
|
|
132
135
|
name: _cleanStringPII(event.name, "trace", this._redactors),
|
|
133
136
|
attributes:
|
|
134
137
|
event?.attributes &&
|
|
135
|
-
|
|
138
|
+
_recursiveObjectClean(event.attributes, "trace", this._redactors),
|
|
136
139
|
});
|
|
137
140
|
return event;
|
|
138
141
|
}),
|
|
@@ -142,13 +145,13 @@ export class PIIExporterDecorator
|
|
|
142
145
|
private _redactLogRecord(log: ReadableLogRecord): ReadableLogRecord {
|
|
143
146
|
return {
|
|
144
147
|
...log,
|
|
145
|
-
body:
|
|
148
|
+
body: _recursiveObjectClean(log.body, "log", this._redactors),
|
|
146
149
|
attributes:
|
|
147
150
|
log.attributes &&
|
|
148
|
-
|
|
151
|
+
_recursiveObjectClean(log.attributes, "log", this._redactors),
|
|
149
152
|
resource: log.resource && {
|
|
150
153
|
...log.resource,
|
|
151
|
-
attributes:
|
|
154
|
+
attributes: _recursiveObjectClean(
|
|
152
155
|
log.resource.attributes,
|
|
153
156
|
"log",
|
|
154
157
|
this._redactors,
|
|
@@ -162,7 +165,7 @@ export class PIIExporterDecorator
|
|
|
162
165
|
resource: {
|
|
163
166
|
attributes:
|
|
164
167
|
metric?.resource?.attributes &&
|
|
165
|
-
|
|
168
|
+
_recursiveObjectClean(
|
|
166
169
|
metric.resource.attributes,
|
|
167
170
|
"metric",
|
|
168
171
|
this._redactors,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { AnyValue, AnyValueMap } from "@opentelemetry/api-logs";
|
|
2
2
|
import { Redactor } from "./redactors/index.js";
|
|
3
|
-
import { AttributeValue } from "@opentelemetry/api";
|
|
4
3
|
|
|
5
4
|
const decoder = new TextDecoder();
|
|
6
5
|
const encoder = new TextEncoder();
|
|
@@ -13,41 +12,44 @@ export type PIISource = "trace" | "log" | "metric";
|
|
|
13
12
|
* @param {string} value - The string to inspect.
|
|
14
13
|
* @returns {boolean} `true` if the string is encoded, `false` otherwise.
|
|
15
14
|
*/
|
|
16
|
-
function _containsEncodedComponents(value: string): boolean {
|
|
15
|
+
export function _containsEncodedComponents(value: string): boolean {
|
|
17
16
|
try {
|
|
18
|
-
|
|
17
|
+
const decodedURIComponent = decodeURIComponent(value);
|
|
18
|
+
if (decodeURI(value) !== decodedURIComponent) {
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
if (value !== decodedURIComponent) {
|
|
23
|
+
return (
|
|
24
|
+
encodeURIComponent(decodedURIComponent) === value ||
|
|
25
|
+
encodeURI(decodedURIComponent) === value
|
|
26
|
+
);
|
|
27
|
+
}
|
|
19
28
|
} catch {
|
|
20
29
|
return false;
|
|
21
30
|
}
|
|
31
|
+
|
|
32
|
+
return false;
|
|
22
33
|
}
|
|
23
34
|
|
|
24
35
|
/**
|
|
25
36
|
* Cleans a string by redacting configured PIIs and emitting metrics for redacted values.
|
|
26
37
|
*
|
|
27
38
|
* If the string is URL-encoded, it will be decoded before redaction.
|
|
28
|
-
* Metrics are emitted for:
|
|
29
|
-
* - each domain found in redacted email addresses.
|
|
30
|
-
* - IPv4|IPv6 addresses redacted.
|
|
31
39
|
*
|
|
32
40
|
* @template T
|
|
33
41
|
*
|
|
34
|
-
* @param {
|
|
42
|
+
* @param {string} value - The input value to sanitize.
|
|
35
43
|
* @param {"trace" | "log"} source - The source context of the input, used in metrics.
|
|
36
44
|
* @param {Redactor[]} redactors - The string processors containing the redaction logic.
|
|
37
45
|
*
|
|
38
|
-
* @returns {
|
|
46
|
+
* @returns {string} The cleaned string with any configured PII replaced by `[REDACTED PII_TYPE]`.
|
|
39
47
|
*/
|
|
40
|
-
export function _cleanStringPII
|
|
41
|
-
value:
|
|
48
|
+
export function _cleanStringPII(
|
|
49
|
+
value: string,
|
|
42
50
|
source: PIISource,
|
|
43
51
|
redactors: Redactor[],
|
|
44
|
-
):
|
|
45
|
-
if (Array.isArray(value)) {
|
|
46
|
-
return value.map((v) =>
|
|
47
|
-
_cleanStringPII<typeof v>(v, source, redactors),
|
|
48
|
-
) as T;
|
|
49
|
-
}
|
|
50
|
-
|
|
52
|
+
): string {
|
|
51
53
|
if (typeof value !== "string") {
|
|
52
54
|
return value;
|
|
53
55
|
}
|
|
@@ -59,37 +61,20 @@ export function _cleanStringPII<T extends AnyValue>(
|
|
|
59
61
|
decodedValue = decodeURIComponent(value);
|
|
60
62
|
kind = "url";
|
|
61
63
|
}
|
|
62
|
-
|
|
63
64
|
return redactors.reduce(
|
|
64
65
|
(redactedValue: string, currentRedactor): string =>
|
|
65
66
|
currentRedactor(redactedValue, source, kind),
|
|
66
67
|
decodedValue,
|
|
67
|
-
) as T;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
export function _cleanObjectPII(
|
|
71
|
-
entry: object,
|
|
72
|
-
source: PIISource,
|
|
73
|
-
redactors: Redactor[],
|
|
74
|
-
): Record<string, AttributeValue> {
|
|
75
|
-
if (!entry) {
|
|
76
|
-
return entry;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return Object.fromEntries(
|
|
80
|
-
Object.entries(entry).map(([k, v]) => [
|
|
81
|
-
k,
|
|
82
|
-
_cleanStringPII(v, source, redactors),
|
|
83
|
-
]),
|
|
84
68
|
);
|
|
85
69
|
}
|
|
86
70
|
|
|
87
|
-
export function
|
|
71
|
+
export function _recursiveObjectClean<T extends AnyValue>(
|
|
88
72
|
value: T,
|
|
73
|
+
source: PIISource,
|
|
89
74
|
redactors: Redactor[],
|
|
90
75
|
): T {
|
|
91
76
|
if (typeof value === "string") {
|
|
92
|
-
return _cleanStringPII(value,
|
|
77
|
+
return _cleanStringPII(value, source, redactors) as T;
|
|
93
78
|
}
|
|
94
79
|
|
|
95
80
|
if (
|
|
@@ -103,7 +88,7 @@ export function _cleanLogBodyPII<T extends AnyValue>(
|
|
|
103
88
|
if (value instanceof Uint8Array) {
|
|
104
89
|
try {
|
|
105
90
|
const decoded = decoder.decode(value);
|
|
106
|
-
const sanitized = _cleanStringPII(decoded,
|
|
91
|
+
const sanitized = _cleanStringPII(decoded, source, redactors);
|
|
107
92
|
return encoder.encode(sanitized) as T;
|
|
108
93
|
} catch {
|
|
109
94
|
return value;
|
|
@@ -111,13 +96,15 @@ export function _cleanLogBodyPII<T extends AnyValue>(
|
|
|
111
96
|
}
|
|
112
97
|
|
|
113
98
|
if (Array.isArray(value)) {
|
|
114
|
-
return value.map((value) =>
|
|
99
|
+
return value.map((value) =>
|
|
100
|
+
_recursiveObjectClean(value, source, redactors),
|
|
101
|
+
) as T;
|
|
115
102
|
}
|
|
116
103
|
|
|
117
104
|
if (typeof value === "object") {
|
|
118
105
|
const sanitized: AnyValueMap = {};
|
|
119
106
|
for (const [key, val] of Object.entries(value)) {
|
|
120
|
-
sanitized[key] =
|
|
107
|
+
sanitized[key] = _recursiveObjectClean(val, source, redactors);
|
|
121
108
|
}
|
|
122
109
|
return sanitized as T;
|
|
123
110
|
}
|
|
@@ -2,9 +2,9 @@ import { _getPIICounterRedactionMetric } from "../../shared-metrics.js";
|
|
|
2
2
|
|
|
3
3
|
// Generous IP address matchers (might match some invalid addresses like 192.168.01.1)
|
|
4
4
|
const IPV4_REGEX =
|
|
5
|
-
/(?<!\d)(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(?!\d)/gi;
|
|
5
|
+
/(?<!\d)(?:%[0-9A-Fa-f]{2})?(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}(?:%[0-9A-Fa-f]{2})?(?!\d)/gi;
|
|
6
6
|
const IPV6_REGEX =
|
|
7
|
-
/(?<![0-9a-f:])((?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|:(?::[0-9A-Fa-f]{1,4}){1,7}:?|(?:[0-9A-Fa-f]{1,4}:){1,4}:(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})){3})(?![0-9a-f:])/gi;
|
|
7
|
+
/(?<![0-9a-f:])(?:%[0-9A-Fa-f]{2})?((?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,7}:|:(?::[0-9A-Fa-f]{1,4}){1,7}|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}|:(?::[0-9A-Fa-f]{1,4}){1,7}:?|(?:[0-9A-Fa-f]{1,4}:){1,4}:(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})(?:\.(?:25[0-5]|2[0-4]\d|1\d\d|\d{1,2})){3})(?:%[0-9A-Fa-f]{2})?(?![0-9a-f:])/gi;
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Redacts all ip addresses in the input string and collects metadata.
|
package/package.json
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { describe, expect, it, vi, beforeEach } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
_cleanStringPII,
|
|
4
|
-
|
|
4
|
+
_containsEncodedComponents,
|
|
5
|
+
_recursiveObjectClean,
|
|
5
6
|
} from "../../lib/internals/redaction/pii-detection.js";
|
|
6
7
|
import * as sharedMetrics from "../../lib/internals/shared-metrics.js";
|
|
7
8
|
import { emailRedactor } from "../../lib/internals/redaction/redactors/email";
|
|
9
|
+
import { ipRedactor } from "../../lib/internals/redaction/redactors/ip";
|
|
8
10
|
|
|
9
11
|
describe("PII Detection Utils", () => {
|
|
10
12
|
const mockMetricAdd = vi.fn();
|
|
@@ -32,7 +34,7 @@ describe("PII Detection Utils", () => {
|
|
|
32
34
|
);
|
|
33
35
|
});
|
|
34
36
|
|
|
35
|
-
it("redacts
|
|
37
|
+
it("redacts email in URL-encoded string", () => {
|
|
36
38
|
const input = "user%40gmail.com";
|
|
37
39
|
const output = _cleanStringPII(input, "log", [emailRedactor]);
|
|
38
40
|
|
|
@@ -46,6 +48,21 @@ describe("PII Detection Utils", () => {
|
|
|
46
48
|
);
|
|
47
49
|
});
|
|
48
50
|
|
|
51
|
+
it("redacts ip in URL-encoded string", () => {
|
|
52
|
+
const input = "%20127.0.0.1";
|
|
53
|
+
const output = _cleanStringPII(input, "log", [ipRedactor]);
|
|
54
|
+
|
|
55
|
+
expect(output).toBe(" [REDACTED IPV4]");
|
|
56
|
+
expect(mockMetricAdd).toHaveBeenCalledWith(
|
|
57
|
+
1,
|
|
58
|
+
expect.objectContaining({
|
|
59
|
+
pii_format: "url",
|
|
60
|
+
pii_type: "IPv4",
|
|
61
|
+
redaction_source: "log",
|
|
62
|
+
}),
|
|
63
|
+
);
|
|
64
|
+
});
|
|
65
|
+
|
|
49
66
|
it("handles strings without PII unchanged", () => {
|
|
50
67
|
const input = "hello world";
|
|
51
68
|
const output = _cleanStringPII(input, "log", [emailRedactor]);
|
|
@@ -54,16 +71,10 @@ describe("PII Detection Utils", () => {
|
|
|
54
71
|
expect(mockMetricAdd).not.toHaveBeenCalled();
|
|
55
72
|
});
|
|
56
73
|
|
|
57
|
-
it("handles array of strings", () => {
|
|
58
|
-
const input = ["one@gmail.com", "two@example.com"];
|
|
59
|
-
const output = _cleanStringPII(input, "log", [emailRedactor]);
|
|
60
|
-
|
|
61
|
-
expect(output).toEqual(["[REDACTED EMAIL]", "[REDACTED EMAIL]"]);
|
|
62
|
-
expect(mockMetricAdd).toHaveBeenCalledTimes(2);
|
|
63
|
-
});
|
|
64
|
-
|
|
65
74
|
it("ignores non-string input", () => {
|
|
75
|
+
// @ts-expect-error
|
|
66
76
|
expect(_cleanStringPII(1234, "trace", [emailRedactor])).toBe(1234);
|
|
77
|
+
// @ts-expect-error
|
|
67
78
|
expect(_cleanStringPII(true, "trace", [emailRedactor])).toBe(true);
|
|
68
79
|
expect(
|
|
69
80
|
_cleanStringPII(undefined, "trace", [emailRedactor]),
|
|
@@ -72,12 +83,22 @@ describe("PII Detection Utils", () => {
|
|
|
72
83
|
});
|
|
73
84
|
});
|
|
74
85
|
|
|
75
|
-
describe("
|
|
86
|
+
describe("_recursiveObjectClean", () => {
|
|
76
87
|
it("cleans string PII", () => {
|
|
77
|
-
const result =
|
|
88
|
+
const result = _recursiveObjectClean("demo@abc.com", "log", [
|
|
89
|
+
emailRedactor,
|
|
90
|
+
]);
|
|
78
91
|
expect(result).toBe("[REDACTED EMAIL]");
|
|
79
92
|
});
|
|
80
93
|
|
|
94
|
+
it("cleans array of strings", () => {
|
|
95
|
+
const input = ["one@gmail.com", "two@example.com"];
|
|
96
|
+
const output = _recursiveObjectClean(input, "log", [emailRedactor]);
|
|
97
|
+
|
|
98
|
+
expect(output).toEqual(["[REDACTED EMAIL]", "[REDACTED EMAIL]"]);
|
|
99
|
+
expect(mockMetricAdd).toHaveBeenCalledTimes(2);
|
|
100
|
+
});
|
|
101
|
+
|
|
81
102
|
it("cleans deeply nested object", () => {
|
|
82
103
|
const input = {
|
|
83
104
|
user: {
|
|
@@ -89,7 +110,7 @@ describe("PII Detection Utils", () => {
|
|
|
89
110
|
status: "active",
|
|
90
111
|
};
|
|
91
112
|
|
|
92
|
-
const result =
|
|
113
|
+
const result = _recursiveObjectClean(input, "log", [emailRedactor]);
|
|
93
114
|
|
|
94
115
|
expect(result).toEqual({
|
|
95
116
|
user: {
|
|
@@ -105,7 +126,7 @@ describe("PII Detection Utils", () => {
|
|
|
105
126
|
it("cleans Uint8Array input", () => {
|
|
106
127
|
const str = "admin@gmail.com";
|
|
107
128
|
const buffer = new TextEncoder().encode(str);
|
|
108
|
-
const result =
|
|
129
|
+
const result = _recursiveObjectClean(buffer, "log", [emailRedactor]);
|
|
109
130
|
const decoded = new TextDecoder().decode(result as Uint8Array);
|
|
110
131
|
|
|
111
132
|
expect(decoded).toBe("[REDACTED EMAIL]");
|
|
@@ -113,7 +134,7 @@ describe("PII Detection Utils", () => {
|
|
|
113
134
|
|
|
114
135
|
it("skips malformed Uint8Array decode", () => {
|
|
115
136
|
const corrupted = new Uint8Array([0xff, 0xfe, 0xfd]);
|
|
116
|
-
const result =
|
|
137
|
+
const result = _recursiveObjectClean(corrupted, "log", [emailRedactor]);
|
|
117
138
|
|
|
118
139
|
// Should return a Uint8Array, but unmodified/redaction should not happen
|
|
119
140
|
expect(result).toBeInstanceOf(Uint8Array);
|
|
@@ -121,8 +142,9 @@ describe("PII Detection Utils", () => {
|
|
|
121
142
|
});
|
|
122
143
|
|
|
123
144
|
it("cleans arrays of values", () => {
|
|
124
|
-
const result =
|
|
145
|
+
const result = _recursiveObjectClean(
|
|
125
146
|
["bob@abc.com", 123, { nested: "jane@example.com" }],
|
|
147
|
+
"log",
|
|
126
148
|
[emailRedactor],
|
|
127
149
|
);
|
|
128
150
|
|
|
@@ -134,10 +156,110 @@ describe("PII Detection Utils", () => {
|
|
|
134
156
|
});
|
|
135
157
|
|
|
136
158
|
it("passes null and boolean through", () => {
|
|
137
|
-
expect(
|
|
138
|
-
expect(
|
|
139
|
-
|
|
140
|
-
|
|
159
|
+
expect(_recursiveObjectClean(null, "log", [emailRedactor])).toBeNull();
|
|
160
|
+
expect(
|
|
161
|
+
_recursiveObjectClean(undefined, "log", [emailRedactor]),
|
|
162
|
+
).toBeUndefined();
|
|
163
|
+
expect(_recursiveObjectClean(true, "log", [emailRedactor])).toBe(true);
|
|
164
|
+
expect(_recursiveObjectClean(false, "log", [emailRedactor])).toBe(false);
|
|
165
|
+
});
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
describe("_containsEncodedComponents", () => {
|
|
169
|
+
describe("should return true for properly URL encoded strings", () => {
|
|
170
|
+
it.each([
|
|
171
|
+
["hello%20world", "Space encoded as %20"],
|
|
172
|
+
["test%2Bvalue", "Plus sign encoded as %2B"],
|
|
173
|
+
["path%2Fto%2Ffile", "Forward slashes encoded"],
|
|
174
|
+
["user%40domain.com", "@ symbol encoded"],
|
|
175
|
+
["100%25%20off", "Percent and space encoded"],
|
|
176
|
+
["a%3Db%26c%3Dd", "Query parameters (a=b&c=d)"],
|
|
177
|
+
["caf%C3%A9", "UTF-8 encoded (café)"],
|
|
178
|
+
["price%3A%20%2410", "Colon, space, dollar ($10)"],
|
|
179
|
+
["%22quoted%22", "Double quotes encoded"],
|
|
180
|
+
["https%3A%2F%2Fexample.com", "Full URL encoded"],
|
|
181
|
+
["file%20name.txt", "filename encoded"],
|
|
182
|
+
["search%3Fq%3Dhello%20world", "Query string encoded"],
|
|
183
|
+
["%3C%3E%26%22%27", "HTML special chars encoded (<>&\"')"],
|
|
184
|
+
["%E2%9C%93", "UTF-8 checkmark (✓) encoded"],
|
|
185
|
+
["test%2b", "Lowercase hex"],
|
|
186
|
+
["%E4%B8%AD%E6%96%87", "Chinese characters encoded"],
|
|
187
|
+
])('should detect "%s" as URL encoded (%s)', (input, description) => {
|
|
188
|
+
expect(_containsEncodedComponents(input)).toBe(true);
|
|
189
|
+
});
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
describe("should return false for non-URL encoded strings", () => {
|
|
193
|
+
it.each([
|
|
194
|
+
["test", "Simple ASCII string"],
|
|
195
|
+
["hello world", "Unencoded space"],
|
|
196
|
+
["user@domain.com", "Unencoded email"],
|
|
197
|
+
["simple123", "Alphanumeric only"],
|
|
198
|
+
["", "Empty string"],
|
|
199
|
+
["25%%", "Literal percent signs"],
|
|
200
|
+
["100% off", "Percent without hex digits"],
|
|
201
|
+
["test%2", "Incomplete percent encoding"],
|
|
202
|
+
["hello%ZZ", "Invalid hex digits"],
|
|
203
|
+
["test%2G", "Invalid hex digit G"],
|
|
204
|
+
["bad%encoding%here", "Percent without hex pairs"],
|
|
205
|
+
["hello%20world and more", "Partially encoded string"],
|
|
206
|
+
["hello%20world%21%20how%20are%20you", "Overly encoded string"],
|
|
207
|
+
["café", "Unicode characters (unencoded)"],
|
|
208
|
+
["hello+world", "Plus sign (form encoding style)"],
|
|
209
|
+
["test%", "Trailing percent"],
|
|
210
|
+
["hello%20world%", "Encoded content with trailing percent"],
|
|
211
|
+
["%", "Single percent"],
|
|
212
|
+
["%%", "Double percent"],
|
|
213
|
+
["normal text with % symbols", "Text with percent but no encoding"],
|
|
214
|
+
["price: $100%", "Currency with percent"],
|
|
215
|
+
["file.txt", "Simple filename"],
|
|
216
|
+
["path/to/file", "Unencoded path"],
|
|
217
|
+
["query?param=value", "Unencoded query string"],
|
|
218
|
+
["hello%world", "Percent without following hex"],
|
|
219
|
+
["test%1", "Single hex digit after percent"],
|
|
220
|
+
["hello%zz", "Non-hex characters after percent"],
|
|
221
|
+
])('should not detect "%s" as URL encoded (%s)', (input, description) => {
|
|
222
|
+
expect(_containsEncodedComponents(input)).toBe(false);
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
|
|
226
|
+
describe("Error handling", () => {
|
|
227
|
+
it.each([
|
|
228
|
+
["%C0%80", "Overlong UTF-8 encoding (security concern)"],
|
|
229
|
+
["%ED%A0%80", "UTF-8 surrogate (invalid)"],
|
|
230
|
+
["%FF%FE", "Invalid UTF-8 sequence"],
|
|
231
|
+
["test%20%ZZ", "Mix of valid and invalid encoding"],
|
|
232
|
+
])('should handle "%s" (%s)', (input, description) => {
|
|
233
|
+
// These should not throw errors
|
|
234
|
+
expect(() => _containsEncodedComponents(input)).not.toThrow();
|
|
235
|
+
|
|
236
|
+
// Most of these should return false due to invalid sequences
|
|
237
|
+
// or security-related encoding issues
|
|
238
|
+
const result = _containsEncodedComponents(input);
|
|
239
|
+
expect(typeof result).toBe("boolean");
|
|
240
|
+
});
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
describe("real-world url examples", () => {
|
|
244
|
+
it.each([
|
|
245
|
+
[
|
|
246
|
+
"https%3A%2F%2Fgoogle.com%2Fsearch%3Fq%3Djavascript",
|
|
247
|
+
"Encoded Google search URL",
|
|
248
|
+
],
|
|
249
|
+
["The%20quick%20brown%20fox", "Sentence with spaces encoded"],
|
|
250
|
+
[
|
|
251
|
+
"/api/user?body=here%20are%20all%20my%20secrets",
|
|
252
|
+
"contextually encoded API path",
|
|
253
|
+
],
|
|
254
|
+
["redirect_uri=https%3A%2F%2Fapp.com%2Fcallback", "OAuth redirect URI"],
|
|
255
|
+
["data%3Atext%2Fplain%3Bbase64%2CSGVsbG8%3D", "Data URL encoded"],
|
|
256
|
+
])('real-world case: "%s" (%s)', (input, description) => {
|
|
257
|
+
const result = _containsEncodedComponents(input);
|
|
258
|
+
|
|
259
|
+
// Verify the function doesn't crash
|
|
260
|
+
expect(typeof result).toBe("boolean");
|
|
261
|
+
expect(result).toBe(true);
|
|
262
|
+
});
|
|
141
263
|
});
|
|
142
264
|
});
|
|
143
265
|
});
|
|
@@ -67,9 +67,13 @@ describe("IP Redaction utils", () => {
|
|
|
67
67
|
${"256.1.1.1"} | ${"256.1.1.1"}
|
|
68
68
|
${"0.0.0.0!"} | ${"[REDACTED IPV4]!"}
|
|
69
69
|
${"text0.0.0.0"} | ${"text[REDACTED IPV4]"}
|
|
70
|
+
${"%A00.0.0.0"} | ${"[REDACTED IPV4]"}
|
|
71
|
+
${"0.0.0.0%20"} | ${"[REDACTED IPV4]"}
|
|
70
72
|
${"0.0.text0.0"} | ${"0.0.text0.0"}
|
|
71
73
|
${"2001:0db8::1"} | ${"[REDACTED IPV6]"}
|
|
72
74
|
${"::1"} | ${"[REDACTED IPV6]"}
|
|
75
|
+
${"%20::1"} | ${"[REDACTED IPV6]"}
|
|
76
|
+
${"::1%A0"} | ${"[REDACTED IPV6]"}
|
|
73
77
|
${"text::1"} | ${"text[REDACTED IPV6]"}
|
|
74
78
|
${"::1text"} | ${"[REDACTED IPV6]text"}
|
|
75
79
|
${"sentence ending with f::1"} | ${"sentence ending with [REDACTED IPV6]"}
|