field-redactor 1.1.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +38 -0
- package/CHANGELOG.md +156 -0
- package/README.md +64 -647
- package/dist/configValidator.d.ts +10 -0
- package/dist/configValidator.d.ts.map +1 -0
- package/dist/configValidator.js +94 -0
- package/dist/configValidator.js.map +1 -0
- package/dist/copyOnWriteHelpers.d.ts +20 -0
- package/dist/copyOnWriteHelpers.d.ts.map +1 -0
- package/dist/copyOnWriteHelpers.js +58 -0
- package/dist/copyOnWriteHelpers.js.map +1 -0
- package/dist/customObjectManager.d.ts +13 -7
- package/dist/customObjectManager.d.ts.map +1 -1
- package/dist/customObjectManager.js +30 -29
- package/dist/customObjectManager.js.map +1 -1
- package/dist/dryRun.d.ts +8 -0
- package/dist/dryRun.d.ts.map +1 -0
- package/dist/dryRun.js +75 -0
- package/dist/dryRun.js.map +1 -0
- package/dist/dryRunAttribution.d.ts +8 -0
- package/dist/dryRunAttribution.d.ts.map +1 -0
- package/dist/dryRunAttribution.js +106 -0
- package/dist/dryRunAttribution.js.map +1 -0
- package/dist/fieldRedactor.d.ts +43 -5
- package/dist/fieldRedactor.d.ts.map +1 -1
- package/dist/fieldRedactor.js +124 -23
- package/dist/fieldRedactor.js.map +1 -1
- package/dist/fieldRedactorConfigBuilder.d.ts +49 -0
- package/dist/fieldRedactorConfigBuilder.d.ts.map +1 -0
- package/dist/fieldRedactorConfigBuilder.js +98 -0
- package/dist/fieldRedactorConfigBuilder.js.map +1 -0
- package/dist/fieldRedactorDeps.d.ts +17 -0
- package/dist/fieldRedactorDeps.d.ts.map +1 -0
- package/dist/fieldRedactorDeps.js +41 -0
- package/dist/fieldRedactorDeps.js.map +1 -0
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -1
- package/dist/index.js.map +1 -1
- package/dist/jsonWalk.d.ts +11 -0
- package/dist/jsonWalk.d.ts.map +1 -0
- package/dist/jsonWalk.js +86 -0
- package/dist/jsonWalk.js.map +1 -0
- package/dist/objectRedactor.d.ts +9 -34
- package/dist/objectRedactor.d.ts.map +1 -1
- package/dist/objectRedactor.js +11 -271
- package/dist/objectRedactor.js.map +1 -1
- package/dist/objectRedactorCow.d.ts +35 -0
- package/dist/objectRedactorCow.d.ts.map +1 -0
- package/dist/objectRedactorCow.js +300 -0
- package/dist/objectRedactorCow.js.map +1 -0
- package/dist/objectRedactorCustomObject.d.ts +49 -0
- package/dist/objectRedactorCustomObject.d.ts.map +1 -0
- package/dist/objectRedactorCustomObject.js +138 -0
- package/dist/objectRedactorCustomObject.js.map +1 -0
- package/dist/objectRedactorHelpers.d.ts +9 -0
- package/dist/objectRedactorHelpers.d.ts.map +1 -0
- package/dist/objectRedactorHelpers.js +58 -0
- package/dist/objectRedactorHelpers.js.map +1 -0
- package/dist/objectRedactorMutation.d.ts +10 -0
- package/dist/objectRedactorMutation.d.ts.map +1 -0
- package/dist/objectRedactorMutation.js +89 -0
- package/dist/objectRedactorMutation.js.map +1 -0
- package/dist/objectRedactorSync.d.ts +2 -0
- package/dist/objectRedactorSync.d.ts.map +1 -0
- package/dist/objectRedactorSync.js +7 -0
- package/dist/objectRedactorSync.js.map +1 -0
- package/dist/objectRedactorTraversal.d.ts +57 -0
- package/dist/objectRedactorTraversal.d.ts.map +1 -0
- package/dist/objectRedactorTraversal.js +576 -0
- package/dist/objectRedactorTraversal.js.map +1 -0
- package/dist/presets.d.ts +22 -0
- package/dist/presets.d.ts.map +1 -0
- package/dist/presets.js +50 -0
- package/dist/presets.js.map +1 -0
- package/dist/primitiveRedactor.d.ts +13 -8
- package/dist/primitiveRedactor.d.ts.map +1 -1
- package/dist/primitiveRedactor.js +32 -27
- package/dist/primitiveRedactor.js.map +1 -1
- package/dist/redactionRules.d.ts +23 -0
- package/dist/redactionRules.d.ts.map +1 -0
- package/dist/redactionRules.js +76 -0
- package/dist/redactionRules.js.map +1 -0
- package/dist/regexUtils.d.ts +3 -0
- package/dist/regexUtils.d.ts.map +1 -0
- package/dist/regexUtils.js +8 -0
- package/dist/regexUtils.js.map +1 -0
- package/dist/secretManager.d.ts +18 -5
- package/dist/secretManager.d.ts.map +1 -1
- package/dist/secretManager.js +49 -11
- package/dist/secretManager.js.map +1 -1
- package/dist/types.d.ts +110 -7
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +13 -6
- package/dist/types.js.map +1 -1
- package/dist/valuePatternMatcher.d.ts +15 -0
- package/dist/valuePatternMatcher.d.ts.map +1 -0
- package/dist/valuePatternMatcher.js +30 -0
- package/dist/valuePatternMatcher.js.map +1 -0
- package/docs/guides/anti-patterns.md +107 -0
- package/docs/guides/metadata-redaction.md +112 -0
- package/docs/guides/migration-1.2-to-1.5.md +106 -0
- package/docs/guides/secret-key-modes.md +108 -0
- package/docs/guides/value-pattern-redaction.md +58 -0
- package/docs/reference/config.md +135 -0
- package/docs/release-notes/2.0.0.md +19 -0
- package/docs/release-notes/2.1.0.md +17 -0
- package/docs/release-notes/2.2.0.md +16 -0
- package/docs/release-notes/2.3.0.md +17 -0
- package/docs/release-notes/2.3.1.md +17 -0
- package/docs/release-notes/2.4.0.md +30 -0
- package/docs/release-notes/2.5.0.md +33 -0
- package/docs/release-notes/2.5.1.md +17 -0
- package/docs/release-notes/README.md +30 -0
- package/docs/release-notes/v1.0.0.md +17 -0
- package/docs/release-notes/v1.1.0.md +18 -0
- package/docs/release-notes/v1.2.0.md +18 -0
- package/docs/release-notes/v1.2.1.md +16 -0
- package/docs/release-notes/v1.2.2.md +19 -0
- package/docs/release-notes/v1.3.0.md +112 -0
- package/docs/release-notes/v1.5.0.md +88 -0
- package/jest.config.js +4 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,680 +1,97 @@
|
|
|
1
1
|
# FieldRedactor
|
|
2
2
|
|
|
3
|
-
A
|
|
3
|
+
A TypeScript library for redacting PII from nested JSON using regex key rules, object **schemas** (sibling-key patterns), and configurable redaction modes.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Install
|
|
6
6
|
|
|
7
|
-
# History
|
|
8
|
-
In many instances, redaction of sensitive data from log output is fairly straightforward and can be accomplished through existing redaction libraries. However, I often encountered scenarios where such simplistic approaches were not sufficient. Take, for example, the following object:
|
|
9
|
-
|
|
10
|
-
```json
|
|
11
|
-
{
|
|
12
|
-
"name": "email",
|
|
13
|
-
"type": "String",
|
|
14
|
-
"value": "foo.bar@example.com"
|
|
15
|
-
}
|
|
16
|
-
```
|
|
17
|
-
|
|
18
|
-
If this object is placed in a data set with many others, and only some of these objects contain PII, redaction becomes quite troublesome. One could redact all `value` fields from these objects - but this merely renders the log output useless for debugging or tracing, as all values are now redacted. The value of the `name` field determines whether or not the `value` field should be redacted. I often found myself writing custom logic to make this determination, which led to brittle code that was difficult to maintain. In the end I decided to write a new, highly configurable plug-and-play solution that could redact sensitive data in various manners while requiring only minor configuration tweaks. Enter FieldRedactor!
|
|
19
|
-
|
|
20
|
-
## Basic Usage
|
|
21
|
-
Basic usage of the FieldRedactor is straightforward but not recommended. Object redaction can be performed either in-place or return the redacted object. If redactor is given a string, primitive, Date, function, or null/undefined value then it returns the value without modification.
|
|
22
|
-
|
|
23
|
-
```typescript
|
|
24
|
-
import { FieldRedactor } from 'field-redactor';
|
|
25
|
-
const myJsonObject = { foo: "bar", fizz: null };
|
|
26
|
-
const fieldRedactor = new FieldRedactor();
|
|
27
|
-
|
|
28
|
-
// return redacted result
|
|
29
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
30
|
-
const primitiveResult = await fieldRedactor.redact("foobar");
|
|
31
|
-
console.log(myJsonObject); // { foo: "bar", fizz: null }
|
|
32
|
-
console.log(result); // { foo: "REDACTED", fizz: null }
|
|
33
|
-
console.log(primitiveResult); // "foobar"
|
|
34
|
-
|
|
35
|
-
// redact in place
|
|
36
|
-
await fieldRedactor.redactInPlace(myJsonObject);
|
|
37
|
-
console.log(myJsonObject); // { foo: "REDACTED", fizz: null }
|
|
38
|
-
```
|
|
39
|
-
> **Note:** `null` and `undefined` values are not redacted by default.
|
|
40
|
-
> **Note:** `null`, `undefined`, `string`, `Date`, `Function`, and primitive value inputs are completely ignored.
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
## Customization
|
|
45
|
-
The true power of this tool comes from its customization. FieldRedactor can be customized to redact only primtive values for certain keys, deeply redact others, while stringifying and fully redacting other keys. The redactor itself can be configured, and "Custom Objects" can be specified to handle certain object shapes in a highly specific and configurable way.
|
|
46
|
-
|
|
47
|
-
### Overview
|
|
48
|
-
| Config Field | Type | Default | Effect |
|
|
49
|
-
|-----------------------|---------------------------------|--------------------------------|-----------------------------------------------------------------------|
|
|
50
|
-
| `redactor` | `(val: any) => Promise<string>` | `(val) => Promise.resolve("REDACTED")` | The function to use when redacting values. |
|
|
51
|
-
| `secretKeys` | `RegExp[]` | `null` | Specifies which values at any level of the JSON object should be redacted. Objects are not deeply redacted, but primitive values in arrays are. If not specified, all values are considered secret. |
|
|
52
|
-
| `deepSecretKeys` | `RegExp[]` | `[]` | Specifies keys at any level of the JSON object to be deeply redacted. All values within matching objects are fully redacted unless matching a custom object. |
|
|
53
|
-
| `fullSecretKeys` | `RegExp[]` | `[]` | Specifies keys at any level of the JSON object to be stringified and fully redacted. Primarily used for objects and arrays. |
|
|
54
|
-
| `customObjects` | `CustomObject[]` | `[]` | Specifies custom objects requiring fine-tuned redaction logic, such as referencing sibling keys. See the "Custom Objects" section for details. |
|
|
55
|
-
| `ignoreBooleans` | `boolean` | `false` | If `true`, booleans will not be redacted even if secret. |
|
|
56
|
-
| `ignoreNullOrUndefined` | `boolean` | `true` | If `true`, `null` and `undefined` values will not be redacted. |
|
|
57
|
-
|
|
58
|
-
### `redactor` Configuration
|
|
59
|
-
Configures the redactor function used when a secret is encountered. Users should typically provide this configuration.
|
|
60
|
-
|
|
61
|
-
#### Details
|
|
62
|
-
- **Type:** `(val: any) => Promise<string>`
|
|
63
|
-
- **Effect:** The function used to redact values.
|
|
64
|
-
- Value can only be null or undefined if `ignoreNullOrUndefined` is set to false.
|
|
65
|
-
- Defaults to `() => Promise.resolve('REDACTED')`.
|
|
66
|
-
|
|
67
|
-
#### Example
|
|
68
|
-
##### Code
|
|
69
|
-
```typescript
|
|
70
|
-
import { FieldRedactor } from 'field-redactor';
|
|
71
|
-
import * as crypto from 'crypto';
|
|
72
|
-
const redactor: Redactor = (val: any) => Promise.resolve(crypto.createHash('sha256').update(val.toString()).digest('hex'));
|
|
73
|
-
const fieldRedactor = new FieldRedactor({
|
|
74
|
-
redactor
|
|
75
|
-
});
|
|
76
|
-
const result = await redactor.redact({foo: "bar"});
|
|
77
|
-
console.log(result);
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
##### Output
|
|
81
7
|
```bash
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
### `secretKeys` Configuration
|
|
86
|
-
Used to specify primitive that should be redacted. If none of `secretKeys`, `deepSecretKeys`, or `fullSecretKeys` specified then all values are considered secret. Users should almost always specify at least one of these three fields. Has the lowest precedence of all configurable secret fields.
|
|
87
|
-
|
|
88
|
-
#### Details
|
|
89
|
-
- **Type:** `RegExp[]`
|
|
90
|
-
- **Default:** All values considered secret
|
|
91
|
-
- If `deepSecretKeys` or `fullSecretKeys` specified, defaults to `[]`
|
|
92
|
-
- **Effect:** Matches keys in objects or child objects and redacts their values, if primitive. Arrays are processed recursively, with primitives redacted.
|
|
93
|
-
- If the value is an object it will be assessed using the same logic, regardless of if its key value is a secret.
|
|
94
|
-
- if an array value is an object it will be assessed using the same logic, regardless of if its key value is a secret.
|
|
95
|
-
- Has lowest precedence when compared to `deepSecretKeys`, `fullSecretKeys`, and `customObjects` configurations.
|
|
96
|
-
|
|
97
|
-
#### Example
|
|
98
|
-
##### Code
|
|
99
|
-
```typescript
|
|
100
|
-
import { FieldRedactor } from 'field-redactor';
|
|
101
|
-
const myJsonObject = {
|
|
102
|
-
timestamp: "2024-12-01T22:07:26.448Z",
|
|
103
|
-
userId: 271,
|
|
104
|
-
contactInfo: {
|
|
105
|
-
email: "foo.bar@example.com",
|
|
106
|
-
firstName: "Foo",
|
|
107
|
-
lastName: "Bar",
|
|
108
|
-
Salutation: "Mr.",
|
|
109
|
-
preference: "email",
|
|
110
|
-
lastUpdated: "2024-12-01T22:07:26.448Z"
|
|
111
|
-
},
|
|
112
|
-
someSecretData: ["fizz", "buzz", { deep: "is not redacted", name: "foobar" }],
|
|
113
|
-
hobbies: ["Basketball", "Baseball", "Tennis"]
|
|
114
|
-
}
|
|
115
|
-
const fieldRedactor = new FieldRedactor({
|
|
116
|
-
secretKeys: [/email/i, /name/i, /someSecretData/i, /contactInfo/i],
|
|
117
|
-
});
|
|
118
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
119
|
-
console.log(result);
|
|
120
|
-
```
|
|
121
|
-
|
|
122
|
-
##### Output
|
|
123
|
-
```json
|
|
124
|
-
{
|
|
125
|
-
"timestamp": "2024-12-01T22:07:26.448Z",
|
|
126
|
-
"userId": 271,
|
|
127
|
-
"contactInfo": {
|
|
128
|
-
"email": "REDACTED",
|
|
129
|
-
"firstName": "REDACTED",
|
|
130
|
-
"lastName": "REDACTED",
|
|
131
|
-
"Salutation": "Mr.",
|
|
132
|
-
"preference": "email",
|
|
133
|
-
"lastUpdated": "2024-12-01T22:07:26.448Z"
|
|
134
|
-
},
|
|
135
|
-
"someSecretData": ["REDACTED", "REDACTED", { "deep": "is not redacted", "name": "REDACTED" }],
|
|
136
|
-
"hobbies": ["Basketball", "Baseball", "Tennis"]
|
|
137
|
-
}
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
> - contactInfo was not redacted despite being a RegExp match as it is not a primitive.
|
|
141
|
-
> - `email`, `firstName`, and `lastName` fields were redacted in contactInfo as they matched RegExes
|
|
142
|
-
> - primitives in `someSecretData` array value were redacted
|
|
143
|
-
> - object in `someSecretData` was evaluated separate from primitives in the same array
|
|
144
|
-
|
|
145
|
-
### `deepSecretKeys` Configuration
|
|
146
|
-
Used to specify secrets which should be deeply redacted. Values which are objects or arrays will have all their values redacted, including child objects and arrays, unless identified as a `fullSecretKey` or `customObject` which take higher precedence.
|
|
147
|
-
|
|
148
|
-
#### Details
|
|
149
|
-
- **Type:** `RegExp[]`
|
|
150
|
-
- **Default:** `[]`
|
|
151
|
-
- **Effect:** Matches keys in objects or child objects and deeply redacts all values, including values in child objects or objects within arrays.
|
|
152
|
-
- Has higher precedence than `secretKeys` but lower precedence than `fullSecretKeys` and `customObjects`.
|
|
153
|
-
|
|
154
|
-
#### Example
|
|
155
|
-
##### Code
|
|
156
|
-
```typescript
|
|
157
|
-
import { FieldRedactor } from 'field-redactor';
|
|
158
|
-
const myJsonObject = {
|
|
159
|
-
timestamp: "2024-12-01T22:07:26.448Z",
|
|
160
|
-
userId: 271,
|
|
161
|
-
contactInfo: {
|
|
162
|
-
email: "foo.bar@example.com",
|
|
163
|
-
firstName: "Foo",
|
|
164
|
-
lastName: "Bar",
|
|
165
|
-
Salutation: "Mr.",
|
|
166
|
-
preference: "email",
|
|
167
|
-
lastUpdated: "2024-12-01T22:07:26.448Z",
|
|
168
|
-
lastUpdatedBy: {
|
|
169
|
-
firstName: "John",
|
|
170
|
-
lastName: "Doe",
|
|
171
|
-
id: "12345"
|
|
172
|
-
}
|
|
173
|
-
},
|
|
174
|
-
someSecretData: ["fizz", "buzz", { deep: "FOO", name: "BAR" }],
|
|
175
|
-
hobbies: ["Basketball", "Baseball", "Tennis", {contactInfo: "foobar"}]
|
|
176
|
-
}
|
|
177
|
-
const fieldRedactor = new FieldRedactor({
|
|
178
|
-
deepSecretKeys: [/someSecretData/i, /contactInfo/i]
|
|
179
|
-
});
|
|
180
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
181
|
-
console.log(result);
|
|
8
|
+
npm install field-redactor
|
|
9
|
+
# or
|
|
10
|
+
yarn add field-redactor
|
|
182
11
|
```
|
|
183
12
|
|
|
184
|
-
|
|
185
|
-
```json
|
|
186
|
-
{
|
|
187
|
-
"timestamp": "2024-12-01T22:07:26.448Z",
|
|
188
|
-
"userId": 271,
|
|
189
|
-
"contactInfo": {
|
|
190
|
-
"email": "REDACTED",
|
|
191
|
-
"firstName": "REDACTED",
|
|
192
|
-
"lastName": "REDACTED",
|
|
193
|
-
"Salutation": "REDACTED",
|
|
194
|
-
"preference": "REDACTED",
|
|
195
|
-
"lastUpdated": "REDACTED",
|
|
196
|
-
"lastUpdatedBy": {
|
|
197
|
-
"firstName": "REDACTED",
|
|
198
|
-
"lastName": "REDACTED",
|
|
199
|
-
"id": "REDACTED"
|
|
200
|
-
}
|
|
201
|
-
},
|
|
202
|
-
"someSecretData": ["REDACTED", "REDACTED", { "deep": "REDACTED", "name": "REDACTED" }],
|
|
203
|
-
"hobbies": ["Basketball", "Baseball", "Tennis", { "contactInfo": "REDACTED" }]
|
|
204
|
-
}
|
|
205
|
-
```
|
|
206
|
-
> - All values in `contactInfo` were redacted, including deep children
|
|
207
|
-
> - All values in `someSecretData` were redacted, including object values
|
|
208
|
-
|
|
209
|
-
### `fullSecretKeys` Configuration
|
|
210
|
-
Specifies values which should be stringified and passed to the redactor function. Has higher precedence than other secret types but lower precedence than `customObjects`. However, if value is stringified and contains a child object which is a custom object then it will, by definition, not be noticed.
|
|
13
|
+
## Quick start
|
|
211
14
|
|
|
212
|
-
#### Details
|
|
213
|
-
- **Type:** `RegExp[]`
|
|
214
|
-
- **Default:** `[]`
|
|
215
|
-
- **Effect:** Matches keys in objects or child objects and stringifies and fully redacts their values.
|
|
216
|
-
- Has higher precedence than `secretKeys` and `deepSecretKeys` but lower precedence than `customObjects`.
|
|
217
|
-
|
|
218
|
-
#### Example
|
|
219
|
-
##### Code
|
|
220
15
|
```typescript
|
|
221
|
-
import { FieldRedactor } from 'field-redactor';
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
237
|
-
const fieldRedactor = new FieldRedactor({
|
|
238
|
-
fullSecretKeys: [/someSecretData/i, /contactInfo/i],
|
|
16
|
+
import { CustomObjectMatchType, FieldRedactor, FieldRedactorConfigBuilder } from 'field-redactor';
|
|
17
|
+
|
|
18
|
+
// Recommended: explicit rules via createSafe or the builder
|
|
19
|
+
const redactor = FieldRedactorConfigBuilder.create()
|
|
20
|
+
.shallow(/email/i, /password/i)
|
|
21
|
+
.remove(/authKey/i)
|
|
22
|
+
.schema(
|
|
23
|
+
{ name: CustomObjectMatchType.Ignore, type: CustomObjectMatchType.Ignore, value: 'name' },
|
|
24
|
+
{ name: 'metadata-entry' }
|
|
25
|
+
)
|
|
26
|
+
.buildSafeRedactor();
|
|
27
|
+
|
|
28
|
+
const result = redactor.redactSync({
|
|
29
|
+
email: 'alice@example.com',
|
|
30
|
+
authKey: 'secret',
|
|
31
|
+
metadata: [{ name: 'email', type: 'String', value: 'alice@example.com' }]
|
|
239
32
|
});
|
|
240
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
241
|
-
console.log(result);
|
|
242
33
|
```
|
|
243
34
|
|
|
244
|
-
|
|
245
|
-
```json
|
|
246
|
-
{
|
|
247
|
-
"timestamp": "2024-12-01T22:07:26.448Z",
|
|
248
|
-
"userId": 271,
|
|
249
|
-
"contactInfo": "REDACTED",
|
|
250
|
-
"someSecretData": "REDACTED",
|
|
251
|
-
"hobbies": ["Basketball", "Baseball", "Tennis"]
|
|
252
|
-
}
|
|
253
|
-
```
|
|
254
|
-
> - Entirity of `contactInfo` was redacted.
|
|
255
|
-
> - Entirity of `someSecretData` was redacted.
|
|
256
|
-
|
|
257
|
-
### `customObjects` Configuration
|
|
258
|
-
**One of the most powerful features of this library and why it was written in the first place.** Combining `secretKeys`, `deepSecretKeys`, `fullSecretKeys`, and `customObjects` yields a powerful and highly customizable redaction tool capable of conditionally redacting a broad variety of input JSON objects correctly according to a single schema configuration.
|
|
35
|
+
`redact()` / `redactSync()` leave the input untouched by default (copy-on-write). Primitives, `Date`, and `null`/`undefined` at the root are returned unchanged.
|
|
259
36
|
|
|
260
|
-
|
|
261
|
-
- **Type:** `CustomObject` (See `CustomObject` Schema Section)
|
|
262
|
-
- **Default:** `[]`
|
|
263
|
-
- **Effect:** Any object that matches the schema will be considered a custom object and redacted based on the schema configuration.
|
|
264
|
-
- Priority supercedes all others. If a `CustomObject` is nested inside a `deepSecretKey` value then the `CustomObject` configuration takes precedence.
|
|
265
|
-
- If a `CustomObject` is found inside another `CustomObject` then the new `CustomObject` takes precedence.
|
|
37
|
+
### Preview changes with `dryRun`
|
|
266
38
|
|
|
267
|
-
#### `CustomObject` Schema
|
|
268
|
-
A custom object takes the following format:
|
|
269
39
|
```typescript
|
|
270
|
-
{
|
|
271
|
-
|
|
272
|
-
}
|
|
40
|
+
const { result, report } = redactor.dryRunSync(payload);
|
|
41
|
+
// report.redactedPaths, report.deletedPaths, report.matchedSchemas, report.pathRules
|
|
273
42
|
```
|
|
274
43
|
|
|
275
|
-
|
|
276
|
-
- **Type:** `string`
|
|
277
|
-
- **Effect:** Specifies the key for a custom object
|
|
278
|
-
- `value`:
|
|
279
|
-
- **Type:** `CustomObjectMatchType | string`
|
|
280
|
-
- **Effect:** Specifies how the value should be redacted
|
|
281
|
-
- `string` - checks if the sibling key with this name has a value which is a `secretKey`, `deepSecretKey`, or `fullSecretKey` and redacts accordingly
|
|
282
|
-
- `CustomObjectMatchType` - Redacts according to the match type. See `CustomObjectMatchType` Enum Section.
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
#### `CustomObjectMatchType` Enum
|
|
286
|
-
| Key | Description |
|
|
287
|
-
| --- | ----------- |
|
|
288
|
-
| `Full` | Stringify value and redact. Functions identical to `fullSecretKeys`. |
|
|
289
|
-
| `Deep` | Redact if primitive, deeply redact all values and children if object or array. Functions identical to `deepSecretKeys`. |
|
|
290
|
-
| `Shallow` | Redact if primitive or array of primitives and revery to normal evaluation if object or array contains object. Functions identical to `secretKeys`. |
|
|
291
|
-
| `Pass` | Do not redact if primitive value and revert to normal rules if an object. |
|
|
292
|
-
| `Ignore` | Do not redact if primitive and skip evaluation entirely if object or array. |
|
|
44
|
+
## Start here
|
|
293
45
|
|
|
294
|
-
#### Example
|
|
295
|
-
```typescript
|
|
296
|
-
import { FieldRedactor } from 'field-redactor';
|
|
297
|
-
|
|
298
|
-
const myCustomObject = {
|
|
299
|
-
name: CustomObjectMatchType.Ignore,
|
|
300
|
-
type: CustomObjectMatchType.Ignore,
|
|
301
|
-
significance: CustomObjectMatchType.Ignore,
|
|
302
|
-
shallowValue: "name",
|
|
303
|
-
deepValue: "type",
|
|
304
|
-
fullValue: "significance"
|
|
305
|
-
shallow: CustomObjectMatchType.Shallow,
|
|
306
|
-
deep: CustomObjectMatchType.Deep,
|
|
307
|
-
full: CustomObjectMatchType.Full,
|
|
308
|
-
pass: CustomObjectMatchType.Pass,
|
|
309
|
-
ignore: CustomObjectMatchType.Ignore
|
|
310
|
-
};
|
|
311
|
-
|
|
312
|
-
const myJsonObject = {
|
|
313
|
-
timestamp: "2024-12-01T22:07:26.448Z",
|
|
314
|
-
userId: 271,
|
|
315
|
-
data: [
|
|
316
|
-
{
|
|
317
|
-
name: "email",
|
|
318
|
-
type: "Secure",
|
|
319
|
-
significance: "meta"
|
|
320
|
-
shallowValue: "foo.bar@example.com",
|
|
321
|
-
deepValue: "foobar",
|
|
322
|
-
fullValue: "hello",
|
|
323
|
-
shallow: "hello",
|
|
324
|
-
deep: "hello",
|
|
325
|
-
full: "hello",
|
|
326
|
-
pass: "hello",
|
|
327
|
-
ignore: "hello"
|
|
328
|
-
},
|
|
329
|
-
{
|
|
330
|
-
name: "email",
|
|
331
|
-
type: "Secure",
|
|
332
|
-
significance: "meta",
|
|
333
|
-
shallowValue: {
|
|
334
|
-
hello: "world",
|
|
335
|
-
email: "foobar"
|
|
336
|
-
},
|
|
337
|
-
deepValue: {
|
|
338
|
-
hello: "world",
|
|
339
|
-
email: "foobar"
|
|
340
|
-
},
|
|
341
|
-
deepValue: {
|
|
342
|
-
hello: "world",
|
|
343
|
-
email: "foobar"
|
|
344
|
-
},
|
|
345
|
-
shallow: {
|
|
346
|
-
hello: "world",
|
|
347
|
-
email: "foobar"
|
|
348
|
-
},
|
|
349
|
-
deep: {
|
|
350
|
-
hello: "world",
|
|
351
|
-
email: "foobar"
|
|
352
|
-
},
|
|
353
|
-
full: {
|
|
354
|
-
hello: "world",
|
|
355
|
-
email: "foobar"
|
|
356
|
-
},
|
|
357
|
-
pass: {
|
|
358
|
-
hello: "world",
|
|
359
|
-
email: "foobar"
|
|
360
|
-
},
|
|
361
|
-
ignore: {
|
|
362
|
-
hello: "world",
|
|
363
|
-
email: "foobar"
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
]
|
|
367
|
-
};
|
|
368
|
-
const fieldRedactor = new FieldRedactor({
|
|
369
|
-
secretKeys: [/email/],
|
|
370
|
-
deepSecretKeys: [/secure/i],
|
|
371
|
-
fullSecretKeys: [/meta/i],
|
|
372
|
-
customObjects: [myCustomObject]
|
|
373
|
-
});
|
|
374
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
375
|
-
console.log(result);
|
|
376
46
|
```
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
{
|
|
381
|
-
"timestamp": "2024-12-01T22:07:26.448Z",
|
|
382
|
-
"userId": 271,
|
|
383
|
-
"data": [
|
|
384
|
-
{
|
|
385
|
-
"name": "email",
|
|
386
|
-
"type": "Secure",
|
|
387
|
-
"shallowValue": "REDACTED",
|
|
388
|
-
"deepValue": "REDACTED",
|
|
389
|
-
"fullValue": "REDACTED",
|
|
390
|
-
"shallow": "REDACTED",
|
|
391
|
-
"deep": "REDACTED",
|
|
392
|
-
"full": "REDACTED",
|
|
393
|
-
"pass": "hello",
|
|
394
|
-
"ignore": "hello"
|
|
395
|
-
},
|
|
396
|
-
{
|
|
397
|
-
"name": "email",
|
|
398
|
-
"type": "Secure",
|
|
399
|
-
"shallowValue": {
|
|
400
|
-
"hello": "world",
|
|
401
|
-
"email": "REDACTED"
|
|
402
|
-
},
|
|
403
|
-
"deepValue": {
|
|
404
|
-
"hello": "REDACTED",
|
|
405
|
-
"email": "REDACTED"
|
|
406
|
-
},
|
|
407
|
-
"fullValue": "REDACTED",
|
|
408
|
-
"shallow": {
|
|
409
|
-
"hello": "world",
|
|
410
|
-
"email": "REDACTED"
|
|
411
|
-
},
|
|
412
|
-
"deep": {
|
|
413
|
-
"hello": "REDACTED",
|
|
414
|
-
"email": "REDACTED"
|
|
415
|
-
},
|
|
416
|
-
"full": "REDACTED",
|
|
417
|
-
"pass": {
|
|
418
|
-
"hello": "world",
|
|
419
|
-
"email": "REDACTED"
|
|
420
|
-
},
|
|
421
|
-
"ignore": {
|
|
422
|
-
"hello": "world",
|
|
423
|
-
"email": "foobar"
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
]
|
|
427
|
-
};
|
|
47
|
+
Do you know which JSON keys are always sensitive?
|
|
48
|
+
├─ yes → Shallow / Deep / Opaque / Remove (see secret key modes guide)
|
|
49
|
+
└─ only sometimes → shaped objects like { name, value }? → Schema rules (see metadata guide)
|
|
428
50
|
```
|
|
429
|
-
> - Both objects in the data array matched the custom object and were redacted accordingly
|
|
430
|
-
> - The first object contains only primitives whereas the second contains objects to highlight the differences
|
|
431
|
-
> - `name` and `type` field were never redacted in either object because `CustomObjectMatchType` was `Ignore`
|
|
432
|
-
> - `shallowValue`, `deepValue`, and `fullValue` had string key specifiers which matched a `shallowKey`, `deepSecretKey`, and `fullSecretKey`, respectively, and were redacted according to those rules.
|
|
433
|
-
> - `shallow`, `deep`, `full`, `pass`, and `ignore` fields were redacted according to their `CustomObjectMatchType` value specified in the schema.
|
|
434
51
|
|
|
435
|
-
|
|
436
|
-
- **Type:** `boolean`
|
|
437
|
-
- **Default:** `false`
|
|
438
|
-
- **Effect:** Specifies if boolean values should be redacted.
|
|
52
|
+
**Precedence:** Schema → Opaque → Deep → Remove → Shallow → Value-pattern
|
|
439
53
|
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
54
|
+
| Concept | Config field |
|
|
55
|
+
| --- | --- |
|
|
56
|
+
| Shallow | `secretKeys` |
|
|
57
|
+
| Deep | `deepSecretKeys` |
|
|
58
|
+
| Opaque | `fullSecretKeys` |
|
|
59
|
+
| Remove | `deleteSecretKeys` |
|
|
60
|
+
| Schema | `customObjects` |
|
|
444
61
|
|
|
445
|
-
|
|
446
|
-
foo: "bar",
|
|
447
|
-
fizz: false,
|
|
448
|
-
buzz: true
|
|
449
|
-
};
|
|
450
|
-
const fieldRedactor = new FieldRedactor({
|
|
451
|
-
ignoreBooleans: true,
|
|
452
|
-
secretKeys: [/foo/, /fizz/, /buzz/]
|
|
453
|
-
});
|
|
62
|
+
Use `FieldRedactor.createSafe({ ... })` or `FieldRedactorConfigBuilder` so you never accidentally redact every field. `new FieldRedactor()` without rules still redacts all values (legacy default).
|
|
454
63
|
|
|
455
|
-
|
|
456
|
-
console.log(result);
|
|
457
|
-
```
|
|
64
|
+
## Documentation
|
|
458
65
|
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
66
|
+
| Guide | Description |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| [Secret key modes](docs/guides/secret-key-modes.md) | Shallow, Deep, Opaque, Remove with examples |
|
|
69
|
+
| [Metadata redaction](docs/guides/metadata-redaction.md) | `{ name, value }` schemas and sibling-key rules |
|
|
70
|
+
| [Anti-patterns](docs/guides/anti-patterns.md) | Common config mistakes and fixes |
|
|
71
|
+
| [Value-pattern redaction](docs/guides/value-pattern-redaction.md) | Detect PII in free-text field values |
|
|
72
|
+
| [Configuration reference](docs/reference/config.md) | Full option table, API, presets, validation |
|
|
73
|
+
| [Migration 1.2 → 1.5](docs/guides/migration-1.2-to-1.5.md) | Upgrade from the previous npm line |
|
|
74
|
+
| [Release notes](docs/release-notes/README.md) | Per-version notes for every published tag |
|
|
467
75
|
|
|
468
|
-
|
|
469
|
-
- **Type:** `boolean`
|
|
470
|
-
- **Default:** `true`
|
|
471
|
-
- **Effect:** Specifies if null or undefined values shold be redacted.
|
|
472
|
-
- *Note: Ensure custom redaction function can appropriately handle null or undefined if set to false.*
|
|
76
|
+
## API surface
|
|
473
77
|
|
|
474
|
-
### Example
|
|
475
|
-
#### Code
|
|
476
78
|
```typescript
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
79
|
+
import {
|
|
80
|
+
FieldRedactor,
|
|
81
|
+
FieldRedactorConfigBuilder,
|
|
82
|
+
FieldRedactorConfigurationError,
|
|
83
|
+
FieldRedactorError,
|
|
84
|
+
presets,
|
|
85
|
+
validateFieldRedactorConfig
|
|
86
|
+
} from 'field-redactor';
|
|
485
87
|
|
|
486
|
-
|
|
487
|
-
console.log(result);
|
|
488
|
-
```
|
|
489
|
-
#### Output
|
|
490
|
-
```json
|
|
491
|
-
{
|
|
492
|
-
"foo": "REDACTED",
|
|
493
|
-
"fizz": "REDACTED",
|
|
494
|
-
"buzz": "REDACTED"
|
|
495
|
-
}
|
|
88
|
+
import type { DryRunPathRule, RedactionRuleLabel, MatchedSchemaReport } from 'field-redactor';
|
|
496
89
|
```
|
|
497
90
|
|
|
498
|
-
|
|
499
|
-
|
|
91
|
+
- **Sync:** `redactSync()`, `redactInPlaceSync()`, `dryRunSync()`
|
|
92
|
+
- **Async:** `redact()`, `redactInPlace()`, `dryRun()` (sync path when no async-only `redactor`)
|
|
93
|
+
- **Presets:** `presets.loggingMetadata()`, `presets.applicationLogging()`, `presets.keyValueEntries()`
|
|
500
94
|
|
|
501
|
-
|
|
502
|
-
```typescript
|
|
503
|
-
import { FieldRedactor } from 'field-redactor';
|
|
504
|
-
const myRedactor = (text: string) => Promise.resolve("REDACTED");
|
|
505
|
-
const metadataCustomObject: CustomObject = {
|
|
506
|
-
name: CustomObjectMatchTypes.Pass,
|
|
507
|
-
type: CustomObjectMatchTypes.Pass,
|
|
508
|
-
id: CustomObjectMatchTypes.Shallow,
|
|
509
|
-
value: "name"
|
|
510
|
-
};
|
|
95
|
+
## Why this library exists
|
|
511
96
|
|
|
512
|
-
|
|
513
|
-
userId: CustomObjectMatchTypes.Pass,
|
|
514
|
-
field: CustomObjectMatchTypes.Pass,
|
|
515
|
-
action: CustomObjectMatchTypes.Pass,
|
|
516
|
-
value: "field"
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
const fieldRedactor: FieldRedactor = new FieldRedactor({
|
|
520
|
-
redactor: myRedactor,
|
|
521
|
-
secretKeys: [/email/, /name/i, /someSecretData/, /children/],
|
|
522
|
-
deepSecretKeys: [/accountInfo/i, /someDeepSecretData/i, /privateInfo/i],
|
|
523
|
-
customObjects: [metadataCustomObject, actionsCustomObject],
|
|
524
|
-
ignoreNullOrUndefined: false
|
|
525
|
-
});
|
|
526
|
-
|
|
527
|
-
const myJsonObjectToRedact = {
|
|
528
|
-
timestamp: "2024-12-01T22:07:26.448Z",
|
|
529
|
-
userId: 271,
|
|
530
|
-
contactInfo: {
|
|
531
|
-
email: "foo.bar@example.com",
|
|
532
|
-
salutation: "Mr.",
|
|
533
|
-
firstName: "Foo",
|
|
534
|
-
lastName: "Bar",
|
|
535
|
-
dob: "1980-01-01",
|
|
536
|
-
backupEmail: undefined,
|
|
537
|
-
preference: "email",
|
|
538
|
-
lastUpdated: "2024-12-01T22:07:26.448Z"
|
|
539
|
-
},
|
|
540
|
-
someSecretData: ["fizz", "buzz", { deep: "is not redacted", name: "foobar" }],
|
|
541
|
-
accountInfo: {
|
|
542
|
-
balance: 123.45,
|
|
543
|
-
institution: "FizzBuz International"
|
|
544
|
-
information: {
|
|
545
|
-
routingNumber: 11111111,
|
|
546
|
-
acctNumber: 222222
|
|
547
|
-
}
|
|
548
|
-
},
|
|
549
|
-
actions: [
|
|
550
|
-
{
|
|
551
|
-
userId: 271,
|
|
552
|
-
field: "email",
|
|
553
|
-
action: "CREATE",
|
|
554
|
-
value: "foo.bar@example.com"
|
|
555
|
-
},
|
|
556
|
-
{
|
|
557
|
-
userId: 271,
|
|
558
|
-
field: "preference",
|
|
559
|
-
action: "UPDATE",
|
|
560
|
-
value: "email"
|
|
561
|
-
}
|
|
562
|
-
],
|
|
563
|
-
metadata: [
|
|
564
|
-
{
|
|
565
|
-
name: "mdn",
|
|
566
|
-
type: "Number",
|
|
567
|
-
id: 12,
|
|
568
|
-
value: 16151112222
|
|
569
|
-
},
|
|
570
|
-
{
|
|
571
|
-
name: "children",
|
|
572
|
-
type: "Array",
|
|
573
|
-
id: 20,
|
|
574
|
-
value: ["John", "Paul","Ringo", "George"]
|
|
575
|
-
},
|
|
576
|
-
{
|
|
577
|
-
name: "traceId",
|
|
578
|
-
type: "String",
|
|
579
|
-
id: 10,
|
|
580
|
-
value: "1234-6587"
|
|
581
|
-
},
|
|
582
|
-
{
|
|
583
|
-
name: "privateInfo",
|
|
584
|
-
type: "Object",
|
|
585
|
-
id: 20,
|
|
586
|
-
value: {
|
|
587
|
-
mySecretThings: {
|
|
588
|
-
a: "foo",
|
|
589
|
-
b: "bar"
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
],
|
|
594
|
-
hobbies: ["Basketball", "Baseball", "Tennis"],
|
|
595
|
-
someDeepSecretData: ["fizz", "buzz", { deep: "is redacted", name: "foobar" }],
|
|
596
|
-
someFullSecretData: {
|
|
597
|
-
foo: "bar"
|
|
598
|
-
},
|
|
599
|
-
someFullSecretData2: ["a", 1, "12"]
|
|
600
|
-
};
|
|
601
|
-
|
|
602
|
-
const result = await fieldRedactor.redact(myJsonObject);
|
|
603
|
-
console.log(result);
|
|
604
|
-
```
|
|
605
|
-
|
|
606
|
-
##### Output
|
|
607
|
-
```json
|
|
608
|
-
{
|
|
609
|
-
"timestamp": "2024-12-01T22:07:26.448Z",
|
|
610
|
-
"userId": 271,
|
|
611
|
-
"contactInfo": {
|
|
612
|
-
"email": "REDACTED",
|
|
613
|
-
"salutation": "Mr.",
|
|
614
|
-
"firstName": "REDACTED",
|
|
615
|
-
"lastName": "REDACTED",
|
|
616
|
-
"dob": "REDACTED",
|
|
617
|
-
"backupEmail": "REDACTED",
|
|
618
|
-
"preference": "email",
|
|
619
|
-
"lastUpdated": "2024-12-01T22:07:26.448Z",
|
|
620
|
-
},
|
|
621
|
-
"someSecretData": ["REDACTED", "REDACTED", { "deep": "is not redacted", "name": "REDACTED" }],
|
|
622
|
-
"accountInfo": {
|
|
623
|
-
"balance": "REDACTED",
|
|
624
|
-
"institution": "REDACTED",
|
|
625
|
-
"information": {
|
|
626
|
-
"routingNumber": "REDACTED",
|
|
627
|
-
"acctNumber": "REDACTED"
|
|
628
|
-
}
|
|
629
|
-
},
|
|
630
|
-
"actions": [
|
|
631
|
-
{
|
|
632
|
-
"userId": 271,
|
|
633
|
-
"field": "email",
|
|
634
|
-
"action": "CREATE",
|
|
635
|
-
"value": "REDACTED"
|
|
636
|
-
},
|
|
637
|
-
{
|
|
638
|
-
"userId": 271,
|
|
639
|
-
"field": "preference",
|
|
640
|
-
"action": "UPDATE",
|
|
641
|
-
"value": "email"
|
|
642
|
-
}
|
|
643
|
-
],
|
|
644
|
-
"metadata": [
|
|
645
|
-
{
|
|
646
|
-
"name": "mdn",
|
|
647
|
-
"type": "Number",
|
|
648
|
-
"id": "REDACTED",
|
|
649
|
-
"value": "REDACTED"
|
|
650
|
-
},
|
|
651
|
-
{
|
|
652
|
-
"name": "children",
|
|
653
|
-
"type": "Array",
|
|
654
|
-
"id": "REDACTED",
|
|
655
|
-
"value": ["REDACTED", "REDACTED", "REDACTED", "REDACTED"]
|
|
656
|
-
},
|
|
657
|
-
{
|
|
658
|
-
"name": "traceId",
|
|
659
|
-
"type": "String",
|
|
660
|
-
"id": 10,
|
|
661
|
-
"value": "1234-6587"
|
|
662
|
-
},
|
|
663
|
-
{
|
|
664
|
-
"name": "privateInfo",
|
|
665
|
-
"type": "Object",
|
|
666
|
-
"id": "REDACTED",
|
|
667
|
-
"value": {
|
|
668
|
-
"mySecretThings": {
|
|
669
|
-
"a": "REDACTED",
|
|
670
|
-
"b": "REDACTED"
|
|
671
|
-
}
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
],
|
|
675
|
-
"hobbies": ["Basketball", "Baseball", "Tennis"],
|
|
676
|
-
"someDeepSecretData": ["REDACTED", "REDACTED", { "deep": "REDACTED", "name": "REDACTED" }],
|
|
677
|
-
"someFullSecretData": "REDACTED",
|
|
678
|
-
"someFullSecretData2": "REDACTED"
|
|
679
|
-
}
|
|
680
|
-
```
|
|
97
|
+
Many log payloads encode sensitivity in sibling fields rather than key names — for example `{ name: "email", value: "user@example.com" }`. FieldRedactor matches object **schemas** and applies rules based on a sibling's value. See [Metadata redaction](docs/guides/metadata-redaction.md) for the full pattern.
|