fluent-plugin-masking 1.2.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +14 -8
- data/lib/fluent/plugin/filter_masking.rb +22 -6
- data/lib/fluent/plugin/version.rb +1 -1
- data/test/fields-to-mask +1 -1
- data/test/test_filter_masking.rb +23 -33
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e8c2ddb483a737dd913e64111218e9484bffb2d94db68c8aefb49ec4efdcfd58
|
4
|
+
data.tar.gz: 3080f077ae6d4437fc3921d34b642d77ffb57e1d4498d6b3028882d3a97a94a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53514f891096b90ce5c8b1957b248ebaf27f6ad25bf6176ac10fcc873e9bd471995c8157deeae57fb9af78487a0018562a973627129d14c6d9a3eabaab2c0d9b
|
7
|
+
data.tar.gz: ca252648f6799d83a5ca72796452ac2a165527275d8387c6cc051a3341381e177d94bb7ab07bef3254016cd4bd7124ec3691b593608d7802acce23eee9cb4eac
|
data/README.md
CHANGED
@@ -14,20 +14,24 @@ Fluentd filter plugin to mask sensitive or privacy records with `*******` in pla
|
|
14
14
|
## Installation
|
15
15
|
Install with gem:
|
16
16
|
|
17
|
-
`gem install fluent-plugin-masking`
|
17
|
+
`fluent-gem install fluent-plugin-masking`
|
18
18
|
|
19
19
|
## Setup
|
20
|
-
In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
|
20
|
+
In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`). if you one or more of the fields will be case insensitive, use the `/i` suffix in your field. see example below.
|
21
21
|
|
22
|
-
|
23
|
-
The JSON fields that are excluded are comma separated.
|
24
|
-
This can be used for logs of registration services or audit log entries which do not need to be masked.
|
25
|
-
|
22
|
+
### Optional configuration
|
23
|
+
- `fieldsToExcludeJSONPaths` - this field receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`) The JSON fields that are excluded are comma separated.
|
24
|
+
This can be used for logs of registration services or audit log entries which do not need to be masked.
|
25
|
+
|
26
|
+
- `handleSpecialEscapedJsonCases` - a boolean value that try to fix special escaped json cases. this feature is currently on alpha stage (default: false). for more details about thoose special cases see [Special Json Cases](#Special-escaped-json-cases-handling)
|
27
|
+
|
28
|
+
An example with optional configuration parameters:
|
26
29
|
```
|
27
30
|
<filter "**">
|
28
31
|
@type masking
|
29
32
|
fieldsToMaskFilePath "/path/to/fields-to-mask-file"
|
30
33
|
fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
|
34
|
+
handleSpecialEscapedJsonCases true
|
31
35
|
</filter>
|
32
36
|
```
|
33
37
|
|
@@ -98,10 +102,12 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
|
|
98
102
|
2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
|
99
103
|
```
|
100
104
|
|
101
|
-
|
102
|
-
### Run Unit Tests
|
105
|
+
## Run Unit Tests
|
103
106
|
```
|
104
107
|
gem install bundler
|
105
108
|
bundle install
|
106
109
|
ruby -r ./test/*.rb
|
107
110
|
```
|
111
|
+
|
112
|
+
## Special escaped json cases handling
|
113
|
+
|
@@ -30,14 +30,20 @@ module Fluent
|
|
30
30
|
end
|
31
31
|
begin
|
32
32
|
recordStr = record.to_s
|
33
|
+
|
34
|
+
if @handleSpecialEscapedJsonCases == true
|
35
|
+
@specialEscapedJsonRegexs.each do | regex, replace |
|
36
|
+
recordStr = recordStr.gsub(regex, replace)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
33
40
|
@fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
|
34
41
|
if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
|
35
42
|
recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement)
|
36
43
|
end
|
37
44
|
end
|
38
|
-
|
45
|
+
|
39
46
|
maskedRecord = strToHash(recordStr)
|
40
|
-
|
41
47
|
rescue Exception => e
|
42
48
|
$log.error "Failed to mask record: #{e}"
|
43
49
|
end
|
@@ -51,6 +57,11 @@ module Fluent
|
|
51
57
|
@fields_to_mask_regex = {}
|
52
58
|
@fields_to_mask_keys = {}
|
53
59
|
@fieldsToExcludeJSONPathsArray = []
|
60
|
+
|
61
|
+
@handleSpecialEscapedJsonCases = false
|
62
|
+
@specialEscapedJsonRegexs = {
|
63
|
+
Regexp.new(/,(( *)(\\*)("*)( *)),/m) => "\1,"
|
64
|
+
}
|
54
65
|
end
|
55
66
|
|
56
67
|
# this method only called ones (on startup time)
|
@@ -58,6 +69,7 @@ module Fluent
|
|
58
69
|
super
|
59
70
|
fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
|
60
71
|
fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
|
72
|
+
handleSpecialCases = conf['handleSpecialEscapedJsonCases']
|
61
73
|
|
62
74
|
if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0
|
63
75
|
fieldsToExcludeJSONPaths.split(",").each do | field |
|
@@ -80,12 +92,12 @@ module Fluent
|
|
80
92
|
if value.end_with? "/i"
|
81
93
|
# case insensitive
|
82
94
|
value = value.delete_suffix('/i')
|
83
|
-
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
|
84
|
-
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"
|
95
|
+
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi) # mask element in hash object
|
96
|
+
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/mi) # mask element in json string using capture groups that count the level of escaping inside the json string
|
85
97
|
else
|
86
98
|
# case sensitive
|
87
|
-
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
|
88
|
-
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"
|
99
|
+
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
|
100
|
+
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
|
89
101
|
end
|
90
102
|
|
91
103
|
@fields_to_mask.push(value)
|
@@ -100,6 +112,10 @@ module Fluent
|
|
100
112
|
end
|
101
113
|
end
|
102
114
|
|
115
|
+
# if true, each record (a json record), will be checked for a special escaped json cases
|
116
|
+
# any found case will be 'gsub' with the right solution
|
117
|
+
@handleSpecialEscapedJsonCases = handleSpecialCases != nil && handleSpecialCases.casecmp("true") == 0
|
118
|
+
|
103
119
|
puts "black list fields:"
|
104
120
|
puts @fields_to_mask
|
105
121
|
end
|
data/test/fields-to-mask
CHANGED
data/test/test_filter_masking.rb
CHANGED
@@ -4,7 +4,6 @@ require "fluent/test/driver/filter"
|
|
4
4
|
require "fluent/test/helpers"
|
5
5
|
require "./lib/fluent/plugin/filter_masking.rb"
|
6
6
|
|
7
|
-
|
8
7
|
MASK_STRING = "*******"
|
9
8
|
|
10
9
|
class YourOwnFilterTest < Test::Unit::TestCase
|
@@ -28,6 +27,13 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
28
27
|
fieldsToMaskFilePath test/fields-to-mask-insensitive
|
29
28
|
]
|
30
29
|
|
30
|
+
# configuration for special json escaped cases
|
31
|
+
CONFIG_SPECIAL_CASES = %[
|
32
|
+
fieldsToMaskFilePath test/fields-to-mask
|
33
|
+
fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
|
34
|
+
handleSpecialEscapedJsonCases true
|
35
|
+
]
|
36
|
+
|
31
37
|
def create_driver(conf = CONFIG)
|
32
38
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
|
33
39
|
end
|
@@ -102,6 +108,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
102
108
|
filtered_records = filter(conf, messages)
|
103
109
|
assert_equal(expected, filtered_records)
|
104
110
|
end
|
111
|
+
|
105
112
|
test 'mask field in hash object with exclude' do
|
106
113
|
conf = CONFIG
|
107
114
|
messages = [
|
@@ -113,6 +120,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
113
120
|
filtered_records = filter(conf, messages)
|
114
121
|
assert_equal(expected, filtered_records)
|
115
122
|
end
|
123
|
+
|
116
124
|
test 'mask field in hash object with nested exclude' do
|
117
125
|
conf = CONFIG
|
118
126
|
messages = [
|
@@ -149,37 +157,6 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
149
157
|
assert_equal(expected, filtered_records)
|
150
158
|
end
|
151
159
|
|
152
|
-
test 'mask field which is inner json string field (should mask the whole object)' do
|
153
|
-
conf = CONFIG
|
154
|
-
messages = [
|
155
|
-
{
|
156
|
-
:body => {
|
157
|
-
:action_name => "some_action",
|
158
|
-
:action_type => "some type",
|
159
|
-
:request => {
|
160
|
-
:body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
|
161
|
-
}
|
162
|
-
},
|
163
|
-
:timestamp => "2020-06-08T16:00:57.341Z"
|
164
|
-
}
|
165
|
-
]
|
166
|
-
|
167
|
-
expected = [
|
168
|
-
{
|
169
|
-
:body => {
|
170
|
-
:action_name => "some_action",
|
171
|
-
:action_type => "some type",
|
172
|
-
:request => {
|
173
|
-
:body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
|
174
|
-
}
|
175
|
-
},
|
176
|
-
:timestamp => "2020-06-08T16:00:57.341Z"
|
177
|
-
}
|
178
|
-
]
|
179
|
-
|
180
|
-
filtered_records = filter(conf, messages)
|
181
|
-
assert_equal(expected, filtered_records)
|
182
|
-
end
|
183
160
|
end
|
184
161
|
|
185
162
|
sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
|
@@ -223,7 +200,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
223
200
|
test 'mask case insensitive and case sensitive field in nested json escaped string' do
|
224
201
|
conf = CONFIG_CASE_INSENSITIVE
|
225
202
|
messages = [
|
226
|
-
{ :body => "{\"firsT_naMe\":\"mickey\",\"
|
203
|
+
{ :body => "{\"firsT_naMe\":\"mickey\",\"last_NAME\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" }
|
227
204
|
]
|
228
205
|
expected = [
|
229
206
|
{ :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
|
@@ -234,4 +211,17 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
234
211
|
|
235
212
|
end
|
236
213
|
|
214
|
+
sub_test_case 'plugin will mask all fields that need masking - special json escaped cases' do
|
215
|
+
test 'mask field in nested json escaped string when one of the values ends with "," (the value for "some_custom" field)' do
|
216
|
+
conf = CONFIG_SPECIAL_CASES
|
217
|
+
messages = [
|
218
|
+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\", \"cookie\":\"some_custom=,,live,default,,2097403972,2.22.242.38,\", \"city\":\"new york\"}" }
|
219
|
+
]
|
220
|
+
expected = [
|
221
|
+
{ :body => "{\"first_name\":\"*******\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\", \"cookie\":\"*******\", \"city\":\"new york\"}" }
|
222
|
+
]
|
223
|
+
filtered_records = filter(conf, messages)
|
224
|
+
assert_equal(expected, filtered_records)
|
225
|
+
end
|
226
|
+
end
|
237
227
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-masking
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shai Moria
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2021-
|
12
|
+
date: 2021-10-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|