fluent-plugin-masking 1.0.8 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/Gemfile.lock +6 -6
- data/README.md +15 -0
- data/fluent-plugin-masking.gemspec +1 -1
- data/lib/fluent/plugin/filter_masking.rb +33 -2
- data/lib/fluent/plugin/helpers.rb +17 -0
- data/lib/fluent/plugin/version.rb +1 -1
- data/test/test_filter_masking.rb +51 -1
- data/test/test_helpers.rb +42 -0
- metadata +11 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5b8b769887786df667374682f71ddbc61f6b575f
|
4
|
+
data.tar.gz: c143faea655e8ec4f621445d66525f827b26e9a0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c964b4e42d7bbbe59fcaf570b1ac8281427ee7d15ea20cb8dcd9320343a306739571e25c2030e3f63cd2465daec6ec757ce7a57c68127c21e7acabba36b3243c
|
7
|
+
data.tar.gz: d9937e8bf71d1431cf9312134618d2bd5c842e203ce9d853cc4e8cb2a08b8c60cfea4b21105c3296ca35dedd2dc3a1cb1c61d871fe90dcf74abdfe64e0f73f29
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-masking (1.
|
4
|
+
fluent-plugin-masking (1.1.2)
|
5
5
|
fluentd (>= 0.14.0)
|
6
6
|
|
7
7
|
GEM
|
@@ -10,7 +10,7 @@ GEM
|
|
10
10
|
concurrent-ruby (1.1.5)
|
11
11
|
cool.io (1.5.4)
|
12
12
|
dig_rb (1.0.1)
|
13
|
-
fluentd (1.
|
13
|
+
fluentd (1.8.0)
|
14
14
|
cool.io (>= 1.4.5, < 2.0.0)
|
15
15
|
dig_rb (~> 1.0.0)
|
16
16
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
@@ -18,7 +18,7 @@ GEM
|
|
18
18
|
serverengine (>= 2.0.4, < 3.0.0)
|
19
19
|
sigdump (~> 0.2.2)
|
20
20
|
strptime (>= 0.2.2, < 1.0.0)
|
21
|
-
tzinfo (
|
21
|
+
tzinfo (>= 1.0, < 3.0)
|
22
22
|
tzinfo-data (~> 1.0)
|
23
23
|
yajl-ruby (~> 1.0)
|
24
24
|
http_parser.rb (0.6.0)
|
@@ -26,7 +26,7 @@ GEM
|
|
26
26
|
power_assert (1.1.5)
|
27
27
|
rake (12.3.3)
|
28
28
|
rr (1.2.1)
|
29
|
-
serverengine (2.
|
29
|
+
serverengine (2.2.0)
|
30
30
|
sigdump (~> 0.2.2)
|
31
31
|
sigdump (0.2.4)
|
32
32
|
strptime (0.2.3)
|
@@ -45,11 +45,11 @@ PLATFORMS
|
|
45
45
|
ruby
|
46
46
|
|
47
47
|
DEPENDENCIES
|
48
|
-
bundler
|
48
|
+
bundler (= 1.17.3)
|
49
49
|
fluent-plugin-masking!
|
50
50
|
rake (~> 12.0)
|
51
51
|
test-unit (>= 3.1.0)
|
52
52
|
test-unit-rr
|
53
53
|
|
54
54
|
BUNDLED WITH
|
55
|
-
|
55
|
+
1.17.3
|
data/README.md
CHANGED
@@ -19,11 +19,15 @@ Install with gem:
|
|
19
19
|
## Setup
|
20
20
|
In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
|
21
21
|
|
22
|
+
In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
|
23
|
+
The JSON fields that are excluded are comma separated.
|
24
|
+
This can be used for logs of registration services or audit log entries which do not need to be masked.
|
22
25
|
This is configured as shown below:
|
23
26
|
```
|
24
27
|
<filter "**">
|
25
28
|
@type masking
|
26
29
|
fieldsToMaskFilePath "/path/to/fields-to-mask-file"
|
30
|
+
fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
|
27
31
|
</filter>
|
28
32
|
```
|
29
33
|
|
@@ -52,6 +56,7 @@ phone
|
|
52
56
|
<filter "**">
|
53
57
|
@type masking
|
54
58
|
fieldsToMaskFilePath "/path/to/fields-to-mask-file"
|
59
|
+
fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
|
55
60
|
</filter>
|
56
61
|
|
57
62
|
<match "**">
|
@@ -83,3 +88,13 @@ This sample result is created from the above configuration file `fluent.conf`. A
|
|
83
88
|
```
|
84
89
|
2019-09-15 16:12:50.359191000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"*******\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"*******\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
|
85
90
|
```
|
91
|
+
|
92
|
+
A sample with exclude in use:
|
93
|
+
```
|
94
|
+
fluentd -c fluent.conf
|
95
|
+
echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\":\"the-dog\", \"password\":\"d0g43u39\"}", "excludeMaskFields"=>"first_name,last_name"}' > /tmp/test.log
|
96
|
+
```
|
97
|
+
|
98
|
+
```
|
99
|
+
2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
|
100
|
+
```
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.required_ruby_version = '>= 2.1'
|
22
22
|
|
23
23
|
spec.add_runtime_dependency "fluentd", ">= 0.14.0"
|
24
|
-
spec.add_development_dependency "bundler"
|
24
|
+
spec.add_development_dependency "bundler", "1.17.3"
|
25
25
|
spec.add_development_dependency "rake", "~> 12.0"
|
26
26
|
spec.add_development_dependency "test-unit", ">= 3.1.0"
|
27
27
|
spec.add_development_dependency "test-unit-rr"
|
@@ -1,8 +1,10 @@
|
|
1
1
|
require 'fluent/filter'
|
2
|
+
require_relative './helpers.rb'
|
2
3
|
|
3
4
|
module Fluent
|
4
5
|
module Plugin
|
5
6
|
class MaskingFilter < Filter
|
7
|
+
include Helpers
|
6
8
|
Fluent::Plugin.register_filter("masking", self) # for "@type masking" in configuration
|
7
9
|
|
8
10
|
MASK_STRING = "*******"
|
@@ -15,11 +17,23 @@ module Fluent
|
|
15
17
|
# error safe method - if any error occurs the original record is returned
|
16
18
|
def maskRecord(record)
|
17
19
|
maskedRecord = record
|
18
|
-
|
20
|
+
excludedFields = []
|
21
|
+
begin
|
22
|
+
@fieldsToExcludeJSONPathsArray.each do | field |
|
23
|
+
field_value = myDig(record, field)
|
24
|
+
if field_value != nil
|
25
|
+
excludedFields = excludedFields + field_value.split(',')
|
26
|
+
end
|
27
|
+
end
|
28
|
+
rescue Exception => e
|
29
|
+
$log.error "Failed to find mask exclude record: #{e}"
|
30
|
+
end
|
19
31
|
begin
|
20
32
|
recordStr = record.to_s
|
21
33
|
@fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
|
22
|
-
|
34
|
+
if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
|
35
|
+
recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement)
|
36
|
+
end
|
23
37
|
end
|
24
38
|
|
25
39
|
maskedRecord = strToHash(recordStr)
|
@@ -35,12 +49,27 @@ module Fluent
|
|
35
49
|
super
|
36
50
|
@fields_to_mask = []
|
37
51
|
@fields_to_mask_regex = {}
|
52
|
+
@fields_to_mask_keys = {}
|
53
|
+
@fieldsToExcludeJSONPathsArray = []
|
38
54
|
end
|
39
55
|
|
40
56
|
# this method only called ones (on startup time)
|
41
57
|
def configure(conf)
|
42
58
|
super
|
43
59
|
fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
|
60
|
+
fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
|
61
|
+
|
62
|
+
if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0
|
63
|
+
fieldsToExcludeJSONPaths.split(",").each do | field |
|
64
|
+
# To save splits we'll save the path as an array
|
65
|
+
splitArray = field.split(".")
|
66
|
+
symArray = []
|
67
|
+
splitArray.each do | pathPortion |
|
68
|
+
symArray.push(pathPortion.to_sym)
|
69
|
+
end
|
70
|
+
@fieldsToExcludeJSONPathsArray.push(symArray)
|
71
|
+
end
|
72
|
+
end
|
44
73
|
|
45
74
|
File.open(fieldsToMaskFilePath, "r") do |f|
|
46
75
|
f.each_line do |line|
|
@@ -54,10 +83,12 @@ module Fluent
|
|
54
83
|
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
|
55
84
|
hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
|
56
85
|
@fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
|
86
|
+
@fields_to_mask_keys[hashObjectRegex] = value
|
57
87
|
|
58
88
|
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
|
59
89
|
innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
|
60
90
|
@fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
|
91
|
+
@fields_to_mask_keys[innerJSONStringRegex] = value
|
61
92
|
end
|
62
93
|
end
|
63
94
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Helpers
|
2
|
+
def myDig(input, path)
|
3
|
+
curr = input
|
4
|
+
for segment in path do
|
5
|
+
if curr != nil && curr.is_a?(Hash)
|
6
|
+
if curr[segment] == nil # segment is not a symbol
|
7
|
+
curr = curr[segment.to_s] # segment as string
|
8
|
+
else
|
9
|
+
curr = curr[segment] # segment as symbol
|
10
|
+
end
|
11
|
+
else
|
12
|
+
return nil
|
13
|
+
end
|
14
|
+
end
|
15
|
+
curr
|
16
|
+
end
|
17
|
+
end
|
data/test/test_filter_masking.rb
CHANGED
@@ -17,6 +17,12 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
17
17
|
# default configuration for tests
|
18
18
|
CONFIG = %[
|
19
19
|
fieldsToMaskFilePath test/fields-to-mask
|
20
|
+
fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
|
21
|
+
]
|
22
|
+
|
23
|
+
# configuration for tests without exclude parameter
|
24
|
+
CONFIG_NO_EXCLUDE = %[
|
25
|
+
fieldsToMaskFilePath test/fields-to-mask
|
20
26
|
]
|
21
27
|
|
22
28
|
def create_driver(conf = CONFIG)
|
@@ -35,7 +41,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
35
41
|
|
36
42
|
sub_test_case 'plugin will mask all fields that need masking' do
|
37
43
|
test 'mask field in hash object' do
|
38
|
-
conf =
|
44
|
+
conf = CONFIG_NO_EXCLUDE
|
39
45
|
messages = [
|
40
46
|
{:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com"}
|
41
47
|
]
|
@@ -93,5 +99,49 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
93
99
|
filtered_records = filter(conf, messages)
|
94
100
|
assert_equal(expected, filtered_records)
|
95
101
|
end
|
102
|
+
test 'mask field in hash object with exclude' do
|
103
|
+
conf = CONFIG
|
104
|
+
messages = [
|
105
|
+
{:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name"}
|
106
|
+
]
|
107
|
+
expected = [
|
108
|
+
{:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name"}
|
109
|
+
]
|
110
|
+
filtered_records = filter(conf, messages)
|
111
|
+
assert_equal(expected, filtered_records)
|
112
|
+
end
|
113
|
+
test 'mask field in hash object with nested exclude' do
|
114
|
+
conf = CONFIG
|
115
|
+
messages = [
|
116
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
|
117
|
+
]
|
118
|
+
expected = [
|
119
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
|
120
|
+
]
|
121
|
+
filtered_records = filter(conf, messages)
|
122
|
+
assert_equal(expected, filtered_records)
|
123
|
+
end
|
124
|
+
test 'mask field in hash object with base and nested exclude' do
|
125
|
+
conf = CONFIG
|
126
|
+
messages = [
|
127
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
|
128
|
+
]
|
129
|
+
expected = [
|
130
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
|
131
|
+
]
|
132
|
+
filtered_records = filter(conf, messages)
|
133
|
+
assert_equal(expected, filtered_records)
|
134
|
+
end
|
135
|
+
test 'mask field in json string with exclude' do
|
136
|
+
conf = CONFIG
|
137
|
+
messages = [
|
138
|
+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
|
139
|
+
]
|
140
|
+
expected = [
|
141
|
+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"*******\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
|
142
|
+
]
|
143
|
+
filtered_records = filter(conf, messages)
|
144
|
+
assert_equal(expected, filtered_records)
|
145
|
+
end
|
96
146
|
end
|
97
147
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require "test/unit"
|
2
|
+
require "./lib/fluent/plugin/helpers.rb"
|
3
|
+
|
4
|
+
class HelpersTest < Test::Unit::TestCase
|
5
|
+
m = Class.new do
|
6
|
+
include Helpers
|
7
|
+
end.new
|
8
|
+
sub_test_case "myDig function" do
|
9
|
+
test "Call function with nil" do
|
10
|
+
t = m.myDig(nil ,[:a])
|
11
|
+
assert_equal(t, nil)
|
12
|
+
end
|
13
|
+
test "Not found" do
|
14
|
+
t = m.myDig({:b => "t"},[:a])
|
15
|
+
assert_equal(t, nil)
|
16
|
+
end
|
17
|
+
test "Found symbol" do
|
18
|
+
t = m.myDig({:a => "t"},[:a])
|
19
|
+
assert_equal(t, "t")
|
20
|
+
end
|
21
|
+
test "Found string when given symbol" do
|
22
|
+
t = m.myDig({"a" => "t"},[:a])
|
23
|
+
assert_equal(t, "t")
|
24
|
+
end
|
25
|
+
test "Found symbol nested" do
|
26
|
+
t = m.myDig({:a => {:b => "t"}},[:a, :b])
|
27
|
+
assert_equal(t, "t")
|
28
|
+
end
|
29
|
+
test "Found string when given symbol nested" do
|
30
|
+
t = m.myDig({"a" => {"b" => "t"}},[:a, :b])
|
31
|
+
assert_equal(t, "t")
|
32
|
+
end
|
33
|
+
test "Found hybrid string/symbol when given symbol nested" do
|
34
|
+
t = m.myDig({"a" => {:b => "t"}},[:a, :b])
|
35
|
+
assert_equal(t, "t")
|
36
|
+
end
|
37
|
+
test "Does not dig in string" do
|
38
|
+
t = m.myDig({"a" => {:b => "t"}},[:a, :b, :c])
|
39
|
+
assert_equal(t, nil)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-masking
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shai Moria
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-12-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -29,16 +29,16 @@ dependencies:
|
|
29
29
|
name: bundler
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
|
-
- -
|
32
|
+
- - '='
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version:
|
34
|
+
version: 1.17.3
|
35
35
|
type: :development
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
|
-
- -
|
39
|
+
- - '='
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version:
|
41
|
+
version: 1.17.3
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: rake
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
@@ -97,9 +97,11 @@ files:
|
|
97
97
|
- Rakefile
|
98
98
|
- fluent-plugin-masking.gemspec
|
99
99
|
- lib/fluent/plugin/filter_masking.rb
|
100
|
+
- lib/fluent/plugin/helpers.rb
|
100
101
|
- lib/fluent/plugin/version.rb
|
101
102
|
- test/fields-to-mask
|
102
103
|
- test/test_filter_masking.rb
|
104
|
+
- test/test_helpers.rb
|
103
105
|
homepage: https://github.com/PayU/fluent-plugin-masking
|
104
106
|
licenses:
|
105
107
|
- Apache-2.0
|
@@ -119,7 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
121
|
- !ruby/object:Gem::Version
|
120
122
|
version: '0'
|
121
123
|
requirements: []
|
122
|
-
|
124
|
+
rubyforge_project:
|
125
|
+
rubygems_version: 2.5.2.3
|
123
126
|
signing_key:
|
124
127
|
specification_version: 4
|
125
128
|
summary: Fluentd filter plugin to mask sensitive or privacy records with `*******`
|
@@ -128,3 +131,4 @@ summary: Fluentd filter plugin to mask sensitive or privacy records with `******
|
|
128
131
|
test_files:
|
129
132
|
- test/fields-to-mask
|
130
133
|
- test/test_filter_masking.rb
|
134
|
+
- test/test_helpers.rb
|