fluent-plugin-masking 1.0.7 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +29 -0
- data/.gitignore +3 -1
- data/README.md +28 -4
- data/fluent-plugin-masking.gemspec +3 -4
- data/lib/fluent/plugin/filter_masking.rb +44 -5
- data/lib/fluent/plugin/helpers.rb +17 -0
- data/lib/fluent/plugin/version.rb +2 -2
- data/test/fields-to-mask +2 -1
- data/test/fields-to-mask-insensitive +5 -0
- data/test/test_filter_masking.rb +144 -4
- data/test/test_helpers.rb +42 -0
- metadata +16 -39
- data/.gitlab-ci.yml +0 -73
- data/Gemfile.lock +0 -55
- data/Rakefile +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3c1d03c89c4b186938fab7c7f41e0217c443dc93c0b4d74fd59a0afe34de7493
|
4
|
+
data.tar.gz: de2fb0a278fdeb128773f96ad1be8ea0e895b611febff6b0fd57ae89668a0ff6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0712e0f59cd1e3d89282772285cd7632a242a1cba129e3c8a42836b0381079698faa536ac24be69768751f91c81155a0d4ff65824d914754e4bf8b175b6aa5eb
|
7
|
+
data.tar.gz: 7c43897e4a4a9a5c5c9cbdf10398efcfe333bc770d0e2c46f6d086ab16cc3c514e934c59210ab213468608c3ec59da6c389bb744b2d27be7409d3ca5df980e0f
|
@@ -0,0 +1,29 @@
|
|
1
|
+
version: 2.1
|
2
|
+
orbs:
|
3
|
+
ruby: circleci/ruby@0.1.2
|
4
|
+
|
5
|
+
executors:
|
6
|
+
v2-5-0:
|
7
|
+
docker:
|
8
|
+
- image: circleci/ruby:2.5.0
|
9
|
+
|
10
|
+
jobs:
|
11
|
+
tests:
|
12
|
+
parameters:
|
13
|
+
ruby-version:
|
14
|
+
type: executor
|
15
|
+
executor: << parameters.ruby-version >>
|
16
|
+
steps:
|
17
|
+
- checkout
|
18
|
+
- run: gem install bundler
|
19
|
+
- run: bundle install
|
20
|
+
- run: ruby -r ./test/*.rb
|
21
|
+
|
22
|
+
workflows:
|
23
|
+
tests:
|
24
|
+
jobs:
|
25
|
+
- tests:
|
26
|
+
matrix:
|
27
|
+
parameters:
|
28
|
+
ruby-version: [v2-5-0]
|
29
|
+
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
# fluent-plugin-masking
|
2
2
|
|
3
|
+
[![Known Vulnerabilities](https://snyk.io//test/github/PayU/fluent-plugin-masking/badge.svg?targetFile=Gemfile.lock)](https://snyk.io//test/github/PayU/fluent-plugin-masking?targetFile=Gemfile.lock) [![Build Status](https://travis-ci.com/PayU/fluent-plugin-masking.svg?branch=master)](https://travis-ci.com/PayU/fluent-plugin-masking)
|
4
|
+
|
3
5
|
## Overview
|
4
6
|
Fluentd filter plugin to mask sensitive or privacy records with `*******` in place of the original value. This data masking plugin protects data such as name, email, phonenumber, address, and any other field you would like to mask.
|
5
7
|
|
6
8
|
## Requirements
|
7
9
|
| fluent-plugin-masking | fluentd | ruby |
|
8
10
|
| --------------------- | ---------- | ------ |
|
9
|
-
| 1.
|
11
|
+
| 1.2.x | >= v0.14.0 | >= 2.5 |
|
10
12
|
|
11
13
|
|
12
14
|
## Installation
|
@@ -17,11 +19,15 @@ Install with gem:
|
|
17
19
|
## Setup
|
18
20
|
In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
|
19
21
|
|
22
|
+
In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
|
23
|
+
The JSON fields that are excluded are comma separated.
|
24
|
+
This can be used for logs of registration services or audit log entries which do not need to be masked.
|
20
25
|
This is configured as shown below:
|
21
26
|
```
|
22
27
|
<filter "**">
|
23
28
|
@type masking
|
24
29
|
fieldsToMaskFilePath "/path/to/fields-to-mask-file"
|
30
|
+
fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
|
25
31
|
</filter>
|
26
32
|
```
|
27
33
|
|
@@ -29,10 +35,9 @@ Example fields-to-mask-file:
|
|
29
35
|
```
|
30
36
|
name
|
31
37
|
email
|
32
|
-
phone
|
38
|
+
phone/i # the '/i' suffix will make sure phone field will be case insensitive
|
33
39
|
```
|
34
40
|
|
35
|
-
|
36
41
|
## Quick Guide
|
37
42
|
|
38
43
|
### Configuration:
|
@@ -50,6 +55,7 @@ phone
|
|
50
55
|
<filter "**">
|
51
56
|
@type masking
|
52
57
|
fieldsToMaskFilePath "/path/to/fields-to-mask-file"
|
58
|
+
fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
|
53
59
|
</filter>
|
54
60
|
|
55
61
|
<match "**">
|
@@ -80,4 +86,22 @@ This sample result is created from the above configuration file `fluent.conf`. A
|
|
80
86
|
|
81
87
|
```
|
82
88
|
2019-09-15 16:12:50.359191000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"*******\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"*******\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
|
83
|
-
```
|
89
|
+
```
|
90
|
+
|
91
|
+
A sample with exclude in use:
|
92
|
+
```
|
93
|
+
fluentd -c fluent.conf
|
94
|
+
echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\":\"the-dog\", \"password\":\"d0g43u39\"}", "excludeMaskFields"=>"first_name,last_name"}' > /tmp/test.log
|
95
|
+
```
|
96
|
+
|
97
|
+
```
|
98
|
+
2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
|
99
|
+
```
|
100
|
+
|
101
|
+
|
102
|
+
### Run Unit Tests
|
103
|
+
```
|
104
|
+
gem install bundler
|
105
|
+
bundle install
|
106
|
+
ruby -r ./test/*.rb
|
107
|
+
```
|
@@ -11,17 +11,16 @@ Gem::Specification.new do |spec|
|
|
11
11
|
spec.email = ["shai.moria@zooz.com", "niv.lipetz@zooz.com"]
|
12
12
|
spec.description = "Fluentd filter plugin to mask sensitive or privacy records in event messages"
|
13
13
|
spec.summary = "Fluentd filter plugin to mask sensitive or privacy records with `*******` in place of the original value. This data masking plugin protects data such as name, email, phonenumber, address, and any other field you would like to mask."
|
14
|
-
spec.homepage = "https://github.com/
|
14
|
+
spec.homepage = "https://github.com/PayU/fluent-plugin-masking"
|
15
15
|
|
16
16
|
spec.files = `git ls-files`.split($\)
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
17
18
|
spec.require_paths = ["lib"]
|
18
19
|
spec.license = "Apache-2.0"
|
19
20
|
|
20
|
-
spec.required_ruby_version = '>= 2.
|
21
|
+
spec.required_ruby_version = '>= 2.5.0'
|
21
22
|
|
22
23
|
spec.add_runtime_dependency "fluentd", ">= 0.14.0"
|
23
|
-
spec.add_development_dependency "bundler"
|
24
|
-
spec.add_development_dependency "rake", "~> 12.0"
|
25
24
|
spec.add_development_dependency "test-unit", ">= 3.1.0"
|
26
25
|
spec.add_development_dependency "test-unit-rr"
|
27
26
|
end
|
@@ -1,8 +1,10 @@
|
|
1
1
|
require 'fluent/filter'
|
2
|
+
require_relative './helpers.rb'
|
2
3
|
|
3
4
|
module Fluent
|
4
5
|
module Plugin
|
5
6
|
class MaskingFilter < Filter
|
7
|
+
include Helpers
|
6
8
|
Fluent::Plugin.register_filter("masking", self) # for "@type masking" in configuration
|
7
9
|
|
8
10
|
MASK_STRING = "*******"
|
@@ -15,11 +17,23 @@ module Fluent
|
|
15
17
|
# error safe method - if any error occurs the original record is returned
|
16
18
|
def maskRecord(record)
|
17
19
|
maskedRecord = record
|
18
|
-
|
20
|
+
excludedFields = []
|
21
|
+
begin
|
22
|
+
@fieldsToExcludeJSONPathsArray.each do | field |
|
23
|
+
field_value = myDig(record, field)
|
24
|
+
if field_value != nil
|
25
|
+
excludedFields = excludedFields + field_value.split(',')
|
26
|
+
end
|
27
|
+
end
|
28
|
+
rescue Exception => e
|
29
|
+
$log.error "Failed to find mask exclude record: #{e}"
|
30
|
+
end
|
19
31
|
begin
|
20
32
|
recordStr = record.to_s
|
21
33
|
@fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
|
22
|
-
|
34
|
+
if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
|
35
|
+
recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement)
|
36
|
+
end
|
23
37
|
end
|
24
38
|
|
25
39
|
maskedRecord = strToHash(recordStr)
|
@@ -35,29 +49,54 @@ module Fluent
|
|
35
49
|
super
|
36
50
|
@fields_to_mask = []
|
37
51
|
@fields_to_mask_regex = {}
|
52
|
+
@fields_to_mask_keys = {}
|
53
|
+
@fieldsToExcludeJSONPathsArray = []
|
38
54
|
end
|
39
55
|
|
40
56
|
# this method only called ones (on startup time)
|
41
57
|
def configure(conf)
|
42
58
|
super
|
43
59
|
fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
|
60
|
+
fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
|
61
|
+
|
62
|
+
if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0
|
63
|
+
fieldsToExcludeJSONPaths.split(",").each do | field |
|
64
|
+
# To save splits we'll save the path as an array
|
65
|
+
splitArray = field.split(".")
|
66
|
+
symArray = []
|
67
|
+
splitArray.each do | pathPortion |
|
68
|
+
symArray.push(pathPortion.to_sym)
|
69
|
+
end
|
70
|
+
@fieldsToExcludeJSONPathsArray.push(symArray)
|
71
|
+
end
|
72
|
+
end
|
44
73
|
|
45
74
|
File.open(fieldsToMaskFilePath, "r") do |f|
|
46
75
|
f.each_line do |line|
|
47
|
-
|
48
76
|
value = line.to_s # make sure it's string
|
49
77
|
value = value.gsub(/\s+/, "") # remove spaces
|
50
78
|
value = value.gsub('\n', '') # remove line breakers
|
51
79
|
|
80
|
+
if value.end_with? "/i"
|
81
|
+
# case insensitive
|
82
|
+
value = value.delete_suffix('/i')
|
83
|
+
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
|
84
|
+
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
|
85
|
+
else
|
86
|
+
# case sensitive
|
87
|
+
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
|
88
|
+
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
|
89
|
+
end
|
90
|
+
|
52
91
|
@fields_to_mask.push(value)
|
53
92
|
|
54
|
-
hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
|
55
93
|
hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
|
56
94
|
@fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
|
95
|
+
@fields_to_mask_keys[hashObjectRegex] = value
|
57
96
|
|
58
|
-
innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
|
59
97
|
innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
|
60
98
|
@fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
|
99
|
+
@fields_to_mask_keys[innerJSONStringRegex] = value
|
61
100
|
end
|
62
101
|
end
|
63
102
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Helpers
|
2
|
+
def myDig(input, path)
|
3
|
+
curr = input
|
4
|
+
for segment in path do
|
5
|
+
if curr != nil && curr.is_a?(Hash)
|
6
|
+
if curr[segment] == nil # segment is not a symbol
|
7
|
+
curr = curr[segment.to_s] # segment as string
|
8
|
+
else
|
9
|
+
curr = curr[segment] # segment as symbol
|
10
|
+
end
|
11
|
+
else
|
12
|
+
return nil
|
13
|
+
end
|
14
|
+
end
|
15
|
+
curr
|
16
|
+
end
|
17
|
+
end
|
@@ -1,3 +1,3 @@
|
|
1
1
|
module FilterMasking
|
2
|
-
VERSION = "1.0
|
3
|
-
end
|
2
|
+
VERSION = "1.2.0"
|
3
|
+
end
|
data/test/fields-to-mask
CHANGED
data/test/test_filter_masking.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# test/plugin/test_filter_your_own.rb
|
2
|
-
|
3
1
|
require "test-unit"
|
4
2
|
require "fluent/test"
|
5
3
|
require "fluent/test/driver/filter"
|
@@ -17,6 +15,17 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
17
15
|
# default configuration for tests
|
18
16
|
CONFIG = %[
|
19
17
|
fieldsToMaskFilePath test/fields-to-mask
|
18
|
+
fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
|
19
|
+
]
|
20
|
+
|
21
|
+
# configuration for tests without exclude parameter
|
22
|
+
CONFIG_NO_EXCLUDE = %[
|
23
|
+
fieldsToMaskFilePath test/fields-to-mask
|
24
|
+
]
|
25
|
+
|
26
|
+
# configuration for tests with case insensitive fields
|
27
|
+
CONFIG_CASE_INSENSITIVE = %[
|
28
|
+
fieldsToMaskFilePath test/fields-to-mask-insensitive
|
20
29
|
]
|
21
30
|
|
22
31
|
def create_driver(conf = CONFIG)
|
@@ -33,9 +42,9 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
33
42
|
d.filtered_records
|
34
43
|
end
|
35
44
|
|
36
|
-
sub_test_case 'plugin will mask all fields that need masking' do
|
45
|
+
sub_test_case 'plugin will mask all fields that need masking - case sensitive fields' do
|
37
46
|
test 'mask field in hash object' do
|
38
|
-
conf =
|
47
|
+
conf = CONFIG_NO_EXCLUDE
|
39
48
|
messages = [
|
40
49
|
{:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com"}
|
41
50
|
]
|
@@ -93,5 +102,136 @@ class YourOwnFilterTest < Test::Unit::TestCase
|
|
93
102
|
filtered_records = filter(conf, messages)
|
94
103
|
assert_equal(expected, filtered_records)
|
95
104
|
end
|
105
|
+
test 'mask field in hash object with exclude' do
|
106
|
+
conf = CONFIG
|
107
|
+
messages = [
|
108
|
+
{:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name"}
|
109
|
+
]
|
110
|
+
expected = [
|
111
|
+
{:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name"}
|
112
|
+
]
|
113
|
+
filtered_records = filter(conf, messages)
|
114
|
+
assert_equal(expected, filtered_records)
|
115
|
+
end
|
116
|
+
test 'mask field in hash object with nested exclude' do
|
117
|
+
conf = CONFIG
|
118
|
+
messages = [
|
119
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
|
120
|
+
]
|
121
|
+
expected = [
|
122
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
|
123
|
+
]
|
124
|
+
filtered_records = filter(conf, messages)
|
125
|
+
assert_equal(expected, filtered_records)
|
126
|
+
end
|
127
|
+
|
128
|
+
test 'mask field in hash object with base and nested exclude' do
|
129
|
+
conf = CONFIG
|
130
|
+
messages = [
|
131
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
|
132
|
+
]
|
133
|
+
expected = [
|
134
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
|
135
|
+
]
|
136
|
+
filtered_records = filter(conf, messages)
|
137
|
+
assert_equal(expected, filtered_records)
|
138
|
+
end
|
139
|
+
|
140
|
+
test 'mask field in json string with exclude' do
|
141
|
+
conf = CONFIG
|
142
|
+
messages = [
|
143
|
+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
|
144
|
+
]
|
145
|
+
expected = [
|
146
|
+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"*******\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
|
147
|
+
]
|
148
|
+
filtered_records = filter(conf, messages)
|
149
|
+
assert_equal(expected, filtered_records)
|
150
|
+
end
|
151
|
+
|
152
|
+
test 'mask field which is inner json string field (should mask the whole object)' do
|
153
|
+
conf = CONFIG
|
154
|
+
messages = [
|
155
|
+
{
|
156
|
+
:body => {
|
157
|
+
:action_name => "some_action",
|
158
|
+
:action_type => "some type",
|
159
|
+
:request => {
|
160
|
+
:body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
|
161
|
+
}
|
162
|
+
},
|
163
|
+
:timestamp => "2020-06-08T16:00:57.341Z"
|
164
|
+
}
|
165
|
+
]
|
166
|
+
|
167
|
+
expected = [
|
168
|
+
{
|
169
|
+
:body => {
|
170
|
+
:action_name => "some_action",
|
171
|
+
:action_type => "some type",
|
172
|
+
:request => {
|
173
|
+
:body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
|
174
|
+
}
|
175
|
+
},
|
176
|
+
:timestamp => "2020-06-08T16:00:57.341Z"
|
177
|
+
}
|
178
|
+
]
|
179
|
+
|
180
|
+
filtered_records = filter(conf, messages)
|
181
|
+
assert_equal(expected, filtered_records)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
|
186
|
+
|
187
|
+
test 'mask field in hash object with camel case' do
|
188
|
+
conf = CONFIG_CASE_INSENSITIVE
|
189
|
+
messages = [
|
190
|
+
{:not_masked_field=>"mickey-the-dog", :Email=>"mickey-the-dog@zooz.com"}
|
191
|
+
]
|
192
|
+
expected = [
|
193
|
+
{:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING}
|
194
|
+
]
|
195
|
+
filtered_records = filter(conf, messages)
|
196
|
+
assert_equal(expected, filtered_records)
|
197
|
+
end
|
198
|
+
|
199
|
+
test 'not mask field in hash object since case not match' do
|
200
|
+
conf = CONFIG_CASE_INSENSITIVE
|
201
|
+
messages = [
|
202
|
+
{:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
|
203
|
+
]
|
204
|
+
expected = [
|
205
|
+
{:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
|
206
|
+
]
|
207
|
+
filtered_records = filter(conf, messages)
|
208
|
+
assert_equal(expected, filtered_records)
|
209
|
+
end
|
210
|
+
|
211
|
+
test 'mask field in hash object with snakecase' do
|
212
|
+
conf = CONFIG_CASE_INSENSITIVE
|
213
|
+
messages = [
|
214
|
+
{:not_masked_field=>"mickey-the-dog", :LaSt_NaMe=>"mickey-the-dog@zooz.com"}
|
215
|
+
]
|
216
|
+
expected = [
|
217
|
+
{:not_masked_field=>"mickey-the-dog", :last_name=>MASK_STRING}
|
218
|
+
]
|
219
|
+
filtered_records = filter(conf, messages)
|
220
|
+
assert_equal(expected, filtered_records)
|
221
|
+
end
|
222
|
+
|
223
|
+
test 'mask case insensitive and case sensitive field in nested json escaped string' do
|
224
|
+
conf = CONFIG_CASE_INSENSITIVE
|
225
|
+
messages = [
|
226
|
+
{ :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" }
|
227
|
+
]
|
228
|
+
expected = [
|
229
|
+
{ :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
|
230
|
+
]
|
231
|
+
filtered_records = filter(conf, messages)
|
232
|
+
assert_equal(expected, filtered_records)
|
233
|
+
end
|
234
|
+
|
96
235
|
end
|
236
|
+
|
97
237
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require "test/unit"
|
2
|
+
require "./lib/fluent/plugin/helpers.rb"
|
3
|
+
|
4
|
+
class HelpersTest < Test::Unit::TestCase
|
5
|
+
m = Class.new do
|
6
|
+
include Helpers
|
7
|
+
end.new
|
8
|
+
sub_test_case "myDig function" do
|
9
|
+
test "Call function with nil" do
|
10
|
+
t = m.myDig(nil ,[:a])
|
11
|
+
assert_equal(t, nil)
|
12
|
+
end
|
13
|
+
test "Not found" do
|
14
|
+
t = m.myDig({:b => "t"},[:a])
|
15
|
+
assert_equal(t, nil)
|
16
|
+
end
|
17
|
+
test "Found symbol" do
|
18
|
+
t = m.myDig({:a => "t"},[:a])
|
19
|
+
assert_equal(t, "t")
|
20
|
+
end
|
21
|
+
test "Found string when given symbol" do
|
22
|
+
t = m.myDig({"a" => "t"},[:a])
|
23
|
+
assert_equal(t, "t")
|
24
|
+
end
|
25
|
+
test "Found symbol nested" do
|
26
|
+
t = m.myDig({:a => {:b => "t"}},[:a, :b])
|
27
|
+
assert_equal(t, "t")
|
28
|
+
end
|
29
|
+
test "Found string when given symbol nested" do
|
30
|
+
t = m.myDig({"a" => {"b" => "t"}},[:a, :b])
|
31
|
+
assert_equal(t, "t")
|
32
|
+
end
|
33
|
+
test "Found hybrid string/symbol when given symbol nested" do
|
34
|
+
t = m.myDig({"a" => {:b => "t"}},[:a, :b])
|
35
|
+
assert_equal(t, "t")
|
36
|
+
end
|
37
|
+
test "Does not dig in string" do
|
38
|
+
t = m.myDig({"a" => {:b => "t"}},[:a, :b, :c])
|
39
|
+
assert_equal(t, nil)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-masking
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shai Moria
|
8
8
|
- Niv Lipetz
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2021-03-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -25,34 +25,6 @@ dependencies:
|
|
25
25
|
- - ">="
|
26
26
|
- !ruby/object:Gem::Version
|
27
27
|
version: 0.14.0
|
28
|
-
- !ruby/object:Gem::Dependency
|
29
|
-
name: bundler
|
30
|
-
requirement: !ruby/object:Gem::Requirement
|
31
|
-
requirements:
|
32
|
-
- - ">="
|
33
|
-
- !ruby/object:Gem::Version
|
34
|
-
version: '0'
|
35
|
-
type: :development
|
36
|
-
prerelease: false
|
37
|
-
version_requirements: !ruby/object:Gem::Requirement
|
38
|
-
requirements:
|
39
|
-
- - ">="
|
40
|
-
- !ruby/object:Gem::Version
|
41
|
-
version: '0'
|
42
|
-
- !ruby/object:Gem::Dependency
|
43
|
-
name: rake
|
44
|
-
requirement: !ruby/object:Gem::Requirement
|
45
|
-
requirements:
|
46
|
-
- - "~>"
|
47
|
-
- !ruby/object:Gem::Version
|
48
|
-
version: '12.0'
|
49
|
-
type: :development
|
50
|
-
prerelease: false
|
51
|
-
version_requirements: !ruby/object:Gem::Requirement
|
52
|
-
requirements:
|
53
|
-
- - "~>"
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
-
version: '12.0'
|
56
28
|
- !ruby/object:Gem::Dependency
|
57
29
|
name: test-unit
|
58
30
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,22 +61,23 @@ executables: []
|
|
89
61
|
extensions: []
|
90
62
|
extra_rdoc_files: []
|
91
63
|
files:
|
64
|
+
- ".circleci/config.yml"
|
92
65
|
- ".gitignore"
|
93
|
-
- ".gitlab-ci.yml"
|
94
66
|
- Gemfile
|
95
|
-
- Gemfile.lock
|
96
67
|
- README.md
|
97
|
-
- Rakefile
|
98
68
|
- fluent-plugin-masking.gemspec
|
99
69
|
- lib/fluent/plugin/filter_masking.rb
|
70
|
+
- lib/fluent/plugin/helpers.rb
|
100
71
|
- lib/fluent/plugin/version.rb
|
101
72
|
- test/fields-to-mask
|
73
|
+
- test/fields-to-mask-insensitive
|
102
74
|
- test/test_filter_masking.rb
|
103
|
-
|
75
|
+
- test/test_helpers.rb
|
76
|
+
homepage: https://github.com/PayU/fluent-plugin-masking
|
104
77
|
licenses:
|
105
78
|
- Apache-2.0
|
106
79
|
metadata: {}
|
107
|
-
post_install_message:
|
80
|
+
post_install_message:
|
108
81
|
rdoc_options: []
|
109
82
|
require_paths:
|
110
83
|
- lib
|
@@ -112,7 +85,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
112
85
|
requirements:
|
113
86
|
- - ">="
|
114
87
|
- !ruby/object:Gem::Version
|
115
|
-
version:
|
88
|
+
version: 2.5.0
|
116
89
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
117
90
|
requirements:
|
118
91
|
- - ">="
|
@@ -120,9 +93,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
120
93
|
version: '0'
|
121
94
|
requirements: []
|
122
95
|
rubygems_version: 3.0.3
|
123
|
-
signing_key:
|
96
|
+
signing_key:
|
124
97
|
specification_version: 4
|
125
98
|
summary: Fluentd filter plugin to mask sensitive or privacy records with `*******`
|
126
99
|
in place of the original value. This data masking plugin protects data such as name,
|
127
100
|
email, phonenumber, address, and any other field you would like to mask.
|
128
|
-
test_files:
|
101
|
+
test_files:
|
102
|
+
- test/fields-to-mask
|
103
|
+
- test/fields-to-mask-insensitive
|
104
|
+
- test/test_filter_masking.rb
|
105
|
+
- test/test_helpers.rb
|
data/.gitlab-ci.yml
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
image: ruby:2.6.3
|
2
|
-
|
3
|
-
# Cache gems in between builds
|
4
|
-
cache:
|
5
|
-
key: ${CI_COMMIT_REF_SLUG}
|
6
|
-
paths:
|
7
|
-
- vendor/ruby
|
8
|
-
|
9
|
-
before_script:
|
10
|
-
- gem install bundler:2.0.2
|
11
|
-
- bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby
|
12
|
-
|
13
|
-
stages:
|
14
|
-
- test
|
15
|
-
- release
|
16
|
-
|
17
|
-
variables:
|
18
|
-
DOCKER_TLS_CERTDIR: ""
|
19
|
-
|
20
|
-
unit-test:
|
21
|
-
tags:
|
22
|
-
- dcos-multi-runner
|
23
|
-
stage: test
|
24
|
-
script:
|
25
|
-
- bundle exec rake test
|
26
|
-
except:
|
27
|
-
- tags
|
28
|
-
|
29
|
-
.release: &release
|
30
|
-
tags:
|
31
|
-
- dcos-multi-runner
|
32
|
-
stage: release
|
33
|
-
before_script:
|
34
|
-
- echo "Setup ssh inside the runner.."
|
35
|
-
- git config --global user.email $GITLAB_USER_EMAIL
|
36
|
-
- git config --global user.name $GITLAB_USER_LOGIN
|
37
|
-
- 'which ssh-agent || ( apk --update add openssh-client )'
|
38
|
-
- eval $(ssh-agent -s)
|
39
|
-
- ssh-add <(echo "$SSH_PRIVATE_KEY")
|
40
|
-
- mkdir -p ~/.ssh
|
41
|
-
- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
|
42
|
-
- echo "Clone the source code.."
|
43
|
-
- GIT_URL=$(echo $CI_PROJECT_URL | cut -d'/' -f3)
|
44
|
-
- git clone git@$GIT_URL:$CI_PROJECT_PATH.git
|
45
|
-
- cd $CI_PROJECT_NAME
|
46
|
-
- git reset --hard $CI_COMMIT_SHA
|
47
|
-
script:
|
48
|
-
- mkdir -p ~/.gem
|
49
|
-
- 'echo -e "---\n:rubygems_api_key: $GEM_HOST_API_KEY" > ~/.gem/credentials && echo "created gem credentials"'
|
50
|
-
- chmod 0600 ~/.gem/credentials
|
51
|
-
- gem install gem-release
|
52
|
-
- gem bump fluent-plugin-masking --version $TYPE --file ./lib/fluent/plugin/version.rb --commit --push --tag
|
53
|
-
- gem build fluent-plugin-masking.gemspec
|
54
|
-
- export VERSION=$(cat ./lib/fluent/plugin/version.rb | grep VERSION | sed -ne 's/[^0-9]*\(\([0-9]\.\)\{0,4\}[0-9][^.]\).*/\1/p' | sed 's/"$//')
|
55
|
-
- gem push fluent-plugin-masking-$VERSION.gem
|
56
|
-
when: manual
|
57
|
-
only:
|
58
|
-
- master
|
59
|
-
|
60
|
-
release:patch:
|
61
|
-
extends: .release
|
62
|
-
variables:
|
63
|
-
TYPE: patch
|
64
|
-
|
65
|
-
release:minor:
|
66
|
-
extends: .release
|
67
|
-
variables:
|
68
|
-
TYPE: minor
|
69
|
-
|
70
|
-
release:major:
|
71
|
-
extends: .release
|
72
|
-
variables:
|
73
|
-
TYPE: major
|
data/Gemfile.lock
DELETED
@@ -1,55 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
fluent-plugin-masking (1.0.6)
|
5
|
-
fluentd (>= 0.14.0)
|
6
|
-
|
7
|
-
GEM
|
8
|
-
remote: https://rubygems.org/
|
9
|
-
specs:
|
10
|
-
cool.io (1.5.4)
|
11
|
-
dig_rb (1.0.1)
|
12
|
-
fluentd (1.6.3)
|
13
|
-
cool.io (>= 1.4.5, < 2.0.0)
|
14
|
-
dig_rb (~> 1.0.0)
|
15
|
-
http_parser.rb (>= 0.5.1, < 0.7.0)
|
16
|
-
msgpack (>= 0.7.0, < 2.0.0)
|
17
|
-
serverengine (>= 2.0.4, < 3.0.0)
|
18
|
-
sigdump (~> 0.2.2)
|
19
|
-
strptime (>= 0.2.2, < 1.0.0)
|
20
|
-
tzinfo (~> 1.0)
|
21
|
-
tzinfo-data (~> 1.0)
|
22
|
-
yajl-ruby (~> 1.0)
|
23
|
-
http_parser.rb (0.6.0)
|
24
|
-
msgpack (1.3.1)
|
25
|
-
power_assert (1.1.5)
|
26
|
-
rake (12.3.3)
|
27
|
-
rr (1.2.1)
|
28
|
-
serverengine (2.1.1)
|
29
|
-
sigdump (~> 0.2.2)
|
30
|
-
sigdump (0.2.4)
|
31
|
-
strptime (0.2.3)
|
32
|
-
test-unit (3.3.3)
|
33
|
-
power_assert
|
34
|
-
test-unit-rr (1.0.5)
|
35
|
-
rr (>= 1.1.1)
|
36
|
-
test-unit (>= 2.5.2)
|
37
|
-
thread_safe (0.3.6)
|
38
|
-
tzinfo (1.2.5)
|
39
|
-
thread_safe (~> 0.1)
|
40
|
-
tzinfo-data (1.2019.2)
|
41
|
-
tzinfo (>= 1.0.0)
|
42
|
-
yajl-ruby (1.4.1)
|
43
|
-
|
44
|
-
PLATFORMS
|
45
|
-
ruby
|
46
|
-
|
47
|
-
DEPENDENCIES
|
48
|
-
bundler
|
49
|
-
fluent-plugin-masking!
|
50
|
-
rake (~> 12.0)
|
51
|
-
test-unit (>= 3.1.0)
|
52
|
-
test-unit-rr
|
53
|
-
|
54
|
-
BUNDLED WITH
|
55
|
-
2.0.2
|