fluent-plugin-masking 1.1.2 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 5b8b769887786df667374682f71ddbc61f6b575f
4
- data.tar.gz: c143faea655e8ec4f621445d66525f827b26e9a0
2
+ SHA256:
3
+ metadata.gz: 3c1d03c89c4b186938fab7c7f41e0217c443dc93c0b4d74fd59a0afe34de7493
4
+ data.tar.gz: de2fb0a278fdeb128773f96ad1be8ea0e895b611febff6b0fd57ae89668a0ff6
5
5
  SHA512:
6
- metadata.gz: c964b4e42d7bbbe59fcaf570b1ac8281427ee7d15ea20cb8dcd9320343a306739571e25c2030e3f63cd2465daec6ec757ce7a57c68127c21e7acabba36b3243c
7
- data.tar.gz: d9937e8bf71d1431cf9312134618d2bd5c842e203ce9d853cc4e8cb2a08b8c60cfea4b21105c3296ca35dedd2dc3a1cb1c61d871fe90dcf74abdfe64e0f73f29
6
+ metadata.gz: 0712e0f59cd1e3d89282772285cd7632a242a1cba129e3c8a42836b0381079698faa536ac24be69768751f91c81155a0d4ff65824d914754e4bf8b175b6aa5eb
7
+ data.tar.gz: 7c43897e4a4a9a5c5c9cbdf10398efcfe333bc770d0e2c46f6d086ab16cc3c514e934c59210ab213468608c3ec59da6c389bb744b2d27be7409d3ca5df980e0f
@@ -0,0 +1,29 @@
1
+ version: 2.1
2
+ orbs:
3
+ ruby: circleci/ruby@0.1.2
4
+
5
+ executors:
6
+ v2-5-0:
7
+ docker:
8
+ - image: circleci/ruby:2.5.0
9
+
10
+ jobs:
11
+ tests:
12
+ parameters:
13
+ ruby-version:
14
+ type: executor
15
+ executor: << parameters.ruby-version >>
16
+ steps:
17
+ - checkout
18
+ - run: gem install bundler
19
+ - run: bundle install
20
+ - run: ruby -r ./test/*.rb
21
+
22
+ workflows:
23
+ tests:
24
+ jobs:
25
+ - tests:
26
+ matrix:
27
+ parameters:
28
+ ruby-version: [v2-5-0]
29
+
data/.gitignore CHANGED
@@ -1,4 +1,6 @@
1
1
  .bundle/
2
2
  vendor/
3
3
  *.gem
4
- .idea/
4
+ .idea/
5
+ Gemfile.lock
6
+ Rakefile
data/README.md CHANGED
@@ -8,7 +8,7 @@ Fluentd filter plugin to mask sensitive or privacy records with `*******` in pla
8
8
  ## Requirements
9
9
  | fluent-plugin-masking | fluentd | ruby |
10
10
  | --------------------- | ---------- | ------ |
11
- | 1.0.x | >= v0.14.0 | >= 2.1 |
11
+ | 1.2.x | >= v0.14.0 | >= 2.5 |
12
12
 
13
13
 
14
14
  ## Installation
@@ -35,10 +35,9 @@ Example fields-to-mask-file:
35
35
  ```
36
36
  name
37
37
  email
38
- phone
38
+ phone/i # the '/i' suffix will make sure phone field will be case insensitive
39
39
  ```
40
40
 
41
-
42
41
  ## Quick Guide
43
42
 
44
43
  ### Configuration:
@@ -98,3 +97,11 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
98
97
  ```
99
98
  2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
100
99
  ```
100
+
101
+
102
+ ### Run Unit Tests
103
+ ```
104
+ gem install bundler
105
+ bundle install
106
+ ruby -r ./test/*.rb
107
+ ```
@@ -18,11 +18,9 @@ Gem::Specification.new do |spec|
18
18
  spec.require_paths = ["lib"]
19
19
  spec.license = "Apache-2.0"
20
20
 
21
- spec.required_ruby_version = '>= 2.1'
21
+ spec.required_ruby_version = '>= 2.5.0'
22
22
 
23
23
  spec.add_runtime_dependency "fluentd", ">= 0.14.0"
24
- spec.add_development_dependency "bundler", "1.17.3"
25
- spec.add_development_dependency "rake", "~> 12.0"
26
24
  spec.add_development_dependency "test-unit", ">= 3.1.0"
27
25
  spec.add_development_dependency "test-unit-rr"
28
26
  end
@@ -73,19 +73,27 @@ module Fluent
73
73
 
74
74
  File.open(fieldsToMaskFilePath, "r") do |f|
75
75
  f.each_line do |line|
76
-
77
76
  value = line.to_s # make sure it's string
78
77
  value = value.gsub(/\s+/, "") # remove spaces
79
78
  value = value.gsub('\n', '') # remove line breakers
80
79
 
80
+ if value.end_with? "/i"
81
+ # case insensitive
82
+ value = value.delete_suffix('/i')
83
+ hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
84
+ innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
85
+ else
86
+ # case sensitive
87
+ hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
88
+ innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
89
+ end
90
+
81
91
  @fields_to_mask.push(value)
82
92
 
83
- hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
84
93
  hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
85
94
  @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
86
95
  @fields_to_mask_keys[hashObjectRegex] = value
87
96
 
88
- innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
89
97
  innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
90
98
  @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
91
99
  @fields_to_mask_keys[innerJSONStringRegex] = value
@@ -1,3 +1,3 @@
1
1
  module FilterMasking
2
- VERSION = "1.1.2"
3
- end
2
+ VERSION = "1.2.0"
3
+ end
data/test/fields-to-mask CHANGED
@@ -3,4 +3,5 @@ first_name
3
3
  last_name
4
4
  street
5
5
  number
6
- password
6
+ password
7
+ json_str_field
@@ -0,0 +1,5 @@
1
+ email/i
2
+ first_name
3
+ last_name/i
4
+ street/i
5
+ number
@@ -1,5 +1,3 @@
1
- # test/plugin/test_filter_your_own.rb
2
-
3
1
  require "test-unit"
4
2
  require "fluent/test"
5
3
  require "fluent/test/driver/filter"
@@ -25,6 +23,11 @@ class YourOwnFilterTest < Test::Unit::TestCase
25
23
  fieldsToMaskFilePath test/fields-to-mask
26
24
  ]
27
25
 
26
+ # configuration for tests with case insensitive fields
27
+ CONFIG_CASE_INSENSITIVE = %[
28
+ fieldsToMaskFilePath test/fields-to-mask-insensitive
29
+ ]
30
+
28
31
  def create_driver(conf = CONFIG)
29
32
  Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
30
33
  end
@@ -39,7 +42,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
39
42
  d.filtered_records
40
43
  end
41
44
 
42
- sub_test_case 'plugin will mask all fields that need masking' do
45
+ sub_test_case 'plugin will mask all fields that need masking - case sensitive fields' do
43
46
  test 'mask field in hash object' do
44
47
  conf = CONFIG_NO_EXCLUDE
45
48
  messages = [
@@ -121,6 +124,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
121
124
  filtered_records = filter(conf, messages)
122
125
  assert_equal(expected, filtered_records)
123
126
  end
127
+
124
128
  test 'mask field in hash object with base and nested exclude' do
125
129
  conf = CONFIG
126
130
  messages = [
@@ -132,6 +136,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
132
136
  filtered_records = filter(conf, messages)
133
137
  assert_equal(expected, filtered_records)
134
138
  end
139
+
135
140
  test 'mask field in json string with exclude' do
136
141
  conf = CONFIG
137
142
  messages = [
@@ -143,5 +148,90 @@ class YourOwnFilterTest < Test::Unit::TestCase
143
148
  filtered_records = filter(conf, messages)
144
149
  assert_equal(expected, filtered_records)
145
150
  end
151
+
152
+ test 'mask field which is inner json string field (should mask the whole object)' do
153
+ conf = CONFIG
154
+ messages = [
155
+ {
156
+ :body => {
157
+ :action_name => "some_action",
158
+ :action_type => "some type",
159
+ :request => {
160
+ :body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
161
+ }
162
+ },
163
+ :timestamp => "2020-06-08T16:00:57.341Z"
164
+ }
165
+ ]
166
+
167
+ expected = [
168
+ {
169
+ :body => {
170
+ :action_name => "some_action",
171
+ :action_type => "some type",
172
+ :request => {
173
+ :body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
174
+ }
175
+ },
176
+ :timestamp => "2020-06-08T16:00:57.341Z"
177
+ }
178
+ ]
179
+
180
+ filtered_records = filter(conf, messages)
181
+ assert_equal(expected, filtered_records)
182
+ end
146
183
  end
184
+
185
+ sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
186
+
187
+ test 'mask field in hash object with camel case' do
188
+ conf = CONFIG_CASE_INSENSITIVE
189
+ messages = [
190
+ {:not_masked_field=>"mickey-the-dog", :Email=>"mickey-the-dog@zooz.com"}
191
+ ]
192
+ expected = [
193
+ {:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING}
194
+ ]
195
+ filtered_records = filter(conf, messages)
196
+ assert_equal(expected, filtered_records)
197
+ end
198
+
199
+ test 'not mask field in hash object since case not match' do
200
+ conf = CONFIG_CASE_INSENSITIVE
201
+ messages = [
202
+ {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
203
+ ]
204
+ expected = [
205
+ {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
206
+ ]
207
+ filtered_records = filter(conf, messages)
208
+ assert_equal(expected, filtered_records)
209
+ end
210
+
211
+ test 'mask field in hash object with snakecase' do
212
+ conf = CONFIG_CASE_INSENSITIVE
213
+ messages = [
214
+ {:not_masked_field=>"mickey-the-dog", :LaSt_NaMe=>"mickey-the-dog@zooz.com"}
215
+ ]
216
+ expected = [
217
+ {:not_masked_field=>"mickey-the-dog", :last_name=>MASK_STRING}
218
+ ]
219
+ filtered_records = filter(conf, messages)
220
+ assert_equal(expected, filtered_records)
221
+ end
222
+
223
+ test 'mask case insensitive and case sensitive field in nested json escaped string' do
224
+ conf = CONFIG_CASE_INSENSITIVE
225
+ messages = [
226
+ { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" }
227
+ ]
228
+ expected = [
229
+ { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
230
+ ]
231
+ filtered_records = filter(conf, messages)
232
+ assert_equal(expected, filtered_records)
233
+ end
234
+
235
+ end
236
+
147
237
  end
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-masking
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shai Moria
8
8
  - Niv Lipetz
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-12-22 00:00:00.000000000 Z
12
+ date: 2021-03-14 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -25,34 +25,6 @@ dependencies:
25
25
  - - ">="
26
26
  - !ruby/object:Gem::Version
27
27
  version: 0.14.0
28
- - !ruby/object:Gem::Dependency
29
- name: bundler
30
- requirement: !ruby/object:Gem::Requirement
31
- requirements:
32
- - - '='
33
- - !ruby/object:Gem::Version
34
- version: 1.17.3
35
- type: :development
36
- prerelease: false
37
- version_requirements: !ruby/object:Gem::Requirement
38
- requirements:
39
- - - '='
40
- - !ruby/object:Gem::Version
41
- version: 1.17.3
42
- - !ruby/object:Gem::Dependency
43
- name: rake
44
- requirement: !ruby/object:Gem::Requirement
45
- requirements:
46
- - - "~>"
47
- - !ruby/object:Gem::Version
48
- version: '12.0'
49
- type: :development
50
- prerelease: false
51
- version_requirements: !ruby/object:Gem::Requirement
52
- requirements:
53
- - - "~>"
54
- - !ruby/object:Gem::Version
55
- version: '12.0'
56
28
  - !ruby/object:Gem::Dependency
57
29
  name: test-unit
58
30
  requirement: !ruby/object:Gem::Requirement
@@ -89,24 +61,23 @@ executables: []
89
61
  extensions: []
90
62
  extra_rdoc_files: []
91
63
  files:
64
+ - ".circleci/config.yml"
92
65
  - ".gitignore"
93
- - ".travis.yml"
94
66
  - Gemfile
95
- - Gemfile.lock
96
67
  - README.md
97
- - Rakefile
98
68
  - fluent-plugin-masking.gemspec
99
69
  - lib/fluent/plugin/filter_masking.rb
100
70
  - lib/fluent/plugin/helpers.rb
101
71
  - lib/fluent/plugin/version.rb
102
72
  - test/fields-to-mask
73
+ - test/fields-to-mask-insensitive
103
74
  - test/test_filter_masking.rb
104
75
  - test/test_helpers.rb
105
76
  homepage: https://github.com/PayU/fluent-plugin-masking
106
77
  licenses:
107
78
  - Apache-2.0
108
79
  metadata: {}
109
- post_install_message:
80
+ post_install_message:
110
81
  rdoc_options: []
111
82
  require_paths:
112
83
  - lib
@@ -114,21 +85,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
114
85
  requirements:
115
86
  - - ">="
116
87
  - !ruby/object:Gem::Version
117
- version: '2.1'
88
+ version: 2.5.0
118
89
  required_rubygems_version: !ruby/object:Gem::Requirement
119
90
  requirements:
120
91
  - - ">="
121
92
  - !ruby/object:Gem::Version
122
93
  version: '0'
123
94
  requirements: []
124
- rubyforge_project:
125
- rubygems_version: 2.5.2.3
126
- signing_key:
95
+ rubygems_version: 3.0.3
96
+ signing_key:
127
97
  specification_version: 4
128
98
  summary: Fluentd filter plugin to mask sensitive or privacy records with `*******`
129
99
  in place of the original value. This data masking plugin protects data such as name,
130
100
  email, phonenumber, address, and any other field you would like to mask.
131
101
  test_files:
132
102
  - test/fields-to-mask
103
+ - test/fields-to-mask-insensitive
133
104
  - test/test_filter_masking.rb
134
105
  - test/test_helpers.rb
data/.travis.yml DELETED
@@ -1,14 +0,0 @@
1
- sudo: false
2
- language: ruby
3
-
4
- rvm:
5
- - 2.6.4
6
- - 2.5.6
7
- - 2.4.7
8
- - 2.3.8
9
-
10
- before_install:
11
- - gem update bundler
12
-
13
- gemfile:
14
- - Gemfile
data/Gemfile.lock DELETED
@@ -1,55 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- fluent-plugin-masking (1.1.2)
5
- fluentd (>= 0.14.0)
6
-
7
- GEM
8
- remote: https://rubygems.org/
9
- specs:
10
- concurrent-ruby (1.1.5)
11
- cool.io (1.5.4)
12
- dig_rb (1.0.1)
13
- fluentd (1.8.0)
14
- cool.io (>= 1.4.5, < 2.0.0)
15
- dig_rb (~> 1.0.0)
16
- http_parser.rb (>= 0.5.1, < 0.7.0)
17
- msgpack (>= 1.2.0, < 2.0.0)
18
- serverengine (>= 2.0.4, < 3.0.0)
19
- sigdump (~> 0.2.2)
20
- strptime (>= 0.2.2, < 1.0.0)
21
- tzinfo (>= 1.0, < 3.0)
22
- tzinfo-data (~> 1.0)
23
- yajl-ruby (~> 1.0)
24
- http_parser.rb (0.6.0)
25
- msgpack (1.3.1)
26
- power_assert (1.1.5)
27
- rake (12.3.3)
28
- rr (1.2.1)
29
- serverengine (2.2.0)
30
- sigdump (~> 0.2.2)
31
- sigdump (0.2.4)
32
- strptime (0.2.3)
33
- test-unit (3.3.3)
34
- power_assert
35
- test-unit-rr (1.0.5)
36
- rr (>= 1.1.1)
37
- test-unit (>= 2.5.2)
38
- tzinfo (2.0.0)
39
- concurrent-ruby (~> 1.0)
40
- tzinfo-data (1.2019.3)
41
- tzinfo (>= 1.0.0)
42
- yajl-ruby (1.4.1)
43
-
44
- PLATFORMS
45
- ruby
46
-
47
- DEPENDENCIES
48
- bundler (= 1.17.3)
49
- fluent-plugin-masking!
50
- rake (~> 12.0)
51
- test-unit (>= 3.1.0)
52
- test-unit-rr
53
-
54
- BUNDLED WITH
55
- 1.17.3
data/Rakefile DELETED
@@ -1,9 +0,0 @@
1
- require "bundler/gem_tasks"
2
- require 'rake/testtask'
3
- Rake::TestTask.new(:test) do |test|
4
- test.libs << 'lib' << 'test'
5
- test.pattern = 'test/**/test_*.rb'
6
- test.verbose = true
7
- end
8
-
9
- task :default => :test