fluent-plugin-masking 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5b8b769887786df667374682f71ddbc61f6b575f
-  data.tar.gz: c143faea655e8ec4f621445d66525f827b26e9a0
+SHA256:
+  metadata.gz: 3c1d03c89c4b186938fab7c7f41e0217c443dc93c0b4d74fd59a0afe34de7493
+  data.tar.gz: de2fb0a278fdeb128773f96ad1be8ea0e895b611febff6b0fd57ae89668a0ff6
 SHA512:
-  metadata.gz: c964b4e42d7bbbe59fcaf570b1ac8281427ee7d15ea20cb8dcd9320343a306739571e25c2030e3f63cd2465daec6ec757ce7a57c68127c21e7acabba36b3243c
-  data.tar.gz: d9937e8bf71d1431cf9312134618d2bd5c842e203ce9d853cc4e8cb2a08b8c60cfea4b21105c3296ca35dedd2dc3a1cb1c61d871fe90dcf74abdfe64e0f73f29
+  metadata.gz: 0712e0f59cd1e3d89282772285cd7632a242a1cba129e3c8a42836b0381079698faa536ac24be69768751f91c81155a0d4ff65824d914754e4bf8b175b6aa5eb
+  data.tar.gz: 7c43897e4a4a9a5c5c9cbdf10398efcfe333bc770d0e2c46f6d086ab16cc3c514e934c59210ab213468608c3ec59da6c389bb744b2d27be7409d3ca5df980e0f

data/.circleci/config.yml

@@ -0,0 +1,29 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2 
+  
+executors:
+  v2-5-0:
+    docker:
+      - image: circleci/ruby:2.5.0
+
+jobs:
+  tests:
+    parameters:
+      ruby-version:
+        type: executor
+    executor: << parameters.ruby-version >>
+    steps:
+      - checkout
+      - run: gem install bundler
+      - run: bundle install
+      - run: ruby -r ./test/*.rb
+
+workflows:
+  tests:
+    jobs:
+      - tests:
+          matrix:
+            parameters:
+              ruby-version: [v2-5-0]
+

data/.gitignore

@@ -1,4 +1,6 @@
 .bundle/
 vendor/
 *.gem
-.idea/
+.idea/
+Gemfile.lock
+Rakefile

data/README.md

@@ -8,7 +8,7 @@ Fluentd filter plugin to mask sensitive or privacy records with `*******` in pla
 ## Requirements
 | fluent-plugin-masking    | fluentd    | ruby   |
 | ---------------------    | ---------- | ------ |
-| 1.0.x                    | 	>= v0.14.0 | >= 2.1 |
+| 1.2.x                    | 	>= v0.14.0 | >= 2.5 |
 
 
 ## Installation
@@ -35,10 +35,9 @@ Example fields-to-mask-file:
 ```
 name
 email
-phone
+phone/i # the '/i' suffix will make sure phone field will be case insensitive
 ```
 
-
 ## Quick Guide
 
 ### Configuration:
@@ -98,3 +97,11 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
 ```
 2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
+
+
+### Run Unit Tests
+```
+gem install bundler
+bundle install
+ruby -r ./test/*.rb
+```

data/fluent-plugin-masking.gemspec

@@ -18,11 +18,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.license = "Apache-2.0"
 
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.5.0'
   
   spec.add_runtime_dependency "fluentd", ">= 0.14.0"
-  spec.add_development_dependency "bundler", "1.17.3"
-  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
   spec.add_development_dependency "test-unit-rr"
 end

data/lib/fluent/plugin/filter_masking.rb

@@ -73,19 +73,27 @@ module Fluent
 
         File.open(fieldsToMaskFilePath, "r") do |f|
           f.each_line do |line|
-
             value = line.to_s # make sure it's string
             value = value.gsub(/\s+/, "") # remove spaces
             value = value.gsub('\n', '') # remove line breakers
 
+            if value.end_with? "/i"
+              # case insensitive
+              value = value.delete_suffix('/i')
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
+            else
+              # case sensitive
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
+            end
+
             @fields_to_mask.push(value)
 
-            hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
             hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
             @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
             @fields_to_mask_keys[hashObjectRegex] = value
 
-            innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
             @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
             @fields_to_mask_keys[innerJSONStringRegex] = value

data/lib/fluent/plugin/version.rb

@@ -1,3 +1,3 @@
 module FilterMasking
-    VERSION = "1.1.2"
-end 
+    VERSION = "1.2.0"
+end

data/test/fields-to-mask

@@ -3,4 +3,5 @@ first_name
 last_name
 street
 number
-password
+password
+json_str_field

data/test/fields-to-mask-insensitive

@@ -0,0 +1,5 @@
+email/i
+first_name
+last_name/i
+street/i
+number

data/test/test_filter_masking.rb

@@ -1,5 +1,3 @@
-# test/plugin/test_filter_your_own.rb
-
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/filter"
@@ -25,6 +23,11 @@ class YourOwnFilterTest < Test::Unit::TestCase
     fieldsToMaskFilePath test/fields-to-mask
   ]
 
+  # configuration for tests with case insensitive fields
+  CONFIG_CASE_INSENSITIVE = %[
+    fieldsToMaskFilePath test/fields-to-mask-insensitive
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
   end
@@ -39,7 +42,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
     d.filtered_records
   end
 
-  sub_test_case 'plugin will mask all fields that need masking' do
+  sub_test_case 'plugin will mask all fields that need masking - case sensitive fields' do
     test 'mask field in hash object' do
       conf = CONFIG_NO_EXCLUDE
       messages = [
@@ -121,6 +124,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in hash object with base and nested exclude' do
       conf = CONFIG
       messages = [
@@ -132,6 +136,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in json string with exclude' do
       conf = CONFIG
       messages = [
@@ -143,5 +148,90 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
+    test 'mask field which is inner json string field (should mask the whole object)' do
+      conf = CONFIG
+      messages = [
+        {
+          :body => { 
+            :action_name => "some_action", 
+            :action_type => "some type",
+            :request => {
+              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
+            }
+          },
+          :timestamp => "2020-06-08T16:00:57.341Z"
+        }
+      ]
+
+      expected = [
+        {
+          :body => { 
+            :action_name => "some_action", 
+            :action_type => "some type",
+            :request => {
+              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
+            }
+          },
+          :timestamp => "2020-06-08T16:00:57.341Z"
+        }
+      ]
+
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
   end
+
+  sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
+
+    test 'mask field in hash object with camel case' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :Email=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'not mask field in hash object since case not match' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :FIRST_NAME=>"mickey-the-dog@zooz.com"}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask field in hash object with snakecase' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :LaSt_NaMe=>"mickey-the-dog@zooz.com"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>MASK_STRING}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask case insensitive and case sensitive field in nested json escaped string' do
+      conf = CONFIG_CASE_INSENSITIVE
+      messages = [
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
+      ]
+      expected = [
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+  end
+
 end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-masking
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Shai Moria
 - Niv Lipetz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-22 00:00:00.000000000 Z
+date: 2021-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -25,34 +25,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.0
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.17.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.17.3
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '12.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -89,24 +61,23 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - README.md
-- Rakefile
 - fluent-plugin-masking.gemspec
 - lib/fluent/plugin/filter_masking.rb
 - lib/fluent/plugin/helpers.rb
 - lib/fluent/plugin/version.rb
 - test/fields-to-mask
+- test/fields-to-mask-insensitive
 - test/test_filter_masking.rb
 - test/test_helpers.rb
 homepage: https://github.com/PayU/fluent-plugin-masking
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -114,21 +85,21 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Fluentd filter plugin to mask sensitive or privacy records with `*******`
   in place of the original value. This data masking plugin protects data such as name,
   email, phonenumber, address, and any other field you would like to mask.
 test_files:
 - test/fields-to-mask
+- test/fields-to-mask-insensitive
 - test/test_filter_masking.rb
 - test/test_helpers.rb

data/.travis.yml

@@ -1,14 +0,0 @@
-sudo: false
-language: ruby
-
-rvm:
-  - 2.6.4
-  - 2.5.6
-  - 2.4.7
-  - 2.3.8
-
-before_install:
-  - gem update bundler
-
-gemfile:
-  - Gemfile

data/Gemfile.lock

@@ -1,55 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-masking (1.1.2)
-      fluentd (>= 0.14.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    concurrent-ruby (1.1.5)
-    cool.io (1.5.4)
-    dig_rb (1.0.1)
-    fluentd (1.8.0)
-      cool.io (>= 1.4.5, < 2.0.0)
-      dig_rb (~> 1.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 1.2.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
-      tzinfo (>= 1.0, < 3.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.3.1)
-    power_assert (1.1.5)
-    rake (12.3.3)
-    rr (1.2.1)
-    serverengine (2.2.0)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.2.3)
-    test-unit (3.3.3)
-      power_assert
-    test-unit-rr (1.0.5)
-      rr (>= 1.1.1)
-      test-unit (>= 2.5.2)
-    tzinfo (2.0.0)
-      concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2019.3)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.4.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (= 1.17.3)
-  fluent-plugin-masking!
-  rake (~> 12.0)
-  test-unit (>= 3.1.0)
-  test-unit-rr
-
-BUNDLED WITH
-   1.17.3

data/Rakefile

@@ -1,9 +0,0 @@
-require "bundler/gem_tasks"
-require 'rake/testtask'
-  Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-task :default => :test