fluent-plugin-masking 1.0.8 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02c98ca4f43cdef94bd6ec5615355f76601d5fa7ec28307ffc7c2748a629a0dd
-  data.tar.gz: a25640c10af2bd1724acb5381cd512eeb129719376824a2be0f98aa3e3d6bd55
+  metadata.gz: '08a7033570c7aaca9dc0a4830f4550ef5e353b9e5195dc3cfcc79eb68880fcae'
+  data.tar.gz: '0845c7ee9ae72c13652cd9a517a71f0289c88f2144cf2afcad18370c1f1fa358'
 SHA512:
-  metadata.gz: 77b885170c3d95b1abcb60df6e0a86958bce0e137718375179cbf17d6a824144034dfa4954c69397b9d1d8f127be5c218c81dc120340c0546def8cd5df27ceff
-  data.tar.gz: 139d6cfec14a8e970653d44bbf85b5ef9de0765f03bd6b0d67ee3fb0c9da2b94f6635c7915e63c64cbb77a6d98e46ffed83821f9a340680e6d7d64bae75db988
+  metadata.gz: 61a8a367ec559746eee72fd7e33a4d62b6f38777a18b3d5560c82dffcee5c65332ef8fbfd39e018b5dec34930e8a13cdbfa41356a796b4afc9c858eac7ce13d2
+  data.tar.gz: 92b95fad73c342b281ce44d451ce0d52ddcce490d1b0753498671320b38ffb840840b02a44a2ab184d7be0fc636aa2f78ba6dbed6e1a3e8c32f4ebcd71ebe3dd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-masking (1.0.7)
+    fluent-plugin-masking (1.1.0)
       fluentd (>= 0.14.0)
 
 GEM
@@ -10,7 +10,7 @@ GEM
     concurrent-ruby (1.1.5)
     cool.io (1.5.4)
     dig_rb (1.0.1)
-    fluentd (1.7.2)
+    fluentd (1.7.4)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
@@ -26,7 +26,7 @@ GEM
     power_assert (1.1.5)
     rake (12.3.3)
     rr (1.2.1)
-    serverengine (2.1.1)
+    serverengine (2.2.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.3)

data/README.md

@@ -19,11 +19,15 @@ Install with gem:
 ## Setup
 In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
 
+In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
+The JSON fields that are excluded are comma separated.  
+This can be used for logs of registration services or audit log entries which do not need to be masked.  
 This is configured as shown below:
 ```
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
+  fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
 </filter>
 ```
 
@@ -52,6 +56,7 @@ phone
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
+  fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
 </filter>
 
 <match "**">
@@ -83,3 +88,13 @@ This sample result is created from the above configuration file `fluent.conf`. A
 ```
 2019-09-15 16:12:50.359191000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"*******\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"*******\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
+
+A sample with exclude in use:
+```
+fluentd -c fluent.conf
+echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\":\"the-dog\", \"password\":\"d0g43u39\"}", "excludeMaskFields"=>"first_name,last_name"}' > /tmp/test.log
+```
+
+```
+2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
+```

data/lib/fluent/plugin/filter_masking.rb

@@ -15,11 +15,19 @@ module Fluent
       # error safe method - if any error occurs the original record is returned
       def maskRecord(record)
         maskedRecord = record
-        
-        begin
+        excludedFields = []
+        @fieldsToExcludeJSONPathsArray.each do | field |
+          field_value = record.dig(*field)
+          if  field_value != nil
+            excludedFields = excludedFields + field_value.split(',')
+          end
+        end
+        begin 
           recordStr = record.to_s
           @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
-            recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
+            if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
+              recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
+            end
           end
 
           maskedRecord = strToHash(recordStr)
@@ -35,12 +43,27 @@ module Fluent
         super
         @fields_to_mask = []
         @fields_to_mask_regex = {}
+        @fields_to_mask_keys = {}
+        @fieldsToExcludeJSONPathsArray = []
       end
 
       # this method only called ones (on startup time)
       def configure(conf)
         super
         fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
+        fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
+
+        if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0 
+          fieldsToExcludeJSONPaths.split(",").each do | field |
+            # To save splits we'll save the path as an array
+            splitArray = field.split(".")
+            symArray = []
+            splitArray.each do | pathPortion |
+              symArray.push(pathPortion.to_sym)
+            end
+            @fieldsToExcludeJSONPathsArray.push(symArray)
+          end
+        end
 
         File.open(fieldsToMaskFilePath, "r") do |f|
           f.each_line do |line|
@@ -54,10 +77,12 @@ module Fluent
             hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
             hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
             @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
+            @fields_to_mask_keys[hashObjectRegex] = value
 
             innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
             @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
+            @fields_to_mask_keys[innerJSONStringRegex] = value
           end
         end
 

data/lib/fluent/plugin/version.rb

@@ -1,3 +1,3 @@
 module FilterMasking
-    VERSION = "1.0.8"
+    VERSION = "1.1.0"
 end 

data/test/test_filter_masking.rb

@@ -17,6 +17,12 @@ class YourOwnFilterTest < Test::Unit::TestCase
   # default configuration for tests
   CONFIG = %[
     fieldsToMaskFilePath test/fields-to-mask
+    fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
+  ]
+
+  # configuration for tests without exclude parameter
+  CONFIG_NO_EXCLUDE = %[
+    fieldsToMaskFilePath test/fields-to-mask
   ]
 
   def create_driver(conf = CONFIG)
@@ -35,7 +41,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
 
   sub_test_case 'plugin will mask all fields that need masking' do
     test 'mask field in hash object' do
-      conf = CONFIG
+      conf = CONFIG_NO_EXCLUDE
       messages = [
         {:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com"}
       ]
@@ -93,5 +99,38 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+    test 'mask field in hash object with exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name"}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+    test 'mask field in hash object with nested exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky",  :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+    test 'mask field in hash object with base and nested exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-masking
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.1.0
 platform: ruby
 authors:
 - Shai Moria
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-22 00:00:00.000000000 Z
+date: 2019-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -119,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd filter plugin to mask sensitive or privacy records with `*******`