fluent-plugin-masking 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3c1d03c89c4b186938fab7c7f41e0217c443dc93c0b4d74fd59a0afe34de7493
-  data.tar.gz: de2fb0a278fdeb128773f96ad1be8ea0e895b611febff6b0fd57ae89668a0ff6
+  metadata.gz: e8c2ddb483a737dd913e64111218e9484bffb2d94db68c8aefb49ec4efdcfd58
+  data.tar.gz: 3080f077ae6d4437fc3921d34b642d77ffb57e1d4498d6b3028882d3a97a94a0
 SHA512:
-  metadata.gz: 0712e0f59cd1e3d89282772285cd7632a242a1cba129e3c8a42836b0381079698faa536ac24be69768751f91c81155a0d4ff65824d914754e4bf8b175b6aa5eb
-  data.tar.gz: 7c43897e4a4a9a5c5c9cbdf10398efcfe333bc770d0e2c46f6d086ab16cc3c514e934c59210ab213468608c3ec59da6c389bb744b2d27be7409d3ca5df980e0f
+  metadata.gz: 53514f891096b90ce5c8b1957b248ebaf27f6ad25bf6176ac10fcc873e9bd471995c8157deeae57fb9af78487a0018562a973627129d14c6d9a3eabaab2c0d9b
+  data.tar.gz: ca252648f6799d83a5ca72796452ac2a165527275d8387c6cc051a3341381e177d94bb7ab07bef3254016cd4bd7124ec3691b593608d7802acce23eee9cb4eac

data/README.md

@@ -14,20 +14,24 @@ Fluentd filter plugin to mask sensitive or privacy records with `*******` in pla
 ## Installation
 Install with gem:
 
-`gem install fluent-plugin-masking`
+`fluent-gem install fluent-plugin-masking`
 
 ## Setup
-In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
+In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`). if you one or more of the fields will be case insensitive, use the `/i` suffix in your field. see example below.
 
-In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
-The JSON fields that are excluded are comma separated.  
-This can be used for logs of registration services or audit log entries which do not need to be masked.  
-This is configured as shown below:
+### Optional configuration
+ - `fieldsToExcludeJSONPaths` - this field receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`) The JSON fields that are excluded are comma separated.  
+This can be used for logs of registration services or audit log entries which do not need to be masked.
+
+- `handleSpecialEscapedJsonCases` - a boolean value that try to fix special escaped json cases. this feature is currently on alpha stage (default: false). for more details about thoose special cases see [Special Json Cases](#Special-escaped-json-cases-handling)
+
+An example with optional configuration parameters:
 ```
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
   fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
+  handleSpecialEscapedJsonCases true
 </filter>
 ```
 
@@ -98,10 +102,12 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
 2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
 
-
-### Run Unit Tests
+## Run Unit Tests
 ```
 gem install bundler
 bundle install
 ruby -r ./test/*.rb
 ```
+
+## Special escaped json cases handling
+

data/lib/fluent/plugin/filter_masking.rb

@@ -30,14 +30,20 @@ module Fluent
         end
         begin
           recordStr = record.to_s
+
+          if @handleSpecialEscapedJsonCases == true
+            @specialEscapedJsonRegexs.each do | regex, replace |
+              recordStr = recordStr.gsub(regex, replace)
+            end
+          end  
+          
           @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
             if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
               recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
             end
           end
-
+          
           maskedRecord = strToHash(recordStr)
-
         rescue Exception => e
           $log.error "Failed to mask record: #{e}"
         end
@@ -51,6 +57,11 @@ module Fluent
         @fields_to_mask_regex = {}
         @fields_to_mask_keys = {}
         @fieldsToExcludeJSONPathsArray = []
+
+        @handleSpecialEscapedJsonCases = false
+        @specialEscapedJsonRegexs = {
+          Regexp.new(/,(( *)(\\*)("*)( *)),/m) => "\1,"
+        }
       end
 
       # this method only called ones (on startup time)
@@ -58,6 +69,7 @@ module Fluent
         super
         fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
         fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
+        handleSpecialCases = conf['handleSpecialEscapedJsonCases']
 
         if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0 
           fieldsToExcludeJSONPaths.split(",").each do | field |
@@ -80,12 +92,12 @@ module Fluent
             if value.end_with? "/i"
               # case insensitive
               value = value.delete_suffix('/i')
-              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
-              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi) # mask element in hash object
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/mi) # mask element in json string using capture groups that count the level of escaping inside the json string
             else
               # case sensitive
-              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
-              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             end
 
             @fields_to_mask.push(value)
@@ -100,6 +112,10 @@ module Fluent
           end
         end
 
+        # if true, each record (a json record), will be checked for a special escaped json cases
+        # any found case will be 'gsub' with the right solution 
+        @handleSpecialEscapedJsonCases = handleSpecialCases != nil && handleSpecialCases.casecmp("true") == 0
+
         puts "black list fields:"
         puts @fields_to_mask
       end

data/lib/fluent/plugin/version.rb

@@ -1,3 +1,3 @@
 module FilterMasking
-    VERSION = "1.2.0"
+    VERSION = "1.3.0"
 end

data/test/fields-to-mask

@@ -4,4 +4,4 @@ last_name
 street
 number
 password
-json_str_field
+cookie

data/test/test_filter_masking.rb

@@ -4,7 +4,6 @@ require "fluent/test/driver/filter"
 require "fluent/test/helpers"
 require "./lib/fluent/plugin/filter_masking.rb"
 
-
 MASK_STRING = "*******"
 
 class YourOwnFilterTest < Test::Unit::TestCase
@@ -28,6 +27,13 @@ class YourOwnFilterTest < Test::Unit::TestCase
     fieldsToMaskFilePath test/fields-to-mask-insensitive
   ]
 
+  # configuration for special json escaped cases
+  CONFIG_SPECIAL_CASES = %[
+    fieldsToMaskFilePath test/fields-to-mask
+    fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
+    handleSpecialEscapedJsonCases true
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
   end
@@ -102,6 +108,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in hash object with exclude' do
       conf = CONFIG
       messages = [
@@ -113,6 +120,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+    
     test 'mask field in hash object with nested exclude' do
       conf = CONFIG
       messages = [
@@ -149,37 +157,6 @@ class YourOwnFilterTest < Test::Unit::TestCase
       assert_equal(expected, filtered_records)
     end
 
-    test 'mask field which is inner json string field (should mask the whole object)' do
-      conf = CONFIG
-      messages = [
-        {
-          :body => { 
-            :action_name => "some_action", 
-            :action_type => "some type",
-            :request => {
-              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
-            }
-          },
-          :timestamp => "2020-06-08T16:00:57.341Z"
-        }
-      ]
-
-      expected = [
-        {
-          :body => { 
-            :action_name => "some_action", 
-            :action_type => "some type",
-            :request => {
-              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
-            }
-          },
-          :timestamp => "2020-06-08T16:00:57.341Z"
-        }
-      ]
-
-      filtered_records = filter(conf, messages)
-      assert_equal(expected, filtered_records)
-    end
   end
 
   sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
@@ -223,7 +200,7 @@ class YourOwnFilterTest < Test::Unit::TestCase
     test 'mask case insensitive and case sensitive field in nested json escaped string' do
       conf = CONFIG_CASE_INSENSITIVE
       messages = [
-        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_NAME\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
       ]
       expected = [
         { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
@@ -234,4 +211,17 @@ class YourOwnFilterTest < Test::Unit::TestCase
 
   end
 
+  sub_test_case 'plugin will mask all fields that need masking - special json escaped cases' do
+    test 'mask field in nested json escaped string when one of the values ends with "," (the value for "some_custom" field)' do
+      conf = CONFIG_SPECIAL_CASES
+      messages = [
+        { :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\", \"cookie\":\"some_custom=,,live,default,,2097403972,2.22.242.38,\", \"city\":\"new york\"}" } 
+      ]
+      expected = [
+        { :body => "{\"first_name\":\"*******\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\", \"cookie\":\"*******\", \"city\":\"new york\"}" }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+  end  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-masking
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Shai Moria
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-14 00:00:00.000000000 Z
+date: 2021-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd