fluent-plugin-sanitizer 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e6b8ad479091eb0d7ce2f1539ec4966648907f638d5d8e5676d408ea0d4c3ad
-  data.tar.gz: 21a0001319b6b8ec60b478bd60897a20fcca5a068879a0c73531ffb360cd576f
+  metadata.gz: 7a773fb4d2f1fb7c00d11982f92f15fc4803135af5a6a5c3433783fa12d0811c
+  data.tar.gz: 8ae5cc7723cab184885ebebd401824700c9ffde8fdf5ab81463f3b67be6aed95
 SHA512:
-  metadata.gz: b30b9bec5a8ccc8bf16fcbd3f66bc7ecce7de97778521530eae226b10653b30505f07de47f37246851e6b727152bbd95488614e8d76c96fd0ac3bec3ac2848f0
-  data.tar.gz: 3f31d9ace5c57a733a956e659e3c80c078e3f447d2b9d705c5d9f1e740be6eb214fd21c6750185f89d920b4cbbec51ac8d8f716fad8021a79286ed9e958bf7d0
+  metadata.gz: e5fe885cf146981c70346b7acd1126eed88cba19f090131f56fcdb19f0ef0f68c1a97d039105e59b997ef9662f2db73d8f6dad8407a056ca9fcdcf9da4dfa261
+  data.tar.gz: 87810f31b6f307779e785291ca1c4ca6d7c17b9cf62235e5a22bfdd99399e1f551c98cb0750add509a69f6ecf2b1134cb593c3175c46abf7b4f20cf894d821e5

data/README.md

@@ -18,10 +18,11 @@ td-agent-gem install fluent-plugin-sanitizer
   - keys (mandatory) :  Name of keys whose values will be masked. You can specify multiple keys. When keys are nested, you can use {parent key}.{child key} like "kubernetes.master_url". 
   - pattern_ipv4 (optional)  : Mask IP addresses in IPv4 format. You can use “true” or “false”. (defalt: false)
   - pattern_fqdn (optional)  : Mask hostname in FQDN style. You can use “true” or “false”. (defalt: false)
-  - pattern_regex (optional)  : Mask value mactches custom regular expression. You need to provide a regular expression in these options.
-  - pattern_regex_prefix (optional) : Define prefix used for masking vales. (default: Regex)
+  - pattern_regex (optional)  : Mask value mactches custom regular expression.
+    - regex_capture_group (optional) : If you define capture group in regular expression, you can specify the name of capture group to be masked.
+    - pattern_regex_prefix (optional) : Define prefix used for masking vales. (default: Regex)
   - pattern_keywords (optional)  : Mask values match custom keywords. You can specify multiple keywords. 
-  - pattern_keywords_prefix (optional) : Define prefix used for masking vales. (default: Keyword)
+    - pattern_keywords_prefix (optional) : Define prefix used for masking vales. (default: Keyword)
 
 You can specify multiple rules in a single configuration. It is also possible to define multiple pattern options in a single rule like the following sample.
   
@@ -129,6 +130,34 @@ In case log messages including sensitive information such as SSN and phone numbe
      }
  }
 ```
+From v0.1.2, "regex_capture_group" option is available. With "regex_capture_group" option, it is possible to mask specific part of original messages. 
+
+**Configuration sample**
+```
+<rule>
+  keys user.email
+  pattern_regex /(?<user>\w+)\@\w+.\w+/
+  regex_capture_group "user"
+  pattern_regex_prefix "USER"
+</rule>
+```
+**Input sample**
+```
+ {
+     "user" : {
+         "email" : "user1@demo.com"
+     }
+ }
+```
+**Output sample**
+```
+ {
+     "user" : {
+         "email" : "USER_321865df6f0ce6bdf3ea16f74623534a@demo.com"
+     }
+ }
+```
+
 ### Tips : Debug how sanitizer works
 When you design custom rules in a configuration file, you might need information about how Sanitizer masks original values into hash values for debugging purposes. You can check that information if you run td-agent/Fluentd with debug option enabled. The debug information is shown in the log file of td-agent/Fluentd like the following log message sample.
 

data/fluent-plugin-sanitizer.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-sanitizer"
-  spec.version = "0.1.2"
+  spec.version = "0.1.3"
   spec.authors = ["TK Kubota"]
   spec.email   = ["tkubota@ctc-america.com"]
 

data/lib/fluent/plugin/filter_sanitizer.rb

@@ -23,44 +23,58 @@ module Fluent
 
       helpers :event_emitter, :record_accessor
 
-      desc "Hash salt to be used to generate hash values with MD5(optional)"
+      desc "Hash salt to be used to generate hash values with specified hash(optional)"
       config_param :hash_salt, :string, default: ""
-      
-      config_section :rule, param_name: :rules, multi: true do 
-        desc "Name of keys whose valuse are to be sanitized"
+
+      desc "Hash scheme to use for generating hash value (supported schemes are md5,sha1,sha256,sha384,sha512) (optional)"
+      config_param :hash_scheme, :enum, list: [:md5, :sha1, :sha256, :sha384, :sha512], default: :md5
+
+      config_section :rule, param_name: :rules, multi: true do
+        desc "Name of keys whose values are to be sanitized"
         config_param :keys, :array, default: []
         desc "Sanitize if values contain IPv4 (optional)"
         config_param :pattern_ipv4, :bool, default: false
-        desc "Sanitize if values contain Hostname in FQDN style (ptional)"
+        desc "Sanitize if values contain Hostname in FQDN style (optional)"
         config_param :pattern_fqdn, :bool, default: false
-        desc "Sanitize if values mactch custom regular expression (optional)"
+        desc "Sanitize if values match custom regular expression (optional)"
         config_param :pattern_regex, :regexp, default: /^$/
-        desc "Prefix for pattern_regex (optional)"
-        config_param :regex_capture_group, :string, default:""
         desc "Target capture group name to be masked (optional)"
+        config_param :regex_capture_group, :string, default:""
+        desc "Prefix for pattern_regex (optional)"
         config_param :pattern_regex_prefix, :string, default: "Regex"
-        desc "Sanitize if values mactch custom keywords (optional)"
+        desc "Sanitize if values match custom keywords (optional)"
         config_param :pattern_keywords, :array, default: []
         desc "Prefix for pattern_keywords (optional)"
         config_param :pattern_keywords_prefix, :string, default: "Keywords"
       end
 
       def configure(conf)
-        super
-      
+        super 
         @salt = conf['hash_salt']
- 
+        @salt = "" if @salt.nil?
+        @hash_scheme = conf['hash_scheme']
+        @sanitize_func =
+          case @hash_scheme
+            when "sha1"
+              Proc.new { |str| Digest::SHA1.hexdigest(@salt + str) }
+            when "sha256"
+              Proc.new { |str| Digest::SHA256.hexdigest(@salt +str) }
+            when "sha384"
+              Proc.new { |str| Digest::SHA384.hexdigest(@salt +str) }
+            when "sha512"
+              Proc.new { |str| Digest::SHA512.hexdigest(@salt +str) }
+            else
+              Proc.new { |str| Digest::MD5.hexdigest(@salt +str) }
+          end
+
         @sanitizerules = []
         @rules.each do |rule|
           if rule.keys.empty?
             raise Fluent::ConfigError, "You need to specify at least one key in rule statement."
           else
-            #keys = record_accessor_create(rule.keys)
             keys = rule.keys
           end
           
-          #record_accessor_create(rule.keys)
-          
           if rule.pattern_ipv4 || !rule.pattern_ipv4
             pattern_ipv4 = rule.pattern_ipv4
           else
@@ -77,16 +91,11 @@ module Fluent
             pattern_regex = rule.pattern_regex
             regex_capture_group = rule.regex_capture_group
           else
-            raise Fluent::ConfigError, "Your need to specify Regexp for pattern_fqdn option."
-          end      
-    
+            raise Fluent::ConfigError, "Your need to specify Regexp for pattern_regex option."
+          end
+
           pattern_keywords = rule.pattern_keywords
 
-          case [pattern_ipv4, pattern_fqdn, pattern_regex, pattern_keywords.empty?]
-          when [false, false, /^$/, true]
-            raise Fluent::ConfigError, "You need to specify at least one pattern option in the rule statement." 
-          end
-      
           regex_prefix = rule.pattern_regex_prefix
           keywords_prefix = rule.pattern_keywords_prefix
 
@@ -98,22 +107,27 @@ module Fluent
         @sanitizerules.each do |keys, pattern_ipv4, pattern_fqdn, pattern_regex, regex_capture_group, pattern_keywords, regex_prefix, keywords_prefix|  
           keys.each do |key|
             accessor = record_accessor_create("$."+key.to_s)
-            if pattern_ipv4 && accessor.call(record)
-              accessor.set(record, sanitize_ipv4_val(accessor.call(record).to_s))
-            end
-            if pattern_fqdn && accessor.call(record)
-              accessor.set(record, sanitize_fqdn_val(accessor.call(record).to_s))
-            end
-            if pattern_regex && accessor.call(record)
-              if regex_capture_group.empty?
-                accessor.set(record, sanitize_regex_val(accessor.call(record).to_s, regex_prefix, pattern_regex))
-              else
-                accessor.set(record, sanitize_regex_val_capture(accessor.call(record).to_s, regex_prefix, pattern_regex, regex_capture_group))
+            begin
+                if pattern_ipv4 && accessor.call(record)
+                  accessor.set(record, sanitize_ipv4_val(accessor.call(record).to_s))
+                end
+                if pattern_fqdn && accessor.call(record)
+                  accessor.set(record, sanitize_fqdn_val(accessor.call(record).to_s))
+                end
+                if !pattern_regex.to_s.eql?("(?-mix:^$)") && accessor.call(record)
+                  if regex_capture_group.empty?
+                    accessor.set(record, sanitize_regex_val(accessor.call(record), regex_prefix, pattern_regex))
+                  else
+                    accessor.set(record, sanitize_regex_val_capture(accessor.call(record), regex_prefix, pattern_regex, regex_capture_group))
+                  end
+                #end
+                end
+                if !pattern_keywords.empty? && accessor.call(record)
+                  accessor.set(record, sanitize_keywords_val(accessor.call(record).to_s, pattern_keywords, keywords_prefix))
+                end
+              rescue => e
+                  log.warn "Skipping this key", error_class: e.class, error: e.message
               end
-            end
-            if !pattern_keywords.empty? && accessor.call(record)
-              accessor.set(record, sanitize_keywords_val(accessor.call(record).to_s, pattern_keywords, keywords_prefix))
-            end
           end
         end
         record
@@ -152,16 +166,32 @@ module Fluent
       end
 
       def sanitize_ipv4(str)
-        return "IPv4_"+Digest::MD5.hexdigest(@salt + str)
+        return "IPv4_"+ @sanitize_func.call(str)
       end
 
       def sanitize_fqdn(str)
-        return "FQDN_"+Digest::MD5.hexdigest(@salt + str)
+        return "FQDN_"+ @sanitize_func.call(str)
+      end
+
+      def sanitize_val(str, prefix)
+        s = prefix + "_" + @sanitize_func.call(str)
+        $log.debug "[pattern_regex] sanitize '#{str}' to '#{s}'" if str != s
+        return s
       end
 
       def sanitize_regex(str, prefix, regex)
-        if str.to_s.match?(regex)
-          return prefix + "_" + Digest::MD5.hexdigest(@salt + str)
+        regex_p = Regexp.new(regex)
+        if str =~ regex_p
+          scans = str.scan(regex).flatten
+          if scans.any?{ |e| e.nil? }
+            return prefix + "_" + @sanitize_func.call(str)
+          else
+            scans.each do |s|
+              mask = prefix + "_" + @sanitize_func.call(str)
+              str = str.gsub(s, mask)
+            end
+          end
+          return str
         else
           $log.debug "[pattern_regex] #{str} does not match given regex #{regex}. skip this rule."
           return str
@@ -169,11 +199,15 @@ module Fluent
       end
 
       def sanitize_regex_capture(str, prefix, regex, capture_group)
-        if str.match?(regex)
+        regex_p = Regexp.new(regex)
+        if str =~ regex_p
           if str.match(regex).names.include?(capture_group)
-            cg = str.match(regex)[capture_group]
-            mask = prefix + "_" + Digest::MD5.hexdigest(@salt + cg)
-            return str.split(cg)[0] + mask + str.split(cg)[1]
+            scans = str.scan(regex).flatten
+            scans.each do |s|
+              mask = prefix + "_" + @sanitize_func.call(str)
+              str = str.gsub(s, mask)
+            end
+            return str
           else
              $log.debug "[pattern_regex] regex pattern matched but capture group '#{capture_group}' does not exist. Skip this rule."
              return str
@@ -185,7 +219,7 @@ module Fluent
       end
       
       def sanitize_keyword(str, prefix)
-        return prefix + "_" + Digest::MD5.hexdigest(@salt + str)
+        return prefix + "_" + @sanitize_func.call(str)
       end
 
       def sanitize_ipv4_port(str)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sanitizer
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - TK Kubota
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-12 00:00:00.000000000 Z
+date: 2022-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler