fluent-plugin-anonymizer 0.4.1 → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a1fc717a156c6b19fea2d19f4522c3b2d8be7dda
+  data.tar.gz: 9c7ff4fda067571c4becc5f29c5be0a5452e9653
+SHA512:
+  metadata.gz: bc870e37f4a1b30b543d53497208e9af52ac1c901bc77ec32c1bb772fb30d9cd4a8ebd2eb1782550757fd4763c93e84248c8e2f2c46826e3c3b688113e1f6a5c
+  data.tar.gz: 0bcc4eaecfceb6f5a00a14d587bd81f77efe17d8500384d73bee0d8f4e63c0e92897fb81aaa7e5809362fdbaf2cf329a85b0361d085ffdae55ab3450993064f7

data/.travis.yml

@@ -1,7 +1,15 @@
+sudo: false
 language: ruby
 
 rvm:
+  - 2.4.0
+  - 2.3.3
   - 2.2
   - 2.1
-  - 2.0.0
-  - 1.9.3
+
+# to avoid travis-ci issue since 2015-12-25
+before_install:
+  - gem update bundler
+
+gemfile:
+  - Gemfile

data/Appraisals

@@ -0,0 +1,7 @@
+appraise "fluentd v0.10" do
+  gem "fluentd", "~> 0.10.0"
+end
+
+appraise "fluentd v0.12" do
+  gem "fluentd", "~> 0.12.0"
+end

data/README.md

@@ -2,37 +2,52 @@
 
 ## Overview
 
-Fluentd filter output plugin to anonymize records with [OpenSSL::HMAC](http://docs.ruby-lang.org/ja/1.9.3/class/OpenSSL=3a=3aHMAC.html) of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.
+Fluentd filter output plugin to anonymize records with [OpenSSL::Digest](https://docs.ruby-lang.org/ja/latest/class/OpenSSL=3a=3aDigest.html) of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.
+
+## Requirements
+
+| fluent-plugin-anonymizer | fluentd    | ruby   |
+|--------------------|------------|--------|
+|  1.0.0            | v0.14.x | >= 2.1 |
+|  0.5.1            | v0.12.x | >= 1.9 |
+
 
 ## Installation
 
-install with gem or fluent-gem command as:
+install with gem or td-agent-gem command as:
 
 `````
-### native gem
-gem install fluent-plugin-anonymizer
+# for system installed fluentd
+$ gem install fluent-plugin-anonymizer
 
-### td-agent gem
-/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-anonymizer
+# for td-agent2 (with fluentd v0.12)
+$ sudo td-agent-gem install fluent-plugin-anonymizer -v 0.5.1
+
+# for td-agent3 (with fluentd v0.14)
+$ sudo td-agent-gem install fluent-plugin-anonymizer -v 1.0.0
 `````
 
+For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/articles/plugin-management)
+
 ## Tutorial
 
-### Output Plugin
+### Filter Plugin
 
 #### configuration
 
-It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, auto-detecting IPv4/IPv6 and rounding number with 24bit(IPv4) or 104bit(IPv6) netmask using `ipaddr_mask_keys` and `ipv4_mask_subnet`, `ipv6_mask_subnet` option.
-
-`````
+```text
 <source>
-  type forward
-  port 24224
+  @type dummy
+  tag raw.dummy
+  dummy [
+  {"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"},
+  {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
+  ]
 </source>
 
-<match test.message>
-  type anonymizer
-  
+<filter raw.**>
+  @type anonymizer
+
   # Specify hashing keys with comma
   sha1_keys         user_id, member_id, mail
   
@@ -43,45 +58,40 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
   ipaddr_mask_keys  host
   ipv4_mask_subnet  24
   ipv6_mask_subnet  104
-  
-  # Set tag rename pattern
-  remove_tag_prefix test.
-  add_tag_prefix    anonymized.
-</match>
+</filter>
 
-<match anonymized.message>
-  type stdout
+<match raw.**>
+  @type stdout
 </match>
-`````
+ ```
 
 #### result
 
-`````
-$ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
-$ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
+This sample result has made with the above configuration into "fluent.conf".
 
-$ tail -f /var/log/td-agent/td-agent.log
-2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
-2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
-`````
+```text
+$ fluentd -c fluent.conf
+2017-02-27 22:59:18.070132000 +0900 raw.dummy: {"host":"10.102.3.0","member_id":"5ab2cebb0537866c4a0cd2e2f3502c0976b788da","mail":"7e9d6dbefa72d56056c8c740b34b5c0bbfec8d87"}
+2017-02-27 22:59:19.079251000 +0900 raw.dummy: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"445514dfcd82b2a8b94ec6763afa6e349e78c5f8","mail":"54608576c8d815a4ffd595a3c1fe72751ed04424"}
+2017-02-27 22:59:20.086747000 +0900 raw.dummy: {"host":"10.102.3.0","member_id":"b14a8f98019ec84c6fe329d5af62c46bb45348f8","mail":"723da8084da3438d9287b44e5a714b70e10a9755"}
+2017-02-27 22:59:21.094767000 +0900 raw.dummy: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"d38ebb9b96c0cbffd4136935c7f6fe9dd05980cd","mail":"b6f9d777831cbecfd2ea806f5f62f79a275bbb82"}
+```
 
-### Filter Plugin
+### Output Plugin
 
 #### configuration
 
-```text
+It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, auto-detecting IPv4/IPv6 and rounding number with 24bit(IPv4) or 104bit(IPv6) netmask using `ipaddr_mask_keys` and `ipv4_mask_subnet`, `ipv6_mask_subnet` option.
+
+`````
 <source>
-  type dummy
-  tag raw.dummy
-  dummy [
-  {"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"},
-  {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
-  ]
+  @type forward
+  port 24224
 </source>
 
-<filter raw.**>
-  type anonymizer
-
+<match test.message>
+  @type anonymizer
+  
   # Specify hashing keys with comma
   sha1_keys         user_id, member_id, mail
   
@@ -92,18 +102,27 @@ $ tail -f /var/log/td-agent/td-agent.log
   ipaddr_mask_keys  host
   ipv4_mask_subnet  24
   ipv6_mask_subnet  104
-</filter>
+  
+  # Set tag rename pattern
+  tag               anonymized.${tag}
+  remove_tag_prefix test.
+</match>
 
-<match raw.**>
-  type stdout
+<match anonymized.message>
+  @type stdout
 </match>
- ```
+`````
 
 #### result
 
-```text
-$ fluentd -c fluent.conf
-```
+`````
+$ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
+$ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
+
+$ tail -f /var/log/td-agent/td-agent.log
+2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
+2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
+`````
 
 ## Parameters
 
@@ -132,6 +151,12 @@ Round number for following one or more keys. It makes easy to aggregate calculat
 * include_tag_key (default: false)
 * tag_key
 
+set one or more option are required for editing tag name using HandleTagNameMixin.
+
+* tag
+
+In the case of using this option [like 'tag anonymized.${tag}' with tag placeholder](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L179), tag will be modified after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
+
 Add original tag name into filtered record using SetTagKeyMixin.
 
 * remove_tag_prefix
@@ -139,12 +164,6 @@ Add original tag name into filtered record using SetTagKeyMixin.
 * add_tag_prefix
 * add_tag_suffix
 
-set one or more option are required for editing tag name using HandleTagNameMixin.
-
-* tag
-
-On using this option [like 'tag anonymized.${tag}' with tag placeholder](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L153), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
-
 ## Notes
 
 * hashing nested value behavior is compatible with [LogStash::Filters::Anonymize](https://github.com/logstash/logstash/blob/master/lib/logstash/filters/anonymize.rb) does. For further details, please check it out the test code at [test_emit_nest_value](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L91).

data/fluent-plugin-anonymizer.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-anonymizer"
-  spec.version       = "0.4.1"
+  spec.version       = "1.0.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.}
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit", "~> 3"
+  spec.add_development_dependency "appraisal"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
 end

data/lib/fluent/plugin/anonymizer.rb

@@ -1,3 +1,4 @@
+require 'fluent/config/error'
 require 'openssl'
 require 'ipaddr'
 
@@ -79,7 +80,7 @@ module Fluent
     def anonymize_value(message, algorithm, salt)
       case algorithm
       when 'md5','sha1','sha256','sha384','sha512'
-        OpenSSL::HMAC.hexdigest(DIGEST[algorithm].call, salt, message.to_s)
+        DIGEST[algorithm].call.update(salt).update(message.to_s).hexdigest
       when 'ipaddr_mask'
         address = IPAddr.new(message)
         subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet

data/lib/fluent/plugin/filter_anonymizer.rb

@@ -1,26 +1,290 @@
-module Fluent
+require 'fluent/plugin/filter'
+require 'openssl'
+require 'uri'
+require 'ipaddr'
+
+# <filter **>
+#   @type mask
+#   # salts will be selected for field names in a deterministic way
+#   salt secret_salt # different salt for each fields?
+#   salt salt_brabra
+#   salts ["s1","s2","s3","s4"]
+#   <mask sha1>
+#     # key user_id
+#     keys ["user_id","session_id","source_ip"]
+#     key_pattern   ^(source|src)_?ip_?(addr|address)?$
+#     value_pattern   @mydomain\.example\.com$
+#     value_in_subnet 192.168.0.0/16 # naming?
+#   </mask>
+#   <mask uri_path>
+#     keys ["url","uri"]
+#     # or key_pattern
+#   </mask>
+#   <mask network>
+#     keys ["dest","destination","dest_ip"]
+#     # or key_pattern
+#     ipv4_mask_bits 24
+#     ipv6_mask_bits 104
+#   </mask>
+# </filter>
+
+module Fluent::Plugin
   class AnonymizerFilter < Filter
-    Plugin.register_filter('anonymizer', self)
+    Fluent::Plugin.register_filter('anonymizer', self)
 
-    config_param :tag, :string, :default => nil
-    config_param :hash_salt, :string, :default => ''
-    config_param :ipv4_mask_subnet, :integer, :default => 24
-    config_param :ipv6_mask_subnet, :integer, :default => 104
+    MASK_METHODS = {
+      md5:    ->(opts){ ->(v,salt){ OpenSSL::Digest.new("md5").update(salt).update(v.to_s).hexdigest } },
+      sha1:   ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha1").update(salt).update(v.to_s).hexdigest } },
+      sha256: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha256").update(salt).update(v.to_s).hexdigest } },
+      sha384: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha384").update(salt).update(v.to_s).hexdigest } },
+      sha512: ->(opts){ ->(v,salt){ OpenSSL::Digest.new("sha512").update(salt).update(v.to_s).hexdigest } },
+      uri_path: ->(opts){ ->(v,salt){
+          begin
+            uri = URI.parse(v)
+            if uri.absolute?
+              uri.path = '/'
+              uri.user = uri.password = uri.query = uri.fragment = nil
+            end
+            uri.to_s
+          rescue
+            v
+          end
+        } },
+      network: ->(opts){ ->(v,salt){
+          begin
+            addr = IPAddr.new(v)
+            if addr.ipv4? && opts.ipv4_mask_bits
+              addr.mask(opts.ipv4_mask_bits).to_s
+            elsif addr.ipv6? && opts.ipv6_mask_bits
+              addr.mask(opts.ipv6_mask_bits).to_s
+            else
+              addr.to_s
+            end
+          rescue
+            v
+          end
+        } },
+    }
 
-    config_set_default :include_tag_key, false
+    config_param :salt, :string, default: nil
+    config_param :salts, :array, default: nil
+    config_section :mask, param_name: :mask_config_list, required: false, multi: true do
+      config_argument :method, :enum, list: MASK_METHODS.keys
+      config_param :salt, :string, default: nil
+
+      config_param :key,             :string, default: nil
+      config_param :keys,            :array,  default: []
+      config_param :key_chain,       :string, default: nil # for.nested.key
+      config_param :key_chains,      :array,  default: []  # ["for.nested.key","can.be.specified","twice.or.more"]
+      config_param :key_pattern,     :string, default: nil
+      config_param :value_pattern,   :string, default: nil
+      config_param :value_in_subnet, :string, default: nil # 192.168.0.0/24
+
+      config_param :mask_array_elements, :bool, default: false
+      config_param :ipv4_mask_bits, :integer, default: nil
+      config_param :ipv6_mask_bits, :integer, default: nil
+    end
+
+    # obsolete configuration parameters
+    config_param :md5_keys,    :string, default: nil
+    config_param :sha1_keys,   :string, default: nil
+    config_param :sha256_keys, :string, default: nil
+    config_param :sha384_keys, :string, default: nil
+    config_param :sha512_keys, :string, default: nil
+    config_param :hash_salt,   :string, default: nil
+    config_param :ipaddr_mask_keys, :string, default: nil
+    config_param :ipv4_mask_subnet, :integer, default: 24
+    config_param :ipv6_mask_subnet, :integer, default: 104
 
     def initialize
       super
-      require 'fluent/plugin/anonymizer'
+      @salt_list = []
+      @salt_map = {}
+      @conversions = []
     end
 
     def configure(conf)
       super
-      @anonymizer = Anonymizer.new(self, conf)
+
+      salt_missing = false
+
+      @salt_list << @salt
+      @salt_list += @salts if @salts
+
+      @masks = []
+      @mask_config_list.each do |c|
+        unless c.salt || @salt_list.size > 0
+          salt_missing = true
+        end
+
+        conv = MASK_METHODS[c.method].call(c)
+        [c.key || nil, *c.keys].compact.each do |key|
+          @masks << masker_for_key(conv, key, c)
+        end
+        [c.key_chain || nil, *c.key_chains].compact.each do |key_chain|
+          @masks << masker_for_key_chain(conv, key_chain.split('.'), c)
+        end
+        @masks << masker_for_key_pattern(conv, c.key_pattern, c) if c.key_pattern
+        @masks << masker_for_value_pattern(conv, c.value_pattern, c) if c.value_pattern
+        @masks << masker_for_value_in_subnet(conv, c.value_in_subnet, c) if c.value_in_subnet
+      end
+
+      # obsolete option handling
+      [[@md5_keys,:md5],[@sha1_keys,:sha1],[@sha256_keys,:sha256],[@sha384_keys,:sha384],[@sha512_keys,:sha512]].each do |param,m|
+        next unless param
+        @salt_list << (@hash_salt || '') if @salt_list.empty? # to suppress ConfigError for salt missing
+        conf = OpenStruct.new
+        conf.salt = @hash_salt || ''
+        conf.mask_array_elements = true
+        conv = MASK_METHODS[m].call(conf)
+        param.split(',').map(&:strip).each do |key|
+          if key.include?('.')
+            @masks << masker_for_key_chain(conv, key.split('.'), conf)
+          else
+            @masks << masker_for_key(conv, key, conf)
+          end
+        end
+      end
+      if @ipaddr_mask_keys
+        @salt_list << (@hash_salt || '') if @salt_list.empty? # to suppress ConfigError for salt missing
+        conf = OpenStruct.new
+        conf.salt = @hash_salt || ''
+        conf.mask_array_elements = true
+        conf.ipv4_mask_bits = @ipv4_mask_subnet
+        conf.ipv6_mask_bits = @ipv6_mask_subnet
+        conv = MASK_METHODS[:network].call(conf)
+        @ipaddr_mask_keys.split(',').map(&:strip).each do |key|
+          if key.include?('.')
+            @masks << masker_for_key_chain(conv, key.split('.'), conf)
+          else
+            @masks << masker_for_key(conv, key, conf)
+          end
+        end
+      end
+
+      if @masks.size < 1
+        raise Fluent::ConfigError, "no anonymizing operations configured"
+      end
+      if salt_missing
+        raise Fluent::ConfigError, "salt (or salts) required, but missing"
+      end
     end
 
     def filter(tag, time, record)
-      record = @anonymizer.anonymize(record)
+      record.update(@masks.reduce(record){|r,mask| mask.call(r)})
+    end
+
+    def salt_determine(key)
+      return @salt_map[key] if @salt_map.has_key?(key)
+      keystr = key.to_s
+      if keystr.empty?
+        @salt_map[key] = @salt_list[0]
+      else
+        @salt_map[key] = @salt_list[(keystr[0].ord + keystr[-1].ord) % @salt_list.size]
+      end
+      @salt_map[key]
+    end
+
+    def mask_value(value, for_each)
+      if for_each && value.is_a?(Array)
+        value.map{|v|
+          yield v
+        }
+      else
+        yield value
+      end
+    end
+
+    def masker_for_key(conv, key, opts)
+      for_each = opts.mask_array_elements
+      salt = opts.salt || salt_determine(key)
+      if for_each
+        ->(record){
+          begin
+            if record.has_key?(key)
+              record[key] = mask_value(record[key], for_each){|v| conv.call(v, salt) }
+            end
+          rescue => e
+            log.error "unexpected error while masking value", error_class: e.class, error: e.message
+          end
+          record
+        }
+      else
+        ->(record){
+          begin
+            if record.has_key?(key)
+              record[key] = conv.call(record[key], salt)
+            end
+          rescue => e
+            log.error "unexpected error while masking value", error_class: e.class, error: e.message
+          end
+          record
+        }
+      end
+    end
+
+    def masker_for_key_chain(conv, key_chain, opts)
+      for_each = opts.mask_array_elements
+      heading = key_chain[0..-2]
+      container_fetcher = ->(record){ heading.reduce(record){|r,c| r && r.has_key?(c) ? r[c] : nil } }
+      tailing = key_chain[-1]
+      ->(record){
+        begin
+          container = container_fetcher.call(record)
+          if container && container.has_key?(tailing)
+            container[tailing] = mask_value(container[tailing], for_each){|v| conv.call(v, opts.salt || salt_determine(tailing)) }
+          end
+        rescue => e
+          log.error "unexpected error while masking value", error_class: e.class, error: e.message
+        end
+        record
+      }
+    end
+
+    def masker_for_key_pattern(conv, pattern, opts)
+      for_each = opts.mask_array_elements
+      regexp = Regexp.new(pattern)
+      ->(record){
+        begin
+          record.each_pair do |key, value|
+            next unless (regexp =~ key.to_s rescue nil)
+            record[key] = mask_value(record[key], for_each){|v| conv.call(v, opts.salt || salt_determine(key)) }
+          end
+        rescue => e
+          log.error "unexpected error while masking value", error_class: e.class, error: e.message
+        end
+        record
+      }
+    end
+
+    def masker_for_value_pattern(conv, pattern, opts)
+      regexp = Regexp.new(pattern)
+      ->(record){
+        begin
+          record.each_pair do |key, value|
+            next unless (regexp =~ value.to_s rescue nil)
+            record[key] = conv.call(value, opts.salt || salt_determine(key))
+          end
+        rescue => e
+          log.error "unexpected error while masking value", error_class: e.class, error: e.message
+        end
+        record
+      }
+    end
+
+    def masker_for_value_in_subnet(conv, network_str, opts)
+      network = IPAddr.new(network_str)
+      ->(record){
+        begin
+          record.each_pair do |key, value|
+            next unless (network.include?(value) rescue nil)
+            record[key] = conv.call(value, opts.salt || salt_determine(key))
+          end
+        rescue => e
+          log.error "unexpected error while masking value", error_class: e.class, error: e.message
+        end
+        record
+      }
     end
   end
 end

data/lib/fluent/plugin/out_anonymizer.rb

@@ -13,10 +13,17 @@ class Fluent::AnonymizerOutput < Fluent::Output
     define_method("router") { Fluent::Engine }
   end
 
-  config_param :tag, :string, :default => nil
-  config_param :hash_salt, :string, :default => ''
-  config_param :ipv4_mask_subnet, :integer, :default => 24
-  config_param :ipv6_mask_subnet, :integer, :default => 104
+  config_param :tag, :string, :default => nil,
+               :desc => 'The output tag.'
+  config_param :hash_salt, :string, :default => '',
+               :desc => <<-DESC
+This salt affects for md5_keys sha1_keys sha256_keys sha384_keys sha512_keys settings.
+It is recommend to set a hash salt to prevent rainbow table attacks.
+DESC
+  config_param :ipv4_mask_subnet, :integer, :default => 24,
+               :desc => 'Anonymize ipv4 addresses by subnet mask.'
+  config_param :ipv6_mask_subnet, :integer, :default => 104,
+               :desc => 'Anonymize ipv6 addresses by subnet mask.'
 
   include Fluent::HandleTagNameMixin
   include Fluent::Mixin::RewriteTagName
@@ -29,6 +36,7 @@ class Fluent::AnonymizerOutput < Fluent::Output
   end
 
   def configure(conf)
+    log.warn "out_anonymizer is now deprecated. It will be removed in a future release. Please consider to use filter_anonymizer."
     super
     @anonymizer = Fluent::Anonymizer.new(self, conf)
   end

data/test/helper.rb

@@ -23,7 +23,10 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_anonymizer'
-require 'fluent/plugin/filter_anonymizer'
+if Fluent.const_defined?(:Filter)
+  require 'fluent/plugin/filter_anonymizer'
+end
+
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_anonymizer.rb

@@ -1,7 +1,9 @@
 require 'helper'
+require 'fluent/test/driver/filter'
 
 class AnonymizerFilterTest < Test::Unit::TestCase
   def setup
+    omit_unless(Fluent.const_defined?(:Filter))
     Fluent::Test.setup
     @time = Fluent::Engine.now
   end
@@ -17,19 +19,66 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     ipv4_mask_subnet  24
   ]
 
-  def create_driver(conf=CONFIG, tag='test')
-    Fluent::Test::FilterTestDriver.new(Fluent::AnonymizerFilter, tag).configure(conf)
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::AnonymizerFilter).configure(conf)
   end
 
   def filter(conf, messages)
     d = create_driver(conf)
-    d.run {
+    d.run(default_tag: 'test') {
       messages.each {|message|
-        d.filter(message, @time)
+        d.feed(@time, message)
       }
     }
-    filtered = d.filtered_as_array
-    filtered.map {|m| m[2] }
+    filtered = d.filtered
+    filtered.map {|m| m.last }
+  end
+
+  require 'ostruct'
+  test 'method md5 works correctly' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:md5].call(OpenStruct.new)
+    digest = conv.call('value1', 'salt')
+    assert_equal 'd21fe9523421f12daad064fd082913fd', digest
+  end
+  test 'method sha1 works correctly' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:sha1].call(OpenStruct.new)
+    digest = conv.call('value2', 'salt')
+    assert_equal 'd2ed8e797065322371012fd8c1a39682987ddb71', digest
+  end
+  test 'method sha256 works correctly' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:sha256].call(OpenStruct.new)
+    digest = conv.call('value3', 'salt')
+    assert_equal 'd70daf9654b8a3ba335f8f9f9638a93e8eba6763a0012ac44a928857871abe82', digest
+  end
+  test 'method sha384 works correctly' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:sha384].call(OpenStruct.new)
+    digest = conv.call('value4', 'salt')
+    assert_equal '646192f8b1ea905238df589a00a10598a53eb245df4ab14b7e9eccf80c37386c99abe5259ccb2ba950003423fa0790ee', digest
+  end
+  test 'method sha512 works correctly' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:sha512].call(OpenStruct.new)
+    digest = conv.call('value5', 'salt')
+    expected = '47c82bfea3783c20e3ba3629f0f827bebf0fa65a9104ada5339e5776e5958f061fe7114bfbe1e9d410aff43c6bee8365adf4fdd072e54ab4fffad820f354f545'
+    assert_equal expected, digest
+  end
+  test 'method uri_path removes path, query parameters, fragment, user and password of uri strings' do
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:uri_path].call(OpenStruct.new)
+    assert_equal '/my/path', conv.call('/my/path', '')
+    assert_equal 'yay/unknown/format', conv.call('yay/unknown/format', '')
+    assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret', '')
+    assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret?a=b', '')
+    assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret#xxx', '')
+    assert_equal 'http://example.com/', conv.call('http://example.com/path/to/secret?a=b#xxx', '')
+    assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/', '')
+    assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/?a=b#xxx', '')
+    assert_equal 'http://example.com/', conv.call('http://tagomoris:secret!@example.com/path/to/secret?a=b#xxx', '')
+  end
+  test 'method network masks ipaddresses with specified mask bit lengths' do
+    conf = OpenStruct.new(ipv4_mask_bits: 24, ipv6_mask_bits: 104)
+    conv = Fluent::Plugin::AnonymizerFilter::MASK_METHODS[:network].call(conf)
+    assert_equal '192.168.1.0', conv.call('192.168.1.1', '')
+    assert_equal '10.110.18.0', conv.call('10.110.18.9', '')
+    assert_equal '2001:db8:0:8d3:0:8a2e::', conv.call('2001:db8:0:8d3:0:8a2e:70:7344', '')
   end
 
   def test_configure
@@ -43,6 +92,91 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     assert_equal 'test_salt_string', d.instance.config['hash_salt']
   end
 
+  test 'masker_for_key generates a lambda for conversion with exact key match' do
+    conf = OpenStruct.new(salt: 's')
+    plugin = create_driver.instance
+    conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
+    masker = plugin.masker_for_key(conv, 'kkk', conf)
+    r = masker.call({"k" => "x", "kk" => "xx", "kkk" => "xxx", "kkkk" => "xxxx"})
+    assert_equal "x", r["k"]
+    assert_equal "xx", r["kk"]
+    assert_equal "XXX:s", r["kkk"]
+    assert_equal "xxxx", r["kkkk"]
+    assert_equal 4, r.size
+  end
+
+  test 'masker_for_key_chain generates a lambda for conversion with recursive key fetching' do
+    conf = OpenStruct.new(salt: 's')
+    plugin = create_driver.instance
+    conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
+    masker = plugin.masker_for_key_chain(conv, 'a.b.c'.split('.'), conf)
+    event = {
+      'a' => {
+        'b' => { 'c' => 'v', 'd' => 'v' }
+      }
+    }
+    r = masker.call(event)
+    assert_equal 'V:s', r['a']['b']['c']
+    assert_equal 'v', r['a']['b']['d']
+  end
+
+  test 'masker_for_key_pattern generates a lambda for conversion with key pattern match' do
+    conf = OpenStruct.new(salt: 's')
+    plugin = create_driver.instance
+    conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
+    masker = plugin.masker_for_key_pattern(conv, '^k+$', conf)
+    r = masker.call({"k" => "x", "kk" => "xx", "kkk" => "xxx", "kkk0" => "xxxx", "f" => "x", "ff" => "xx"})
+    assert_equal "X:s", r["k"]
+    assert_equal "XX:s", r["kk"]
+    assert_equal "XXX:s", r["kkk"]
+    assert_equal "xxxx", r["kkk0"]
+    assert_equal "x", r["f"]
+    assert_equal "xx", r["ff"]
+    assert_equal 6, r.size
+  end
+
+  test 'masker_for_value_pattern' do
+    conf = OpenStruct.new(salt: 's')
+    plugin = create_driver.instance
+    conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
+    masker = plugin.masker_for_value_pattern(conv, '^x+$', conf)
+    r = masker.call({"k" => "x", "kk" => "x0", "kkk" => "xx0", "kkk0" => "xxxx", "f" => "x", "ff" => "xx"})
+    assert_equal "X:s", r["k"]
+    assert_equal "x0", r["kk"]
+    assert_equal "xx0", r["kkk"]
+    assert_equal "XXXX:s", r["kkk0"]
+    assert_equal "X:s", r["f"]
+    assert_equal "XX:s", r["ff"]
+    assert_equal 6, r.size
+  end
+
+  test 'masker_for_value_in_subnet' do
+    conf = OpenStruct.new(salt: 's')
+    plugin = create_driver.instance
+    conv = ->(v,salt){ "#{v.upcase}:#{salt}" } # it's dummy for test
+    masker = plugin.masker_for_value_in_subnet(conv, '192.168.0.0/16', conf)
+    r = masker.call({"k1" => "x", "k2" => "192.169.1.1", "k3" => "192.168.128.13", "f1" => "x", "f2" => "10.0.12.1", "f3" => "192.168.1.1"})
+    assert_equal "x", r["k1"]
+    assert_equal "192.169.1.1", r["k2"]
+    assert_equal "192.168.128.13:s", r["k3"]
+    assert_equal "x", r["f1"]
+    assert_equal "10.0.12.1", r["f2"]
+    assert_equal "192.168.1.1:s", r["f3"]
+  end
+
+  test 'filter plugin can mask specified fields' do
+    plugin = create_driver(<<-CONF).instance
+      <mask md5>
+        salt testing
+        key  test
+      </mask>
+CONF
+    r = plugin.filter('tag', Time.now.to_i, {"test" => "value", "name" => "fluentd plugin"})
+    assert_equal 2, r.size
+    assert_equal "fluentd plugin", r["name"]
+    assert_equal "6255093f2e4204e24df48ddd7f4a8abe", r["test"]
+  end
+
   def test_filter
     messages = [
       {
@@ -56,11 +190,11 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     ]
     expected = {
       'host'            => '10.102.3.0',
-      'data_for_md5'    => 'e738cbde82a514dc60582cd467c240ed',
-      'data_for_sha1'   => '69cf099459c06b852ede96d39b710027727d13c6',
-      'data_for_sha256' => '804d83b8c6a3e01498d40677652b084333196d8e548ee5a8710fbd0e1e115527',
-      'data_for_sha384' => '6c90c389bbdfc210416b9318df3f526b4f218f8a8df3a67020353c35da22dc154460b18f22a8009a747b3ef2975acae7',
-      'data_for_sha512' => 'cdbb897e6f3a092161bdb51164eb2996b75b00555f568219628ff15cd2929865d217af5dff9c32ddc908b75a89baec96b3e9a0da120e919f5246de0f1bc54c58'
+      'data_for_md5'    => '9138bd41172f5485f7b6eee3afcd0d62',
+      'data_for_sha1'   => 'ee98db51658d38580b1cf788db19ad06e51a32f7',
+      'data_for_sha256' => 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd',
+      'data_for_sha384' => '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131',
+      'data_for_sha512' => 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20'
     }
     filtered = filter(CONFIG, messages)
     assert_equal(expected, filtered[0])
@@ -85,9 +219,9 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     expected = {
       'host'      => '10.102.0.0',
       'host2'     => '10.102.0.0',
-      'member_id' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7',
-      'mail'      => 'd7b728209f5dd8df10cecbced30394c3c7fc2c82',
-      'telephone' => 'a67f73c395105a358a03a0f127bf64b5495e7841',
+      'member_id' => '8cb2237d0679ca88db6464eac60da96345513964',
+      'mail'      => '914fec35ce8bfa1a067581032f26b053591ee38a',
+      'telephone' => 'ce164718b94212332187eb8420903b46b334d609',
       'action'    => 'signup'
     }
     filtered = filter(conf, messages)
@@ -118,9 +252,9 @@ class AnonymizerFilterTest < Test::Unit::TestCase
         'host1' => '10.102.0.0'
       },
       'nested' => {
-        'data' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7',
+        'data' => '8cb2237d0679ca88db6464eac60da96345513964',
         'nested' => {
-          'data' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7'
+          'data' => '8cb2237d0679ca88db6464eac60da96345513964'
         }
       }
     }
@@ -142,8 +276,8 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     ]
     expected = {
       'host' => '10.102.3.0',
-      'array' => ["c1628fc0d473cb21b15607c10bdcad19d1a42e24", "ea87abc249f9f2d430edb816514bffeffd3e698e"],
-      'hash' => '28fe85deb0d1d39ee14c49c62bc4773b0338247b'
+      'array' => ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"],
+      'hash' => '1a1903d78aed9403649d61cb21ba6b489249761b'
     }
     filtered = filter(conf, messages)
     assert_equal(expected, filtered[0])
@@ -169,4 +303,3 @@ class AnonymizerFilterTest < Test::Unit::TestCase
     assert_equal(expected, filtered)
   end
 end
-

data/test/plugin/test_out_anonymizer.rb

@@ -49,11 +49,11 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal 1, emits.length
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.3.0', emits[0][2]['host']
-    assert_equal 'e738cbde82a514dc60582cd467c240ed', emits[0][2]['data_for_md5']
-    assert_equal '69cf099459c06b852ede96d39b710027727d13c6', emits[0][2]['data_for_sha1']
-    assert_equal '804d83b8c6a3e01498d40677652b084333196d8e548ee5a8710fbd0e1e115527', emits[0][2]['data_for_sha256']
-    assert_equal '6c90c389bbdfc210416b9318df3f526b4f218f8a8df3a67020353c35da22dc154460b18f22a8009a747b3ef2975acae7', emits[0][2]['data_for_sha384']
-    assert_equal 'cdbb897e6f3a092161bdb51164eb2996b75b00555f568219628ff15cd2929865d217af5dff9c32ddc908b75a89baec96b3e9a0da120e919f5246de0f1bc54c58', emits[0][2]['data_for_sha512']
+    assert_equal '9138bd41172f5485f7b6eee3afcd0d62', emits[0][2]['data_for_md5']
+    assert_equal 'ee98db51658d38580b1cf788db19ad06e51a32f7', emits[0][2]['data_for_sha1']
+    assert_equal 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd', emits[0][2]['data_for_sha256']
+    assert_equal '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131', emits[0][2]['data_for_sha384']
+    assert_equal 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20', emits[0][2]['data_for_sha512']
   end
 
   def test_emit_multi_keys
@@ -79,9 +79,9 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.0.0', emits[0][2]['host']
     assert_equal '10.102.0.0', emits[0][2]['host2']
-    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
-    assert_equal 'd7b728209f5dd8df10cecbced30394c3c7fc2c82', emits[0][2]['mail']
-    assert_equal 'a67f73c395105a358a03a0f127bf64b5495e7841', emits[0][2]['telephone']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
+    assert_equal '914fec35ce8bfa1a067581032f26b053591ee38a', emits[0][2]['mail']
+    assert_equal 'ce164718b94212332187eb8420903b46b334d609', emits[0][2]['telephone']
     assert_equal 'signup', emits[0][2]['action']
   end
 
@@ -110,8 +110,8 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal 1, emits.length
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.0.0', emits[0][2]['hosts']['host1']
-    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['nested']['data']
-    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['nested']['nested']['data']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['nested']['data']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['nested']['nested']['data']
   end
 
   def test_emit_nest_value
@@ -132,8 +132,8 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal 1, emits.length
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.3.0', emits[0][2]['host']
-    assert_equal ["c1628fc0d473cb21b15607c10bdcad19d1a42e24", "ea87abc249f9f2d430edb816514bffeffd3e698e"], emits[0][2]['array']
-    assert_equal '28fe85deb0d1d39ee14c49c62bc4773b0338247b', emits[0][2]['hash']
+    assert_equal ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"], emits[0][2]['array']
+    assert_equal '1a1903d78aed9403649d61cb21ba6b489249761b', emits[0][2]['hash']
   end
 
   def test_emit_ipv6
@@ -170,7 +170,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     emits = d1.emits
     assert_equal 1, emits.length
     assert_equal 'anonymized.message', emits[0][0] # tag
-    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
   end
 
   def test_emit_tag_placeholder
@@ -187,7 +187,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     emits = d1.emits
     assert_equal 1, emits.length
     assert_equal 'anonymized.access', emits[0][0] # tag
-    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
   end
 end
 

metadata

@@ -1,94 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anonymizer
 version: !ruby/object:Gem::Version
-  version: 0.4.1
-  prerelease: 
+  version: 1.0.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-01 00:00:00.000000000 Z
+date: 2017-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: appraisal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-rewrite-tag-name
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: 
@@ -98,8 +101,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
+- Appraisals
 - Gemfile
 - LICENSE
 - README.md
@@ -114,27 +118,26 @@ files:
 homepage: https://github.com/y-ken/fluent-plugin-anonymizer
 licenses:
 - Apache License, Version 2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512
   algorithms. This data masking plugin protects privacy data such as UserID, Email,
   Phone number, IPv4/IPv6 address and so on.