fluent-plugin-anonymizer 0.3.0 → 0.4.0

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 
 rvm:
+  - 2.2
   - 2.1
   - 2.0.0
   - 1.9.3

data/README.md

@@ -18,6 +18,8 @@ gem install fluent-plugin-anonymizer
 
 ## Tutorial
 
+### Output Plugin
+
 #### configuration
 
 It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, auto-detecting IPv4/IPv6 and rounding number with 24bit(IPv4) or 104bit(IPv6) netmask using `ipaddr_mask_keys` and `ipv4_mask_subnet`, `ipv6_mask_subnet` option.
@@ -63,6 +65,46 @@ $ tail -f /var/log/td-agent/td-agent.log
 2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
 `````
 
+### Filter Plugin
+
+#### configuration
+
+```text
+<source>
+  type dummy
+  tag raw.dummy
+  dummy [
+  {"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"},
+  {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
+  ]
+</source>
+
+<filter raw.**>
+  type anonymizer
+
+  # Specify hashing keys with comma
+  sha1_keys         user_id, member_id, mail
+  
+  # Set hash salt with any strings for more security
+  hash_salt         mysaltstring
+  
+  # Specify rounding address keys with comma and subnet mask
+  ipaddr_mask_keys  host
+  ipv4_mask_subnet  24
+  ipv6_mask_subnet  104
+</filter>
+
+<match raw.**>
+  type stdout
+</match>
+ ```
+
+#### result
+
+```text
+$ fluentd -c fluent.conf
+```
+
 ## Parameters
 
 * `md5_keys` `sha1_keys` `sha256_keys` `sha384_keys` `sha512_keys`

data/fluent-plugin-anonymizer.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-anonymizer"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
   spec.summary       = %q{Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.}
@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "test-unit", "~> 3"
   spec.add_runtime_dependency "fluentd"
   spec.add_runtime_dependency "fluent-mixin-rewrite-tag-name"
 end

data/lib/fluent/plugin/anonymizer.rb

@@ -0,0 +1,98 @@
+require 'openssl'
+require 'ipaddr'
+
+module Fluent
+  class Anonymizer
+
+    attr_reader :log
+
+    HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipaddr_mask)
+    DIGEST = {
+      "md5" => Proc.new { OpenSSL::Digest.new('md5') },
+      "sha1" => Proc.new { OpenSSL::Digest.new('sha1') },
+      "sha256" => Proc.new { OpenSSL::Digest.new('sha256') },
+      "sha384" => Proc.new { OpenSSL::Digest.new('sha384') },
+      "sha512" => Proc.new { OpenSSL::Digest.new('sha512') }
+    }
+
+    def initialize(plugin, conf)
+      @log = plugin.log
+      @hash_salt = plugin.hash_salt
+      @ipv4_mask_subnet = plugin.ipv4_mask_subnet
+      @ipv6_mask_subnet = plugin.ipv6_mask_subnet
+
+      @hash_keys = {}
+      conf.keys.select{|k| k =~ /_keys$/}.each do |key|
+        hash_algorithm_name = key.sub('_keys','')
+        raise Fluent::ConfigError, "anonymizer: unsupported key #{hash_algorithm_name}" unless HASH_ALGORITHM.include?(hash_algorithm_name)
+        conf[key].gsub(' ', '').split(',').each do |record_key|
+          @hash_keys.store(record_key.split('.'), hash_algorithm_name)
+        end
+      end
+
+      if @hash_keys.empty?
+        raise Fluent::ConfigError, "anonymizer: missing hash keys setting."
+      end
+      log.info "anonymizer: adding anonymize rules for each field. #{@hash_keys}"
+
+      if plugin.is_a?(Fluent::Output)
+        unless have_tag_option?(plugin)
+          raise Fluent::ConfigError, "anonymizer: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
+        end
+      end
+    end
+
+    def anonymize(record)
+      @hash_keys.each do |hash_key, hash_algorithm|
+        record = anonymize_record(record, hash_key, hash_algorithm)
+      end
+      record
+    end
+
+    private
+
+    def anonymize_record(record, key, hash_algorithm)
+      if record.has_key?(key.first)
+        if key.size == 1
+          record[key.first] = anonymize_values(record[key.first], hash_algorithm)
+        else
+          record[key.first] = anonymize_record(record[key.first], key[1..-1], hash_algorithm)
+        end
+      end
+      record
+    end
+
+    def anonymize_values(data, hash_algorithm)
+      begin
+        if data.is_a?(Array)
+          data = data.collect { |v| anonymize_value(v, hash_algorithm, @hash_salt) }
+        else
+          data = anonymize_value(data, hash_algorithm, @hash_salt)
+        end
+      rescue => e
+        log.error "anonymizer: failed to anonymize record. :message=>#{e.message} :data=>#{data}"
+        log.error e.backtrace.join("\n")
+      end
+      data
+    end
+
+    def anonymize_value(message, algorithm, salt)
+      case algorithm
+      when 'md5','sha1','sha256','sha384','sha512'
+        OpenSSL::HMAC.hexdigest(DIGEST[algorithm].call, salt, message.to_s)
+      when 'ipaddr_mask'
+        address = IPAddr.new(message)
+        subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet
+        address.mask(subnet).to_s
+      else
+        log.warn "anonymizer: unknown algorithm #{algorithm} has called."
+      end
+    end
+
+    def have_tag_option?(plugin)
+      plugin.tag ||
+        plugin.remove_tag_prefix || plugin.remove_tag_suffix ||
+        plugin.add_tag_prefix || plugin.add_tag_suffix
+    end
+  end
+end

data/lib/fluent/plugin/filter_anonymizer.rb

@@ -0,0 +1,26 @@
+module Fluent
+  class AnonymizerFilter < Filter
+    Plugin.register_filter('anonymizer', self)
+
+    config_param :tag, :string, :default => nil
+    config_param :hash_salt, :string, :default => ''
+    config_param :ipv4_mask_subnet, :integer, :default => 24
+    config_param :ipv6_mask_subnet, :integer, :default => 104
+
+    config_set_default :include_tag_key, false
+
+    def initialize
+      super
+      require 'fluent/plugin/anonymizer'
+    end
+
+    def configure(conf)
+      super
+      @anonymizer = Anonymizer.new(self, conf)
+    end
+
+    def filter(tag, time, record)
+      record = @anonymizer.anonymize(record)
+    end
+  end
+end

data/lib/fluent/plugin/out_anonymizer.rb

@@ -8,7 +8,6 @@ class Fluent::AnonymizerOutput < Fluent::Output
     define_method(:log) { $log }
   end
 
-  HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipaddr_mask)
   config_param :tag, :string, :default => nil
   config_param :hash_salt, :string, :default => ''
   config_param :ipv4_mask_subnet, :integer, :default => 24
@@ -19,90 +18,24 @@ class Fluent::AnonymizerOutput < Fluent::Output
   include Fluent::SetTagKeyMixin
   config_set_default :include_tag_key, false
 
-  DIGEST = {
-    "md5" => Proc.new { OpenSSL::Digest.new('md5') },
-    "sha1" => Proc.new { OpenSSL::Digest.new('sha1') },
-    "sha256" => Proc.new { OpenSSL::Digest.new('sha256') },
-    "sha384" => Proc.new { OpenSSL::Digest.new('sha384') },
-    "sha512" => Proc.new { OpenSSL::Digest.new('sha512') }
-  }
-
   def initialize
-    require 'openssl'
-    require 'ipaddr'
+    require 'fluent/plugin/anonymizer'
     super
   end
 
   def configure(conf)
     super
-
-    @hash_keys = Hash.new
-    conf.keys.select{|k| k =~ /_keys$/}.each do |key|
-      hash_algorithm_name = key.sub('_keys','')
-      raise Fluent::ConfigError, "anonymizer: unsupported key #{hash_algorithm_name}" unless HASH_ALGORITHM.include?(hash_algorithm_name)
-      conf[key].gsub(' ', '').split(',').each do |record_key|
-        @hash_keys.store(record_key.split('.'), hash_algorithm_name)
-      end
-    end
-
-    if @hash_keys.count < 1
-      raise Fluent::ConfigError, "anonymizer: missing hash keys setting."
-    end
-    log.info "anonymizer: adding anonymize rules for each field. #{@hash_keys}"
-
-    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
-      raise Fluent::ConfigError, "anonymizer: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
-    end
+    @anonymizer = Fluent::Anonymizer.new(self, conf)
   end
 
   def emit(tag, es, chain)
     es.each do |time, record|
-      @hash_keys.each do |hash_key, hash_algorithm|
-        record = filter_anonymize_record(record, hash_key, hash_algorithm)
-      end
+      record = @anonymizer.anonymize(record)
       emit_tag = tag.dup
       filter_record(emit_tag, time, record)
       Fluent::Engine.emit(emit_tag, time, record)
     end
     chain.next
   end
-
-  def filter_anonymize_record(record, key, hash_algorithm)
-    if record.has_key?(key.first)
-      if key.size == 1
-        record[key.first] = filter_anonymize_value(record[key.first], hash_algorithm)
-      else
-        record[key.first] = filter_anonymize_record(record[key.first], key[1..-1], hash_algorithm)
-      end
-    end
-    return record
-  end
-
-  def filter_anonymize_value(data, hash_algorithm)
-    begin
-      if data.is_a?(Array)
-        data = data.collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
-      else
-        data = anonymize(data, hash_algorithm, @hash_salt)
-      end
-    rescue StandardError => e
-      log.error "anonymizer: failed to anonymize record. :message=>#{e.message} :data=>#{data}"
-      log.error e.backtrace.join("\n")
-    end
-    data
-  end
-
-  def anonymize(message, algorithm, salt)
-    case algorithm
-    when 'md5','sha1','sha256','sha384','sha512'
-      OpenSSL::HMAC.hexdigest(DIGEST[algorithm].call, salt, message.to_s)
-    when 'ipaddr_mask'
-      address = IPAddr.new(message)
-      subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet
-      address.mask(subnet).to_s
-    else
-      log.warn "anonymizer: unknown algorithm #{algorithm} has called."
-    end
-  end
 end
 

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_anonymizer'
+require 'fluent/plugin/filter_anonymizer'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_anonymizer.rb

@@ -0,0 +1,172 @@
+require 'helper'
+
+class AnonymizerFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  CONFIG = %[
+    md5_keys          data_for_md5
+    sha1_keys         data_for_sha1
+    sha256_keys       data_for_sha256
+    sha384_keys       data_for_sha384
+    sha512_keys       data_for_sha512
+    hash_salt         test_salt_string
+    ipaddr_mask_keys  host
+    ipv4_mask_subnet  24
+  ]
+
+  def create_driver(conf=CONFIG, tag='test')
+    Fluent::Test::FilterTestDriver.new(Fluent::AnonymizerFilter, tag).configure(conf)
+  end
+
+  def filter(conf, messages)
+    d = create_driver(conf)
+    d.run {
+      messages.each {|message|
+        d.filter(message, @time)
+      }
+    }
+    filtered = d.filtered_as_array
+    filtered.map {|m| m[2] }
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('unknown_keys')
+    }
+    d = create_driver(CONFIG)
+    assert_equal 'test_salt_string', d.instance.config['hash_salt']
+  end
+
+  def test_filter
+    messages = [
+      {
+        'host'            => '10.102.3.80',
+        'data_for_md5'    => '12345',
+        'data_for_sha1'   => '12345',
+        'data_for_sha256' => '12345',
+        'data_for_sha384' => '12345',
+        'data_for_sha512' => '12345'
+      }
+    ]
+    expected = {
+      'host'            => '10.102.3.0',
+      'data_for_md5'    => 'e738cbde82a514dc60582cd467c240ed',
+      'data_for_sha1'   => '69cf099459c06b852ede96d39b710027727d13c6',
+      'data_for_sha256' => '804d83b8c6a3e01498d40677652b084333196d8e548ee5a8710fbd0e1e115527',
+      'data_for_sha384' => '6c90c389bbdfc210416b9318df3f526b4f218f8a8df3a67020353c35da22dc154460b18f22a8009a747b3ef2975acae7',
+      'data_for_sha512' => 'cdbb897e6f3a092161bdb51164eb2996b75b00555f568219628ff15cd2929865d217af5dff9c32ddc908b75a89baec96b3e9a0da120e919f5246de0f1bc54c58'
+    }
+    filtered = filter(CONFIG, messages)
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_multi_keys
+    conf = %[
+      sha1_keys         member_id, mail, telephone
+      ipaddr_mask_keys  host, host2
+      ipv4_mask_subnet  16
+    ]
+    messages = [
+      {
+        'host'      => '10.102.3.80',
+        'host2'     => '10.102.3.80',
+        'member_id' => '12345',
+        'mail'      => 'example@example.com',
+        'telephone' => '00-0000-0000',
+        'action'    => 'signup'
+      }
+    ]
+    expected = {
+      'host'      => '10.102.0.0',
+      'host2'     => '10.102.0.0',
+      'member_id' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7',
+      'mail'      => 'd7b728209f5dd8df10cecbced30394c3c7fc2c82',
+      'telephone' => 'a67f73c395105a358a03a0f127bf64b5495e7841',
+      'action'    => 'signup'
+    }
+    filtered = filter(conf, messages)
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_nested_keys
+    conf = %[
+      sha1_keys         nested.data,nested.nested.data
+      ipaddr_mask_keys  hosts.host1
+      ipv4_mask_subnet  16
+    ]
+    messages = [
+      {
+        'hosts' => {
+          'host1' => '10.102.3.80',
+        },
+        'nested' => {
+          'data' => '12345',
+          'nested' => {
+            'data' => '12345'
+          }
+        }
+      }
+    ]
+    expected = {
+      'hosts' => {
+        'host1' => '10.102.0.0'
+      },
+      'nested' => {
+        'data' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7',
+        'nested' => {
+          'data' => '774472f0dc892f0b3299cae8dadacd0a74ba59d7'
+        }
+      }
+    }
+    filtered = filter(conf, messages)
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_nest_value
+    conf = %[
+      sha1_keys         array,hash
+      ipaddr_mask_keys  host
+    ]
+    messages = [
+      {
+        'host' => '10.102.3.80',
+        'array' => ['1000', '2000'],
+        'hash' => {'foo' => '1000', 'bar' => '2000'},
+      }
+    ]
+    expected = {
+      'host' => '10.102.3.0',
+      'array' => ["c1628fc0d473cb21b15607c10bdcad19d1a42e24", "ea87abc249f9f2d430edb816514bffeffd3e698e"],
+      'hash' => '28fe85deb0d1d39ee14c49c62bc4773b0338247b'
+    }
+    filtered = filter(conf, messages)
+    assert_equal(expected, filtered[0])
+  end
+
+  def test_filter_ipv6
+    conf = %[
+      ipaddr_mask_keys  host
+      ipv4_mask_subnet  24
+      ipv6_mask_subnet  104
+    ]
+    messages = [
+      { 'host' => '10.102.3.80' },
+      { 'host' => '0:0:0:0:0:FFFF:129.144.52.38' },
+      { 'host' => '2001:db8:0:8d3:0:8a2e:70:7344' }
+    ]
+    expected = [
+      { 'host' => '10.102.3.0' },
+      { 'host' => '::ffff:129.0.0.0' },
+      { 'host' => '2001:db8:0:8d3:0:8a2e::' }
+    ]
+    filtered = filter(conf, messages)
+    assert_equal(expected, filtered)
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anonymizer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-08 00:00:00.000000000 Z
+date: 2015-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -89,8 +105,11 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-anonymizer.gemspec
+- lib/fluent/plugin/anonymizer.rb
+- lib/fluent/plugin/filter_anonymizer.rb
 - lib/fluent/plugin/out_anonymizer.rb
 - test/helper.rb
+- test/plugin/test_filter_anonymizer.rb
 - test/plugin/test_out_anonymizer.rb
 homepage: https://github.com/y-ken/fluent-plugin-anonymizer
 licenses:
@@ -121,4 +140,5 @@ summary: Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1
   Phone number, IPv4/IPv6 address and so on.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_anonymizer.rb
 - test/plugin/test_out_anonymizer.rb