fluent-plugin-anonymizer 0.1.0 → 0.2.0

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 
 rvm:
+  - 2.1.0
   - 2.0.0
   - 1.9.3

data/README.md

@@ -2,10 +2,12 @@
 
 ## Overview
 
-Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IPv4/IPv6 address and so on.
+Fluentd filter output plugin to anonymize records with [OpenSSL::HMAC](http://docs.ruby-lang.org/ja/1.9.3/class/OpenSSL=3a=3aHMAC.html) of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.
 
 ## Installation
 
+install with gem or fluent-gem command as:
+
 `````
 ### native gem
 gem install fluent-plugin-anonymizer
@@ -28,10 +30,19 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
 
 <match test.message>
   type anonymizer
+  
+  # Specify hashing keys with comma
   sha1_keys         user_id, member_id, mail
+  
+  # Set hash salt with any strings for more security
+  hash_salt         mysaltstring
+  
+  # Specify rounding address keys with comma and subnet mask
   ipaddr_mask_keys  host
   ipv4_mask_subnet  24
   ipv6_mask_subnet  104
+  
+  # Set tag rename pattern
   remove_tag_prefix test.
   add_tag_prefix    anonymized.
 </match>
@@ -48,8 +59,8 @@ $ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}
 $ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
 
 $ tail -f /var/log/td-agent/td-agent.log
-2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
-2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
+2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
+2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"61f6c1b5f19e0a7f73dd52a23534085bf01f2c67","mail":"eeb890d74b8c1c4cd1e35a3ea62166e0b770f4f4"}
 `````
 
 ## Parameters
@@ -88,13 +99,18 @@ Add original tag name into filtered record using SetTagKeyMixin.
 
 set one or more option are required for editing tag name using HandleTagNameMixin.
 
+* tag
+
+On using this option [like 'tag anonymized.${tag}' with tag placeholder](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L153), it will be overwrite after these options affected. which are remove_tag_prefix, remove_tag_suffix, add_tag_prefix and add_tag_suffix.
+
 ## Notes
 
-* hashing nested value behavior is compatible with [LogStash::Filters::Anonymize](https://github.com/logstash/logstash/blob/master/lib/logstash/filters/anonymize.rb) does. For further details, please check it out the test code at [test_emit_nest_value](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L98).
+* hashing nested value behavior is compatible with [LogStash::Filters::Anonymize](https://github.com/logstash/logstash/blob/master/lib/logstash/filters/anonymize.rb) does. For further details, please check it out the test code at [test_emit_nest_value](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L91).
 
 ## Blog Articles
 
-* http://y-ken.hatenablog.com/entry/fluent-plugin-anonymizer-has-released
+* 個人情報を難読化するfluent-plugin-anonymizerをリリースしました #fluentd - Y-Ken Studio  
+http://y-ken.hatenablog.com/entry/fluent-plugin-anonymizer-has-released
 
 ## TODO
 

data/fluent-plugin-anonymizer.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-anonymizer"
-  spec.version       = "0.1.0"
+  spec.version       = "0.2.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
-  spec.summary       = %q{Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IPv4/IPv6 address and so on.}
+  spec.summary       = %q{Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as UserID, Email, Phone number, IPv4/IPv6 address and so on.}
   spec.homepage      = "https://github.com/y-ken/fluent-plugin-anonymizer"
   spec.license       = "Apache License, Version 2.0"
 

data/lib/fluent/plugin/out_anonymizer.rb

@@ -2,6 +2,7 @@ class Fluent::AnonymizerOutput < Fluent::Output
   Fluent::Plugin.register_output('anonymizer', self)
   
   HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipaddr_mask)
+  config_param :tag, :string, :default => nil
   config_param :hash_salt, :string, :default => ''
   config_param :ipv4_mask_subnet, :integer, :default => 24
   config_param :ipv6_mask_subnet, :integer, :default => 104
@@ -12,15 +13,15 @@ class Fluent::AnonymizerOutput < Fluent::Output
   config_set_default :include_tag_key, false
 
   DIGEST = {
-    "md5" => Proc.new { Digest::MD5 },
-    "sha1" => Proc.new { Digest::SHA1 },
-    "sha256" => Proc.new { Digest::SHA256 },
-    "sha384" => Proc.new { Digest::SHA384 },
-    "sha512" => Proc.new { Digest::SHA512 }
+    "md5" => Proc.new { OpenSSL::Digest.new('md5') },
+    "sha1" => Proc.new { OpenSSL::Digest.new('sha1') },
+    "sha256" => Proc.new { OpenSSL::Digest.new('sha256') },
+    "sha384" => Proc.new { OpenSSL::Digest.new('sha384') },
+    "sha512" => Proc.new { OpenSSL::Digest.new('sha512') }
   }
 
   def initialize
-    require 'digest/sha2'
+    require 'openssl'
     require 'ipaddr'
     super
   end
@@ -42,7 +43,7 @@ class Fluent::AnonymizerOutput < Fluent::Output
     end
     $log.info "anonymizer: adding anonymize rules for each field. #{@hash_keys}"
 
-    if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
+    if ( !@tag && !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
       raise Fluent::ConfigError, "anonymizer: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
     end
   end
@@ -53,13 +54,25 @@ class Fluent::AnonymizerOutput < Fluent::Output
         next unless record.include?(hash_key)
         record[hash_key] = filter_anonymize_record(record[hash_key], hash_algorithm)
       end
-      t = tag.dup
-      filter_record(t, time, record)
-      Fluent::Engine.emit(t, time, record)
+      emit_tag = tag.dup
+      filter_record(emit_tag, time, record)
+      emit_tag = rewrite_tag(@tag, emit_tag) if @tag
+      Fluent::Engine.emit(emit_tag, time, record)
     end
     chain.next
   end
 
+  def rewrite_tag(rewritetag, tag)
+    placeholder = {
+      '${tag}' => tag,
+      '__TAG__' => tag
+    }
+    return rewritetag.gsub(/(\${[a-z_]+(\[[0-9]+\])?}|__[A-Z_]+__)/) do
+      $log.warn "anonymizer: unknown placeholder found. :placeholder=>#{$1} :tag=>#{tag} :rewritetag=>#{rewritetag}" unless placeholder.include?($1)
+      placeholder[$1]
+    end
+  end
+
   def filter_anonymize_record(data, hash_algorithm)
     begin
       if data.is_a?(Array)
@@ -77,7 +90,7 @@ class Fluent::AnonymizerOutput < Fluent::Output
   def anonymize(message, algorithm, salt)
     case algorithm
     when 'md5','sha1','sha256','sha384','sha512'
-      DIGEST[algorithm].call.hexdigest(salt + message.to_s)
+      OpenSSL::HMAC.hexdigest(DIGEST[algorithm].call, salt, message.to_s)
     when 'ipaddr_mask'
       address = IPAddr.new(message)
       subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet

data/test/plugin/test_out_anonymizer.rb

@@ -18,29 +18,6 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     add_tag_prefix    anonymized.
   ]
 
-  CONFIG_MULTI_KEYS = %[
-    sha1_keys         member_id, mail, telephone
-    ipaddr_mask_keys  host
-    ipv4_mask_subnet  16
-    remove_tag_prefix input.
-    add_tag_prefix    anonymized.
-  ]
-
-  CONFIG_NEST_VALUE = %[
-    sha1_keys         array,hash
-    ipaddr_mask_keys  host
-    remove_tag_prefix input.
-    add_tag_prefix    anonymized.
-  ]
-
-  CONFIG_IPV6 = %[
-    ipaddr_mask_keys  host
-    ipv4_mask_subnet  24
-    ipv6_mask_subnet  104
-    remove_tag_prefix input.
-    add_tag_prefix    anonymized.
-  ]
-
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::AnonymizerOutput, tag).configure(conf)
   end
@@ -74,18 +51,25 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     p emits[0]
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.3.0', emits[0][2]['host']
-    assert_equal '9138bd41172f5485f7b6eee3afcd0d62', emits[0][2]['data_for_md5']
-    assert_equal 'ee98db51658d38580b1cf788db19ad06e51a32f7', emits[0][2]['data_for_sha1']
-    assert_equal 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd', emits[0][2]['data_for_sha256']
-    assert_equal '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131', emits[0][2]['data_for_sha384']
-    assert_equal 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20', emits[0][2]['data_for_sha512']
+    assert_equal 'e738cbde82a514dc60582cd467c240ed', emits[0][2]['data_for_md5']
+    assert_equal '69cf099459c06b852ede96d39b710027727d13c6', emits[0][2]['data_for_sha1']
+    assert_equal '804d83b8c6a3e01498d40677652b084333196d8e548ee5a8710fbd0e1e115527', emits[0][2]['data_for_sha256']
+    assert_equal '6c90c389bbdfc210416b9318df3f526b4f218f8a8df3a67020353c35da22dc154460b18f22a8009a747b3ef2975acae7', emits[0][2]['data_for_sha384']
+    assert_equal 'cdbb897e6f3a092161bdb51164eb2996b75b00555f568219628ff15cd2929865d217af5dff9c32ddc908b75a89baec96b3e9a0da120e919f5246de0f1bc54c58', emits[0][2]['data_for_sha512']
   end
 
   def test_emit_multi_keys
-    d1 = create_driver(CONFIG_MULTI_KEYS, 'input.access')
+    d1 = create_driver(%[
+      sha1_keys         member_id, mail, telephone
+      ipaddr_mask_keys  host, host2
+      ipv4_mask_subnet  16
+      remove_tag_prefix input.
+      add_tag_prefix    anonymized.
+    ], 'input.access')
     d1.run do
       d1.emit({
         'host' => '10.102.3.80',
+        'host2' => '10.102.3.80',
         'member_id' => '12345',
         'mail' => 'example@example.com',
         'telephone' => '00-0000-0000',
@@ -97,14 +81,20 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     p emits[0]
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.0.0', emits[0][2]['host']
-    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
-    assert_equal '914fec35ce8bfa1a067581032f26b053591ee38a', emits[0][2]['mail']
-    assert_equal 'ce164718b94212332187eb8420903b46b334d609', emits[0][2]['telephone']
+    assert_equal '10.102.0.0', emits[0][2]['host2']
+    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
+    assert_equal 'd7b728209f5dd8df10cecbced30394c3c7fc2c82', emits[0][2]['mail']
+    assert_equal 'a67f73c395105a358a03a0f127bf64b5495e7841', emits[0][2]['telephone']
     assert_equal 'signup', emits[0][2]['action']
   end
 
   def test_emit_nest_value
-    d1 = create_driver(CONFIG_NEST_VALUE, 'input.access')
+    d1 = create_driver(%[
+      sha1_keys         array,hash
+      ipaddr_mask_keys  host
+      remove_tag_prefix input.
+      add_tag_prefix    anonymized.
+    ], 'input.access')
     d1.run do
       d1.emit({
         'host' => '10.102.3.80',
@@ -117,12 +107,18 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     p emits[0]
     assert_equal 'anonymized.access', emits[0][0] # tag
     assert_equal '10.102.3.0', emits[0][2]['host']
-    assert_equal ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"], emits[0][2]['array']
-    assert_equal '1a1903d78aed9403649d61cb21ba6b489249761b', emits[0][2]['hash']
+    assert_equal ["c1628fc0d473cb21b15607c10bdcad19d1a42e24", "ea87abc249f9f2d430edb816514bffeffd3e698e"], emits[0][2]['array']
+    assert_equal '28fe85deb0d1d39ee14c49c62bc4773b0338247b', emits[0][2]['hash']
   end
 
   def test_emit_ipv6
-    d1 = create_driver(CONFIG_IPV6, 'input.access')
+    d1 = create_driver(%[
+      ipaddr_mask_keys  host
+      ipv4_mask_subnet  24
+      ipv6_mask_subnet  104
+      remove_tag_prefix input.
+      add_tag_prefix    anonymized.
+    ], 'input.access')
     d1.run do
       d1.emit({'host' => '10.102.3.80'})
       d1.emit({'host' => '0:0:0:0:0:FFFF:129.144.52.38'})
@@ -136,4 +132,39 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal '::ffff:129.0.0.0', emits[1][2]['host']
     assert_equal '2001:db8:0:8d3:0:8a2e::', emits[2][2]['host']
   end
+
+  def test_emit_tag_static
+    d1 = create_driver(%[
+      sha1_keys         member_id
+      tag               anonymized.message
+    ], 'input.access')
+    d1.run do
+      d1.emit({
+        'member_id' => '12345',
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'anonymized.message', emits[0][0] # tag
+    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
+  end
+
+  def test_emit_tag_placeholder
+    d1 = create_driver(%[
+      sha1_keys         member_id
+      tag               anonymized.${tag}
+      remove_tag_prefix input.
+    ], 'input.access')
+    d1.run do
+      d1.emit({
+        'member_id' => '12345',
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'anonymized.access', emits[0][0] # tag
+    assert_equal '774472f0dc892f0b3299cae8dadacd0a74ba59d7', emits[0][2]['member_id']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anonymizer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-07 00:00:00.000000000 Z
+date: 2014-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,9 +100,9 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512
-  algorithms. This data masking plugin protects privacy data such as ID, email, phone
-  number, IPv4/IPv6 address and so on.
+summary: Fluentd filter output plugin to anonymize records with HMAC of MD5/SHA1/SHA256/SHA384/SHA512
+  algorithms. This data masking plugin protects privacy data such as UserID, Email,
+  Phone number, IPv4/IPv6 address and so on.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_anonymizer.rb