fluent-plugin-anonymizer 0.0.3 → 0.1.0

data/README.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IP address and so on.
+Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IPv4/IPv6 address and so on.
 
 ## Installation
 
@@ -18,7 +18,7 @@ gem install fluent-plugin-anonymizer
 
 #### configuration
 
-It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, rounding number with 24bit netmask with `ipv4_mask_keys` and `ipv4_mask_subnet` option.
+It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, auto-detecting IPv4/IPv6 and rounding number with 24bit(IPv4) or 104bit(IPv6) netmask using `ipaddr_mask_keys` and `ipv4_mask_subnet`, `ipv6_mask_subnet` option.
 
 `````
 <source>
@@ -29,8 +29,9 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
 <match test.message>
   type anonymizer
   sha1_keys         user_id, member_id, mail
-  ipv4_mask_keys    host
+  ipaddr_mask_keys  host
   ipv4_mask_subnet  24
+  ipv6_mask_subnet  104
   remove_tag_prefix test.
   add_tag_prefix    anonymized.
 </match>
@@ -44,9 +45,11 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
 
 `````
 $ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
+$ echo '{"host":"2001:db8:0:8d3:0:8a2e:70:7344","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
 
 $ tail -f /var/log/td-agent/td-agent.log
-2013-11-19 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
+2014-01-06 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
+2014-01-06 18:30:22 +0900 anonymized.message: {"host":"2001:db8:0:8d3:0:8a2e::","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
 `````
 
 ## Parameters
@@ -61,8 +64,9 @@ This salt affects for `md5_keys` `sha1_keys` `sha256_keys` `sha384_keys` `sha512
 It is recommend to set a hash salt to prevent rainbow table attacks.
 
 
-* `ipv4_mask_keys`
+* `ipaddr_mask_keys`
 * `ipv4_mask_subnet` (default: 24)
+* `ipv6_mask_subnet` (default: 104)
 
 Round number for following one or more keys. It makes easy to aggregate calculation. 
 

data/fluent-plugin-anonymizer.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-anonymizer"
-  spec.version       = "0.0.3"
+  spec.version       = "0.1.0"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
-  spec.summary       = %q{Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IP address and so on.}
+  spec.summary       = %q{Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IPv4/IPv6 address and so on.}
   spec.homepage      = "https://github.com/y-ken/fluent-plugin-anonymizer"
   spec.license       = "Apache License, Version 2.0"
 

data/lib/fluent/plugin/out_anonymizer.rb

@@ -1,9 +1,10 @@
 class Fluent::AnonymizerOutput < Fluent::Output
   Fluent::Plugin.register_output('anonymizer', self)
   
-  HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipv4_mask)
+  HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipaddr_mask)
   config_param :hash_salt, :string, :default => ''
   config_param :ipv4_mask_subnet, :integer, :default => 24
+  config_param :ipv6_mask_subnet, :integer, :default => 104
 
   include Fluent::HandleTagNameMixin
 
@@ -39,6 +40,7 @@ class Fluent::AnonymizerOutput < Fluent::Output
     if @hash_keys.count < 1
       raise Fluent::ConfigError, "anonymizer: missing hash keys setting."
     end
+    $log.info "anonymizer: adding anonymize rules for each field. #{@hash_keys}"
 
     if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
       raise Fluent::ConfigError, "anonymizer: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
@@ -59,10 +61,15 @@ class Fluent::AnonymizerOutput < Fluent::Output
   end
 
   def filter_anonymize_record(data, hash_algorithm)
-    if data.is_a?(Array)
-      data = data.collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
-    else
-      data = anonymize(data, hash_algorithm, @hash_salt)
+    begin
+      if data.is_a?(Array)
+        data = data.collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
+      else
+        data = anonymize(data, hash_algorithm, @hash_salt)
+      end
+    rescue StandardError => e
+      $log.error "anonymizer: failed to anonymize record. :message=>#{e.message} :data=>#{data}"
+      $log.error e.backtrace.join("\n")
     end
     data
   end
@@ -71,8 +78,10 @@ class Fluent::AnonymizerOutput < Fluent::Output
     case algorithm
     when 'md5','sha1','sha256','sha384','sha512'
       DIGEST[algorithm].call.hexdigest(salt + message.to_s)
-    when 'ipv4_mask'
-      IPAddr.new(message).mask(@ipv4_mask_subnet).to_s
+    when 'ipaddr_mask'
+      address = IPAddr.new(message)
+      subnet = address.ipv4? ? @ipv4_mask_subnet : @ipv6_mask_subnet
+      address.mask(subnet).to_s
     else
       $log.warn "anonymizer: unknown algorithm #{algorithm} has called."
     end

data/test/plugin/test_out_anonymizer.rb

@@ -12,7 +12,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     sha384_keys       data_for_sha384
     sha512_keys       data_for_sha512
     hash_salt         test_salt_string
-    ipv4_mask_keys    host
+    ipaddr_mask_keys  host
     ipv4_mask_subnet  24
     remove_tag_prefix input.
     add_tag_prefix    anonymized.
@@ -20,7 +20,7 @@ class AnonymizerOutputTest < Test::Unit::TestCase
 
   CONFIG_MULTI_KEYS = %[
     sha1_keys         member_id, mail, telephone
-    ipv4_mask_keys    host
+    ipaddr_mask_keys  host
     ipv4_mask_subnet  16
     remove_tag_prefix input.
     add_tag_prefix    anonymized.
@@ -28,7 +28,15 @@ class AnonymizerOutputTest < Test::Unit::TestCase
 
   CONFIG_NEST_VALUE = %[
     sha1_keys         array,hash
-    ipv4_mask_keys    host
+    ipaddr_mask_keys  host
+    remove_tag_prefix input.
+    add_tag_prefix    anonymized.
+  ]
+
+  CONFIG_IPV6 = %[
+    ipaddr_mask_keys  host
+    ipv4_mask_subnet  24
+    ipv6_mask_subnet  104
     remove_tag_prefix input.
     add_tag_prefix    anonymized.
   ]
@@ -112,4 +120,20 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"], emits[0][2]['array']
     assert_equal '1a1903d78aed9403649d61cb21ba6b489249761b', emits[0][2]['hash']
   end
+
+  def test_emit_ipv6
+    d1 = create_driver(CONFIG_IPV6, 'input.access')
+    d1.run do
+      d1.emit({'host' => '10.102.3.80'})
+      d1.emit({'host' => '0:0:0:0:0:FFFF:129.144.52.38'})
+      d1.emit({'host' => '2001:db8:0:8d3:0:8a2e:70:7344'})
+    end
+    emits = d1.emits
+    assert_equal 3, emits.length
+    p emits
+    assert_equal 'anonymized.access', emits[0][0] # tag
+    assert_equal '10.102.3.0', emits[0][2]['host']
+    assert_equal '::ffff:129.0.0.0', emits[1][2]['host']
+    assert_equal '2001:db8:0:8d3:0:8a2e::', emits[2][2]['host']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anonymizer
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-03 00:00:00.000000000 Z
+date: 2014-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,7 +102,7 @@ signing_key:
 specification_version: 3
 summary: Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512
   algorithms. This data masking plugin protects privacy data such as ID, email, phone
-  number, IP address and so on.
+  number, IPv4/IPv6 address and so on.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_anonymizer.rb