fluent-plugin-anonymizer 0.0.1 → 0.0.2

data/README.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-Fluentd filter output plugin to anonymize records. This data masking plugin protects privacy data such as IP address, ID, email, phone number and so on.
+Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IP address and so on.
 
 ## Installation
 
@@ -27,7 +27,7 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
 </source>
 
 <match test.message>
-  type anonymize
+  type anonymizer
   sha1_keys         user_id, member_id, mail
   ipv4_mask_keys    host
   ipv4_mask_subnet  24
@@ -46,10 +46,10 @@ It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. F
 $ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
 
 $ tail -f /var/log/td-agent/td-agent.log
-2013-11-19 18:30:21 +0900 anonymized.message: {"host":"10.102.0.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
+2013-11-19 18:30:21 +0900 anonymized.message: {"host":"10.102.3.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
 `````
 
-### Params
+## Parameters
 
 * `md5_keys` `sha1_keys` `sha256_keys` `sha384_keys` `sha512_keys`
 
@@ -83,6 +83,10 @@ Add original tag name into filtered record using SetTagKeyMixin function.
 
 Edit tag format using HandleTagNameMixin function.
 
+## Notes
+
+* hashing nested value behavior is compatible with [LogStash::Filters::Anonymize](https://github.com/logstash/logstash/blob/master/lib/logstash/filters/anonymize.rb) does. For further details, please check it out the test code at [test_emit_nest_value](https://github.com/y-ken/fluent-plugin-anonymizer/blob/master/test/plugin/test_out_anonymizer.rb#L98).
+
 ## Blog Articles
 
 * http://y-ken.hatenablog.com/entry/fluent-plugin-anonymizer-has-released

data/fluent-plugin-anonymizer.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-anonymizer"
-  spec.version       = "0.0.1"
+  spec.version       = "0.0.2"
   spec.authors       = ["Kentaro Yoshida"]
   spec.email         = ["y.ken.studio@gmail.com"]
-  spec.summary       = %q{Fluentd filter output plugin to anonymize records. This data masking plugin protects privacy data such as IP address, ID, email, phone number and so on.}
+  spec.summary       = %q{Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512 algorithms. This data masking plugin protects privacy data such as ID, email, phone number, IP address and so on.}
   spec.homepage      = "https://github.com/y-ken/fluent-plugin-anonymizer"
   spec.license       = "Apache License, Version 2.0"
 

data/lib/fluent/plugin/out_anonymizer.rb

@@ -47,23 +47,23 @@ class Fluent::AnonymizerOutput < Fluent::Output
 
   def emit(tag, es, chain)
     es.each do |time, record|
-      record = filter_anonymize_record(record)
+      @hash_keys.each do |hash_key, hash_algorithm|
+        next unless record.include?(hash_key)
+        record[hash_key] = filter_anonymize_record(record[hash_key], hash_algorithm)
+      end
       filter_record(tag, time, record)
       Fluent::Engine.emit(tag, time, record)
     end
     chain.next
   end
 
-  def filter_anonymize_record(record)
-    @hash_keys.each do |hash_key, hash_algorithm|
-      next unless record.include?(hash_key)
-      if record[hash_key].is_a?(Array)
-        record[hash_key] = record[hash_key].collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
-      else
-        record[hash_key] = anonymize(record[hash_key], hash_algorithm, @hash_salt)
-      end
+  def filter_anonymize_record(data, hash_algorithm)
+    if data.is_a?(Array)
+      data = data.collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
+    else
+      data = anonymize(data, hash_algorithm, @hash_salt)
     end
-    return record
+    data
   end
 
   def anonymize(message, algorithm, salt)

data/test/plugin/test_out_anonymizer.rb

@@ -26,6 +26,13 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     add_tag_prefix    anonymized.
   ]
 
+  CONFIG_NEST_VALUE = %[
+    sha1_keys         array,hash
+    ipv4_mask_keys    host
+    remove_tag_prefix input.
+    add_tag_prefix    anonymized.
+  ]
+
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::AnonymizerOutput, tag).configure(conf)
   end
@@ -87,4 +94,22 @@ class AnonymizerOutputTest < Test::Unit::TestCase
     assert_equal 'ce164718b94212332187eb8420903b46b334d609', emits[0][2]['telephone']
     assert_equal 'signup', emits[0][2]['action']
   end
+
+  def test_emit_nest_value
+    d1 = create_driver(CONFIG_NEST_VALUE, 'input.access')
+    d1.run do
+      d1.emit({
+        'host' => '10.102.3.80',
+        'array' => ['1000', '2000'],
+        'hash' => {'foo' => '1000', 'bar' => '2000'},
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'anonymized.access', emits[0][0] # tag
+    assert_equal '10.102.3.0', emits[0][2]['host']
+    assert_equal ["e3cbba8883fe746c6e35783c9404b4bc0c7ee9eb", "a4ac914c09d7c097fe1f4f96b897e625b6922069"], emits[0][2]['array']
+    assert_equal '1a1903d78aed9403649d61cb21ba6b489249761b', emits[0][2]['hash']
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-anonymizer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-19 00:00:00.000000000 Z
+date: 2013-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,8 +100,9 @@ rubyforge_project:
 rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: Fluentd filter output plugin to anonymize records. This data masking plugin
-  protects privacy data such as IP address, ID, email, phone number and so on.
+summary: Fluentd filter output plugin to anonymize records with MD5/SHA1/SHA256/SHA384/SHA512
+  algorithms. This data masking plugin protects privacy data such as ID, email, phone
+  number, IP address and so on.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_anonymizer.rb