fluent-plugin-anonymizer 0.0.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+vendor/*

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-anonymizer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2013- Kentaro Yoshida
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,100 @@
+# fluent-plugin-anonymizer [![Build Status](https://travis-ci.org/y-ken/fluent-plugin-anonymizer.png?branch=master)](https://travis-ci.org/y-ken/fluent-plugin-anonymizer)
+
+## Overview
+
+Fluentd filter output plugin to anonymize records. This data masking plugin protects privacy data such as IP address, ID, email, phone number and so on.
+
+## Installation
+
+`````
+### native gem
+gem install fluent-plugin-anonymizer
+
+### td-agent gem
+/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-anonymizer
+`````
+
+## Tutorial
+
+#### configuration
+
+It is a sample to hash record with sha1 for `user_id`, `member_id` and `mail`. For IP address, rounding number with 24bit netmask with `ipv4_mask_keys` and `ipv4_mask_subnet` option.
+
+`````
+<source>
+  type forward
+  port 24224
+</source>
+
+<match test.message>
+  type anonymize
+  sha1_keys         user_id, member_id, mail
+  ipv4_mask_keys    host
+  ipv4_mask_subnet  24
+  remove_tag_prefix test.
+  add_rag_prefix    anonymized.
+</match>
+
+<match anonymized.message>
+  type stdout
+</match>
+`````
+
+#### result
+
+`````
+$ echo '{"host":"10.102.3.80","member_id":"12345", "mail":"example@example.com"}' | fluent-cat test.message
+
+$ tail -f /var/log/td-agent/td-agent.log
+2013-11-19 18:30:21 +0900 anonymized.message: {"host":"10.102.0.0","member_id":"8cb2237d0679ca88db6464eac60da96345513964","mail":"914fec35ce8bfa1a067581032f26b053591ee38a"}
+`````
+
+### Params
+
+* `md5_keys` `sha1_keys` `sha256_keys` `sha384_keys` `sha512_keys`
+
+Specify which hash algorithm to be used for following one or more keys.
+
+* `hash_salt` (default: none)
+
+This salt affects for `md5_keys` `sha1_keys` `sha256_keys` `sha384_keys` `sha512_keys` settings.  
+It is recommend to set a hash salt to prevent rainbow table attacks.
+
+
+* `ipv4_mask_keys`
+* `ipv4_mask_subnet` (default: 24)
+
+Round number for following one or more keys. It makes easy to aggregate calculation. 
+
+| ipv4_mask_subnet |      input      |    output     |
+|------------------|-----------------|---------------|
+|               24 | 192.168.200.100 | 192.168.200.0 |
+|               16 | 192.168.200.100 | 192.168.0.0   |
+|                8 | 192.168.200.100 | 192.0.0.0     |
+
+* include_tag_key (default: false)
+
+Add original tag name into filtered record using SetTagKeyMixin function.
+
+* remove_tag_prefix
+* remove_tag_suffix
+* add_tag_prefix
+* add_tag_suffix
+
+Edit tag format using HandleTagNameMixin function.
+
+## Blog Articles
+
+* http://y-ken.hatenablog.com/entry/fluent-plugin-anonymizer-has-released
+
+## TODO
+
+Pull requests are very welcome!!
+
+## Copyright
+
+Copyright © 2013- Kentaro Yoshida ([@yoshi_ken](https://twitter.com/yoshi_ken))
+
+## License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-anonymizer.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-anonymizer"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Kentaro Yoshida"]
+  spec.email         = ["y.ken.studio@gmail.com"]
+  spec.summary       = %q{Fluentd filter output plugin to anonymize records. This data masking plugin protects privacy data such as IP address, ID, email, phone number and so on.}
+  spec.homepage      = "https://github.com/y-ken/fluent-plugin-anonymizer"
+  spec.license       = "Apache License, Version 2.0"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd"
+end

data/lib/fluent/plugin/out_anonymizer.rb

@@ -0,0 +1,79 @@
+class Fluent::AnonymizerOutput < Fluent::Output
+  Fluent::Plugin.register_output('anonymizer', self)
+  
+  HASH_ALGORITHM = %w(md5 sha1 sha256 sha384 sha512 ipv4_mask)
+  config_param :hash_salt, :string, :default => ''
+  config_param :ipv4_mask_subnet, :integer, :default => 24
+
+  include Fluent::HandleTagNameMixin
+
+  include Fluent::SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  DIGEST = {
+    "md5" => Proc.new { Digest::MD5 },
+    "sha1" => Proc.new { Digest::SHA1 },
+    "sha256" => Proc.new { Digest::SHA256 },
+    "sha384" => Proc.new { Digest::SHA384 },
+    "sha512" => Proc.new { Digest::SHA512 }
+  }
+
+  def initialize
+    require 'digest/sha2'
+    require 'ipaddr'
+    super
+  end
+  
+  def configure(conf)
+    super
+
+    @hash_keys = Hash.new
+    conf.keys.select{|k| k =~ /_keys$/}.each do |key|
+      hash_algorithm_name = key.sub('_keys','')
+      raise Fluent::ConfigError, "anonymizer: unsupported key #{hash_algorithm_name}" unless HASH_ALGORITHM.include?(hash_algorithm_name)
+      conf[key].gsub(' ', '').split(',').each do |record_key|
+        @hash_keys.store(record_key, hash_algorithm_name)
+      end
+    end
+
+    if @hash_keys.count < 1
+      raise Fluent::ConfigError, "anonymizer: missing hash keys setting."
+    end
+
+    if ( !@remove_tag_prefix && !@remove_tag_suffix && !@add_tag_prefix && !@add_tag_suffix )
+      raise Fluent::ConfigError, "anonymizer: missing remove_tag_prefix, remove_tag_suffix, add_tag_prefix or add_tag_suffix."
+    end
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      record = filter_anonymize_record(record)
+      filter_record(tag, time, record)
+      Fluent::Engine.emit(tag, time, record)
+    end
+    chain.next
+  end
+
+  def filter_anonymize_record(record)
+    @hash_keys.each do |hash_key, hash_algorithm|
+      next unless record.include?(hash_key)
+      if record[hash_key].is_a?(Array)
+        record[hash_key] = record[hash_key].collect { |v| anonymize(v, hash_algorithm, @hash_salt) }
+      else
+        record[hash_key] = anonymize(record[hash_key], hash_algorithm, @hash_salt)
+      end
+    end
+    return record
+  end
+
+  def anonymize(message, algorithm, salt)
+    case algorithm
+    when 'md5','sha1','sha256','sha384','sha512'
+      DIGEST[algorithm].call.hexdigest(salt + message.to_s)
+    when 'ipv4_mask'
+      IPAddr.new(message).mask(@ipv4_mask_subnet).to_s
+    else
+      $log.warn "anonymizer: unknown algorithm #{algorithm} has called."
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_anonymizer'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_anonymizer.rb

@@ -0,0 +1,90 @@
+require 'helper'
+
+class AnonymizerOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    md5_keys          data_for_md5
+    sha1_keys         data_for_sha1
+    sha256_keys       data_for_sha256
+    sha384_keys       data_for_sha384
+    sha512_keys       data_for_sha512
+    hash_salt         test_salt_string
+    ipv4_mask_keys    host
+    ipv4_mask_subnet  24
+    remove_tag_prefix input.
+    add_tag_prefix    anonymized.
+  ]
+
+  CONFIG_MULTI_KEYS = %[
+    sha1_keys         member_id, mail, telephone
+    ipv4_mask_keys    host
+    ipv4_mask_subnet  16
+    remove_tag_prefix input.
+    add_tag_prefix    anonymized.
+  ]
+
+  def create_driver(conf=CONFIG,tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::AnonymizerOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('unknown_keys')
+    }
+    d = create_driver(CONFIG)
+    puts d.instance.inspect
+    assert_equal 'test_salt_string', d.instance.config['hash_salt']
+  end
+
+  def test_emit
+    d1 = create_driver(CONFIG, 'input.access')
+    d1.run do
+      d1.emit({
+        'host' => '10.102.3.80',
+        'data_for_md5' => '12345',
+        'data_for_sha1' => '12345',
+        'data_for_sha256' => '12345',
+        'data_for_sha384' => '12345',
+        'data_for_sha512' => '12345'
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'anonymized.access', emits[0][0] # tag
+    assert_equal '10.102.3.0', emits[0][2]['host']
+    assert_equal '9138bd41172f5485f7b6eee3afcd0d62', emits[0][2]['data_for_md5']
+    assert_equal 'ee98db51658d38580b1cf788db19ad06e51a32f7', emits[0][2]['data_for_sha1']
+    assert_equal 'd53d15615b19597b0f95a984a132ed5164ba9676bf3cb28e018d28feaa2ea6fd', emits[0][2]['data_for_sha256']
+    assert_equal '6e9cd6d84ea371a72148b418f1a8cb2534da114bc2186d36ec6f14fd5c237b6f2e460f409dda89b7e42a14b7da8a8131', emits[0][2]['data_for_sha384']
+    assert_equal 'adcf4e5d1e52f57f67d8b0cd85051158d7362103d7ed4cb6302445c2708eff4b17cb309cf5d09fd5cf76615c75652bd29d1707ce689a28e8700afd7a7439ef20', emits[0][2]['data_for_sha512']
+  end
+
+  def test_emit_multi_keys
+    d1 = create_driver(CONFIG_MULTI_KEYS, 'input.access')
+    d1.run do
+      d1.emit({
+        'host' => '10.102.3.80',
+        'member_id' => '12345',
+        'mail' => 'example@example.com',
+        'telephone' => '00-0000-0000',
+        'action' => 'signup'
+      })
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    p emits[0]
+    assert_equal 'anonymized.access', emits[0][0] # tag
+    assert_equal '10.102.0.0', emits[0][2]['host']
+    assert_equal '8cb2237d0679ca88db6464eac60da96345513964', emits[0][2]['member_id']
+    assert_equal '914fec35ce8bfa1a067581032f26b053591ee38a', emits[0][2]['mail']
+    assert_equal 'ce164718b94212332187eb8420903b46b334d609', emits[0][2]['telephone']
+    assert_equal 'signup', emits[0][2]['action']
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-anonymizer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kentaro Yoshida
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- y.ken.studio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-anonymizer.gemspec
+- lib/fluent/plugin/out_anonymizer.rb
+- test/helper.rb
+- test/plugin/test_out_anonymizer.rb
+homepage: https://github.com/y-ken/fluent-plugin-anonymizer
+licenses:
+- Apache License, Version 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fluentd filter output plugin to anonymize records. This data masking plugin
+  protects privacy data such as IP address, ID, email, phone number and so on.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_anonymizer.rb