fluent-plugin-record-modifier 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 087c5bc2ce61f49e401036aed71936646a5d3675
-  data.tar.gz: a717f25a685785416486c95cf0bcbba468a6d27c
+  metadata.gz: 344506e1d7deb2d3211df17f02386283fa8a30ab
+  data.tar.gz: 44e6d422823ddf0c686abe066b7da36fc96e4e19
 SHA512:
-  metadata.gz: 4aadefcd8c62d51b31d68e6557961145676f58ead37f989f93b875a05f76c5394d32ec803337a9317f7b11ebe2a6a94b8bfff4a3fc756ad46749d162355c5fa7
-  data.tar.gz: f29f6542bba329efb6afc28dd74d9d6506b64431ff229ea47d76f3d21150d8a764d3f635e806399384c946be3d085c5394759181dc2756641f221dd731aaf34e
+  metadata.gz: 45f15173d92fa1d91e20c62337ffe757852bf42bd006c7847459ca61521cd21cc63b604715489cfec05555f8ce95ba91d182816e470c64771bb3e81cecacad17
+  data.tar.gz: 7c63568717b13b17bdec90bf43c6edf30884639dd9d586efe85bd90f02787d99a7cbbd26484f6ab2676bdddb2facae032148307b0a35aec6c36428f0e602f3ca

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.3.0 - 2015/06/09
+
+  * Add record_modifier filter
+  * Change fluentd dependency version
+
+
 Release 0.2.0 - 2014/09/29
 
   * Update fluent-mixin-config-placeholders to v0.3.0

data/Rakefile

@@ -1,12 +1,11 @@
-
 require 'bundler'
-Bundler::GemHelper.install_tasks
+require 'bundler/gem_tasks'
 
 require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/*.rb']
+  test.test_files = FileList['test/test_*.rb']
   test.verbose = true
 end
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/fluent-plugin-record-modifier.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
 end

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -0,0 +1,86 @@
+require 'fluent/mixin/config_placeholders'
+
+module Fluent
+  class RecordModifierFilter < Filter
+    Fluent::Plugin.register_filter('record_modifier', self)
+
+    config_param :char_encoding, :string, :default => nil
+    config_param :remove_keys, :string, :default => nil
+
+    include SetTagKeyMixin
+    include Fluent::Mixin::ConfigPlaceholders
+
+    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id include_tag_key tag_key char_encoding remove_keys)
+
+    def configure(conf)
+      super
+
+      @map = {}
+      conf.each_pair { |k, v|
+        unless BUILTIN_CONFIGURATIONS.include?(k)
+          conf.has_key?(k)
+          @map[k] = v
+        end
+      }
+
+      if @char_encoding
+        from, to = @char_encoding.split(':', 2)
+        @from_enc = Encoding.find(from)
+        @to_enc = Encoding.find(to) if to
+
+        m = if @to_enc
+              method(:convert_encoding)
+            else
+              method(:set_encoding)
+            end
+
+        (class << self; self; end).module_eval do
+          define_method(:change_encoding, m)
+        end
+      end
+
+      if @remove_keys
+        @remove_keys = @remove_keys.split(',').map {|e| e.strip }
+      end
+    end
+
+    def filter(tag, time, record)
+      filter_record(tag, time, record)
+      modify_record(record)
+    end
+
+    private
+
+    def modify_record(record)
+      @map.each_pair { |k, v|
+        record[k] = v
+      }
+
+      if @remove_keys
+        @remove_keys.each { |v|
+          record.delete(v)
+        }
+      end
+
+      record = change_encoding(record) if @char_encoding
+      record
+    end
+
+    def set_encoding(record)
+      record.each_pair { |k, v|
+        if v.is_a?(String)
+          v.force_encoding(@from_enc)
+        end
+      }
+    end
+
+    def convert_encoding(record)
+      record.each_pair { |k, v|
+        if v.is_a?(String)
+          v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
+          v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
+        end
+      }
+    end
+  end if defined?(Filter)
+end

data/test/test_filter_record_modifier.rb

@@ -0,0 +1,111 @@
+require 'fluent/test'
+require 'fluent/plugin/filter_record_modifier'
+require 'test/unit'
+
+exit unless defined?(Fluent::Filter)
+
+class RecordModifierFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @tag = 'test.tag'
+  end
+
+  CONFIG = %[
+    type record_modifier
+
+    gen_host ${hostname}
+    foo bar
+    include_tag_key
+    tag_key included_tag
+    remove_keys hoge
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::FilterTestDriver.new(Fluent::RecordModifierFilter, @tag).configure(conf)
+  end
+
+  def get_hostname
+    require 'socket'
+    Socket.gethostname.chomp
+  end
+
+  def test_configure
+    d = create_driver
+    map = d.instance.instance_variable_get(:@map)
+
+    assert_equal get_hostname, map['gen_host']
+    assert_equal 'bar', map['foo']
+  end
+
+  def test_format
+    d = create_driver
+
+    d.run do
+      d.emit("a" => 1)
+      d.emit("a" => 2)
+    end
+
+    mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag}
+    assert_equal [
+      {"a" => 1}.merge(mapped),
+      {"a" => 2}.merge(mapped),
+    ], d.filtered_as_array.map { |e| e.last }
+  end
+
+  def test_set_char_encoding
+    d = create_driver %[
+      type record_modifier
+
+      char_encoding utf-8
+    ]
+
+
+    d.run do
+      d.emit("k" => 'v'.force_encoding('BINARY'))
+    end
+
+    assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.filtered_as_array.map { |e| e.last }
+  end
+
+  def test_convert_char_encoding
+    d = create_driver %[
+      type record_modifier
+
+      char_encoding utf-8:cp932
+    ]
+
+    d.run do
+      d.emit("k" => 'v'.force_encoding('utf-8'))
+    end
+
+    assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.filtered_as_array.map { |e| e.last }
+  end
+
+  def test_remove_one_key
+    d = create_driver %[
+      type record_modifier
+
+      remove_keys k1
+    ]
+
+    d.run do
+      d.emit("k1" => 'v', "k2" => 'v')
+    end
+
+    assert_equal [{"k2" => 'v'}], d.filtered_as_array.map { |e| e.last }
+  end
+
+  def test_remove_multiple_keys
+    d = create_driver %[
+      type record_modifier
+
+      remove_keys k1, k2, k3
+    ]
+
+    d.run do
+      d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v')
+    end
+
+    assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
+  end
+end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-29 00:00:00.000000000 Z
+date: 2015-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.10.17
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 0.10.17
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
@@ -65,8 +71,10 @@ files:
 - Rakefile
 - VERSION
 - fluent-plugin-record-modifier.gemspec
+- lib/fluent/plugin/filter_record_modifier.rb
 - lib/fluent/plugin/out_record_modifier.rb
-- test/out_record_modifier.rb
+- test/test_filter_record_modifier.rb
+- test/test_out_record_modifier.rb
 homepage: https://github.com/repeatedly/fluent-plugin-record-modifier
 licenses:
 - MIT
@@ -92,4 +100,5 @@ signing_key:
 specification_version: 4
 summary: Output filter plugin for modifying each event record
 test_files:
-- test/out_record_modifier.rb
+- test/test_filter_record_modifier.rb
+- test/test_out_record_modifier.rb