fluent-plugin-record-modifier 0.1.1 → 0.1.2

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - rbx-19mode
+
+branches:
+  only:
+    - master
+
+script: bundle exec rake test

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.1.2 - 2013/11/22
+
+  * Add char_encoding parameter to handle charactor encoding in event record
+    https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding
+
+
 Release 0.1.1 - 2013/04/24
 
   * Fix SetTagKeyMixin

data/README.md

@@ -33,6 +33,25 @@ then you got new record like below:
 {"message":"hello world!", "gen_host":"oreore-mac.local", "foo":"bar"}
 ```
 
+### char_encoding
+
+Fluentd including some plugins treats the logs as a BINARY by default to forward.
+But an user sometimes processes the logs depends on their requirements, e.g. handling char encoding correctly.
+
+`char_encoding` parameter is useful for this case.
+
+```conf
+<match pattern>
+  type record_modifier
+
+  # set UTF-8 encoding information to string.
+  char_encoding utf-8
+
+  # change char encoding from 'UTF-8' to 'EUC-JP'
+  char_encoding utf-8:euc-jp
+</match>
+```
+
 ### Mixins
 
 * [SetTagKeyMixin](https://github.com/fluent/fluentd/blob/master/lib/fluent/mixin.rb#L181)

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/fluent/plugin/out_record_modifier.rb

@@ -5,11 +5,12 @@ module Fluent
     Fluent::Plugin.register_output('record_modifier', self)
 
     config_param :tag, :string
+    config_param :char_encoding, :string, :default => nil
 
     include SetTagKeyMixin
     include Fluent::Mixin::ConfigPlaceholders
 
-    BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key)
+    BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding)
 
     def configure(conf)
       super
@@ -21,6 +22,22 @@ module Fluent
           @map[k] = v
         end
       }
+
+      if @char_encoding
+        from, to = @char_encoding.split(':', 2)
+        @from_enc = Encoding.find(from)
+        @to_enc = Encoding.find(to) if to
+
+        m = if @to_enc
+              method(:convert_encoding)
+            else
+              method(:set_encoding)
+            end
+
+        (class << self; self; end).module_eval do
+          define_method(:change_encoding, m)
+        end
+      end
     end
 
     def emit(tag, es, chain)
@@ -39,7 +56,25 @@ module Fluent
         record[k] = v
       }
 
+      record = change_encoding(record) if @char_encoding
       record
     end
+
+    def set_encoding(record)
+      record.each_pair { |k, v|
+        if v.is_a?(String)
+          v.force_encoding(@from_enc)
+        end
+      }
+    end
+
+    def convert_encoding(record)
+      record.each_pair { |k, v|
+        if v.is_a?(String)
+          v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
+          v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
+        end
+      }
+    end
   end
 end

data/test/out_record_modifier.rb

@@ -48,4 +48,35 @@ class RecordModifierOutputTest < Test::Unit::TestCase
       {"a" => 2}.merge(mapped),
     ], d.records
   end
+
+  def test_set_char_encoding
+    d = create_driver %[
+      type record_modifier
+
+      tag foo.filtered
+      char_encoding utf-8
+    ]
+
+
+    d.run do
+      d.emit("k" => 'v'.force_encoding('BINARY'))
+    end
+
+    assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.records
+  end
+
+  def test_convert_char_encoding
+    d = create_driver %[
+      type record_modifier
+
+      tag foo.filtered
+      char_encoding utf-8:cp932
+    ]
+
+    d.run do
+      d.emit("k" => 'v'.force_encoding('utf-8'))
+    end
+
+    assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.records
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-24 00:00:00.000000000 Z
+date: 2013-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,6 +65,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .travis.yml
 - ChangeLog
 - Gemfile
 - README.md
@@ -87,7 +88,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2957918703603277385
+      hash: -3676841215308348521
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -96,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2957918703603277385
+      hash: -3676841215308348521
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23