fluent-plugin-record-reformer 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 224873397bd762ffdf8b4256c727f0d274c0f39b
-  data.tar.gz: 3c2f1c849c49d690c1e583ac03ecd3817ba670ea
+  metadata.gz: a717722e65d7aece7073303d2231c6d441362612
+  data.tar.gz: b8417dd2ca5425ea7dbc692598ce7190ccacff64
 SHA512:
-  metadata.gz: b8dbc917cf34ea266ec33e4a39c59e87536856bf43af2ea1889714b71eb3d1145dc03f45789d7af99b6c89f9d36b581fd171ca94188c1afe25d19bcf3153464a
-  data.tar.gz: 9c47cd0a33f18d0b3f62ecc43077df7c1a2710aca5e110d39225a4c2a66e89823826ba3685f9f444f921098897fb67c1b71941575d3580d578e0817f9c4ee9ae
+  metadata.gz: 4d76f1d8acf8fb44942b49c5453dbf571667ab4e8f04805efa13c5431b5288b5f60215433c06bda677fd02b6225045b7948f95a831708a5939a0004508f59b8f
+  data.tar.gz: c1d3de259f79c65ea8c833da2d66eab150841285001b2ce65ebe996b239f081f04b978499e4510483d5a2c09c0ea3e26889c7b9708f248f43bb651efa6683a20

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.9.1 (2017/07/26)
+
+Enhancements:
+
+* Support multi process workers of v0.14.12.
+
 ## 0.9.0 (2017/02/21)
 
 Enhancements:

data/{example.conf → example/example.conf}

@@ -1,11 +1,11 @@
 <source>
-  type dummy
+  @type dummy
   tag dummy
   dummy {"message":"foo","time":1432732710,"members":["Alice"]}
 </source>
 
 <match dummy>
-  type record_reformer
+  @type record_reformer
   renew_time_key time
   tag reformed.${tag}
   enable_ruby true
@@ -16,5 +16,5 @@
 </match>
 
 <match reformed.**>
-  type stdout
+  @type stdout
 </match>

data/example/workers.conf

@@ -0,0 +1,5 @@
+<system>
+  workers 2 # v0.4.12
+</system>
+
+@include example.conf

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.9.0"
+  gem.version     = "0.9.1"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer/core.rb

@@ -82,6 +82,10 @@ module Fluent
       @hostname = Socket.gethostname
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def process(tag, es)
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)

data/test/helper.rb

@@ -29,6 +29,27 @@ module Fluent
   end
 end
 
+## v0.14
+# d = create_driver(conf, syntax: :v1)
+# d.run(default_tag: @tag) do
+#   d.feed(time, record)
+# end
+# d.events
+#
+## v0.12
+# d = create_driver(conf, use_v1, default_tag: @tag)
+# d.run do
+#   d.emit(record, time)
+# end
+# d.emits
+#
+## Ours
+# d = create_driver(conf, syntax: :v1, default_tag: @tag)
+# d.run do
+#   d.feed(time, record)
+# end
+# d.events
+
 require 'fluent/version'
 major, minor, patch = Fluent::VERSION.split('.').map(&:to_i)
 if major > 0 || (major == 0 && minor >= 14)
@@ -44,32 +65,36 @@ if major > 0 || (major == 0 && minor >= 14)
       @tag = tag
     end
 
-    def configure(conf, use_v1)
-      super(conf, syntax: use_v1 ? :v1 : :v0)
-    end
-
     def run(&block)
       super(default_tag: @tag, &block)
     end
-
-    def emit(record, time)
-      feed(time, record)
-    end
-
-    def emits
-      events
-    end
   end
 
-  def create_driver(conf, use_v1)
-    RecordReformerOutputTestDriver.new(Fluent::Plugin::RecordReformerOutput, @tag).configure(conf, use_v1)
+  def create_driver(conf, syntax: :v1, default_tag: nil)
+    default_tag ||= @tag
+    RecordReformerOutputTestDriver.new(Fluent::Plugin::RecordReformerOutput, default_tag).configure(conf, syntax: syntax)
   end
-else
+else # <= v0.12e
   def event_time(str)
     Time.parse(str)
   end
 
-  def create_driver(conf, use_v1)
-    Fluent::Test::OutputTestDriver.new(Fluent::RecordReformerOutput, @tag).configure(conf, use_v1)
+  class RecordReformerOutputTestDriver < Fluent::Test::OutputTestDriver
+    def configure(conf, syntax: :v1)
+      syntax == :v1 ? super(conf, true) : super(conf, false)
+    end
+
+    def feed(time, record)
+      emit(record, time)
+    end
+
+    def events
+      emits
+    end
+  end
+
+  def create_driver(conf, syntax: :v1, default_tag: nil)
+    default_tag ||= @tag
+    RecordReformerOutputTestDriver.new(Fluent::RecordReformerOutput, default_tag).configure(conf, syntax: :v1)
   end
 end

data/test/test_out_record_reformer.rb

@@ -5,20 +5,20 @@ require 'fluent/plugin/out_record_reformer'
 Fluent::Test.setup
 
 class RecordReformerOutputTest < Test::Unit::TestCase
-  def emit(config, use_v1, msgs = [''])
-    d = create_driver(config, use_v1)
+  def feed(config, msgs: [''], syntax: :v1)
+    d = create_driver(config, syntax: syntax, default_tag: @tag)
     d.run do
       records = msgs.map do |msg|
         next msg if msg.is_a?(Hash)
         { 'eventType0' => 'bar', 'message' => msg }
       end
       records.each do |record|
-        d.emit(record, @time)
+        d.feed(@time, record)
       end
     end
 
     @instance = d.instance
-    d.emits
+    d.events
   end
 
   setup do
@@ -42,23 +42,23 @@ class RecordReformerOutputTest < Test::Unit::TestCase
     message ${hostname} ${tag_parts.last} ${URI.escape(message)}
   ]
 
-  [true, false].each do |use_v1|
+  [:v1, :v0].each do |syntax|
     sub_test_case 'configure' do
       test 'typical usage' do
         assert_nothing_raised do
-          create_driver(CONFIG, use_v1)
+          create_driver(CONFIG, syntax: syntax)
         end
       end
 
       test "tag is not specified" do
         assert_raise(Fluent::ConfigError) do
-          create_driver('', use_v1)
+          create_driver('', syntax: syntax)
         end
       end
 
       test "keep_keys must be specified together with renew_record true" do
         assert_raise(Fluent::ConfigError) do
-          create_driver(%[keep_keys a], use_v1)
+          create_driver(%[keep_keys a], syntax: syntax)
         end
       end
     end
@@ -66,9 +66,9 @@ class RecordReformerOutputTest < Test::Unit::TestCase
     sub_test_case "test options" do
       test 'typical usage' do
         msgs = ['1', '2']
-        emits = emit(CONFIG, use_v1, msgs)
-        assert_equal 2, emits.size
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(CONFIG, syntax: syntax, msgs: msgs)
+        assert_equal 2, events.size
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal('bar', record['eventType0'])
           assert_equal(@hostname, record['hostname'])
@@ -81,8 +81,8 @@ class RecordReformerOutputTest < Test::Unit::TestCase
       test '(obsolete) output_tag' do
         config = %[output_tag reformed.${tag}]
         msgs = ['1']
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
         end
       end
@@ -99,8 +99,8 @@ class RecordReformerOutputTest < Test::Unit::TestCase
         </record>
         ]
         msgs = ['1', '2']
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal('bar', record['eventType0'])
           assert_equal(@hostname, record['hostname'])
@@ -112,8 +112,8 @@ class RecordReformerOutputTest < Test::Unit::TestCase
 
       test 'remove_keys' do
         config = CONFIG + %[remove_keys eventType0,message]
-        emits = emit(config, use_v1)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_not_include(record, 'eventType0')
           assert_equal(@hostname, record['hostname'])
@@ -126,8 +126,8 @@ class RecordReformerOutputTest < Test::Unit::TestCase
       test 'renew_record' do
         config = CONFIG + %[renew_record true]
         msgs = ['1', '2']
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_not_include(record, 'eventType0')
           assert_equal(@hostname, record['hostname'])
@@ -148,8 +148,8 @@ class RecordReformerOutputTest < Test::Unit::TestCase
     </record>
 EOC
         msgs = times.map{|t| t.to_s }
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal(times[i].to_i, time)
           assert_true(record.has_key?('event_time_key'))
@@ -168,8 +168,8 @@ EOC
 EOC
         times = [ Time.at(event_time("2010-05-04 03:02:02")), Time.at(event_time("2010-05-04 03:02:03")) ]
         msgs = times.map{|t| t.to_s }
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal(times[i].to_i, time)
           assert_false(record.has_key?('event_time_key'))
@@ -179,8 +179,8 @@ EOC
       test 'keep_keys' do
         config = %[tag reformed.${tag}\nrenew_record true\nkeep_keys eventType0,message]
         msgs = ['1', '2']
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal('bar', record['eventType0'])
           assert_equal(msgs[i], record['message'])
@@ -196,8 +196,8 @@ EOC
           </record>
         ]
         msgs = ['1', '2']
-        emits = emit(config, use_v1, msgs)
-        emits.each_with_index do |(tag, time, record), i|
+        events = feed(config, syntax: syntax, msgs: msgs)
+        events.each_with_index do |(tag, time, record), i|
           assert_equal("reformed.#{@tag}", tag)
           assert_equal("#{@hostname}  ", record['message'])
         end
@@ -215,8 +215,8 @@ EOC
               message ${hostname}
             </record>
           ]
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(@hostname, record['message'])
           end
         end
@@ -229,8 +229,8 @@ EOC
               message ${tag}
             </record>
           ]
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(@tag, record['message'])
           end
         end
@@ -244,8 +244,8 @@ EOC
             </record>
           ]
           expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(expected, record['message'])
           end
         end
@@ -259,8 +259,8 @@ EOC
             </record>
           ]
           expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(expected, record['message'])
           end
         end
@@ -275,8 +275,8 @@ EOC
           ]
           @tag = 'prefix.test.tag.suffix'
           expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix"
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(expected, record['message'])
           end
         end
@@ -289,8 +289,8 @@ EOC
               message ${time}
             </record>
           ]
-          emits = emit(config, use_v1)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax)
+          events.each do |(tag, time, record)|
             assert_equal(Time.at(time).localtime.to_s, record['message'])
           end
         end
@@ -306,8 +306,8 @@ EOC
             </record>
           ]
           msgs = ['1', '2']
-          emits = emit(config, use_v1, msgs)
-          emits.each_with_index do |(tag, time, record), i|
+          events = feed(config, syntax: syntax, msgs: msgs)
+          events.each_with_index do |(tag, time, record), i|
             assert_not_include(record, 'eventType0')
             assert_equal("bar", record['eventtype'])
             assert_equal("bar #{msgs[i]}", record['message'])
@@ -326,8 +326,8 @@ EOC
             </record>
           ]
           records = [{'tag' => 'tag', 'time' => 'time'}]
-          emits = emit(config, use_v1, records)
-          emits.each do |(tag, time, record)|
+          events = feed(config, syntax: syntax, msgs: records)
+          events.each do |(tag, time, record)|
             assert_not_equal('tag', record['new_tag'])
             assert_equal(@tag, record['new_tag'])
             assert_not_equal('time', record['new_time'])
@@ -346,7 +346,7 @@ EOC
             </record>
           ]
           msgs = ['1', '2']
-          es = emit(config, use_v1, msgs)
+          es = feed(config, syntax: syntax, msgs: msgs)
           es.each_with_index do |(tag, time, record), i|
             assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, record['hash_field'])
           end
@@ -361,7 +361,7 @@ EOC
             </record>
           ]
           msgs = ['1', '2']
-          es = emit(config, use_v1, msgs)
+          es = feed(config, syntax: syntax, msgs: msgs)
           es.each_with_index do |(tag, time, record), i|
             assert_equal([@hostname, @tag], record['array_field'])
           end
@@ -376,13 +376,13 @@ EOC
             </record>
           ]
           msgs = ['1', '2']
-          es = emit(config, use_v1, msgs)
+          es = feed(config, syntax: syntax, msgs: msgs)
           es.each_with_index do |(tag, time, record), i|
             assert_equal([{"tag" => @tag}], record['mixed_field'])
           end
         end
 
-        if use_v1 == true
+        if syntax == :v1
           # works with only v1 config
           test "keys with placeholders with enable_ruby #{enable_ruby}" do
             config = %[
@@ -395,7 +395,7 @@ EOC
               </record>
             ]
             msgs = ['1', '2']
-            es = emit(config, use_v1, msgs)
+            es = feed(config, syntax: syntax, msgs: msgs)
             es.each_with_index do |(tag, time, record), i|
               assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, record)
             end
@@ -444,7 +444,7 @@ EOC
               :with_suffix => "#{nil.to_s}-suffix" },
           ]
           actual_results = []
-          es = emit(config, use_v1, msgs)
+          es = feed(config, syntax: syntax, msgs: msgs)
           es.each_with_index do |(tag, time, record), i|
             actual_results << {
               :single      => record["single"],
@@ -498,7 +498,7 @@ EOC
               :with_suffix => "#{nil.to_s}-suffix" },
           ]
           actual_results = []
-          es = emit(config, use_v1, msgs)
+          es = feed(config, syntax: syntax, msgs: msgs)
           es.each_with_index do |(tag, time, record), i|
             actual_results << {
               :single      => record["single"],
@@ -520,12 +520,12 @@ EOC
               _foo_bar   ${record["foo.bar"]}
             </record>
           ]
-          d = create_driver(config, use_v1)
+          d = create_driver(config, syntax: syntax)
           record = {
             "foo.bar"    => "foo.bar",
             "@timestamp" => 10,
           }
-          es = emit(config, use_v1, [record])
+          es = feed(config, syntax: syntax, msgs: [record])
           es.each_with_index do |(tag, time, r), i|
             assert { r['_timestamp'] == record['@timestamp'] }
             assert { r['_foo_bar'] == record['foo.bar'] }
@@ -541,10 +541,10 @@ EOC
             message ${unknown}
           </record>
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         mock(d.instance.log).warn("record_reformer: unknown placeholder `${unknown}` found")
-        d.run { d.emit({}, @time) }
-        assert_equal 1, d.emits.size
+        d.run { d.feed(@time, {}) }
+        assert_equal 1, d.events.size
       end
 
       test 'failed to expand record field (enable_ruby yes)' do
@@ -555,12 +555,12 @@ EOC
             message ${unknown['bar']}
           </record>
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
-        d.run { d.emit({}, @time) }
-        # emit, but nil value
-        assert_equal 1, d.emits.size
-        d.emits.each do |(tag, time, record)|
+        d.run { d.feed(@time, {}) }
+        # feed, but nil value
+        assert_equal 1, d.events.size
+        d.events.each do |(tag, time, record)|
           assert_nil(record['message'])
         end
       end
@@ -570,11 +570,11 @@ EOC
           tag ${unknown['bar']}
           enable_ruby yes
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
-        d.run { d.emit({}, @time) }
-        # nil tag message should not be emitted
-        assert_equal 0, d.emits.size
+        d.run { d.feed(@time, {}) }
+        # nil tag message should not be feedted
+        assert_equal 0, d.events.size
       end
 
       test 'expand fields starting with @ (enable_ruby no)' do
@@ -585,10 +585,10 @@ EOC
             foo ${@timestamp}
           </record>
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         message = {"@timestamp" => "foo"}
-        d.run { d.emit(message, @time) }
-        d.emits.each do |(tag, time, record)|
+        d.run { d.feed(@time, message) }
+        d.events.each do |(tag, time, record)|
           assert_equal message["@timestamp"], record["foo"]
         end
       end
@@ -603,10 +603,10 @@ EOC
             foo ${record.map{|k,v|v}}
           </record>
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         message = {"@timestamp" => "foo"}
-        d.run { d.emit(message, @time) }
-        d.emits.each do |(tag, time, record)|
+        d.run { d.feed(@time, message) }
+        d.events.each do |(tag, time, record)|
           assert_equal [message["@timestamp"]], record["foo"]
         end
       end
@@ -619,16 +619,16 @@ EOC
             foo ${__send__("@timestamp")}
           </record>
         ]
-        d = create_driver(config, use_v1)
+        d = create_driver(config, syntax: syntax)
         message = {"@timestamp" => "foo"}
-        d.run { d.emit(message, @time) }
-        d.emits.each do |(tag, time, record)|
+        d.run { d.feed(@time, message) }
+        d.events.each do |(tag, time, record)|
           assert_equal message["@timestamp"], record["foo"]
         end
       end
     end
 
-    test "compatibility test (enable_ruby yes) (use_v1 #{use_v1})" do
+    test "compatibility test (enable_ruby yes) (syntax: #{syntax})" do
       config = %[
         tag tag
         enable_ruby yes
@@ -642,7 +642,7 @@ EOC
           _foo_bar   ${__send__('foo.bar')}
         </record>
       ]
-      d = create_driver(config, use_v1)
+      d = create_driver(config, syntax: syntax)
       record = {
         "number"     => "-",
         "hex"        => "0x10",
@@ -650,8 +650,8 @@ EOC
         "@timestamp" => 10,
         "message"    => "10",
       }
-      es = emit(config, use_v1, [record])
-      es.each_with_index do |(tag, time, r), i|
+      events = feed(config, syntax: syntax, msgs: [record])
+      events.each_with_index do |(tag, time, r), i|
         assert { r['_message'] == "prefix-#{record['message']}-suffix" }
         assert { r['_time'] == Time.at(@time) }
         assert { r['_number'] == 0 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-21 00:00:00.000000000 Z
+date: 2017-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -124,7 +124,8 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- example.conf
+- example/example.conf
+- example/workers.conf
 - fluent-plugin-record-reformer.gemspec
 - lib/fluent/plugin/out_record_reformer.rb
 - lib/fluent/plugin/out_record_reformer/core.rb
@@ -153,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to add or replace fields of a event record