fluentd 0.14.3 → 0.14.4

data/lib/fluent/winsvc.rb

@@ -30,7 +30,7 @@ begin
   def read_fluentdopt
     require 'win32/Registry'
     Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc") do |reg|
-      reg.read("fluentdopt")[1]
+      reg.read("fluentdopt")[1] rescue ""
     end
   end
 
@@ -47,7 +47,6 @@ begin
     @pid = 0
 
     def service_main
-      opt = read_fluentdopt
       @pid = service_main_start
       while running?
         sleep 10

data/test/compat/test_calls_super.rb

@@ -32,12 +32,14 @@ class CompatCallsSuperTest < Test::Unit::TestCase
   end
   class DummyGoodFilter < Fluent::Filter
     def configure(conf); super; end
+    def filter(tag, time, record); end
     def start; super; end
     def before_shutdown; super; end
     def shutdown; super; end
   end
   class DummyBadFilter < Fluent::Filter
     def configure(conf); super; end
+    def filter(tag, time, record); end
     def start; end
     def before_shutdown; end
     def shutdown; end

data/test/config/test_configurable.rb

@@ -342,6 +342,12 @@ module ConfigurableSpec
       config_param :size_of_something, :size, default: 128
     end
   end
+  class UnRecommended
+    include Fluent::Configurable
+    attr_accessor :log
+    config_param :key1, :string, default: 'deprecated', deprecated: "key1 will be removed."
+    config_param :key2, :string, default: 'obsoleted', obsoleted: "key2 has been removed."
+  end
 end
 
 module Fluent::Config
@@ -1102,5 +1108,87 @@ module Fluent::Config
         end
       end
     end
+    sub_test_case 'non-required options for config_param' do
+      test 'desc must be a string if specified' do
+        assert_raise ArgumentError.new("key: desc must be a String, but Symbol") do
+          class InvalidDescClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', desc: :invalid_description
+          end
+        end
+      end
+      test 'alias must be a symbol if specified' do
+        assert_raise ArgumentError.new("key: alias must be a Symbol, but String") do
+          class InvalidAliasClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', alias: 'yay'
+          end
+        end
+      end
+      test 'deprecated must be a string if specified' do
+        assert_raise ArgumentError.new("key: deprecated must be a String, but TrueClass") do
+          class InvalidDeprecatedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', deprecated: true
+          end
+        end
+      end
+      test 'obsoleted must be a string if specified' do
+        assert_raise ArgumentError.new("key: obsoleted must be a String, but TrueClass") do
+          class InvalidObsoletedClass
+            include Fluent::Configurable
+            config_param :key, :string, default: '', obsoleted: true
+          end
+        end
+      end
+      test 'value_type for hash must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfHashClass
+            include Fluent::Configurable
+            config_param :key, :hash, value_type: 'yay'
+          end
+        end
+      end
+      test 'value_type for array must be a symbol' do
+        assert_raise ArgumentError.new("key: value_type must be a Symbol, but String") do
+          class InvalidValueTypeOfArrayClass
+            include Fluent::Configurable
+            config_param :key, :array, value_type: 'yay'
+          end
+        end
+      end
+    end
+    sub_test_case 'enum parameters' do
+      test 'list must be specified as an array of symbols'
+    end
+    sub_test_case 'deprecated/obsoleted parameters' do
+      test 'both cannot be specified at once' do
+        assert_raise ArgumentError.new("param1: both of deprecated and obsoleted cannot be specified at once") do
+          class Buggy1
+            include Fluent::Configurable
+            config_param :param1, :string, default: '', deprecated: 'yay', obsoleted: 'foo!'
+          end
+        end
+      end
+
+      test 'warned if deprecated parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+        obj.configure(config_element('ROOT', '', {'key1' => 'yay'}, []))
+
+        assert_equal 'yay', obj.key1
+        first_log = obj.log.logs.first
+        assert{ first_log && first_log.include?("[warn]") && first_log.include?("'key1' parameter is deprecated: key1 will be removed.") }
+      end
+
+      test 'error raised if obsoleted parameter is configured' do
+        obj = ConfigurableSpec::UnRecommended.new
+        obj.log = Fluent::Test::TestLogger.new
+
+        assert_raise Fluent::ObsoletedParameterError.new("'key2' parameter is already removed: key2 has been removed.") do
+          obj.configure(config_element('ROOT', '', {'key2' => 'yay'}, []))
+        end
+      end
+    end
   end
 end

data/test/config/test_types.rb

@@ -72,9 +72,9 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
       assert_equal :value, Config::ENUM_TYPE.call('value', {list: [:val, :value, :v]})
-      assert_raises(Fluent::ConfigError){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
-      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {}) }
-      assert_raises(RuntimeError){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
+      assert_raises(Fluent::ConfigError.new("valid options are val,value,v but got x")){ Config::ENUM_TYPE.call('x', {list: [:val, :value, :v]}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {}) }
+      assert_raises(RuntimeError.new("Plugin BUG: config type 'enum' requires :list of symbols")){ Config::ENUM_TYPE.call('val', {list: ["val", "value", "v"]}) }
     end
 
     test 'integer' do
@@ -138,6 +138,8 @@ class TestConfigTypes < ::Test::Unit::TestCase
 
       assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}))
       assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h',               {value_type: :time}))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
     test 'array' do
@@ -162,6 +164,8 @@ class TestConfigTypes < ::Test::Unit::TestCase
       }
       assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options))
       assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options))
+
+      assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::ARRAY_TYPE.call("1,2", {value_type: :foo}) }
     end
   end
 end

data/test/plugin/test_filter.rb

@@ -30,6 +30,27 @@ module FluentPluginFilterTest
       record
     end
   end
+  class NumDoublePluginWithTime < Fluent::Plugin::Filter
+    def filter_with_time(tag, time, record)
+      r = record.dup
+      r["num"] = r["num"].to_i * 2
+      [time, r]
+    end
+  end
+  class IgnoreForNumPluginWithTime < Fluent::Plugin::Filter
+    def filter_with_time(tag, time, record)
+      if record["num"].is_a? Numeric
+        nil
+      else
+        [time, record]
+      end
+    end
+  end
+  class InvalidPlugin < Fluent::Plugin::Filter
+    # Because of implemnting `filter_with_time` and `filter` methods
+    def filter_with_time(tag, time, record); end
+    def filter(tag, time, record); end
+  end
 end
 
 class FilterPluginTest < Test::Unit::TestCase
@@ -39,6 +60,10 @@ class FilterPluginTest < Test::Unit::TestCase
     end
   end
 
+  setup do
+    @p = nil
+  end
+
   teardown do
     if @p
       @p.stop unless @p.stopped?
@@ -53,12 +78,24 @@ class FilterPluginTest < Test::Unit::TestCase
   sub_test_case 'for basic dummy plugin' do
     setup do
       Fluent::Test.setup
-      @p = FluentPluginFilterTest::DummyPlugin.new
+    end
+
+    test 'plugin does not define #filter raises error' do
+      assert_raise NotImplementedError do
+        FluentPluginFilterTest::DummyPlugin.new
+      end
+    end
+  end
+
+  sub_test_case 'normal filter plugin' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::NumDoublePlugin.new
     end
 
     test 'has healthy lifecycle' do
       assert !@p.configured?
-      @p.configure(config_element())
+      @p.configure(config_element)
       assert @p.configured?
 
       assert !@p.started?
@@ -94,7 +131,7 @@ class FilterPluginTest < Test::Unit::TestCase
       assert @p.respond_to?(:plugin_id_configured?)
       assert @p.respond_to?(:plugin_id)
 
-      @p.configure(config_element())
+      @p.configure(config_element)
 
       assert !@p.plugin_id_configured?
       assert @p.plugin_id
@@ -128,30 +165,13 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
-    test 'plugin does not define #filter raises error' do
-      es = [
-        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
-        [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}],
-        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
-      ]
-      assert_raise NotImplementedError do
-        @p.filter_stream('testing', es)
-      end
-    end
-  end
-
-  sub_test_case 'normal filter plugin' do
-    setup do
-      Fluent::Test.setup
-      @p = FluentPluginFilterTest::NumDoublePlugin.new
-    end
-
     test 'filters events correctly' do
       test_es = [
         [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
         [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}],
         [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
       ]
+      @p.configure(config_element)
       es = @p.filter_stream('testing', test_es)
       assert es.is_a? Fluent::EventStream
 
@@ -186,6 +206,7 @@ class FilterPluginTest < Test::Unit::TestCase
         [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Ignored, yay!"}],
         [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
       ]
+      @p.configure(config_element)
       es = @p.filter_stream('testing', test_es)
       assert es.is_a? Fluent::EventStream
 
@@ -213,8 +234,7 @@ class FilterPluginTest < Test::Unit::TestCase
 
     test 'has router and can emit events to error streams' do
       assert @p.has_router?
-
-      @p.configure(config_element())
+      @p.configure(config_element)
       assert @p.router
 
       @p.router = DummyRouter.new([])
@@ -252,4 +272,82 @@ class FilterPluginTest < Test::Unit::TestCase
       assert_equal "Value of num is Number!", error_emits[0][3].message
     end
   end
+
+  sub_test_case 'filter plugins that is implmented `filter_with_time`' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::NumDoublePluginWithTime.new
+    end
+
+    test 'filters events correctly' do
+      test_es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      es = @p.filter_stream('testing', test_es)
+      assert es.is_a? Fluent::EventStream
+
+      ary = []
+      es.each do |time, r|
+        ary << [time, r]
+      end
+
+      assert_equal 3, ary.size
+
+      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]
+      assert_equal "Hello filters!", ary[0][1]["message"]
+      assert_equal 2, ary[0][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0]
+      assert_equal 4, ary[1][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0]
+      assert_equal 6, ary[2][1]["num"]
+    end
+  end
+
+  sub_test_case 'filter plugin that is implmented `filter_with_time` and returns nil for some records' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::IgnoreForNumPluginWithTime.new
+    end
+
+    test 'filter_stream ignores records which #filter_with_time return nil' do
+      test_es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Ignored, yay!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      @p.configure(config_element)
+      es = @p.filter_stream('testing', test_es)
+      assert es.is_a? Fluent::EventStream
+
+      ary = []
+      es.each do |time, r|
+        ary << [time, r]
+      end
+
+      assert_equal 2, ary.size
+
+      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]
+      assert_equal "Hello filters!", ary[0][1]["message"]
+      assert_equal "1", ary[0][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]
+      assert_equal "3", ary[1][1]["num"]
+    end
+  end
+
+  sub_test_case 'filter plugins that is implmented both `filter_with_time` and `filter`' do
+    setup do
+      Fluent::Test.setup
+    end
+
+    test 'raises DuplicatedImplementError' do
+      assert_raise do
+        FluentPluginFilterTest::InvalidPlugin.new
+      end
+    end
+  end
 end

data/test/plugin/test_filter_record_transformer.rb

@@ -174,12 +174,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       d = create_driver(config)
       yield d if block_given?
       d.run {
-        msgs.each do |msg|
-          record = {
-            'eventType0' => 'bar',
-            'message'    => msg,
-          }
-          record = record.merge(msg) if msg.is_a?(Hash)
+        records = msgs.map do |msg|
+          next msg if msg.is_a?(Hash)
+          { 'eventType0' => 'bar', 'message' => msg }
+        end
+        records.each do |record|
           d.feed(@tag, @time, record)
         end
       }
@@ -273,6 +272,23 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         end
       end
 
+      test "Prevent overwriting reserved keys such as tag with enable_ruby #{enable_ruby}" do
+        config = %[
+          enable_ruby #{enable_ruby}
+          <record>
+            new_tag ${tag}
+            new_record_tag ${record["tag"]}
+          </record>
+        ]
+        records = [{'tag' => 'tag', 'time' => 'time'}]
+        filtered = filter(config, records)
+        filtered.each_with_index do |(_t, r), i|
+          assert_not_equal('tag', r['new_tag'])
+          assert_equal(@tag, r['new_tag'])
+          assert_equal('tag', r['new_record_tag'])
+        end
+      end
+
       test "hash values with placeholders with enable_ruby #{enable_ruby}" do
         config = %[
           enable_ruby #{enable_ruby}

data/test/plugin/test_in_debug_agent.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_debug_agent'
 require 'fileutils'
 
@@ -13,7 +13,7 @@ class DebugAgentInputTest < Test::Unit::TestCase
   TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/in_debug_agent")
 
   def create_driver(conf = '')
-    Fluent::Test::InputTestDriver.new(Fluent::DebugAgentInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput).configure(conf)
   end
 
   def test_unix_path_writable

data/test/plugin/test_in_forward.rb

@@ -29,6 +29,7 @@ class ForwardInputTest < Test::Unit::TestCase
     port #{PORT}
     bind 127.0.0.1
   ]
+  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
 
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
@@ -260,7 +261,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        option = d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
         assert_equal option['size'], events.length
       end
     end
@@ -282,7 +283,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -311,7 +312,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -338,7 +339,7 @@ class ForwardInputTest < Test::Unit::TestCase
     # d.run => send_data
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -360,7 +361,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     # d.run => send_data
     d.run do
-      d.instance.send(:on_message, data, 1000000000, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      d.instance.send(:on_message, data, 1000000000, PEERADDR)
     end
 
     # check emitted data
@@ -476,7 +477,6 @@ class ForwardInputTest < Test::Unit::TestCase
 
     assert_equal events, d.emits
     assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
-
   end
 
   def test_not_respond_to_message_not_requiring_ack
@@ -583,5 +583,53 @@ class ForwardInputTest < Test::Unit::TestCase
     @responses << res if try_to_receive_response
   end
 
+  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
+  test 'message protocol with source_hostname_key' do
+    execute_test { |events|
+      events.each { |tag, time, record|
+        send_data [tag, time, record].to_msgpack
+      }
+    }
+  end
+
+  test 'forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ['tag1', entries].to_msgpack
+    }
+  end
+
+  test 'packed forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = ''
+      events.each { |tag, time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    }
+  end
+
+  def execute_test(&block)
+    d = create_driver(CONFIG + 'source_hostname_key source')
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      block.call(events)
+    end
+
+    d.emits.each { |tag, _time, record|
+      assert_true record.has_key?('source')
+    }
+  end
+
   # TODO heartbeat
 end