fluentd 1.8.1 → 1.9.0.rc1

data/lib/fluent/test/driver/base.rb

@@ -22,6 +22,7 @@ require 'fluent/clock'
 require 'serverengine/socket_manager'
 require 'fileutils'
 require 'timeout'
+require 'logger'
 
 module Fluent
   module Test
@@ -135,16 +136,16 @@ module Fluent
           # same with above
         end
 
-        def instance_shutdown
+        def instance_shutdown(log: Logger.new($stdout))
           instance_hook_before_stopped
 
           show_errors_if_exists = ->(label, block){
             begin
               block.call
             rescue => e
-              puts "unexpected error while #{label}, #{e.class}:#{e.message}"
+              log.error "unexpected error while #{label}, #{e.class}:#{e.message}"
               e.backtrace.each do |bt|
-                puts "\t#{bt}"
+                log.error "\t#{bt}"
               end
             end
           }

data/lib/fluent/variable_store.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  # VariableStore provides all pluigns with the way to shared variable without using class variable
+  # it's for safe reloading mechanism
+  class VariableStore
+    @data = {}
+
+    class << self
+      def fetch_or_build(namespace, default_value: {})
+        @data[namespace] ||= default_value
+      end
+
+      def try_to_reset
+        @data, old = {}, @data
+
+        begin
+          yield
+        rescue
+          @data = old
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.8.1'
+  VERSION = '1.9.0.rc1'
 
 end

data/test/config/test_system_config.rb

@@ -68,7 +68,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(1, sc.workers)
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
@@ -98,7 +98,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       if k == 'log_level'
         assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k))
       else
@@ -117,7 +117,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(:json, sc.log.format)
       assert_equal('%Y', sc.log.time_format)
     end
@@ -136,7 +136,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new(log_level: level)
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(level, sc.log_level)
     end
   end

data/test/plugin/test_in_http.rb

@@ -208,6 +208,38 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_exact_match_for_expect
+    d = create_driver(CONFIG)
+    records = [{ "a" => 1}, { "a" => 2 }]
+    tag = "tag1"
+    res_codes = []
+
+    d.run(expect_records: 0, timeout: 5) do
+      res = post("/#{tag}", { "json" => records.to_json }, { 'Expect' => 'something' })
+      res_codes << res.code
+    end
+    assert_equal ["417"], res_codes
+  end
+
+  def test_exact_match_for_expect_with_other_header
+    d = create_driver(CONFIG)
+
+    records = [{ "a" => 1}, { "a" => 2 }]
+    tag = "tag1"
+    res_codes = []
+
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", { "json" => records.to_json, 'x-envoy-expected-rq-timeout-ms' => 4 })
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+
+    assert_equal "tag1", d.events[0][0]
+    assert_equal 1, d.events[0][2]["a"]
+    assert_equal "tag1", d.events[1][0]
+    assert_equal 2, d.events[1][2]["a"]
+  end
+
   def test_multi_json_with_add_remote_addr
     d = create_driver(CONFIG + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -446,7 +478,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[0][1]
     assert_equal_event_time time, d.events[1][1]
   end
-  
+
   def test_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -716,7 +748,7 @@ class HttpInputTest < Test::Unit::TestCase
       end
     end
   end
-  
+
   def test_cors_allowed_exclude_empty_string
     d = create_driver(CONFIG + 'cors_allow_origins ["", "http://*.foo.com"]')
 
@@ -736,7 +768,7 @@ class HttpInputTest < Test::Unit::TestCase
       end
     end
   end
-  
+
   def test_cors_allowed_wildcard_preflight_for_subdomain
     d = create_driver(CONFIG + 'cors_allow_origins ["http://*.foo.com"]')
 

data/test/plugin/test_out_http.rb

@@ -117,7 +117,7 @@ class HTTPOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
     FileUtils.rm_rf(TMP_DIR)
 
-    @@result = Result.new(nil, nil, {}, nil)    
+    @@result = Result.new(nil, nil, {}, nil)
     @@http_server_thread ||= Thread.new do
       run_http_server
     end
@@ -230,6 +230,9 @@ class HTTPOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_retryable_response
+    old_report_on_exception = Thread.report_on_exception
+    Thread.report_on_exception = false # thread finished as invalid state since RetryableResponse raises.
+
     d = create_driver("endpoint #{base_endpoint}/503")
     assert_raise(Fluent::Plugin::HTTPOutput::RetryableResponse) do
       d.run(default_tag: 'test.http', shutdown: false) do
@@ -238,7 +241,10 @@ class HTTPOutputTest < Test::Unit::TestCase
         }
       end
     end
-    d.instance_shutdown
+
+    d.instance_shutdown(log: $log)
+  ensure
+    Thread.report_on_exception = old_report_on_exception
   end
 
   def test_write_with_disabled_unrecoverable

data/test/plugin/test_output.rb

@@ -855,7 +855,7 @@ class OutputTest < Test::Unit::TestCase
     sub_test_case 'configure secondary' do
       test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
         o = create_output(:buffered)
-        mock(o.log).warn("secondary type should be same with primary one",
+        mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                          { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
@@ -864,7 +864,7 @@ class OutputTest < Test::Unit::TestCase
 
       test "don't warn if primary type is the same as secondary type" do
         o = Fluent::Plugin::TestOutput.new
-        mock(o.log).warn("secondary type should be same with primary one",
+        mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                          { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
         o.configure(config_element('ROOT','',{'name' => "cool2"},
@@ -876,7 +876,7 @@ class OutputTest < Test::Unit::TestCase
 
       test "don't warn if primary type is different from secondary type and both don't have custom_format" do
         o = create_output(:standard)
-        mock(o.log).warn("secondary type should be same with primary one",
+        mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
                          { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -153,7 +153,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       i = create_output()
       i.configure(config_element('ROOT','',{},[priconf,secconf]))
       logs = i.log.out.logs
-      assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } }
+      assert{ logs.any?{|l| l.include?("Use different plugin for secondary. Check the plugin works with primary like secondary_file") } }
     end
 
     test 'secondary plugin lifecycle is kicked by primary' do
@@ -162,7 +162,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       i = create_output()
       i.configure(config_element('ROOT','',{},[priconf,secconf]))
       logs = i.log.out.logs
-      assert{ logs.any?{|l| l.include?("secondary type should be same with primary one") } }
+      assert{ logs.any?{|l| l.include?("Use different plugin for secondary. Check the plugin works with primary like secondary_file") } }
 
       assert i.secondary.configured?
 

data/test/test_config.rb

@@ -151,16 +151,16 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal before_size, match_conf.unused.size
   end
 
-  def write_config(path, data)
+  def write_config(path, data, encoding: 'utf-8')
     FileUtils.mkdir_p(File.dirname(path))
-    File.open(path, "w") {|f| f.write data }
+    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
   def test_inline
     prepare_config
     opts = {
-        :config_path => "#{TMP_DIR}/config_test_1.conf",
-        :inline_config => "<source>\n  type http\n  port 2222\n </source>"
+      :config_path => "#{TMP_DIR}/config_test_1.conf",
+      :inline_config => "<source>\n  type http\n  port 2222\n </source>"
     }
     assert_nothing_raised do
       Fluent::Supervisor.new(opts)
@@ -175,5 +175,27 @@ class ConfigTest < Test::Unit::TestCase
     logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
     $log = Fluent::Log.new(logger)
   end
-end
 
+  sub_test_case '.build' do
+    test 'read config' do
+      write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build.conf")
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with encoding' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "#てすと\nkey value", encoding: 'shift_jis')
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", encoding: 'shift_jis')
+      assert_equal('value', c['key'])
+    end
+
+    test 'read config with additional_config' do
+      write_config("#{TMP_DIR}/build/config_build2.conf", "key value")
+
+      c = Fluent::Config.build(config_path: "#{TMP_DIR}/build/config_build2.conf", additional_config: 'key2 value2')
+      assert_equal('value', c['key'])
+      assert_equal('value2', c['key2'])
+    end
+  end
+end

data/test/test_engine.rb

@@ -0,0 +1,203 @@
+require_relative 'helper'
+require 'fluent/engine'
+require 'fluent/config'
+require 'fluent/input'
+require 'fluent/system_config'
+
+class EngineTest < ::Test::Unit::TestCase
+  class DummyEngineTestOutput < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTest2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_engine_test2', self)
+    def write(chunk); end
+  end
+
+  class DummyEngineTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineTest2Input < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_test2', self)
+    def multi_workers_ready?; true; end
+  end
+
+  class DummyEngineClassVarTestInput < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_engine_class_var_test', self)
+    @@test = nil
+    def multi_workers_ready?; true; end
+  end
+
+  sub_test_case '#reload_config' do
+    test 'reload new configuration' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.once
+        agent
+      end
+
+      engine.reload_config(new_conf)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test "doesn't start RootAgent when supervisor is true" do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test2
+        </source>
+        <match>
+          @type dummy_engine_test2
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      engine.reload_config(new_conf, supervisor: true)
+
+      assert_kind_of DummyEngineTest2Input, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTest2Output, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when conf is invalid' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      new_conf_data = <<-CONF
+        <source>
+          @type
+        </source>
+      CONF
+
+      new_conf = Fluent::Config.parse(new_conf_data, '(test)', '(test_dir)', true)
+
+      agent = Fluent::RootAgent.new(log: $log, system_config: system_config)
+      stub(Fluent::RootAgent).new do
+        stub(agent).start.never
+        agent
+      end
+
+      assert_raise(Fluent::ConfigError.new("Missing '@type' parameter on <source> directive")) do
+        engine.reload_config(new_conf)
+      end
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+
+    test 'raise an error when unreloadable exists' do
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+      system_config = Fluent::SystemConfig.create(conf)
+
+      engine = Fluent::EngineClass.new
+      engine.init(system_config)
+      engine.configure(conf)
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+
+      conf_data = <<-CONF
+        <source>
+          @type dummy_engine_class_var_test
+        </source>
+        <match>
+          @type dummy_engine_test
+        </match>
+      CONF
+
+      new_conf = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
+
+      e = assert_raise(Fluent::ConfigError) do
+        engine.reload_config(new_conf)
+      end
+      assert e.message.match?('Unreloadable plugin plugin: dummy_engine_class_var_test')
+
+      assert_kind_of DummyEngineTestInput, engine.root_agent.inputs[0]
+      assert_kind_of DummyEngineTestOutput, engine.root_agent.outputs[0]
+    end
+  end
+end