fluentd 1.8.1-x86-mingw32 → 1.9.0-x86-mingw32

data/lib/fluent/system_config.rb

@@ -27,7 +27,7 @@ module Fluent
       :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value
+      :strict_config_value, :enable_msgpack_time_support
     ]
 
     config_param :workers,   :integer, default: 1
@@ -42,6 +42,7 @@ module Fluent
     config_param :enable_get_dump, :bool, default: nil
     config_param :process_name,    :string, default: nil
     config_param :strict_config_value, :bool, default: nil
+    config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end

data/lib/fluent/test/driver/base.rb

@@ -22,6 +22,7 @@ require 'fluent/clock'
 require 'serverengine/socket_manager'
 require 'fileutils'
 require 'timeout'
+require 'logger'
 
 module Fluent
   module Test
@@ -135,16 +136,16 @@ module Fluent
           # same with above
         end
 
-        def instance_shutdown
+        def instance_shutdown(log: Logger.new($stdout))
           instance_hook_before_stopped
 
           show_errors_if_exists = ->(label, block){
             begin
               block.call
             rescue => e
-              puts "unexpected error while #{label}, #{e.class}:#{e.message}"
+              log.error "unexpected error while #{label}, #{e.class}:#{e.message}"
               e.backtrace.each do |bt|
-                puts "\t#{bt}"
+                log.error "\t#{bt}"
               end
             end
           }

data/lib/fluent/variable_store.rb

@@ -0,0 +1,40 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  # VariableStore provides all pluigns with the way to shared variable without using class variable
+  # it's for safe reloading mechanism
+  class VariableStore
+    @data = {}
+
+    class << self
+      def fetch_or_build(namespace, default_value: {})
+        @data[namespace] ||= default_value
+      end
+
+      def try_to_reset
+        @data, old = {}, @data
+
+        begin
+          yield
+        rescue
+          @data = old
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.8.1'
+  VERSION = '1.9.0'
 
 end

data/test/config/test_config_parser.rb

@@ -362,55 +362,56 @@ module Fluent::Config
     # port from test_config.rb
     sub_test_case '@include parsing' do
       TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"
+      TMP_DIR_WITH_SPACES = File.dirname(__FILE__) + "/tmp/folder with spaces/v1_config#{ENV['TEST_ENV_NUMBER']}"
 
       def write_config(path, data)
         FileUtils.mkdir_p(File.dirname(path))
         File.open(path, "w") { |f| f.write data }
       end
 
-      def prepare_config
-        write_config "#{TMP_DIR}/config_test_1.conf", %[
+      def prepare_config(tmp_dir)
+        write_config "#{tmp_dir}/config_test_1.conf", %[
         k1 root_config
         include dir/config_test_2.conf  #
-        @include #{TMP_DIR}/config_test_4.conf
-        include file://#{TMP_DIR}/config_test_5.conf
+        @include #{tmp_dir}/config_test_4.conf
+        include file://#{tmp_dir}/config_test_5.conf
         @include config.d/*.conf
       ]
-        write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+        write_config "#{tmp_dir}/dir/config_test_2.conf", %[
         k2 relative_path_include
         @include ../config_test_3.conf
       ]
-        write_config "#{TMP_DIR}/config_test_3.conf", %[
+        write_config "#{tmp_dir}/config_test_3.conf", %[
         k3 relative_include_in_included_file
       ]
-        write_config "#{TMP_DIR}/config_test_4.conf", %[
+        write_config "#{tmp_dir}/config_test_4.conf", %[
         k4 absolute_path_include
       ]
-        write_config "#{TMP_DIR}/config_test_5.conf", %[
+        write_config "#{tmp_dir}/config_test_5.conf", %[
         k5 uri_include
       ]
-        write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+        write_config "#{tmp_dir}/config.d/config_test_6.conf", %[
         k6 wildcard_include_1
         <elem1 name>
           include normal_parameter
         </elem1>
       ]
-        write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+        write_config "#{tmp_dir}/config.d/config_test_7.conf", %[
         k7 wildcard_include_2
       ]
-        write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+        write_config "#{tmp_dir}/config.d/config_test_8.conf", %[
         <elem2 name>
           @include ../dir/config_test_9.conf
         </elem2>
       ]
-        write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+        write_config "#{tmp_dir}/dir/config_test_9.conf", %[
         k9 embedded
         <elem3 name>
           nested nested_value
           include hoge
         </elem3>
       ]
-        write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+        write_config "#{tmp_dir}/config.d/00_config_test_8.conf", %[
         k8 wildcard_include_3
         <elem4 name>
           include normal_parameter
@@ -418,9 +419,11 @@ module Fluent::Config
       ]
       end
 
-      test 'parses @include / include correctly' do
-        prepare_config
-        c = read_config("#{TMP_DIR}/config_test_1.conf")
+      data("TMP_DIR without spaces" => TMP_DIR,
+           "TMP_DIR with spaces" => TMP_DIR_WITH_SPACES)
+      test 'parses @include / include correctly' do |data|
+        prepare_config(data)
+        c = read_config("#{data}/config_test_1.conf")
         assert_equal('root_config', c['k1'])
         assert_equal('relative_path_include', c['k2'])
         assert_equal('relative_include_in_included_file', c['k3'])

data/test/config/test_system_config.rb

@@ -68,7 +68,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(1, sc.workers)
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
@@ -76,6 +76,7 @@ module Fluent::Config
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_nil(sc.enable_msgpack_time_support)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
     end
@@ -89,6 +90,7 @@ module Fluent::Config
       'suppress_config_dump' => ['suppress_config_dump', true],
       'without_source' => ['without_source', true],
       'strict_config_value' => ['strict_config_value', true],
+      'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
     )
     test "accepts parameters" do |(k, v)|
       conf = parse_text(<<-EOS)
@@ -98,7 +100,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       if k == 'log_level'
         assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k))
       else
@@ -117,7 +119,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(:json, sc.log.format)
       assert_equal('%Y', sc.log.time_format)
     end
@@ -136,7 +138,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new(log_level: level)
       sc = Fluent::SystemConfig.new(conf)
-      sc.overwrite_variables(s.for_system_config)
+      sc.overwrite_variables(**s.for_system_config)
       assert_equal(level, sc.log_level)
     end
   end

data/test/plugin/test_in_exec.rb

@@ -28,7 +28,7 @@ class ExecInputTest < Test::Unit::TestCase
 
   TSV_CONFIG = %[
     command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
-    run_interval 1s
+    run_interval 0.3
     <parse>
       @type tsv
       keys time, tag, k1
@@ -43,7 +43,7 @@ class ExecInputTest < Test::Unit::TestCase
 
   JSON_CONFIG = %[
     command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
-    run_interval 1s
+    run_interval 0.3
     <parse>
       @type json
     </parse>
@@ -56,7 +56,7 @@ class ExecInputTest < Test::Unit::TestCase
 
   MSGPACK_CONFIG = %[
     command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
-    run_interval 1s
+    run_interval 0.3
     <parse>
       @type msgpack
     </parse>
@@ -70,7 +70,7 @@ class ExecInputTest < Test::Unit::TestCase
   # here document for not de-quoting backslashes
   REGEXP_CONFIG = %[
     command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
-    run_interval 1s
+    run_interval 0.3
     tag regex_tag
 ] + <<'EOC'
     <parse>
@@ -143,7 +143,7 @@ EOC
       time_key time
       tag_key tag
       time_format %Y-%m-%d %H:%M:%S
-      run_interval 1s
+      run_interval 0.3
   ]
 
   JSON_CONFIG_COMPAT = %[
@@ -151,7 +151,7 @@ EOC
       format json
       tag_key tag
       time_key time
-      run_interval 1s
+      run_interval 0.3
   ]
 
   MSGPACK_CONFIG_COMPAT = %[
@@ -159,14 +159,14 @@ EOC
       format msgpack
       tag_key tagger
       time_key datetime
-      run_interval 1s
+      run_interval 0.3
   ]
 
   REGEXP_CONFIG_COMPAT = %[
       command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
       format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
       tag regex_tag
-      run_interval 1s
+      run_interval 0.3
   ]
 
   sub_test_case 'with traditional configuration' do
@@ -235,7 +235,7 @@ EOC
     d.run(expect_emits: 2, timeout: 10)
 
     assert{ d.events.length > 0 }
-    d.events.each_with_index {|event, i|
+    d.events.each {|event|
       assert_equal_event_time(time, event[1])
       assert_equal [tag, time, record], event
     }

data/test/plugin/test_in_forward.rb

@@ -231,10 +231,9 @@ class ForwardInputTest < Test::Unit::TestCase
         ["tag2", time_i, {"a"=>2}],
       ]
 
-      d.run(expect_records: records.length, timeout: 20) do
+      d.run(expect_records: records.length, timeout: 20, shutdown: true) do
         records.each {|tag, _time, record|
           send_data [tag, _time, record].to_json + "\n"
-        sleep 1
         }
       end
 
@@ -727,7 +726,7 @@ class ForwardInputTest < Test::Unit::TestCase
         events.each {|tag, _time, record|
           op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
           expected_acks << op['chunk']
-          send_data [tag, _time, record, op].to_msgpack, try_to_receive_response: true, **options
+          send_data([tag, _time, record, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
         }
       end
 
@@ -769,7 +768,7 @@ class ForwardInputTest < Test::Unit::TestCase
         }
         op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
         expected_acks << op['chunk']
-        send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
+        send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
       end
 
       assert_equal events, d.events
@@ -809,7 +808,7 @@ class ForwardInputTest < Test::Unit::TestCase
         }
         op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
         expected_acks << op['chunk']
-        send_data ["tag1", entries, op].to_msgpack, try_to_receive_response: true, **options
+        send_data(["tag1", entries, op].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
       end
 
       assert_equal events, d.events
@@ -849,7 +848,7 @@ class ForwardInputTest < Test::Unit::TestCase
         events.each {|tag, _time, record|
           op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
           expected_acks << op['chunk']
-          send_data [tag, _time, record, op].to_json, try_to_receive_response: true, **options
+          send_data([tag, _time, record, op].to_json, try_to_receive_response: true, response_timeout: 1, **options)
         }
       end
 
@@ -885,7 +884,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run(expect_records: events.size, timeout: 20) do
         events.each {|tag, _time, record|
-          send_data [tag, _time, record].to_msgpack, try_to_receive_response: true, **options
+          send_data([tag, _time, record].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
         }
       end
 
@@ -922,7 +921,7 @@ class ForwardInputTest < Test::Unit::TestCase
         events.each {|tag, _time, record|
           entries << [_time, record]
         }
-        send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
+        send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
       end
 
       assert_equal events, d.events
@@ -958,7 +957,7 @@ class ForwardInputTest < Test::Unit::TestCase
         events.each {|tag, _time, record|
           [_time, record].to_msgpack(entries)
         }
-        send_data ["tag1", entries].to_msgpack, try_to_receive_response: true, **options
+        send_data(["tag1", entries].to_msgpack, try_to_receive_response: true, response_timeout: 1, **options)
       end
 
       assert_equal events, d.events
@@ -994,7 +993,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run(expect_records: events.size, timeout: 20) do
         events.each {|tag, _time, record|
-          send_data [tag, _time, record].to_json, try_to_receive_response: true, **options
+          send_data([tag, _time, record].to_json, try_to_receive_response: true, response_timeout: 1, **options)
         }
       end
 
@@ -1016,7 +1015,7 @@ class ForwardInputTest < Test::Unit::TestCase
   # nil: socket read timeout
   def read_data(io, timeout, &block)
     res = ''
-    select_timeout = 2
+    select_timeout = 0.5
     clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
     timeout_at = Process.clock_gettime(clock_id) + timeout
     begin

data/test/plugin/test_in_http.rb

@@ -208,6 +208,38 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_exact_match_for_expect
+    d = create_driver(CONFIG)
+    records = [{ "a" => 1}, { "a" => 2 }]
+    tag = "tag1"
+    res_codes = []
+
+    d.run(expect_records: 0, timeout: 5) do
+      res = post("/#{tag}", { "json" => records.to_json }, { 'Expect' => 'something' })
+      res_codes << res.code
+    end
+    assert_equal ["417"], res_codes
+  end
+
+  def test_exact_match_for_expect_with_other_header
+    d = create_driver(CONFIG)
+
+    records = [{ "a" => 1}, { "a" => 2 }]
+    tag = "tag1"
+    res_codes = []
+
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", { "json" => records.to_json, 'x-envoy-expected-rq-timeout-ms' => 4 })
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+
+    assert_equal "tag1", d.events[0][0]
+    assert_equal 1, d.events[0][2]["a"]
+    assert_equal "tag1", d.events[1][0]
+    assert_equal 2, d.events[1][2]["a"]
+  end
+
   def test_multi_json_with_add_remote_addr
     d = create_driver(CONFIG + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -446,7 +478,7 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[0][1]
     assert_equal_event_time time, d.events[1][1]
   end
-  
+
   def test_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -716,7 +748,7 @@ class HttpInputTest < Test::Unit::TestCase
       end
     end
   end
-  
+
   def test_cors_allowed_exclude_empty_string
     d = create_driver(CONFIG + 'cors_allow_origins ["", "http://*.foo.com"]')
 
@@ -736,7 +768,7 @@ class HttpInputTest < Test::Unit::TestCase
       end
     end
   end
-  
+
   def test_cors_allowed_wildcard_preflight_for_subdomain
     d = create_driver(CONFIG + 'cors_allow_origins ["http://*.foo.com"]')