fluentd 1.11.3-x86-mingw32 → 1.12.2-x86-mingw32

data/test/plugin_helper/test_compat_parameters.rb

@@ -10,6 +10,11 @@ class CompatParameterTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
     @i = nil
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   teardown do
@@ -226,7 +231,7 @@ class CompatParameterTest < Test::Unit::TestCase
       t = event_time('2016-06-24 16:05:01') # localtime
       iso8601str = Time.at(t.to_i).iso8601
       formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
-      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
     end
 
     test 'plugin helper setups time injecting as unix time (integer from epoch)' do
@@ -260,7 +265,7 @@ class CompatParameterTest < Test::Unit::TestCase
       t = event_time('2016-06-24 16:05:01') # localtime
       iso8601str = Time.at(t.to_i).iso8601
       formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
-      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
     end
   end
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -132,7 +132,9 @@ class HttpHelperTest < Test::Unit::TestCase
         error = e
       end
 
-      resp = Response.new(response.status.to_s, response.body.read, response.headers)
+      if response
+        resp = Response.new(response.status.to_s, response.body.read, response.headers)
+      end
     end
 
     if error

data/test/plugin_helper/test_inject.rb

@@ -176,6 +176,48 @@ class InjectHelperTest < Test::Unit::TestCase
       assert_equal record.merge({"timedata" => float_time}), @d.inject_values_to_record('tag', time, record)
     end
 
+    test 'injects time as unix time millis into specified key' do
+      time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
+      time_subsecond = 320_101_224
+      time = Fluent::EventTime.new(time_in_unix, time_subsecond)
+      unixtime_millis = 1466464211320
+
+      @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime_millis"))
+      @d.start
+
+      record = {"key1" => "value1", "key2" => 2}
+      assert_equal record.merge({"timedata" => unixtime_millis}), @d.inject_values_to_record('tag', time, record)
+      assert_equal record.merge({"timedata" => time_in_unix * 1_000}), @d.inject_values_to_record('tag', time_in_unix, record)
+    end
+
+    test 'injects time as unix time micros into specified key' do
+      time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
+      time_subsecond = 320_101_224
+      time = Fluent::EventTime.new(time_in_unix, time_subsecond)
+      unixtime_micros = 1466464211320101
+
+      @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime_micros"))
+      @d.start
+
+      record = {"key1" => "value1", "key2" => 2}
+      assert_equal record.merge({"timedata" => unixtime_micros}), @d.inject_values_to_record('tag', time, record)
+      assert_equal record.merge({"timedata" => time_in_unix * 1_000_000}), @d.inject_values_to_record('tag', time_in_unix, record)
+    end
+
+    test 'injects time as unix time nanos into specified key' do
+      time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
+      time_subsecond = 320_101_224
+      time = Fluent::EventTime.new(time_in_unix, time_subsecond)
+      unixtime_nanos = 1466464211320101224
+
+      @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime_nanos"))
+      @d.start
+
+      record = {"key1" => "value1", "key2" => 2}
+      assert_equal record.merge({"timedata" => unixtime_nanos}), @d.inject_values_to_record('tag', time, record)
+      assert_equal record.merge({"timedata" => time_in_unix * 1_000_000_000}), @d.inject_values_to_record('tag', time_in_unix, record)
+    end
+
     test 'injects time as unix time into specified key' do
       time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
       time_subsecond = 320_101_224

data/test/plugin_helper/test_server.rb

@@ -832,8 +832,9 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     chain_cert.sign(root_key, "sha256")
 
     server_cert, server_key, _ = CertUtil.cert_option_generate_pair(create_server_options, chain_cert.subject)
-    server_cert.add_extension OpenSSL::X509::Extension.new('basicConstraints', OpenSSL::ASN1.Sequence([OpenSSL::ASN1::Boolean(false)]))
-    server_cert.add_extension OpenSSL::X509::Extension.new('nsCertType', 'server')
+    factory = OpenSSL::X509::ExtensionFactory.new
+    server_cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE'))
+    server_cert.add_extension(factory.create_extension('nsCertType', 'server'))
     server_cert.sign(chain_key, "sha256")
 
     # write chained cert
@@ -1494,8 +1495,13 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test "can't connect with different TLS version" do
         @d.server_create_tls(:s, PORT, tls_options: @tls_options) do |data, conn|
         end
+        if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+          version = :'TLS1_3'
+        else
+          version = :'TLS1_1'
+        end
         assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {
-          open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: :'TLS1_1') do |sock|
+          open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: version) do |sock|
           end
         }
       end
@@ -1503,14 +1509,21 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       test "can specify multiple TLS versions by min_version/max_version" do
         omit "min_version=/max_version= is not supported" unless Fluent::TLS::MIN_MAX_AVAILABLE
 
-        opts = @tls_options.merge(min_version: :'TLS1_1', max_version: :'TLSv1_2')
+        min_version = :'TLS1_2'
+        if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+          max_version = :'TLS1_3'
+        else
+          max_version = :'TLS1_2'
+        end
+
+        opts = @tls_options.merge(min_version: min_version, max_version: max_version)
         @d.server_create_tls(:s, PORT, tls_options: opts) do |data, conn|
         end
         assert_raise(OpenSSL::SSL::SSLError, Errno::ECONNRESET) {
           open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: :'TLS1') do |sock|
           end
         }
-        [:'TLS1_1', :'TLS1_2'].each { |ver|
+        [min_version, max_version].each { |ver|
           assert_nothing_raised {
             open_tls_session('127.0.0.1', PORT, cert_path: @cert_path, version: ver) do |sock|
             end

data/test/test_capability.rb

@@ -0,0 +1,74 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/capability'
+
+class FluentCapabilityTest < ::Test::Unit::TestCase
+  setup do
+    @capability = Fluent::Capability.new(:current_process)
+    omit "Fluent::Capability class is not usable on this environment" unless @capability.usable?
+  end
+
+  sub_test_case "check capability" do
+    test "effective" do
+      @capability.clear(:both)
+      assert_true @capability.update(:add, :effective, :dac_read_search)
+      assert_equal CapNG::Result::PARTIAL, @capability.have_capabilities?(:caps)
+      assert_nothing_raised do
+        @capability.apply(:caps)
+      end
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)
+      assert_true @capability.have_capability?(:effective, :dac_read_search)
+      assert_false @capability.have_capability?(:inheritable, :dac_read_search)
+      assert_false @capability.have_capability?(:permitted, :dac_read_search)
+    end
+
+    test "inheritable" do
+      @capability.clear(:both)
+      capabilities = [:chown, :dac_override]
+      assert_equal [true, true], @capability.update(:add, :inheritable, capabilities)
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:caps)
+      assert_nothing_raised do
+        @capability.apply(:caps)
+      end
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)
+      capabilities.each do |capability|
+        assert_false @capability.have_capability?(:effective, capability)
+        assert_true @capability.have_capability?(:inheritable, capability)
+        assert_false @capability.have_capability?(:permitted, capability)
+      end
+    end
+
+    test "permitted" do
+      @capability.clear(:both)
+      capabilities = [:fowner, :fsetid, :kill]
+      assert_equal [true, true, true], @capability.update(:add, :permitted, capabilities)
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:caps)
+      assert_nothing_raised do
+        @capability.apply(:caps)
+      end
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)
+      capabilities.each do |capability|
+        assert_false @capability.have_capability?(:effective, capability)
+        assert_false @capability.have_capability?(:inheritable, capability)
+        assert_true @capability.have_capability?(:permitted, capability)
+      end
+    end
+
+    test "effective/inheritable/permitted" do
+      @capability.clear(:both)
+      capabilities = [:setpcap, :net_admin, :net_raw, :sys_boot, :sys_time]
+      update_type = CapNG::Type::EFFECTIVE | CapNG::Type::INHERITABLE | CapNG::Type::PERMITTED
+      assert_equal [true, true, true, true, true], @capability.update(:add, update_type, capabilities)
+      assert_equal CapNG::Result::PARTIAL, @capability.have_capabilities?(:caps)
+      assert_nothing_raised do
+        @capability.apply(:caps)
+      end
+      assert_equal CapNG::Result::NONE, @capability.have_capabilities?(:bounds)
+      capabilities.each do |capability|
+        assert_true @capability.have_capability?(:effective, capability)
+        assert_true @capability.have_capability?(:inheritable, capability)
+        assert_true @capability.have_capability?(:permitted, capability)
+      end
+    end
+  end
+end

data/test/test_event.rb

@@ -401,6 +401,22 @@ module EventTest
         i += 1
       }
     end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'size_after_any' do
+      @es.any?
+
+      assert_equal 2, @es.size
+    end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'each_after_any' do
+      @es.any?
+
+      count = 0
+      @es.each { |time, record| count += 1 }
+      assert_equal 2, count
+    end
   end
 
   class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase

data/test/test_formatter.rb

@@ -53,6 +53,11 @@ module FormatterTest
     def setup
       @formatter = Fluent::Test::FormatterTestDriver.new('out_file')
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     def configure(conf)
@@ -63,28 +68,28 @@ module FormatterTest
       configure({})
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time
       configure('output_time' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{tag}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_tag
       configure('output_tag' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time_and_tag
       configure('output_tag' => 'false', 'output_time' => 'false')
       formatted = @formatter.format('tag', @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time_and_tag_against_string_literal_configure
@@ -95,7 +100,7 @@ module FormatterTest
       ])
       formatted = @formatter.format('tag', @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
   end
 
@@ -105,6 +110,11 @@ module FormatterTest
     def setup
       @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::JSONFormatter)
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     data('oj' => 'oj', 'yajl' => 'yajl')
@@ -112,7 +122,7 @@ module FormatterTest
       @formatter.configure('json_parser' => data)
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
   end
 
@@ -138,6 +148,11 @@ module FormatterTest
     def setup
       @formatter = TextFormatter::LabeledTSVFormatter.new
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     def test_config_params
@@ -157,7 +172,7 @@ module FormatterTest
       @formatter.configure({})
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message:awesome\tgreeting:hello\n", formatted)
+      assert_equal("message:awesome\tgreeting:hello#{@newline}", formatted)
     end
 
     def test_format_with_customized_delimiters
@@ -167,7 +182,37 @@ module FormatterTest
       )
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message=awesome,greeting=hello\n", formatted)
+      assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
+    end
+
+    def record_with_tab
+      {'message' => "awe\tsome", 'greeting' => "hello\t"}
+    end
+
+    def test_format_suppresses_tab
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
+    end
+
+    def test_format_suppresses_tab_custom_replacement
+      @formatter.configure(
+        'replacement'      => 'X',
+      )
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
+    end
+
+    def test_format_suppresses_custom_delimiter
+      @formatter.configure(
+        'delimiter'       => 'w',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
     end
   end
 
@@ -260,6 +305,14 @@ module FormatterTest
 
   class SingleValueFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
+    def setup
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
+    end
+
 
     def test_config_params
       formatter = TextFormatter::SingleValueFormatter.new
@@ -271,8 +324,9 @@ module FormatterTest
 
     def test_format
       formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
-      assert_equal("awesome\n", formatted)
+      assert_equal("awesome#{@newline}", formatted)
     end
 
     def test_format_without_newline
@@ -287,7 +341,7 @@ module FormatterTest
       formatter.configure('message_key' => 'foobar')
       formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
 
-      assert_equal("foo\n", formatted)
+      assert_equal("foo#{@newline}", formatted)
     end
   end
 

data/test/test_output.rb

@@ -230,6 +230,11 @@ module FluentOutputTest
       setup do
         @time = Time.parse("2011-01-02 13:14:15 UTC")
         Timecop.freeze(@time)
+        @newline = if Fluent.windows?
+                     "\r\n"
+                   else
+                     "\n"
+                   end
       end
 
       teardown do
@@ -265,7 +270,7 @@ module FluentOutputTest
         ])
         time = Time.parse("2016-11-08 12:00:00 UTC").to_i
         d.emit({"a" => 1}, time)
-        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}\n]
+        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}#{@newline}]
         d.run
       end
     end

data/test/test_supervisor.rb

@@ -8,6 +8,10 @@ require 'net/http'
 require 'uri'
 require 'fileutils'
 
+if Fluent.windows?
+  require 'win32/event'
+end
+
 class SupervisorTest < ::Test::Unit::TestCase
   class DummyServer
     include Fluent::ServerModule
@@ -107,6 +111,32 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
   end
 
+  def test_main_process_command_handlers
+    omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
+
+    create_info_dummy_logger
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+    r, w = IO.pipe
+    $stdin = r
+    sv.send(:install_main_process_signal_handlers)
+
+    begin
+      w.write("GRACEFUL_RESTART\n")
+      w.flush
+    ensure
+      $stdin = STDIN
+    end
+
+    sleep 1
+
+    info_msg = '[info]: force flushing buffered events' + "\n"
+    assert{ $log.out.logs.first.end_with?(info_msg) }
+  ensure
+    $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
+  end
+
   def test_supervisor_signal_handler
     omit "Windows cannot handle signals" if Fluent.windows?
 
@@ -128,6 +158,60 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
   end
 
+  def test_windows_shutdown_event
+    omit "Only for Windows platform" unless Fluent.windows?
+
+    server = DummyServer.new
+    def server.config
+      {:signame => "TestFluentdEvent"}
+    end
+
+    mock(server).stop(true)
+    stub(Process).kill.times(0)
+
+    server.install_windows_event_handler
+    begin
+      sleep 0.1 # Wait for starting windows event thread
+      event = Win32::Event.open("TestFluentdEvent")
+      event.set
+      event.close
+    ensure
+      server.stop_windows_event_thread
+    end
+
+    debug_msg = '[debug]: Got Win32 event "TestFluentdEvent"'
+    logs = $log.out.logs
+    assert{ logs.any?{|log| log.include?(debug_msg) } }
+  ensure
+    $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
+  end
+
+  def test_supervisor_event_handler
+    omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
+
+    create_debug_dummy_logger
+
+    server = DummyServer.new
+    def server.config
+      {:signame => "TestFluentdEvent"}
+    end
+    server.install_windows_event_handler
+    begin
+      sleep 0.1 # Wait for starting windows event thread
+      event = Win32::Event.open("TestFluentdEvent_USR1")
+      event.set
+      event.close
+    ensure
+      server.stop_windows_event_thread
+    end
+
+    debug_msg = '[debug]: Got Win32 event "TestFluentdEvent_USR1"'
+    logs = $log.out.logs
+    assert{ logs.any?{|log| log.include?(debug_msg) } }
+  ensure
+    $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
+  end
+
   def test_rpc_server
     omit "Windows cannot handle signals" if Fluent.windows?
 
@@ -150,7 +234,7 @@ class SupervisorTest < ::Test::Unit::TestCase
     server.run_rpc_server
 
     sv.send(:install_main_process_signal_handlers)
-    Net::HTTP.get URI.parse('http://0.0.0.0:24447/api/plugins.flushBuffers')
+    response = Net::HTTP.get(URI.parse('http://127.0.0.1:24447/api/plugins.flushBuffers'))
     info_msg = '[info]: force flushing buffered events' + "\n"
 
     server.stop_rpc_server
@@ -159,11 +243,45 @@ class SupervisorTest < ::Test::Unit::TestCase
     # This test will be passed in such environment.
     pend unless $log.out.logs.first
 
+    assert_equal('{"ok":true}', response)
     assert{ $log.out.logs.first.end_with?(info_msg) }
   ensure
     $log.out.reset if $log.out.is_a?(Fluent::Test::DummyLogDevice)
   end
 
+  def test_rpc_server_windows
+    omit "Only for windows platform" unless Fluent.windows?
+
+    create_info_dummy_logger
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+    conf_data = <<-EOC
+  <system>
+    rpc_endpoint 0.0.0.0:24447
+  </system>
+    EOC
+    conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
+    sys_conf = sv.__send__(:build_system_config, conf)
+
+    server = DummyServer.new
+    def server.config
+      {
+        :signame => "TestFluentdEvent",
+        :worker_pid => 5963,
+      }
+    end
+    server.rpc_endpoint = sys_conf.rpc_endpoint
+
+    server.run_rpc_server
+
+    mock(server).restart(true) { nil }
+    response = Net::HTTP.get(URI.parse('http://127.0.0.1:24447/api/plugins.flushBuffers'))
+
+    server.stop_rpc_server
+    assert_equal('{"ok":true}', response)
+  end
+
   def test_load_config
     tmp_dir = "#{TMP_DIR}/dir/test_load_config.conf"
     conf_info_str = %[
@@ -399,6 +517,37 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal Fluent::Log::LEVEL_ERROR, $log.level
   end
 
+  def test_enable_shared_socket
+    server = DummyServer.new
+    begin
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+      server.before_run
+      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread
+      assert_not_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+    ensure
+      server.after_run
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+    end
+  end
+
+  def test_disable_shared_socket
+    server = DummyServer.new
+    def server.config
+      {
+        :disable_shared_socket => true,
+      }
+    end
+    begin
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+      server.before_run
+      sleep 0.1 if Fluent.windows? # Wait for starting windows event thread
+      assert_nil(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+    ensure
+      server.after_run
+      ENV.delete('SERVERENGINE_SOCKETMANAGER_PATH')
+    end
+  end
+
   def create_debug_dummy_logger
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::DEBUG