fluentd 1.11.0-x64-mingw32 → 1.11.5-x64-mingw32

data/test/plugin/test_output.rb

@@ -378,6 +378,18 @@ class OutputTest < Test::Unit::TestCase
       assert { logs.any? { |log| log.include?("${chunk_id} is not allowed in this plugin") } }
     end
 
+    test '#extract_placeholders does not log for ${chunk_id} placeholder' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+      tmpl = "/mypath/${chunk_id}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.log.out.logs.clear
+      @i.extract_placeholders(tmpl, c)
+      logs = @i.log.out.logs
+      assert { logs.none? { |log| log.include?("${chunk_id}") } }
+    end
+
     test '#extract_placeholders logs warn message with not replaced key' do
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
       tmpl = "/mypath/${key1}/test"

data/test/plugin_helper/test_cert_option.rb

@@ -15,4 +15,11 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
     certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/cert-with-CRLF.pem")
     assert_equal(1, certs.length)
   end
+
+  test 'raise an error for broken certificates_from_file file' do
+    d = Dummy.new
+    assert_raise Fluent::ConfigError do
+      certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
+    end
+  end
 end

data/test/plugin_helper/test_child_process.rb

@@ -818,5 +818,20 @@ class ChildProcessTest < Test::Unit::TestCase
       end
       assert File.exist?(@temp_path)
     end
+
+    test 'execute child process writing data to stdout which is unread' do
+      callback_called = false
+      exit_status = nil
+      prog = "echo writing to stdout"
+      callback = ->(status){ exit_status = status; callback_called = true }
+      Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+        @d.child_process_execute(:out_exec_process, prog, stderr: :connect, immediate: true, parallel: true, mode: [], wait_timeout: 1, on_exit_callback: callback)
+        sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until callback_called
+      end
+      assert callback_called
+      assert exit_status
+      assert exit_status.success?
+      assert_equal 0, exit_status.exitstatus
+    end
   end
 end

data/test/plugin_helper/test_compat_parameters.rb

@@ -10,6 +10,11 @@ class CompatParameterTest < Test::Unit::TestCase
   setup do
     Fluent::Test.setup
     @i = nil
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   teardown do
@@ -226,7 +231,7 @@ class CompatParameterTest < Test::Unit::TestCase
       t = event_time('2016-06-24 16:05:01') # localtime
       iso8601str = Time.at(t.to_i).iso8601
       formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
-      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
     end
 
     test 'plugin helper setups time injecting as unix time (integer from epoch)' do
@@ -260,7 +265,7 @@ class CompatParameterTest < Test::Unit::TestCase
       t = event_time('2016-06-24 16:05:01') # localtime
       iso8601str = Time.at(t.to_i).iso8601
       formatted = @i.f.format('tag.test', t, @i.inject_values_to_record('tag.test', t, {"value" => 1}))
-      assert_equal "value%1,tag%tag.test,time%#{iso8601str}\n", formatted
+      assert_equal "value%1,tag%tag.test,time%#{iso8601str}#{@default_newline}", formatted
     end
   end
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -278,6 +278,7 @@ class HttpHelperTest < Test::Unit::TestCase
             driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: PORT, logger: NULL_LOGGER) do |s|
               s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
             end
+            omit "TLS connection should be aborted due to `Errno::ECONNABORTED`. Need to debug." if Fluent.windows?
 
             resp = secure_get("https://127.0.0.1:#{PORT}/example/hello", verify: false)
             assert_equal('200', resp.code)
@@ -293,6 +294,8 @@ class HttpHelperTest < Test::Unit::TestCase
           'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem'],
           'without passphrase' => [nil, 'cert.pem', 'cert-key.pem'])
         test 'load self-signed cert/key pair, verified from clients using cert files' do |(passphrase, cert, private_key)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
           cert_path = File.join(CERT_DIR, cert)
           private_key_path = File.join(CERT_DIR, private_key)
           opt = { 'insecure' => 'false', 'private_key_path' => private_key_path, 'cert_path' => cert_path }
@@ -315,6 +318,8 @@ class HttpHelperTest < Test::Unit::TestCase
           'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem', 'ca-cert-pass.pem'],
           'without passphrase' => [nil, 'cert.pem', 'cert-key.pem', 'ca-cert.pem'])
         test 'load cert by private CA cert file, verified from clients using CA cert file' do |(passphrase, cert, cert_key, ca_cert)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
           cert_path = File.join(CERT_CA_DIR, cert)
           private_key_path = File.join(CERT_CA_DIR, cert_key)
 

data/test/plugin_helper/test_inject.rb

@@ -176,6 +176,19 @@ class InjectHelperTest < Test::Unit::TestCase
       assert_equal record.merge({"timedata" => float_time}), @d.inject_values_to_record('tag', time, record)
     end
 
+    test 'injects time as unix time millis into specified key' do
+      time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
+      time_subsecond = 320_101_224
+      time = Fluent::EventTime.new(time_in_unix, time_subsecond)
+      unixtime_millis = 1466464211320
+
+      @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime_millis"))
+      @d.start
+
+      record = {"key1" => "value1", "key2" => 2}
+      assert_equal record.merge({"timedata" => unixtime_millis}), @d.inject_values_to_record('tag', time, record)
+    end
+
     test 'injects time as unix time into specified key' do
       time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
       time_subsecond = 320_101_224

data/test/plugin_helper/test_server.rb

@@ -1233,6 +1233,40 @@ class ServerPluginHelperTest < Test::Unit::TestCase
         waiting(10){ sleep 0.1 until received.bytesize == 8 }
         assert_equal "yay\nfoo\n", received
       end
+
+      test 'set ciphers' do
+        cert_path = File.join(@server_cert_dir, "cert.pem")
+        private_key_path = File.join(@certs_dir, "server.key.pem")
+        create_server_pair_signed_by_self(cert_path, private_key_path, nil)
+        tls_options = {
+          protocol: :tls,
+          version: :TLSv1_2,
+          ciphers: 'SHA256',
+          insecure: false,
+          cert_path: cert_path,
+          private_key_path: private_key_path,
+        }
+        conf = @d.server_create_transport_section_object(tls_options)
+        ctx = @d.cert_option_create_context(conf.version, conf.insecure, conf.ciphers, conf)
+        matched = false
+        ctx.ciphers.each do |cipher|
+          cipher_name, tls_version = cipher
+          # OpenSSL 1.0.2: "TLSv1/SSLv3"
+          # OpenSSL 1.1.1: "TLSv1.2"
+          if tls_version == "TLSv1/SSLv3" || tls_version == "TLSv1.2"
+            matched = true
+            unless cipher_name.match(/#{conf.ciphers}/)
+              matched = false
+              break
+            end
+          end
+        end
+
+        error_msg = build_message("Unexpected ciphers for #{conf.version}",
+                                  "<?>\nwas expected to include only <?> ciphers for #{conf.version}",
+                                  ctx.ciphers, conf.ciphers)
+        assert(matched, error_msg)
+      end
     end
   end
 

data/test/plugin_helper/test_socket.rb

@@ -128,4 +128,12 @@ class SocketHelperTest < Test::Unit::TestCase
       client.close
     end
   end
+
+  test 'with empty cert file' do
+    cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')
+
+    assert_raise Fluent::ConfigError do
+      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, cert_path: cert_path)
+    end
+  end
 end

data/test/test_formatter.rb

@@ -53,6 +53,11 @@ module FormatterTest
     def setup
       @formatter = Fluent::Test::FormatterTestDriver.new('out_file')
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     def configure(conf)
@@ -63,28 +68,28 @@ module FormatterTest
       configure({})
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time
       configure('output_time' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{tag}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_tag
       configure('output_tag' => 'false')
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time_and_tag
       configure('output_tag' => 'false', 'output_time' => 'false')
       formatted = @formatter.format('tag', @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
 
     def test_format_without_time_and_tag_against_string_literal_configure
@@ -95,7 +100,7 @@ module FormatterTest
       ])
       formatted = @formatter.format('tag', @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
   end
 
@@ -105,6 +110,11 @@ module FormatterTest
     def setup
       @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::JSONFormatter)
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     data('oj' => 'oj', 'yajl' => 'yajl')
@@ -112,7 +122,7 @@ module FormatterTest
       @formatter.configure('json_parser' => data)
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("#{Yajl.dump(record)}\n", formatted)
+      assert_equal("#{Yajl.dump(record)}#{@newline}", formatted)
     end
   end
 
@@ -138,6 +148,11 @@ module FormatterTest
     def setup
       @formatter = TextFormatter::LabeledTSVFormatter.new
       @time = Engine.now
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
     end
 
     def test_config_params
@@ -157,7 +172,7 @@ module FormatterTest
       @formatter.configure({})
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message:awesome\tgreeting:hello\n", formatted)
+      assert_equal("message:awesome\tgreeting:hello#{@newline}", formatted)
     end
 
     def test_format_with_customized_delimiters
@@ -167,7 +182,7 @@ module FormatterTest
       )
       formatted = @formatter.format(tag, @time, record)
 
-      assert_equal("message=awesome,greeting=hello\n", formatted)
+      assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
     end
   end
 
@@ -260,6 +275,14 @@ module FormatterTest
 
   class SingleValueFormatterTest < ::Test::Unit::TestCase
     include FormatterTest
+    def setup
+      @newline = if Fluent.windows?
+                   "\r\n"
+                 else
+                   "\n"
+                 end
+    end
+
 
     def test_config_params
       formatter = TextFormatter::SingleValueFormatter.new
@@ -271,8 +294,9 @@ module FormatterTest
 
     def test_format
       formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
-      assert_equal("awesome\n", formatted)
+      assert_equal("awesome#{@newline}", formatted)
     end
 
     def test_format_without_newline
@@ -287,7 +311,7 @@ module FormatterTest
       formatter.configure('message_key' => 'foobar')
       formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
 
-      assert_equal("foo\n", formatted)
+      assert_equal("foo#{@newline}", formatted)
     end
   end
 

data/test/test_log.rb

@@ -411,6 +411,50 @@ class LogTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "ignore_same_log_interval" do
+    teardown do
+      Thread.current[:last_same_log] = nil
+    end
+
+    def test_same_message
+      message = "This is test"
+      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
+      log = Fluent::Log.new(logger, {ignore_same_log_interval: 5})
+
+      log.error message
+      10.times { |i|
+        Timecop.freeze(@timestamp + i + 1)
+        log.error message
+      }
+
+      expected = [
+        "2016-04-21 02:58:41 +0000 [error]: This is test\n",
+        "2016-04-21 02:58:47 +0000 [error]: This is test\n"
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+
+    def test_different_message
+      message = "This is test"
+      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
+      log = Fluent::Log.new(logger, {ignore_same_log_interval: 10})
+
+      log.error message
+      3.times { |i|
+        Timecop.freeze(@timestamp + i)
+        log.error message
+        log.error message
+        log.info "Hello! " + message
+      }
+
+      expected = [
+        "2016-04-21 02:58:41 +0000 [error]: This is test\n",
+        "2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
   def test_dup
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE

data/test/test_match.rb

@@ -101,6 +101,17 @@ class MatchTest < Test::Unit::TestCase
     assert_or_not_match('a.b.** a.c', 'a.c.d')
   end
 
+  def test_regex_pattern
+    assert_glob_match('/a/', 'a')
+    assert_glob_not_match('/a/', 'abc')
+    assert_glob_match('/a.*/', 'abc')
+    assert_glob_not_match('/b.*/', 'abc')
+    assert_glob_match('/a\..*/', 'a.b.c')
+    assert_glob_not_match('/(?!a\.).*/', 'a.b.c')
+    assert_glob_not_match('/a\..*/', 'b.b.c')
+    assert_glob_match('/(?!a\.).*/', 'b.b.c')
+  end
+
   #def test_character_class
   #  assert_match('[a]', 'a')
   #  assert_match('[ab]', 'a')

data/test/test_output.rb

@@ -230,6 +230,11 @@ module FluentOutputTest
       setup do
         @time = Time.parse("2011-01-02 13:14:15 UTC")
         Timecop.freeze(@time)
+        @newline = if Fluent.windows?
+                     "\r\n"
+                   else
+                     "\n"
+                   end
       end
 
       teardown do
@@ -265,7 +270,7 @@ module FluentOutputTest
         ])
         time = Time.parse("2016-11-08 12:00:00 UTC").to_i
         d.emit({"a" => 1}, time)
-        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}\n]
+        d.expect_format %[2016-11-08T12:00:00Z\ttest\t{"a":1,"time":"2016-11-08T12:00:00Z"}#{@newline}]
         d.run
       end
     end

data/test/test_static_config_analysis.rb

@@ -6,7 +6,7 @@ require 'fluent/plugin/out_forward'
 require 'fluent/plugin/out_stdout'
 require 'fluent/plugin/out_exec'
 require 'fluent/plugin/in_forward'
-require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/in_sample'
 require 'fluent/plugin/filter_grep'
 require 'fluent/plugin/filter_stdout'
 require 'fluent/plugin/filter_parser'
@@ -74,7 +74,7 @@ class StaticConfigAnalysisTest < ::Test::Unit::TestCase
       c = Fluent::Config.parse(conf_data, '(test)', '(test_dir)', true)
       ret = Fluent::StaticConfigAnalysis.call(c)
       assert_equal [Fluent::Plugin::ExecOutput, Fluent::Plugin::StdoutOutput, Fluent::Plugin::ForwardOutput], ret.outputs.map(&:plugin).map(&:class)
-      assert_equal [Fluent::Plugin::DummyInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
+      assert_equal [Fluent::Plugin::SampleInput, Fluent::Plugin::ForwardInput], ret.inputs.map(&:plugin).map(&:class)
       assert_equal [Fluent::Plugin::ParserFilter, Fluent::Plugin::StdoutFilter, Fluent::Plugin::GrepFilter], ret.filters.map(&:plugin).map(&:class)
       assert_equal 1, ret.labels.size
       assert_equal '@test', ret.labels[0].name

data/test/test_supervisor.rb

@@ -8,6 +8,10 @@ require 'net/http'
 require 'uri'
 require 'fileutils'
 
+if Fluent.windows?
+  require 'win32/event'
+end
+
 class SupervisorTest < ::Test::Unit::TestCase
   class DummyServer
     include Fluent::ServerModule
@@ -128,6 +132,28 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
   end
 
+  def test_windows_shutdown_event
+    omit "Only for Windows platform" unless Fluent.windows?
+
+    server = DummyServer.new
+    def server.config
+      {:signame => "TestFluentdEvent", :worker_pid => {0 => 1234}}
+    end
+
+    mock(server).stop(true)
+    stub(Process).kill.times(0)
+
+    server.before_run
+    server.install_windows_event_handler
+    sleep 0.1 # Wait for starting windows event thread
+    event = Win32::Event.open("TestFluentdEvent")
+    event.set
+    event.close
+    # Wait for stopping windows event thread. Should larger than 1 sec
+    # because the thread is awaked every 1 sec.
+    sleep 1.1
+  end
+
   def test_rpc_server
     omit "Windows cannot handle signals" if Fluent.windows?