fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

data/test/plugin/test_out_copy.rb

@@ -156,5 +156,36 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_ERROR_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_error
+    d = create_driver(IGNORE_ERROR_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
 end
 

data/test/plugin/test_out_file.rb

@@ -304,7 +304,11 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal r4, d.formatted[3]
       assert_equal r5, d.formatted[4]
 
-      read_gunzip = ->(path){ File.open(path){|fio| Zlib::GzipReader.open(fio){|io| io.read } } }
+      read_gunzip = ->(path){
+        File.open(path){ |fio|
+          Zlib::GzipReader.new(StringIO.new(fio.read)).read
+        }
+      }
       assert_equal r1 + r2, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
       assert_equal r3, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
       assert_equal r4, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
@@ -370,7 +374,7 @@ class FileOutputTest < Test::Unit::TestCase
     result = ''
     File.open(path, "rb") { |io|
       loop do
-        gzr = Zlib::GzipReader.new(io)
+        gzr = Zlib::GzipReader.new(StringIO.new(io.read))
         result << gzr.read
         unused = gzr.unused
         gzr.finish
@@ -557,6 +561,108 @@ class FileOutputTest < Test::Unit::TestCase
     check_gzipped_result(path, formatted_lines * 3)
   end
 
+  test 'append when JST' do
+    with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
+      time = event_time("2011-01-02 03:14:15+09:00")
+      formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}\n] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone Asia/Tokyo
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
+  test 'append when UTC-02 but timekey_zone is +0900' do
+    with_timezone("UTC-02") do # +0200
+      time = event_time("2011-01-02 17:14:15+02:00")
+      formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}\n] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone +0900
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      # Rotated at 2011-01-02 17:00:00+02:00
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
+  test '${chunk_id}' do
+    time = event_time("2011-01-02 13:14:15 UTC")
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    write_once = ->(){
+      d = create_driver %[
+        path #{TMP_DIR}/out_file_chunk_id_${chunk_id}
+        utc
+        append true
+        <buffer>
+          timekey_use_utc true
+        </buffer>
+      ]
+      d.run(default_tag: 'test'){
+        d.feed(time, {"a"=>1})
+        d.feed(time, {"a"=>2})
+      }
+      d.instance.last_written_path
+    }
+
+    path = write_once.call
+    if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).20110102.log/
+      unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
+      assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
+    else
+      flunk "chunk_id is not included in the path"
+    end
+  end
+
   test 'symlink' do
     omit "Windows doesn't support symlink" if Fluent.windows?
     conf = CONFIG + %[

data/test/plugin/test_out_forward.rb

@@ -9,6 +9,8 @@ require 'fluent/plugin/in_forward'
 class ForwardOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
     @d = nil
   end
 
@@ -16,6 +18,8 @@ class ForwardOutputTest < Test::Unit::TestCase
     @d.instance_shutdown if @d
   end
 
+  TMP_DIR = File.join(__dir__, "../tmp/out_forward#{ENV['TEST_ENV_NUMBER']}")
+
   TARGET_HOST = '127.0.0.1'
   TARGET_PORT = unused_port
   CONFIG = %[
@@ -153,6 +157,27 @@ EOL
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
 
+  data('CA cert'     => 'tls_ca_cert_path',
+       'non CA cert' => 'tls_cert_path')
+  test 'configure tls_cert_path/tls_ca_cert_path' do |param|
+    dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
+    FileUtils.touch(dummy_cert_path)
+    conf = %[
+      send_timeout 5
+      transport tls
+      tls_insecure_mode true
+      #{param} #{dummy_cert_path}
+      <server>
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+      </server>
+    ]
+
+    @d = d = create_driver(conf)
+    # In the plugin, tls_ca_cert_path is used for both cases
+    assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress
@@ -217,6 +242,18 @@ EOL
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'verify_connection_at_startup is disabled in default' do
+    @d = d = create_driver(CONFIG)
+    assert_false d.instance.verify_connection_at_startup
+  end
+
+  test 'verify_connection_at_startup can be enabled' do
+    @d = d = create_driver(CONFIG + %[
+      verify_connection_at_startup true
+    ])
+    assert_true d.instance.verify_connection_at_startup
+  end
+
   test 'send tags in str (utf-8 strings)' do
     target_input_driver = create_target_input_driver
 
@@ -486,6 +523,7 @@ EOL
         retry_type periodic
         retry_wait 30s
         flush_at_shutdown false # suppress errors in d.instance_shutdown
+        flush_thread_interval 30s
       </buffer>
     ])
 
@@ -501,7 +539,7 @@ EOL
     target_input_driver.end_if{ d.instance.rollback_count > 0 }
     target_input_driver.end_if{ !node.available }
     target_input_driver.run(expect_records: 2, timeout: 25) do
-      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
         delayed_commit_timeout_value = d.instance.delayed_commit_timeout
         d.feed([[time, records[0]], [time,records[1]]])
       end
@@ -530,6 +568,7 @@ EOL
         retry_type periodic
         retry_wait 30s
         flush_at_shutdown false # suppress errors in d.instance_shutdown
+        flush_thread_interval 30s
       </buffer>
     ])
 
@@ -545,7 +584,7 @@ EOL
     target_input_driver.end_if{ d.instance.rollback_count > 0 }
     target_input_driver.end_if{ !node.available }
     target_input_driver.run(expect_records: 2, timeout: 25) do
-      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
         delayed_commit_timeout_value = d.instance.delayed_commit_timeout
         d.feed([[time, records[0]], [time,records[1]]])
       end
@@ -664,6 +703,51 @@ EOL
     assert_equal(['test', time, records[1]], events[1])
   end
 
+  # This test is not 100% but test failed with previous Node implementation which has race condition
+  test 'Node with security is thread-safe on multi threads' do
+    input_conf = TARGET_CONFIG + %[
+                   <security>
+                     self_hostname in.localhost
+                     shared_key fluentd-sharedkey
+                     <client>
+                       host 127.0.0.1
+                     </client>
+                   </security>
+                 ]
+    target_input_driver = create_target_input_driver(conf: input_conf)
+    output_conf = %[
+      send_timeout 51
+      <security>
+        self_hostname localhost
+        shared_key fluentd-sharedkey
+      </security>
+      <server>
+        name test
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+        shared_key fluentd-sharedkey
+      </server>
+    ]
+    @d = d = create_driver(output_conf)
+
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+    target_input_driver.run(timeout: 15) do
+      d.run(shutdown: false) do
+        node = d.instance.nodes.first
+        arr = []
+        4.times {
+          arr << Thread.new {
+            node.send_data('test', chunk) rescue nil
+          }
+        }
+        arr.each { |a| a.join }
+      end
+    end
+
+    logs = d.logs
+    assert_false(logs.any? { |log| log.include?("invalid format for PONG message") || log.include?("shared key mismatch") }, "'#{logs.last.strip}' happens")
+  end
+
   def create_target_input_driver(response_stub: nil, disconnect: false, conf: TARGET_CONFIG)
     require 'fluent/plugin/in_forward'
 
@@ -727,4 +811,113 @@ EOL
       i.configure(conf)
     end
   end
+
+  sub_test_case 'verify_connection_at_startup' do
+    test 'nodes are not available' do
+      @d = d = create_driver(CONFIG + %[
+        verify_connection_at_startup true
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+        </buffer>
+      ])
+      assert_raise Fluent::UnrecoverableError do
+        d.instance_start
+      end
+      d.instance_shutdown
+    end
+
+    test 'nodes_shared_key_miss_match' do
+      input_conf = TARGET_CONFIG + %[
+                   <security>
+                     self_hostname in.localhost
+                     shared_key fluentd-sharedkey
+                   </security>
+                 ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+      output_conf = %[
+        send_timeout 30
+        heartbeat_type transport
+        transport tls
+        tls_verify_hostname false
+        verify_connection_at_startup true
+        require_ack_response true
+        ack_response_timeout 5s
+        <security>
+          self_hostname localhost
+          shared_key key_miss_match
+        </security>
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+          flush_thread_interval 31s
+        </buffer>
+
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      target_input_driver.run(expect_records: 1, timeout: 15) do
+        assert_raise Fluent::UnrecoverableError do
+          d.instance_start
+        end
+        d.instance_shutdown
+      end
+    end
+
+    test 'nodes_shared_key_match' do
+      input_conf = TARGET_CONFIG + %[
+                       <security>
+                         self_hostname in.localhost
+                         shared_key fluentd-sharedkey
+                         <client>
+                           host 127.0.0.1
+                         </client>
+                       </security>
+                     ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+
+      output_conf = %[
+          send_timeout 51
+          verify_connection_at_startup true
+          <security>
+            self_hostname localhost
+            shared_key fluentd-sharedkey
+          </security>
+          <server>
+            name test
+            host #{TARGET_HOST}
+            port #{TARGET_PORT}
+            shared_key fluentd-sharedkey
+          </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      time = event_time("2011-01-02 13:14:15 UTC")
+      records = [
+          {"a" => 1},
+          {"a" => 2}
+      ]
+
+      target_input_driver.run(expect_records: 2, timeout: 15) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
+        end
+      end
+
+      events = target_input_driver.events
+      assert{ events != [] }
+      assert_equal(['test', time, records[0]], events[0])
+      assert_equal(['test', time, records[1]], events[1])
+    end
+  end
 end

data/test/plugin/test_out_secondary_file.rb

@@ -260,6 +260,20 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
       assert_equal "#{TMP_DIR}/dump.bin", path
     end
 
+    test 'path with ${chunk_id}' do
+      d = create_driver %[
+        directory #{TMP_DIR}
+        basename out_file_chunk_id_${chunk_id}
+      ]
+      path = d.instance.write(@c)
+      if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).0/
+        unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
+        assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
+      else
+        flunk "chunk_id is not included in the path"
+      end
+    end
+
     data(
       invalid_tag: [/tag/, '${tag}'],
       invalid_tag0: [/tag\[0\]/, '${tag[0]}'],