fluentd 0.14.10-x64-mingw32 → 0.14.11-x64-mingw32

data/test/plugin/test_in_syslog.rb

@@ -3,21 +3,8 @@ require 'fluent/test/driver/input'
 require 'fluent/plugin/in_syslog'
 
 class SyslogInputTest < Test::Unit::TestCase
-  class << self
-    def startup
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
-    end
-
-    def shutdown
-      @server.close
-    end
-  end
-
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/socket_util'
   end
 
   PORT = unused_port
@@ -37,41 +24,45 @@ class SyslogInputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)
   end
 
-  def test_configure
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
-
-    configs.each_pair { |k, v|
-      d = create_driver(v)
-      assert_equal PORT, d.instance.port
-      assert_equal k, d.instance.bind
-    }
+  data(
+    ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
+    ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
+  )
+  def test_configure(data)
+    bind_addr, config, family = data
+    omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
+
+    d = create_driver(config)
+    assert_equal PORT, d.instance.port
+    assert_equal bind_addr, d.instance.bind
   end
 
-  def test_time_format
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
-
-    configs.each_pair { |k, v|
-      d = create_driver(v)
+  data(
+    ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
+    ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
+  )
+  def test_time_format(data)
+    bind_addr, config, family = data
+    omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
 
-      tests = [
-        {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
-        {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},
-      ]
-      d.run(expect_emits: 2) do
-        u = Fluent::SocketUtil.create_udp_socket(k)
-        u.connect(k, PORT)
-        tests.each {|test|
-          u.send(test['msg'], 0)
-        }
-      end
+    d = create_driver(config)
 
-      events = d.events
-      assert(events.size > 0)
-      events.each_index {|i|
-        assert_equal_event_time(tests[i]['expected'], events[i][1])
+    tests = [
+      {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
+      {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},
+    ]
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new(family)
+      u.connect(bind_addr, PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
       }
+    end
+
+    events = d.events
+    assert(events.size > 0)
+    events.each_index {|i|
+      assert_equal_event_time(tests[i]['expected'], events[i][1])
     }
   end
 
@@ -179,7 +170,7 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.events, tests, {host: host})
   end
 
-  def test_msg_size_with_include_priority
+  def test_msg_size_with_priority_key
     d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
     tests = create_test_case
 
@@ -196,7 +187,7 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.events, tests, {priority: priority})
   end
 
-  def test_msg_size_with_include_facility
+  def test_msg_size_with_facility_key
     d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
     tests = create_test_case
 
@@ -213,6 +204,43 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.events, tests, {facility: facility})
   end
 
+  def test_msg_size_with_source_address_key
+    d = create_driver([CONFIG, 'source_address_key source_address'].join("\n"))
+    tests = create_test_case
+
+    address = nil
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      address = u.peeraddr[3]
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {address: address})
+  end
+
+  def test_msg_size_with_source_hostname_key
+    d = create_driver([CONFIG, 'source_hostname_key source_hostname'].join("\n"))
+    tests = create_test_case
+
+    hostname = nil
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new
+      u.do_not_reverse_lookup = false
+      u.connect('127.0.0.1', PORT)
+      hostname = u.peeraddr[2]
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {hostname: hostname})
+  end
+
   def create_test_case(large_message: false)
     # actual syslog message has "\n"
     if large_message
@@ -234,6 +262,8 @@ class SyslogInputTest < Test::Unit::TestCase
       assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
       assert_equal(tests[i]['expected'], events[i][2]['message'])
       assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
+      assert_equal(options[:address], events[i][2]['source_address']) if options[:address]
+      assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]
       assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }

data/test/plugin/test_out_file.rb

@@ -136,7 +136,6 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'configured as secondary with primary using chunk_key_tag and not using chunk_key_time' do
       require 'fluent/plugin/out_null'
-      port = unused_port
       conf = config_element('match', '**', {
         }, [
           config_element('buffer', 'tag', {

data/test/plugin/test_out_forward.rb

@@ -35,16 +35,25 @@ class ForwardOutputTest < Test::Unit::TestCase
 
   def create_driver(conf=CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
-      attr_reader :responses, :exceptions
+      attr_reader :response_chunk_ids, :exceptions, :sent_chunk_ids
 
       def initialize
         super
-        @responses = []
+        @sent_chunk_ids = []
+        @response_chunk_ids = []
         @exceptions = []
       end
 
+      def try_write(chunk)
+        retval = super
+        @sent_chunk_ids << chunk.unique_id
+        retval
+      end
+
       def read_ack_from_sock(sock, unpacker)
-        @responses << super
+        retval = super
+        @response_chunk_ids << retval
+        retval
       rescue => e
         @exceptions << e
         raise e
@@ -302,7 +311,7 @@ EOL
     assert_equal ['test', time, records[0]], events[0]
     assert_equal ['test', time, records[1]], events[1]
 
-    assert_empty d.instance.responses # not attempt to receive responses, so it's empty
+    assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
     assert_empty d.instance.exceptions
   end
 
@@ -329,7 +338,7 @@ EOL
     assert_equal ['test', time, records[0]], events[0]
     assert_equal ['test', time, records[1]], events[1]
 
-    assert_empty d.instance.responses # not attempt to receive responses, so it's empty
+    assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
     assert_empty d.instance.exceptions
   end
 
@@ -354,7 +363,7 @@ EOL
       {"a" => 2}
     ]
     target_input_driver.run(expect_records: 2) do
-      d.end_if{ d.instance.responses.length > 0 }
+      d.end_if{ d.instance.response_chunk_ids.length > 0 }
       d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
         d.feed([[time, records[0]], [time,records[1]]])
       end
@@ -364,7 +373,8 @@ EOL
     assert_equal ['test', time, records[0]], events[0]
     assert_equal ['test', time, records[1]], events[1]
 
-    assert_equal 1, d.instance.responses.length
+    assert_equal 1, d.instance.response_chunk_ids.size
+    assert_equal d.instance.sent_chunk_ids.first, d.instance.response_chunk_ids.first
     assert_empty d.instance.exceptions
   end
 
@@ -400,7 +410,7 @@ EOL
       end
     end
 
-    assert_equal 1, delayed_commit_timeout_value
+    assert_equal (1 + 2), delayed_commit_timeout_value
 
     events = target_input_driver.events
     assert_equal ['test', time, records[0]], events[0]
@@ -409,7 +419,6 @@ EOL
     assert{ d.instance.rollback_count > 0 }
 
     logs = d.instance.log.logs
-    assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
     assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
   end
 
@@ -445,7 +454,7 @@ EOL
       end
     end
 
-    assert_equal 5, delayed_commit_timeout_value
+    assert_equal (5 + 2), delayed_commit_timeout_value
 
     events = target_input_driver.events
     assert_equal ['test', time, records[0]], events[0]
@@ -454,7 +463,6 @@ EOL
     assert{ d.instance.rollback_count > 0 }
 
     logs = d.instance.log.logs
-    assert{ logs.any?{|log| log.include?("failed to flush the buffer chunk, timeout to commit.") } }
     assert{ logs.any?{|log| log.include?("no response from node. regard it as unavailable.") } }
   end
 

data/test/plugin/test_output.rb

@@ -799,4 +799,48 @@ class OutputTest < Test::Unit::TestCase
       assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
     end
   end
+
+  sub_test_case 'slow_flush_log_threshold' do
+    def invoke_slow_flush_log_threshold_test(i)
+      i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.start
+      i.after_start
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+      i.force_flush
+
+      waiting(4) { Thread.pass until i.test_finished? }
+
+      yield
+    ensure
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#write flush took longer time than slow_flush_log_threshold' do
+      i = create_output(:buffered)
+      write_called = false
+      i.register(:write) { |chunk| sleep 1 }
+      i.define_singleton_method(:test_finished?) { write_called }
+      i.define_singleton_method(:try_flush) { super(); write_called = true }
+
+      invoke_slow_flush_log_threshold_test(i) {
+        assert write_called
+        assert_equal 1, i.log.out.logs.select { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold: elapsed_time/ }.size
+      }
+    end
+
+    test '#try_write flush took longer time than slow_flush_log_threshold' do
+      i = create_output(:delayed)
+      try_write_called = false
+      i.register(:try_write){ |chunk| sleep 1 }
+      i.define_singleton_method(:test_finished?) { try_write_called }
+      i.define_singleton_method(:try_flush) { super(); try_write_called = true }
+
+      invoke_slow_flush_log_threshold_test(i) {
+        assert try_write_called
+        assert_equal 1, i.log.out.logs.select { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold: elapsed_time/ }.size
+      }
+    end
+  end
 end

data/test/plugin/test_storage_local.rb

@@ -1,8 +1,296 @@
 require_relative '../helper'
 require 'fluent/plugin/storage_local'
+require 'fluent/plugin/input'
+require 'fluent/system_config'
+require 'fileutils'
 
 class LocalStorageTest < Test::Unit::TestCase
-  test 'syntax' do
-    assert true
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/storage_local#{ENV['TEST_ENV_NUMBER']}")
+
+  class MyInput < Fluent::Plugin::Input
+    helpers :storage
+    config_section :storage do
+      config_set_default :@type, 'local'
+    end
+  end
+
+  setup do
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+    Fluent::Test.setup
+    @d = MyInput.new
+  end
+
+  teardown do
+    @d.stop unless @d.stopped?
+    @d.before_shutdown unless @d.before_shutdown?
+    @d.shutdown unless @d.shutdown?
+    @d.after_shutdown unless @d.after_shutdown?
+    @d.close unless @d.closed?
+    @d.terminate unless @d.terminated?
+  end
+
+  sub_test_case 'without any configuration' do
+    test 'works as on-memory storage' do
+      conf = config_element()
+
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_nil @p.path
+      assert @p.store.empty?
+
+      assert_nil @p.get('key1')
+      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
+
+      @p.put('key1', '1')
+      assert_equal '1', @p.get('key1')
+
+      @p.update('key1') do |v|
+        (v.to_i * 2).to_s
+      end
+      assert_equal '2', @p.get('key1')
+
+      @p.save # on-memory storage does nothing...
+
+      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
+
+      # re-create to reload storage contents
+      @d = MyInput.new
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert @p.store.empty?
+    end
+
+    test 'warns about on-memory storage if autosave is true' do
+      @d.configure(config_element())
+      @d.start
+      @p = @d.storage_create()
+
+      logs = @d.log.out.logs
+      assert{ logs.any?{|log| log.include?("[warn]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.") } }
+    end
+
+    test 'show info log about on-memory storage if autosave is false' do
+      @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'autosave' => 'false'})]))
+      @d.start
+      @p = @d.storage_create()
+
+      logs = @d.log.out.logs
+      assert{ logs.any?{|log| log.include?("[info]: both of Plugin @id and path for <storage> are not specified. Using on-memory store.") } }
+    end
+  end
+
+  sub_test_case 'configured with path' do
+    test 'works as storage which stores data on disk' do
+      storage_path = File.join(TMP_DIR, 'my_store.json')
+      conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path})])
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal storage_path, @p.path
+      assert @p.store.empty?
+
+      assert_nil @p.get('key1')
+      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
+
+      @p.put('key1', '1')
+      assert_equal '1', @p.get('key1')
+
+      @p.update('key1') do |v|
+        (v.to_i * 2).to_s
+      end
+      assert_equal '2', @p.get('key1')
+
+      @p.save # stores all data into file
+
+      assert File.exist?(storage_path)
+
+      @p.put('key2', 4)
+
+      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
+
+      assert_equal({'key1' => '2', 'key2' => 4}, File.open(storage_path){|f| JSON.parse(f.read) })
+
+      # re-create to reload storage contents
+      @d = MyInput.new
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_false @p.store.empty?
+
+      assert_equal '2', @p.get('key1')
+      assert_equal 4, @p.get('key2')
+    end
+  end
+
+  sub_test_case 'configured with root-dir and plugin id' do
+    test 'works as storage which stores data under root dir' do
+      root_dir = File.join(TMP_DIR, 'root')
+      expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')
+      conf = config_element('ROOT', '', {'@id' => 'local_storage_test'})
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
+        @d.configure(conf)
+      end
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal expected_storage_path, @p.path
+      assert @p.store.empty?
+
+      assert_nil @p.get('key1')
+      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
+
+      @p.put('key1', '1')
+      assert_equal '1', @p.get('key1')
+
+      @p.update('key1') do |v|
+        (v.to_i * 2).to_s
+      end
+      assert_equal '2', @p.get('key1')
+
+      @p.save # stores all data into file
+
+      assert File.exist?(expected_storage_path)
+
+      @p.put('key2', 4)
+
+      @d.stop; @d.before_shutdown; @d.shutdown; @d.after_shutdown; @d.close; @d.terminate
+
+      assert_equal({'key1' => '2', 'key2' => 4}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
+
+      # re-create to reload storage contents
+      @d = MyInput.new
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
+        @d.configure(conf)
+      end
+      @d.start
+      @p = @d.storage_create()
+
+      assert_false @p.store.empty?
+
+      assert_equal '2', @p.get('key1')
+      assert_equal 4, @p.get('key2')
+    end
+  end
+
+  sub_test_case 'persistent specified' do
+    test 'works well with path' do
+      omit "It's hard to test on Windows" if Fluent.windows?
+
+      storage_path = File.join(TMP_DIR, 'my_store.json')
+      conf = config_element('ROOT', '', {}, [config_element('storage', '', {'path' => storage_path, 'persistent' => 'true'})])
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal storage_path, @p.path
+      assert @p.store.empty?
+
+      assert_nil @p.get('key1')
+      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
+
+      @p.put('key1', '1')
+      assert_equal({'key1' => '1'}, File.open(storage_path){|f| JSON.parse(f.read) })
+
+      @p.update('key1') do |v|
+        (v.to_i * 2).to_s
+      end
+      assert_equal({'key1' => '2'}, File.open(storage_path){|f| JSON.parse(f.read) })
+    end
+
+    test 'works well with root-dir and plugin id' do
+      omit "It's hard to test on Windows" if Fluent.windows?
+
+      root_dir = File.join(TMP_DIR, 'root')
+      expected_storage_path = File.join(root_dir, 'worker0', 'local_storage_test', 'storage.json')
+      conf = config_element('ROOT', '', {'@id' => 'local_storage_test'}, [config_element('storage', '', {'persistent' => 'true'})])
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => root_dir) do
+        @d.configure(conf)
+      end
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal expected_storage_path, @p.path
+      assert @p.store.empty?
+
+      assert_nil @p.get('key1')
+      assert_equal 'EMPTY', @p.fetch('key1', 'EMPTY')
+
+      @p.put('key1', '1')
+      assert_equal({'key1' => '1'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
+
+      @p.update('key1') do |v|
+        (v.to_i * 2).to_s
+      end
+      assert_equal({'key1' => '2'}, File.open(expected_storage_path){|f| JSON.parse(f.read) })
+    end
+
+    test 'raises error if it is configured to use on-memory storage' do
+      assert_raise Fluent::ConfigError.new("Plugin @id or path for <storage> required when 'persistent' is true") do
+        @d.configure(config_element('ROOT', '', {}, [config_element('storage', '', {'persistent' => 'true'})]))
+      end
+    end
+  end
+
+  sub_test_case 'with various configurations' do
+    test 'mode and dir_mode controls permissions of stored data' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')
+      storage_conf = {
+        'path' => storage_path,
+        'mode' => '0600',
+        'dir_mode' => '0777',
+      }
+      conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal storage_path, @p.path
+      assert @p.store.empty?
+
+      @p.put('key1', '1')
+      assert_equal '1', @p.get('key1')
+
+      @p.save # stores all data into file
+
+      assert File.exist?(storage_path)
+      assert_equal 0600, (File.stat(storage_path).mode % 01000)
+      assert_equal 0777, (File.stat(File.dirname(storage_path)).mode % 01000)
+    end
+
+    test 'pretty_print controls to write data in files as human-easy-to-read' do
+      storage_path = File.join(TMP_DIR, 'dir', 'my_store.json')
+      storage_conf = {
+        'path' => storage_path,
+        'pretty_print' => 'true',
+      }
+      conf = config_element('ROOT', '', {}, [config_element('storage', '', storage_conf)])
+      @d.configure(conf)
+      @d.start
+      @p = @d.storage_create()
+
+      assert_equal storage_path, @p.path
+      assert @p.store.empty?
+
+      @p.put('key1', '1')
+      assert_equal '1', @p.get('key1')
+
+      @p.save # stores all data into file
+
+      expected_pp_json = <<JSON.chomp
+{
+  "key1": "1"
+}
+JSON
+      assert_equal expected_pp_json, File.read(storage_path)
+    end
   end
 end