fluentd 1.13.3 → 1.16.5

data/test/plugin_helper/test_server.rb

@@ -16,14 +16,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   setup do
     @port = unused_port
-    @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-      FileUtils.rm_f @socket_manager_path
+    if Fluent.windows?
+      @socket_manager_server = ServerEngine::SocketManager::Server.open
+      @socket_manager_path = @socket_manager_server.path
+    else
+      @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+        FileUtils.rm_f @socket_manager_path
+      end
+      @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     end
-    @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
 
     @d = Dummy.new
+    @d.under_plugin_development = true
     @d.start
     @d.after_start
   end
@@ -37,7 +43,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     (@d.terminated? || @d.terminate) rescue nil
 
     @socket_manager_server.close
-    if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
       FileUtils.rm_f @socket_manager_path
     end
   end
@@ -64,6 +70,24 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       end
       assert d.plugin_id
       assert d.log
+      assert_equal 0, d.transport_config.linger_timeout
+    end
+
+    test 'can change linger_timeout option' do
+      d = Dummy.new
+
+      transport_opts = {
+        'linger_timeout' => 1,
+      }
+      transport_conf = config_element('transport', 'tcp', transport_opts)
+      conf = config_element('source', 'tag.*', {}, [transport_conf])
+
+      assert_nothing_raised do
+        d.configure(conf)
+      end
+      assert d.plugin_id
+      assert d.log
+      assert_equal 1, d.transport_config.linger_timeout
     end
   end
 
@@ -771,6 +795,50 @@ class ServerPluginHelperTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case 'over max_bytes' do
+      data("cut off on Non-Windows", { max_bytes: 32, records: ["a" * 40], expected: ["a" * 32] }, keep: true) unless Fluent.windows?
+      data("drop on Windows", { max_bytes: 32, records: ["a" * 40], expected: [] }, keep: true) if Fluent.windows?
+      test 'with sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data, sock|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+
+      test 'without sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+    end
   end
 
   module CertUtil
@@ -1261,7 +1329,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
           # OpenSSL 1.1.1: "TLSv1.2"
           if tls_version == "TLSv1/SSLv3" || tls_version == "TLSv1.2"
             matched = true
-            unless cipher_name.match(/#{conf.ciphers}/)
+            unless cipher_name.match?(/#{conf.ciphers}/)
               matched = false
               break
             end
@@ -1552,6 +1620,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   def open_client(proto, addr, port)
     client = case proto
+             when :udp
+               c = UDPSocket.open
+               c.connect(addr, port)
+               c
              when :tcp
                TCPSocket.open(addr, port)
              when :tls

data/test/plugin_helper/test_timer.rb

@@ -90,8 +90,8 @@ class TimerTest < Test::Unit::TestCase
     assert{ counter1 >= 4 && counter1 <= 5 }
     assert{ counter2 == 2 }
     msg = "Unexpected error raised. Stopping the timer. title=:t2"
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") })
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") })
 
     d1.shutdown; d1.close; d1.terminate
   end

data/test/test_config.rb

@@ -10,11 +10,18 @@ class ConfigTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
 
-  def read_config(path)
+  def read_config(path, use_yaml: false)
     path = File.expand_path(path)
-    File.open(path) { |io|
-      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
-    }
+    if use_yaml
+      context = Kernel.binding
+
+      s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+      Fluent::Config::YamlParser::Parser.new(s).build.to_element
+    else
+      File.open(path) { |io|
+        Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+      }
+    end
   end
 
   def prepare_config
@@ -151,30 +158,190 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal before_size, match_conf.unused.size
   end
 
-  def write_config(path, data, encoding: 'utf-8')
-    FileUtils.mkdir_p(File.dirname(path))
-    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
-  end
+  sub_test_case "yaml config" do
+    def test_included
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - source:
+            $type: dummy
+            tag: tag.dummy
+        - source:
+            $type: tcp
+            tag: tag.tcp
+            parse:
+              $arg:
+                - why.parse.section.doesnot.have.arg
+                - huh
+              $type: none
+        - match:
+            $tag: tag.*
+            $type: stdout
+            buffer:
+              $type: memory
+              flush_interval: 1s
+        - !include fluent-included.yaml
+      EOS
+      write_config "#{TMP_DIR}/fluent-included.yaml", <<-EOS
+      - label:
+          $name: '@FLUENT_LOG'
+          config:
+            - match:
+                $type: "null"
+                $tag: "**"
+                buffer:
+                  $type: memory
+                  flush_mode: interval
+                  flush_interval: 1s
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      dummy_source_conf = root_conf.elements.first
+      tcp_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+      label_conf = root_conf.elements[3]
+      fluent_log_conf = label_conf.elements.first
+      fluent_log_buffer_conf = fluent_log_conf.elements.first
 
-  def test_inline
-    prepare_config
-    opts = {
-      :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
-      :use_v1_config => false
-    }
-    assert_nothing_raised do
-      Fluent::Supervisor.new(opts)
+      assert_equal(
+        [
+          'dummy',
+          'tag.dummy',
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'stdout',
+          'tag.*',
+          'null',
+          '**',
+          '@FLUENT_LOG',
+          'memory',
+          'interval',
+          '1s',
+        ],
+        [
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          match_conf['@type'],
+          match_conf.arg,
+          fluent_log_conf['@type'],
+          fluent_log_conf.arg,
+          label_conf.arg,
+          fluent_log_buffer_conf['@type'],
+          fluent_log_buffer_conf['flush_mode'],
+          fluent_log_buffer_conf['flush_interval'],
+        ])
+    end
+
+    def test_included_glob
+      write_config "#{TMP_DIR}/config.yaml", <<-EOS
+      config:
+        - !include "include/*.yaml"
+      EOS
+      write_config "#{TMP_DIR}/include/02_source2.yaml", <<-EOS
+      - source:
+          $type: dummy
+          tag: tag.dummy
+      EOS
+      write_config "#{TMP_DIR}/include/01_source1.yaml", <<-EOS
+      - source:
+          $type: tcp
+          tag: tag.tcp
+          parse:
+            $arg:
+              - why.parse.section.doesnot.have.arg
+              - huh
+            $type: none
+      EOS
+      write_config "#{TMP_DIR}/include/03_match1.yaml", <<-EOS
+      - match:
+          $tag: tag.*
+          $type: stdout
+          buffer:
+            $type: memory
+            flush_interval: 1s
+      EOS
+      root_conf = read_config("#{TMP_DIR}/config.yaml", use_yaml: true)
+      tcp_source_conf = root_conf.elements.first
+      dummy_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+
+      assert_equal(
+        [
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'dummy',
+          'tag.dummy',
+          'stdout',
+          'tag.*',
+        ],
+        [
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          match_conf['@type'],
+          match_conf.arg,
+        ])
+    end
+
+    def test_check_not_fetchd
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - match:
+            $arg: dummy
+            $type: rewrite
+            add_prefix:    filtered
+            rule:
+              key:     path
+              pattern: "^[A-Z]+"
+              replace: true
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      match_conf = root_conf.elements.first
+      rule_conf  = match_conf.elements.first
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[key pattern replace], not_fetched
+
+      # accessing should delete
+      match_conf['type']
+      rule_conf['key']
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[pattern replace], not_fetched
+
+      # repeatedly accessing should not grow memory usage
+      before_size = match_conf.unused.size
+      10.times { match_conf['type'] }
+      assert_equal before_size, match_conf.unused.size
     end
-    create_warn_dummy_logger
   end
 
-  def create_warn_dummy_logger
-    dl_opts = {}
-    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
-    logdev = Fluent::Test::DummyLogDevice.new
-    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
-    $log = Fluent::Log.new(logger)
+  def write_config(path, data, encoding: 'utf-8')
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
   sub_test_case '.build' do

data/test/test_event_router.rb

@@ -326,6 +326,23 @@ class EventRouterTest < ::Test::Unit::TestCase
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
       end
+
+      test 'can pass records modified by filters to handle_emits_error' do
+        filter = Class.new(FluentTestFilter) {
+          def filter_stream(_tag, es); end
+        }.new
+        event_router.add_rule('test', filter)
+        event_router.add_rule('test', error_output)
+
+        time = Engine.now
+        modified_es = OneEventStream.new(time, 'modified_label' => 'modified_value')
+
+        assert_rr do
+          stub(filter).filter_stream { modified_es }
+          mock(emit_handler).handle_emits_error('test', modified_es, is_a(RuntimeError))
+          event_router.emit('test', time, 'pre_label' => 'pre_value')
+        end
+      end
     end
   end
 end

data/test/test_file_wrapper.rb

@@ -0,0 +1,53 @@
+require_relative 'helper'
+require 'fluent/file_wrapper'
+
+class FileWrapperTest < Test::Unit::TestCase
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"
+
+  def setup
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  def teardown
+    FileUtils.rm_rf(TMP_DIR)
+  end
+
+  sub_test_case 'WindowsFile exceptions' do
+    test 'nothing raised' do
+      begin
+        path = "#{TMP_DIR}/test_windows_file.txt"
+        file1 = file2 = nil
+        file1 = File.open(path, "wb") do |f|
+        end
+        assert_nothing_raised do
+          file2 = Fluent::WindowsFile.new(path)
+        ensure
+          file2.close
+        end
+      ensure
+        file1.close if file1
+      end
+    end
+
+    test 'Errno::ENOENT raised' do
+      path = "#{TMP_DIR}/nofile.txt"
+      file = nil
+      assert_raise(Errno::ENOENT) do
+        file = Fluent::WindowsFile.new(path)
+      ensure
+        file.close if file
+      end
+    end
+
+    test 'Errno::ENOENT raised on DeletePending' do
+      path = "#{TMP_DIR}/deletepending.txt"
+      file = Fluent::WindowsFile.new(path, mode='w')
+      File.delete(path)
+      assert_raise(Errno::ENOENT) do
+        file.stat
+      ensure
+        file.close if file
+      end
+    end
+  end
+end if Fluent.windows?

data/test/test_formatter.rb

@@ -18,7 +18,7 @@ module FormatterTest
 
     def test_call
       formatter = Formatter.new
-      formatter.configure({})
+      formatter.configure(config_element())
       assert_raise NotImplementedError do
         formatter.format('tag', Engine.now, {})
       end
@@ -130,7 +130,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::MessagePackFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)
       @time = Engine.now
     end
 
@@ -146,7 +146,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::LabeledTSVFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)
       @time = Engine.now
       @newline = if Fluent.windows?
                    "\r\n"
@@ -156,16 +156,16 @@ module FormatterTest
     end
 
     def test_config_params
-      assert_equal "\t", @formatter.delimiter
-      assert_equal  ":", @formatter.label_delimiter
+      assert_equal "\t", @formatter.instance.delimiter
+      assert_equal  ":", @formatter.instance.label_delimiter
 
       @formatter.configure(
         'delimiter'       => ',',
         'label_delimiter' => '=',
       )
 
-      assert_equal ",", @formatter.delimiter
-      assert_equal "=", @formatter.label_delimiter
+      assert_equal ",", @formatter.instance.delimiter
+      assert_equal "=", @formatter.instance.label_delimiter
     end
 
     def test_format
@@ -220,14 +220,14 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::CsvFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)
       @time = Engine.now
     end
 
     def test_config_params
-      assert_equal ',', @formatter.delimiter
-      assert_equal true, @formatter.force_quotes
-      assert_nil @formatter.fields
+      assert_equal ',', @formatter.instance.delimiter
+      assert_equal true, @formatter.instance.force_quotes
+      assert_nil @formatter.instance.fields
     end
 
     data(
@@ -237,8 +237,8 @@ module FormatterTest
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
       @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
-      assert_equal expected, @formatter.delimiter
-      assert_equal ['a', 'b', 'c'], @formatter.fields
+      assert_equal expected, @formatter.instance.delimiter
+      assert_equal ['a', 'b', 'c'], @formatter.instance.fields
     end
 
     def test_format
@@ -299,7 +299,7 @@ module FormatterTest
       'blank' => 'one,,two,three')
     def test_config_params_with_fields(data)
       @formatter.configure('fields' => data)
-      assert_equal %w(one two three), @formatter.fields
+      assert_equal %w(one two three), @formatter.instance.fields
     end
   end
 
@@ -313,31 +313,34 @@ module FormatterTest
                  end
     end
 
+    def create_driver(klass_or_str)
+      Fluent::Test::FormatterTestDriver.new(klass_or_str)
+    end
 
     def test_config_params
-      formatter = TextFormatter::SingleValueFormatter.new
-      assert_equal "message", formatter.message_key
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
+      assert_equal "message", formatter.instance.message_key
 
       formatter.configure('message_key' => 'foobar')
-      assert_equal "foobar", formatter.message_key
+      assert_equal "foobar", formatter.instance.message_key
     end
 
     def test_format
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome#{@newline}", formatted)
     end
 
     def test_format_without_newline
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure('add_newline' => 'false')
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome", formatted)
     end
 
     def test_format_with_message_key
-      formatter = TextFormatter::SingleValueFormatter.new
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
       formatter.configure('message_key' => 'foobar')
       formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
 
@@ -349,7 +352,7 @@ module FormatterTest
     include FormatterTest
 
     def test_unknown_format
-      assert_raise ConfigError do
+      assert_raise NotFoundPluginError do
         Fluent::Plugin.new_formatter('unknown')
       end
     end