fluentd 1.14.5 → 1.15.1

data/test/plugin_helper/test_server.rb

@@ -64,6 +64,24 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       end
       assert d.plugin_id
       assert d.log
+      assert_equal 0, d.transport_config.linger_timeout
+    end
+
+    test 'can change linger_timeout option' do
+      d = Dummy.new
+
+      transport_opts = {
+        'linger_timeout' => 1,
+      }
+      transport_conf = config_element('transport', 'tcp', transport_opts)
+      conf = config_element('source', 'tag.*', {}, [transport_conf])
+
+      assert_nothing_raised do
+        d.configure(conf)
+      end
+      assert d.plugin_id
+      assert d.log
+      assert_equal 1, d.transport_config.linger_timeout
     end
   end
 

data/test/test_config.rb

@@ -10,11 +10,18 @@ class ConfigTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
 
-  def read_config(path)
+  def read_config(path, use_yaml: false)
     path = File.expand_path(path)
-    File.open(path) { |io|
-      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
-    }
+    if use_yaml
+      context = Kernel.binding
+
+      s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+      Fluent::Config::YamlParser::Parser.new(s).build.to_element
+    else
+      File.open(path) { |io|
+        Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+      }
+    end
   end
 
   def prepare_config
@@ -151,6 +158,130 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal before_size, match_conf.unused.size
   end
 
+  sub_test_case "yaml config" do
+    def test_included
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - source:
+            $type: dummy
+            tag: tag.dummy
+        - source:
+            $type: tcp
+            tag: tag.tcp
+            parse:
+              $arg:
+                - why.parse.section.doesnot.have.arg
+                - huh
+              $type: none
+        - match:
+            $tag: tag.*
+            $type: stdout
+            buffer:
+              $type: memory
+              flush_interval: 1s
+        - !include fluent-included.yaml
+      EOS
+      write_config "#{TMP_DIR}/fluent-included.yaml", <<-EOS
+      - label:
+          $name: '@FLUENT_LOG'
+          config:
+            - match:
+                $type: "null"
+                $tag: "**"
+                buffer:
+                  $type: memory
+                  flush_mode: interval
+                  flush_interval: 1s
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      dummy_source_conf = root_conf.elements.first
+      tcp_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+      label_conf = root_conf.elements[3]
+      fluent_log_conf = label_conf.elements.first
+      fluent_log_buffer_conf = fluent_log_conf.elements.first
+
+      assert_equal(
+        [
+          'dummy',
+          'tag.dummy',
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'stdout',
+          'tag.*',
+          'null',
+          '**',
+          '@FLUENT_LOG',
+          'memory',
+          'interval',
+          '1s',
+        ],
+        [
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          match_conf['@type'],
+          match_conf.arg,
+          fluent_log_conf['@type'],
+          fluent_log_conf.arg,
+          label_conf.arg,
+          fluent_log_buffer_conf['@type'],
+          fluent_log_buffer_conf['flush_mode'],
+          fluent_log_buffer_conf['flush_interval'],
+        ])
+    end
+
+    def test_check_not_fetchd
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - match:
+            $arg: dummy
+            $type: rewrite
+            add_prefix:    filtered
+            rule:
+              key:     path
+              pattern: "^[A-Z]+"
+              replace: true
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      match_conf = root_conf.elements.first
+      rule_conf  = match_conf.elements.first
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[key pattern replace], not_fetched
+
+      # accessing should delete
+      match_conf['type']
+      rule_conf['key']
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[pattern replace], not_fetched
+
+      # repeatedly accessing should not grow memory usage
+      before_size = match_conf.unused.size
+      10.times { match_conf['type'] }
+      assert_equal before_size, match_conf.unused.size
+    end
+  end
+
   def write_config(path, data, encoding: 'utf-8')
     FileUtils.mkdir_p(File.dirname(path))
     File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }

data/test/test_event_router.rb

@@ -326,6 +326,23 @@ class EventRouterTest < ::Test::Unit::TestCase
           event_router.emit('test', Engine.now, 'k' => 'v')
         end
       end
+
+      test 'can pass records modified by filters to handle_emits_error' do
+        filter = Class.new(FluentTestFilter) {
+          def filter_stream(_tag, es); end
+        }.new
+        event_router.add_rule('test', filter)
+        event_router.add_rule('test', error_output)
+
+        time = Engine.now
+        modified_es = OneEventStream.new(time, 'modified_label' => 'modified_value')
+
+        assert_rr do
+          stub(filter).filter_stream { modified_es }
+          mock(emit_handler).handle_emits_error('test', modified_es, is_a(RuntimeError))
+          event_router.emit('test', time, 'pre_label' => 'pre_value')
+        end
+      end
     end
   end
 end

data/test/test_supervisor.rb

@@ -90,6 +90,93 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 2, counter_client.timeout
   end
 
+  sub_test_case "yaml config" do
+    def parse_yaml(yaml)
+      context = Kernel.binding
+
+      config = nil
+      Tempfile.open do |file|
+        file.puts(yaml)
+        file.flush
+        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(file))
+        config = Fluent::Config::YamlParser::Parser.new(s).build.to_element
+      end
+      config
+    end
+
+    def test_system_config
+      opts = Fluent::Supervisor.default_options
+      sv = Fluent::Supervisor.new(opts)
+      conf_data = <<-EOC
+      system:
+        rpc_endpoint: 127.0.0.1:24445
+        suppress_repeated_stacktrace: true
+        suppress_config_dump: true
+        without_source: true
+        enable_get_dump: true
+        process_name: "process_name"
+        log_level: info
+        root_dir: !fluent/s "#{TMP_ROOT_DIR}"
+        log:
+          format: json
+          time_format: "%Y"
+        counter_server:
+          bind: 127.0.0.1
+          port: 24321
+          scope: server1
+          backup_path: /tmp/backup
+        counter_client:
+          host: 127.0.0.1
+          port: 24321
+          timeout: 2
+      EOC
+      conf = parse_yaml(conf_data)
+      sys_conf = sv.__send__(:build_system_config, conf)
+
+    counter_client = sys_conf.counter_client
+    counter_server = sys_conf.counter_server
+    assert_equal(
+      [
+        '127.0.0.1:24445',
+        true,
+        true,
+        true,
+        true,
+        "process_name",
+        2,
+        TMP_ROOT_DIR,
+        :json,
+        '%Y',
+        '127.0.0.1',
+        24321,
+        'server1',
+        '/tmp/backup',
+        '127.0.0.1',
+        24321,
+        2,
+      ],
+      [
+        sys_conf.rpc_endpoint,
+        sys_conf.suppress_repeated_stacktrace,
+        sys_conf.suppress_config_dump,
+        sys_conf.without_source,
+        sys_conf.enable_get_dump,
+        sys_conf.process_name,
+        sys_conf.log_level,
+        sys_conf.root_dir,
+        sys_conf.log.format,
+        sys_conf.log.time_format,
+        counter_server.bind,
+        counter_server.port,
+        counter_server.scope,
+        counter_server.backup_path,
+        counter_client.host,
+        counter_client.port,
+        counter_client.timeout,
+      ])
+    end
+  end
+
   def test_main_process_signal_handlers
     omit "Windows cannot handle signals" if Fluent.windows?
 
@@ -213,16 +300,65 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
   end
 
-  def test_rpc_server
+  data("Normal", {raw_path: "C:\\Windows\\Temp\\sigdump.log", expected: "C:\\Windows\\Temp\\sigdump-#{$$}.log"})
+  data("UNIX style", {raw_path: "/Windows/Temp/sigdump.log", expected: "/Windows/Temp/sigdump-#{$$}.log"})
+  data("No extension", {raw_path: "C:\\Windows\\Temp\\sigdump", expected: "C:\\Windows\\Temp\\sigdump-#{$$}"})
+  data("Multi-extension", {raw_path: "C:\\Windows\\Temp\\sig.dump.bk", expected: "C:\\Windows\\Temp\\sig.dump-#{$$}.bk"})
+  def test_fluentsigdump_get_path_with_pid(data)
+    p data
+    path = Fluent::FluentSigdump.get_path_with_pid(data[:raw_path])
+    assert_equal(data[:expected], path)
+  end
+
+  def test_supervisor_event_dump_windows
+    omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
+
+    server = DummyServer.new
+    def server.config
+      {:signame => "TestFluentdEvent"}
+    end
+    server.install_windows_event_handler
+
+    assert_rr do
+      # Have to use mock because `Sigdump.dump` seems to be somehow incompatible with RR.
+      # The `mock(server).restart(true) { nil }` line in `test_rpc_server_windows` cause the next error.
+      # Failure: test_supervisor_event_dump_windows(SupervisorTest):
+      #   class()
+      #   Called 0 times.
+      #   Expected 1 times.
+      # .../Ruby26-x64/lib/ruby/gems/2.6.0/gems/sigdump-0.2.4/lib/sigdump.rb:74:in `block in dump_object_count'
+      #     73: ObjectSpace.each_object {|o|
+      #     74:   c = o.class <-- HERE!
+      mock(Sigdump).dump(anything)
+
+      begin
+        sleep 0.1 # Wait for starting windows event thread
+        event = Win32::Event.open("TestFluentdEvent_CONT")
+        event.set
+        event.close
+        sleep 1.0 # Wait for dumping
+      ensure
+        server.stop_windows_event_thread
+      end
+    end
+  end
+
+  data(:ipv4 => ["0.0.0.0", "127.0.0.1", false],
+       :ipv6 => ["[::]", "[::1]", true],
+       :localhost_ipv4 => ["localhost", "127.0.0.1", false])
+  def test_rpc_server(data)
     omit "Windows cannot handle signals" if Fluent.windows?
 
+    bindaddr, localhost, ipv6 = data
+    omit "IPv6 is not supported on this environment" if ipv6 && !ipv6_enabled?
+
     create_info_dummy_logger
 
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
     conf_data = <<-EOC
   <system>
-    rpc_endpoint 0.0.0.0:24447
+    rpc_endpoint "#{bindaddr}:24447"
   </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
@@ -235,7 +371,7 @@ class SupervisorTest < ::Test::Unit::TestCase
     server.run_rpc_server
 
     sv.send(:install_main_process_signal_handlers)
-    response = Net::HTTP.get(URI.parse('http://127.0.0.1:24447/api/plugins.flushBuffers'))
+    response = Net::HTTP.get(URI.parse("http://#{localhost}:24447/api/plugins.flushBuffers"))
     info_msg = '[info]: force flushing buffered events' + "\n"
 
     server.stop_rpc_server
@@ -250,16 +386,45 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log.out.is_a?(Fluent::Test::DummyLogDevice)
   end
 
-  def test_rpc_server_windows
+  data(:no_port => ["127.0.0.1"],
+       :invalid_addr => ["*:24447"])
+  def test_invalid_rpc_endpoint(data)
+    endpoint = data[0]
+
+    opts = Fluent::Supervisor.default_options
+    sv = Fluent::Supervisor.new(opts)
+    conf_data = <<-EOC
+  <system>
+    rpc_endpoint "#{endpoint}"
+  </system>
+    EOC
+    conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
+    sys_conf = sv.__send__(:build_system_config, conf)
+
+    server = DummyServer.new
+    server.rpc_endpoint = sys_conf.rpc_endpoint
+
+    assert_raise(Fluent::ConfigError.new("Invalid rpc_endpoint: #{endpoint}")) do
+      server.run_rpc_server
+    end
+  end
+
+  data(:ipv4 => ["0.0.0.0", "127.0.0.1", false],
+       :ipv6 => ["[::]", "[::1]", true],
+       :localhost_ipv4 => ["localhost", "127.0.0.1", true])
+  def test_rpc_server_windows(data)
     omit "Only for windows platform" unless Fluent.windows?
 
+    bindaddr, localhost, ipv6 = data
+    omit "IPv6 is not supported on this environment" if ipv6 && !ipv6_enabled?
+
     create_info_dummy_logger
 
     opts = Fluent::Supervisor.default_options
     sv = Fluent::Supervisor.new(opts)
     conf_data = <<-EOC
   <system>
-    rpc_endpoint 0.0.0.0:24447
+    rpc_endpoint "#{bindaddr}:24447"
   </system>
     EOC
     conf = Fluent::Config.parse(conf_data, "(test)", "(test_dir)", true)
@@ -277,7 +442,7 @@ class SupervisorTest < ::Test::Unit::TestCase
     server.run_rpc_server
 
     mock(server).restart(true) { nil }
-    response = Net::HTTP.get(URI.parse('http://127.0.0.1:24447/api/plugins.flushBuffers'))
+    response = Net::HTTP.get(URI.parse("http://#{localhost}:24447/api/plugins.flushBuffers"))
 
     server.stop_rpc_server
     assert_equal('{"ok":true}', response)
@@ -522,6 +687,31 @@ class SupervisorTest < ::Test::Unit::TestCase
                       logger.instance_variable_get(:@rotate_size)])
       end
     end
+
+    def test_override_default_log_rotate_with_yaml_config
+      Tempfile.open do |file|
+        config = <<-EOS
+          system:
+            log:
+              rotate_age: 3
+              rotate_size: 300
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml,
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
   end
 
   def test_inline_config

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.5
+  version: 1.15.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-09 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,7 +84,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.5
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -94,7 +94,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.5
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -507,6 +507,11 @@ files:
 - lib/fluent/config/section.rb
 - lib/fluent/config/types.rb
 - lib/fluent/config/v1_parser.rb
+- lib/fluent/config/yaml_parser.rb
+- lib/fluent/config/yaml_parser/fluent_value.rb
+- lib/fluent/config/yaml_parser/loader.rb
+- lib/fluent/config/yaml_parser/parser.rb
+- lib/fluent/config/yaml_parser/section_builder.rb
 - lib/fluent/configurable.rb
 - lib/fluent/counter.rb
 - lib/fluent/counter/base_socket.rb
@@ -579,6 +584,7 @@ files:
 - lib/fluent/plugin/in_sample.rb
 - lib/fluent/plugin/in_syslog.rb
 - lib/fluent/plugin/in_tail.rb
+- lib/fluent/plugin/in_tail/group_watch.rb
 - lib/fluent/plugin/in_tail/position_file.rb
 - lib/fluent/plugin/in_tcp.rb
 - lib/fluent/plugin/in_udp.rb
@@ -959,7 +965,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.3
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector