fluentd 1.14.6 → 1.15.0

data/test/plugin/test_out_forward.rb

@@ -431,10 +431,12 @@ EOL
     ]
 
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run do
-        emit_events.each do |tag, t, record|
-          d.feed(tag, t, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run do
+          emit_events.each do |tag, t, record|
+            d.feed(tag, t, record)
+          end
         end
       end
     end
@@ -461,10 +463,12 @@ EOL
     ]
 
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -491,10 +495,12 @@ EOL
       {"a" => 2}
     ]
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -549,10 +555,12 @@ EOL
     ]
     # not attempt to receive responses
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -575,10 +583,12 @@ EOL
     ]
     # not attempt to receive responses
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end

data/test/plugin/test_parser_syslog.rb

@@ -644,7 +644,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
 
   data('regexp' => :regexp, 'string' => :string)
   def test_parser_engine(engine)
-    d = @parser.configure({'parser_engine' => engine.to_s})
+    @parser.configure({'parser_engine' => engine.to_s})
     assert_equal(engine, @parser.instance.parser_engine)
   end
 end

data/test/plugin_helper/test_cert_option.rb

@@ -19,7 +19,7 @@ class CertOptionPluginHelperTest < Test::Unit::TestCase
   test 'raise an error for broken certificates_from_file file' do
     d = Dummy.new
     assert_raise Fluent::ConfigError do
-      certs = d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
+      d.cert_option_certificates_from_file("test/plugin_helper/data/cert/empty.pem")
     end
   end
 end

data/test/plugin_helper/test_child_process.rb

@@ -321,15 +321,21 @@ class ChildProcessTest < Test::Unit::TestCase
     ary = []
     arguments = ["okay"]
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+      start_time = Fluent::Clock.now
       @d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
         ary << io.read.split("\n").map(&:chomp).join
       end
-      sleep 2.9 # 2sec(second invocation) + 0.9sec
+      1.upto(2) do |i|
+        sleep 0.1 while ary.size < i
+        elapsed = Fluent::Clock.now - start_time
+        assert_equal(i, ary.size)
+        assert_true(elapsed > i && elapsed < i + 0.5,
+                    "actual elapsed: #{elapsed}")
+      end
       assert_equal [], @d.log.out.logs
       @d.stop
       assert_equal [], @d.log.out.logs
       @d.shutdown; @d.close; @d.terminate
-      assert_equal 2, ary.size
     end
   end
 
@@ -337,12 +343,18 @@ class ChildProcessTest < Test::Unit::TestCase
     ary = []
     arguments = ["okay"]
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
+      start_time = Fluent::Clock.now
       @d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
         ary << io.read.split("\n").map(&:chomp).join
       end
-      sleep 1.9 # 1sec(second invocation) + 0.9sec
+      0.upto(1) do |i|
+        sleep 0.1 while ary.size < i + 1
+        elapsed = Fluent::Clock.now - start_time
+        assert_equal(i + 1, ary.size)
+        assert_true(elapsed > i && elapsed < i + 0.5,
+                    "actual elapsed: #{elapsed}")
+      end
       @d.stop; @d.shutdown; @d.close; @d.terminate
-      assert_equal 2, ary.size
       assert_equal [], @d.log.out.logs
     end
   end

data/test/test_config.rb

@@ -10,11 +10,18 @@ class ConfigTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/tmp/config#{ENV['TEST_ENV_NUMBER']}"
 
-  def read_config(path)
+  def read_config(path, use_yaml: false)
     path = File.expand_path(path)
-    File.open(path) { |io|
-      Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
-    }
+    if use_yaml
+      context = Kernel.binding
+
+      s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(path))
+      Fluent::Config::YamlParser::Parser.new(s).build.to_element
+    else
+      File.open(path) { |io|
+        Fluent::Config::Parser.parse(io, File.basename(path), File.dirname(path))
+      }
+    end
   end
 
   def prepare_config
@@ -151,6 +158,130 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal before_size, match_conf.unused.size
   end
 
+  sub_test_case "yaml config" do
+    def test_included
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - source:
+            $type: dummy
+            tag: tag.dummy
+        - source:
+            $type: tcp
+            tag: tag.tcp
+            parse:
+              $arg:
+                - why.parse.section.doesnot.have.arg
+                - huh
+              $type: none
+        - match:
+            $tag: tag.*
+            $type: stdout
+            buffer:
+              $type: memory
+              flush_interval: 1s
+        - !include fluent-included.yaml
+      EOS
+      write_config "#{TMP_DIR}/fluent-included.yaml", <<-EOS
+      - label:
+          $name: '@FLUENT_LOG'
+          config:
+            - match:
+                $type: "null"
+                $tag: "**"
+                buffer:
+                  $type: memory
+                  flush_mode: interval
+                  flush_interval: 1s
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      dummy_source_conf = root_conf.elements.first
+      tcp_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+      label_conf = root_conf.elements[3]
+      fluent_log_conf = label_conf.elements.first
+      fluent_log_buffer_conf = fluent_log_conf.elements.first
+
+      assert_equal(
+        [
+          'dummy',
+          'tag.dummy',
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'stdout',
+          'tag.*',
+          'null',
+          '**',
+          '@FLUENT_LOG',
+          'memory',
+          'interval',
+          '1s',
+        ],
+        [
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          match_conf['@type'],
+          match_conf.arg,
+          fluent_log_conf['@type'],
+          fluent_log_conf.arg,
+          label_conf.arg,
+          fluent_log_buffer_conf['@type'],
+          fluent_log_buffer_conf['flush_mode'],
+          fluent_log_buffer_conf['flush_interval'],
+        ])
+    end
+
+    def test_check_not_fetchd
+      write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
+      config:
+        - match:
+            $arg: dummy
+            $type: rewrite
+            add_prefix:    filtered
+            rule:
+              key:     path
+              pattern: "^[A-Z]+"
+              replace: true
+      EOS
+      root_conf  = read_config("#{TMP_DIR}/config_test_not_fetched.yaml", use_yaml: true)
+      match_conf = root_conf.elements.first
+      rule_conf  = match_conf.elements.first
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix key pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[key pattern replace], not_fetched
+
+      # accessing should delete
+      match_conf['type']
+      rule_conf['key']
+
+      not_fetched = []; root_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; match_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[@type $arg add_prefix pattern replace], not_fetched
+
+      not_fetched = []; rule_conf.check_not_fetched {|key, e| not_fetched << key }
+      assert_equal %w[pattern replace], not_fetched
+
+      # repeatedly accessing should not grow memory usage
+      before_size = match_conf.unused.size
+      10.times { match_conf['type'] }
+      assert_equal before_size, match_conf.unused.size
+    end
+  end
+
   def write_config(path, data, encoding: 'utf-8')
     FileUtils.mkdir_p(File.dirname(path))
     File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }

data/test/test_supervisor.rb

@@ -90,6 +90,93 @@ class SupervisorTest < ::Test::Unit::TestCase
     assert_equal 2, counter_client.timeout
   end
 
+  sub_test_case "yaml config" do
+    def parse_yaml(yaml)
+      context = Kernel.binding
+
+      config = nil
+      Tempfile.open do |file|
+        file.puts(yaml)
+        file.flush
+        s = Fluent::Config::YamlParser::Loader.new(context).load(Pathname.new(file))
+        config = Fluent::Config::YamlParser::Parser.new(s).build.to_element
+      end
+      config
+    end
+
+    def test_system_config
+      opts = Fluent::Supervisor.default_options
+      sv = Fluent::Supervisor.new(opts)
+      conf_data = <<-EOC
+      system:
+        rpc_endpoint: 127.0.0.1:24445
+        suppress_repeated_stacktrace: true
+        suppress_config_dump: true
+        without_source: true
+        enable_get_dump: true
+        process_name: "process_name"
+        log_level: info
+        root_dir: !fluent/s "#{TMP_ROOT_DIR}"
+        log:
+          format: json
+          time_format: "%Y"
+        counter_server:
+          bind: 127.0.0.1
+          port: 24321
+          scope: server1
+          backup_path: /tmp/backup
+        counter_client:
+          host: 127.0.0.1
+          port: 24321
+          timeout: 2
+      EOC
+      conf = parse_yaml(conf_data)
+      sys_conf = sv.__send__(:build_system_config, conf)
+
+    counter_client = sys_conf.counter_client
+    counter_server = sys_conf.counter_server
+    assert_equal(
+      [
+        '127.0.0.1:24445',
+        true,
+        true,
+        true,
+        true,
+        "process_name",
+        2,
+        TMP_ROOT_DIR,
+        :json,
+        '%Y',
+        '127.0.0.1',
+        24321,
+        'server1',
+        '/tmp/backup',
+        '127.0.0.1',
+        24321,
+        2,
+      ],
+      [
+        sys_conf.rpc_endpoint,
+        sys_conf.suppress_repeated_stacktrace,
+        sys_conf.suppress_config_dump,
+        sys_conf.without_source,
+        sys_conf.enable_get_dump,
+        sys_conf.process_name,
+        sys_conf.log_level,
+        sys_conf.root_dir,
+        sys_conf.log.format,
+        sys_conf.log.time_format,
+        counter_server.bind,
+        counter_server.port,
+        counter_server.scope,
+        counter_server.backup_path,
+        counter_client.host,
+        counter_client.port,
+        counter_client.timeout,
+      ])
+    end
+  end
+
   def test_main_process_signal_handlers
     omit "Windows cannot handle signals" if Fluent.windows?
 
@@ -213,6 +300,49 @@ class SupervisorTest < ::Test::Unit::TestCase
     $log.out.reset if $log && $log.out && $log.out.respond_to?(:reset)
   end
 
+  data("Normal", {raw_path: "C:\\Windows\\Temp\\sigdump.log", expected: "C:\\Windows\\Temp\\sigdump-#{$$}.log"})
+  data("UNIX style", {raw_path: "/Windows/Temp/sigdump.log", expected: "/Windows/Temp/sigdump-#{$$}.log"})
+  data("No extension", {raw_path: "C:\\Windows\\Temp\\sigdump", expected: "C:\\Windows\\Temp\\sigdump-#{$$}"})
+  data("Multi-extension", {raw_path: "C:\\Windows\\Temp\\sig.dump.bk", expected: "C:\\Windows\\Temp\\sig.dump-#{$$}.bk"})
+  def test_fluentsigdump_get_path_with_pid(data)
+    p data
+    path = Fluent::FluentSigdump.get_path_with_pid(data[:raw_path])
+    assert_equal(data[:expected], path)
+  end
+
+  def test_supervisor_event_dump_windows
+    omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
+
+    server = DummyServer.new
+    def server.config
+      {:signame => "TestFluentdEvent"}
+    end
+    server.install_windows_event_handler
+
+    assert_rr do
+      # Have to use mock because `Sigdump.dump` seems to be somehow incompatible with RR.
+      # The `mock(server).restart(true) { nil }` line in `test_rpc_server_windows` cause the next error.
+      # Failure: test_supervisor_event_dump_windows(SupervisorTest):
+      #   class()
+      #   Called 0 times.
+      #   Expected 1 times.
+      # .../Ruby26-x64/lib/ruby/gems/2.6.0/gems/sigdump-0.2.4/lib/sigdump.rb:74:in `block in dump_object_count'
+      #     73: ObjectSpace.each_object {|o|
+      #     74:   c = o.class <-- HERE!
+      mock(Sigdump).dump(anything)
+
+      begin
+        sleep 0.1 # Wait for starting windows event thread
+        event = Win32::Event.open("TestFluentdEvent_CONT")
+        event.set
+        event.close
+        sleep 1.0 # Wait for dumping
+      ensure
+        server.stop_windows_event_thread
+      end
+    end
+  end
+
   data(:ipv4 => ["0.0.0.0", "127.0.0.1", false],
        :ipv6 => ["[::]", "[::1]", true],
        :localhost_ipv4 => ["localhost", "127.0.0.1", false])
@@ -557,6 +687,31 @@ class SupervisorTest < ::Test::Unit::TestCase
                       logger.instance_variable_get(:@rotate_size)])
       end
     end
+
+    def test_override_default_log_rotate_with_yaml_config
+      Tempfile.open do |file|
+        config = <<-EOS
+          system:
+            log:
+              rotate_age: 3
+              rotate_size: 300
+        EOS
+        file.puts(config)
+        file.flush
+        opts = Fluent::Supervisor.default_options.merge(
+          log_path: "#{TMP_DIR}/test.log", config_path: file.path, config_file_type: :yaml,
+        )
+        sv = Fluent::Supervisor.new(opts)
+
+        log = sv.instance_variable_get(:@log)
+        log.init(:standalone, 0)
+        logger = $log.instance_variable_get(:@logger)
+
+        assert_equal([3, 300],
+                     [logger.instance_variable_get(:@rotate_age),
+                      logger.instance_variable_get(:@rotate_size)])
+      end
+    end
   end
 
   def test_inline_config

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.6
+  version: 1.15.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-31 00:00:00.000000000 Z
+date: 2022-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,7 +84,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.5
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -94,7 +94,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.5
+        version: 2.3.0
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -507,6 +507,11 @@ files:
 - lib/fluent/config/section.rb
 - lib/fluent/config/types.rb
 - lib/fluent/config/v1_parser.rb
+- lib/fluent/config/yaml_parser.rb
+- lib/fluent/config/yaml_parser/fluent_value.rb
+- lib/fluent/config/yaml_parser/loader.rb
+- lib/fluent/config/yaml_parser/parser.rb
+- lib/fluent/config/yaml_parser/section_builder.rb
 - lib/fluent/configurable.rb
 - lib/fluent/counter.rb
 - lib/fluent/counter/base_socket.rb
@@ -579,6 +584,7 @@ files:
 - lib/fluent/plugin/in_sample.rb
 - lib/fluent/plugin/in_syslog.rb
 - lib/fluent/plugin/in_tail.rb
+- lib/fluent/plugin/in_tail/group_watch.rb
 - lib/fluent/plugin/in_tail/position_file.rb
 - lib/fluent/plugin/in_tcp.rb
 - lib/fluent/plugin/in_udp.rb
@@ -959,7 +965,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector