fluentd 1.14.0 → 1.14.4

data/test/plugin/test_in_tail.rb

@@ -99,7 +99,7 @@ class TailInputTest < Test::Unit::TestCase
                           })
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
-  CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
+  CONFIG_DISABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
   CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
   CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
   COMMON_FOLLOW_INODE_CONFIG = config_element("ROOT", "", {
@@ -199,7 +199,7 @@ class TailInputTest < Test::Unit::TestCase
 
     sub_test_case "log throttling per file" do
       test "w/o watcher timer is invalid" do
-        conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        conf = CONFIG_DISABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
         assert_raise(Fluent::ConfigError) do
           create_driver(conf)
         end
@@ -215,7 +215,7 @@ class TailInputTest < Test::Unit::TestCase
 
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
-        create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
+        create_driver(CONFIG_DISABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
       end
     end
 
@@ -570,9 +570,9 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, events[3][2])
     end
 
-    data(flat: CONFIG_ENABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
-         parse: CONFIG_ENABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
-    def test_emit_with_enable_watch_timer(data)
+    data(flat: CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG,
+         parse: CONFIG_DISABLE_WATCH_TIMER + PARSE_SINGLE_LINE_CONFIG)
+    def test_emit_without_watch_timer(data)
       config = data
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
         f.puts "test1"
@@ -596,6 +596,38 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, events[1][2])
     end
 
+    # https://github.com/fluent/fluentd/pull/3541#discussion_r740197711
+    def test_watch_wildcard_path_without_watch_timer
+      omit "need inotify" unless Fluent.linux?
+
+      config = config_element("ROOT", "", {
+                                "path" => "#{TMP_DIR}/tail*.txt",
+                                "tag" => "t1",
+                              })
+      config = config + CONFIG_DISABLE_WATCH_TIMER + SINGLE_LINE_CONFIG
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(config, false)
+
+      d.run(expect_emits: 1, timeout: 1) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+      end
+
+      assert_equal(
+        [
+          {"message" => "test3"},
+          {"message" => "test4"},
+        ],
+        d.events.collect { |event| event[2] })
+    end
+
     data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
          parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
     def test_emit_with_disable_stat_watcher(data)
@@ -619,6 +651,23 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test3"}, events[0][2])
       assert_equal({"message" => "test4"}, events[1][2])
     end
+
+    def test_always_read_from_head_on_detecting_a_new_file
+      d = create_driver(SINGLE_LINE_CONFIG)
+
+      d.run(expect_emits: 1, timeout: 3) do
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+          f.puts "test1\ntest2\n"
+        }
+      end
+
+      assert_equal(
+        [
+          {"message" => "test1"},
+          {"message" => "test2"},
+        ],
+        d.events.collect { |event| event[2] })
+    end
   end
 
   class TestWithSystem < self
@@ -1524,11 +1573,18 @@ class TailInputTest < Test::Unit::TestCase
     plugin.instance_eval do
       @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
       @loop = Coolio::Loop.new
+      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+      opened_file_metrics.configure(config_element('metrics', '', {}))
+      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+      closed_file_metrics.configure(config_element('metrics', '', {}))
+      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+      rotated_file_metrics.configure(config_element('metrics', '', {}))
+      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       ex_paths.each do |target_info|
-        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       end
 
       plugin.refresh_watchers
@@ -1542,7 +1598,7 @@ class TailInputTest < Test::Unit::TestCase
       path = "test/plugin/data/2010/01/20100102-030406.log"
       inode = Fluent::FileWrapper.stat(path).ino
       target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       plugin.refresh_watchers
 
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1651,6 +1707,41 @@ class TailInputTest < Test::Unit::TestCase
       mock(plugin.router).emit_stream('pre.foo.bar.log.post', anything).once
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
+
+    data(
+      small: ["128", 128],
+      KiB: ["1k", 1024]
+    )
+    test 'max_line_size' do |(label, size)|
+      config = config_element("", "", {
+                                "tag" => "max_line_size",
+                                "path" => "#{TMP_DIR}/with_long_lines.txt",
+                                "format" => "none",
+                                "read_from_head" => true,
+                                "max_line_size" => label,
+                                "log_level" => "debug"
+                              })
+      File.open("#{TMP_DIR}/with_long_lines.txt", "w+") do |f|
+        f.puts "foo"
+        f.puts "x" * size # 'x' * size + \n > @max_line_size
+        f.puts "bar"
+      end
+      d = create_driver(config, false)
+      timestamp = Time.parse("Mon Nov 29 11:22:33 UTC 2021")
+      Timecop.freeze(timestamp)
+      d.run(expect_records: 2)
+      assert_equal([
+                     [{"message" => "foo"},{"message" => "bar"}],
+                     [
+                       "2021-11-29 11:22:33 +0000 [warn]: received line length is longer than #{size}\n",
+                       "2021-11-29 11:22:33 +0000 [debug]: skipped line: #{'x' * size}\n"
+                     ]
+                   ],
+                   [
+                     d.events.collect { |event| event.last },
+                     d.logs[-2..]
+                   ])
+    end
   end
 
   # Ensure that no fatal exception is raised when a file is missing and that
@@ -1885,13 +1976,22 @@ class TailInputTest < Test::Unit::TestCase
       config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
 
       d = create_driver(config, false)
+      d.instance.instance_eval do
+        opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+        opened_file_metrics.configure(config_element('metrics', '', {}))
+        closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+        closed_file_metrics.configure(config_element('metrics', '', {}))
+        rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+        rotated_file_metrics.configure(config_element('metrics', '', {}))
+        @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      end
 
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
         f.puts "test1"
         f.puts "test2"
       }
       target_info = create_target_info("#{TMP_DIR}/tail.txt")
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
       d.run(shutdown: false)
       assert d.instance.instance_variable_get(:@tails)[target_info]
 

data/test/plugin/test_input.rb

@@ -73,7 +73,7 @@ class InputTest < Test::Unit::TestCase
 
     @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-    assert{ @p.log.object_id != original_logger.object_id }
+    assert(@p.log.object_id != original_logger.object_id)
     assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
   end
 

data/test/plugin/test_out_exec.rb

@@ -243,7 +243,7 @@ class ExecOutputTest < Test::Unit::TestCase
   sub_test_case 'when executed process dies unexpectedly' do
     setup do
       @gen_config = ->(num){ <<EOC
-    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
+    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write(f.read); STDOUT.flush} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
     <inject>
       tag_key tag
       time_key time
@@ -265,7 +265,7 @@ EOC
       expect_path = "#{TMP_DIR}/fail_out"
 
       d.end_if{ File.exist?(expect_path) }
-      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: true, shutdown: false) do
         d.feed(time, records[0])
         d.feed(time, records[1])
       end
@@ -281,7 +281,8 @@ EOC
       assert{ d.instance.buffer.queue.empty? }
       assert{ d.instance.dequeued_chunks.empty? }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
 
     test 'flushed chunk will be taken back after child process unexpectedly exits' do
@@ -304,7 +305,8 @@ EOC
 
       assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
   end
 end

data/test/plugin/test_out_exec_filter.rb

@@ -524,6 +524,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal pid_list[1], events[1][2]['child_pid']
     assert_equal pid_list[0], events[2][2]['child_pid']
     assert_equal pid_list[1], events[3][2]['child_pid']
+
+  ensure
+    d.run(start: false, shutdown: true)
   end
 
   # child process exits per 3 lines
@@ -597,6 +600,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal 2, logs.select { |l| l.include?('child process exits with error code') }.size
     assert_equal 2, logs.select { |l| l.include?('respawning child process') }.size
 
+  ensure
     d.run(start: false, shutdown: true)
   end
 end

data/test/plugin/test_out_file.rb

@@ -394,6 +394,11 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal expect, result
   end
 
+  def check_result(path, expect)
+    result = File.read(path, mode: "rb")
+    assert_equal expect, result
+  end
+
   sub_test_case 'write' do
     test 'basic case' do
       d = create_driver
@@ -535,20 +540,27 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
-  test 'append' do
+  data(
+    "with compression" => true,
+    "without compression" => false,
+  )
+  test 'append' do |compression|
     time = event_time("2011-01-02 13:14:15 UTC")
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
-      d = create_driver %[
+      config = %[
         path #{TMP_DIR}/out_file_test
-        compress gz
         utc
         append true
         <buffer>
           timekey_use_utc true
         </buffer>
       ]
+      if compression
+        config << "        compress gz"
+      end
+      d = create_driver(config)
       d.run(default_tag: 'test'){
         d.feed(time, {"a"=>1})
         d.feed(time, {"a"=>2})
@@ -556,17 +568,21 @@ class FileOutputTest < Test::Unit::TestCase
       d.instance.last_written_path
     }
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
+    log_file_name = "out_file_test.20110102.log"
+    if compression
+      log_file_name << ".gz"
+    end
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    1.upto(3) do |i|
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/#{log_file_name}", path
+      expect = formatted_lines * i
+      if compression
+        check_gzipped_result(path, expect)
+      else
+        check_result(path, expect)
+      end
+    end
   end
 
   test 'append when JST' do

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -775,7 +775,7 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:13').to_i, {"name" => "moris", "age" => 36, "message" => "data2"} ], written[1]
       assert_equal [ 'test.tag.1', event_time('2016-04-13 18:33:32').to_i, {"name" => "moris", "age" => 36, "message" => "data3"} ], written[2]
 
-      assert{ @i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") } }
+      assert(@i.log.out.logs.any?{|l| l.include?("[warn]: retry succeeded by secondary.") })
     end
 
     test 'exponential backoff interval will be initialized when switched to secondary' do

data/test/plugin_helper/test_child_process.rb

@@ -319,12 +319,12 @@ class ChildProcessTest < Test::Unit::TestCase
 
   test 'can execute external command many times, which finishes immediately' do
     ary = []
-    arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
+    arguments = ["okay"]
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
-      @d.child_process_execute(:t5, "ruby", arguments: arguments, interval: 1, mode: [:read]) do |io|
+      @d.child_process_execute(:t5, "echo", arguments: arguments, interval: 1, mode: [:read]) do |io|
         ary << io.read.split("\n").map(&:chomp).join
       end
-      sleep 2.5 # 2sec(second invocation) + 0.5sec
+      sleep 2.9 # 2sec(second invocation) + 0.9sec
       assert_equal [], @d.log.out.logs
       @d.stop
       assert_equal [], @d.log.out.logs
@@ -335,12 +335,12 @@ class ChildProcessTest < Test::Unit::TestCase
 
   test 'can execute external command many times, with leading once executed immediately' do
     ary = []
-    arguments = ["-e", "puts 'okay'; STDOUT.flush rescue nil"]
+    arguments = ["okay"]
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
-      @d.child_process_execute(:t6, "ruby", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
+      @d.child_process_execute(:t6, "echo", arguments: arguments, interval: 1, immediate: true, mode: [:read]) do |io|
         ary << io.read.split("\n").map(&:chomp).join
       end
-      sleep 1.5 # 2sec(second invocation) + 0.5sec
+      sleep 1.9 # 1sec(second invocation) + 0.9sec
       @d.stop; @d.shutdown; @d.close; @d.terminate
       assert_equal 2, ary.size
       assert_equal [], @d.log.out.logs
@@ -722,14 +722,14 @@ class ChildProcessTest < Test::Unit::TestCase
       read_data_list = []
       exit_status_list = []
 
-      args = ['-e', 'puts "yay"']
+      args = ['yay']
       cb = ->(status){ exit_status_list << status }
 
       Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
-        @d.child_process_execute(:st1, "ruby", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
+        @d.child_process_execute(:st1, "echo", arguments: args, immediate: true, interval: 1, mode: [:read], on_exit_callback: cb) do |readio|
           read_data_list << readio.read.chomp
         end
-        sleep 2
+        sleep 2.5
       end
 
       assert { read_data_list.size >= 2 }

data/test/plugin_helper/test_timer.rb

@@ -90,8 +90,8 @@ class TimerTest < Test::Unit::TestCase
     assert{ counter1 >= 4 && counter1 <= 5 }
     assert{ counter2 == 2 }
     msg = "Unexpected error raised. Stopping the timer. title=:t2"
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") } }
-    assert{ d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") } }
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?(msg) && line.include?("abort!!!!!!") })
+    assert(d1.log.out.logs.any?{|line| line.include?("[error]:") && line.include?("Timer detached. title=:t2") })
 
     d1.shutdown; d1.close; d1.terminate
   end

data/test/test_time_parser.rb

@@ -337,4 +337,26 @@ class TimeParserTest < ::Test::Unit::TestCase
       assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
     end
   end
+
+  # https://github.com/fluent/fluentd/issues/3195
+  test 'change timezone without zone specifier in a format' do
+    expected = 1607457600 # 2020-12-08T20:00:00Z
+    time1 = time2 = nil
+
+    with_timezone("UTC-05") do # EST
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'string',
+                                               'time_format' => '%Y-%m-%dT%H:%M:%SZ',
+                                               'utc' => true}))
+      parser = i.time_parser_create
+
+      time1 = parser.parse('2020-12-08T20:00:00Z').to_i
+      time2 = with_timezone("UTC-04") do # EDT
+        # to avoid using cache, increment 1 sec
+        parser.parse('2020-12-08T20:00:01Z').to_i
+      end
+    end
+
+    assert_equal([expected, expected + 1], [time1, time2])
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.14.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2022-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,7 +107,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -117,7 +117,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: sigdump
   requirement: !ruby/object:Gem::Requirement
@@ -172,7 +172,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -182,7 +182,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.4
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
@@ -395,6 +395,7 @@ files:
 - MAINTAINERS.md
 - README.md
 - Rakefile
+- SECURITY.md
 - bin/fluent-binlog-reader
 - bin/fluent-ca-generate
 - bin/fluent-cap-ctl
@@ -944,7 +945,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector