fluentd 1.11.2 → 1.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f6935bfa737b73c4be0d7bb1c5a689b644c2044da43386db71460b174bbf409
-  data.tar.gz: 9144f8027f0d372c99ecbf8059c29fb6efb8cb780d7d4dcafcbf33de5822b22d
+  metadata.gz: 9fac73611b9d1c9b470912f6de8294795e5e8809d1730fa9f692aae4777ffdc2
+  data.tar.gz: 88c9fe2ccf19fa1a1e82924072182ed7c0c688e70f25dcdff8245d386644faee
 SHA512:
-  metadata.gz: 23612a8c3dd10c3e2b7b32bba4169f063eed5b50ba1c33f9fbf298fe4846eedfbc5605fb216ecaf1aff27f36c6852e8e3f9957a4a191106f27b1caf955364467
-  data.tar.gz: fa945c1f5c57d2ddc35109f5b10aca233fe4268970765dce1dd5e5798457942062cdd803a82317a968df5d1d11835da46a97eb8e8824f4d08979728c6693ac85
+  metadata.gz: d8ee0bff2c2fb43bd8cbcf727de8cf31ab885ecd4e4eae299c2588559d665a4c309a8b575cedadd6534a51962934d6f643eda5baa6056759345814fb1ab084ee
+  data.tar.gz: 40c86faa9b747e65e32ffa004f5f1e8b5563fb81f5d686174da99866ad0c8789d91d465e78d5e4fff891359b05a71db4e204f2f91eed737c6cc5804999644ab7

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # v1.11
 
+## Release v1.11.3 - 2020/09/30
+
+### Enhancement
+
+* in_exec: Add `connect_mode` parameter to read stderr
+  https://github.com/fluent/fluentd/pull/3108
+* parser_json: Improve the performance
+  https://github.com/fluent/fluentd/pull/3109
+* log: Add `ignore_same_log_interval` parameter
+  https://github.com/fluent/fluentd/pull/3119
+* Upgrade win32 gems
+  https://github.com/fluent/fluentd/pull/3100
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3094
+  https://github.com/fluent/fluentd/pull/3118
+
+### Bug fix
+
+* buffer: Fix calculation of timekey stats
+  https://github.com/fluent/fluentd/pull/3018
+* buffer: fix binmode usage for prevent gc
+  https://github.com/fluent/fluentd/pull/3138
+
 ## Release v1.11.2 - 2020/08/04
 
 ### Enhancement

data/fluentd.gemspec

@@ -32,11 +32,11 @@ Gem::Specification.new do |gem|
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
-    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
-    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
-    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
-    gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
+    gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end
 
   gem.add_development_dependency("rake", ["~> 13.0"])

data/lib/fluent/log.rb

@@ -113,6 +113,7 @@ module Fluent
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
+      @ignore_same_log_interval = opts[:ignore_same_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -141,7 +142,8 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
-                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval,
+                             ignore_same_log_interval: @ignore_same_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -151,7 +153,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -428,6 +430,27 @@ module Fluent
       (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
     end
 
+    def ignore_same_log?(time, message)
+      cached_log = Thread.current[:last_same_log]
+      if cached_log.nil?
+        Thread.current[:last_same_log] = {message => time}
+        return false
+      end
+
+      prev_time = cached_log[message]
+      if prev_time
+        if (time - prev_time) <= @ignore_same_log_interval
+          true
+        else
+          cached_log[message] = time
+          false
+        end
+      else
+        cached_log[message] = time
+        false
+      end
+    end
+
     def suppress_stacktrace?(backtrace)
       cached_log = Thread.current[:last_repeated_stacktrace]
       return false if cached_log.nil?
@@ -507,7 +530,11 @@ module Fluent
         end
       }
 
-      if @ignore_repeated_log_interval
+      if @ignore_same_log_interval
+        if ignore_same_log?(time, message)
+          return nil, nil
+        end
+      elsif @ignore_repeated_log_interval
         if ignore_repeated_log?(:last_repeated_log, time, message)
           return nil, nil
         else
@@ -569,6 +596,9 @@ module Fluent
       if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
         @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
       end
+      if logger.instance_variable_defined?(:@ignore_same_log_interval)
+        @ignore_same_log_interval = logger.instance_variable_get(:@ignore_same_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format

data/lib/fluent/plugin/buffer.rb

@@ -192,6 +192,7 @@ module Fluent
 
         @stage_size = @queue_size = 0
         @timekeys = Hash.new(0)
+        @enable_update_timekeys = false
         @mutex = Mutex.new
       end
 
@@ -207,24 +208,23 @@ module Fluent
         end
       end
 
+      def enable_update_timekeys
+        @enable_update_timekeys = true
+      end
+
       def start
         super
 
         @stage, @queue = resume
         @stage.each_pair do |metadata, chunk|
           @stage_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(metadata.timekey)
-          end
         end
         @queue.each do |chunk|
           @queued_num[chunk.metadata] ||= 0
           @queued_num[chunk.metadata] += 1
           @queue_size += chunk.bytesize
-          if chunk.metadata && chunk.metadata.timekey
-            add_timekey(chunk.metadata.timekey)
-          end
         end
+        update_timekeys
         log.debug "buffer started", instance: self.object_id, stage_size: @stage_size, queue_size: @queue_size
       end
 
@@ -273,12 +273,9 @@ module Fluent
         Metadata.new(timekey, tag, variables)
       end
 
+      # Keep this method for existing code
       def metadata(timekey: nil, tag: nil, variables: nil)
-        meta = Metadata.new(timekey, tag, variables)
-        if (t = meta.timekey)
-          add_timekey(t)
-        end
-        meta
+        Metadata.new(timekey, tag, variables)
       end
 
       def timekeys
@@ -472,9 +469,23 @@ module Fluent
         end
       end
 
+      def update_timekeys
+        synchronize do
+          chunks = @stage.values
+          chunks.concat(@queue)
+          @timekeys = chunks.each_with_object({}) do |chunk, keys|
+            if chunk.metadata && chunk.metadata.timekey
+              t = chunk.metadata.timekey
+              keys[t] = keys.fetch(t, 0) + 1
+            end
+          end
+        end
+      end
+
       # At flush_at_shutdown, all staged chunks should be enqueued for buffer flush. Set true to force_enqueue for it.
       def enqueue_all(force_enqueue = false)
         log.on_trace { log.trace "enqueueing all chunks in buffer", instance: self.object_id }
+        update_timekeys if @enable_update_timekeys
 
         if block_given?
           synchronize{ @stage.keys }.each do |metadata|
@@ -553,10 +564,6 @@ module Fluent
           log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
         end
 
-        if metadata && metadata.timekey
-          del_timekey(metadata.timekey)
-        end
-
         nil
       end
 
@@ -788,11 +795,11 @@ module Fluent
           'total_queued_size' => stage_size + queue_size,
         }
 
-        if (m = timekeys.min)
+        tkeys = timekeys
+        if (m = tkeys.min)
           stats['oldest_timekey'] = m
         end
-
-        if (m = timekeys.max)
+        if (m = tkeys.max)
           stats['newest_timekey'] = m
         end
 
@@ -809,24 +816,6 @@ module Fluent
           !@queue.empty?
         end
       end
-
-      def add_timekey(t)
-        @mutex.synchronize do
-          @timekeys[t] += 1
-        end
-        nil
-      end
-
-      def del_timekey(t)
-        @mutex.synchronize do
-          if @timekeys[t] <= 1
-            @timekeys.delete(t)
-          else
-            @timekeys[t] -= 1
-          end
-        end
-        nil
-      end
     end
   end
 end

data/lib/fluent/plugin/buffer/chunk.rb

@@ -206,8 +206,9 @@ module Fluent
                 output_io = if chunk_io.is_a?(StringIO)
                               StringIO.new
                             else
-                              Tempfile.new('decompressed-data').binmode
+                              Tempfile.new('decompressed-data')
                             end
+                output_io.binmode if output_io.is_a?(Tempfile)
                 decompress(input_io: chunk_io, output_io: output_io)
                 output_io.seek(0, IO::SEEK_SET)
                 yield output_io

data/lib/fluent/plugin/in_exec.rb

@@ -25,6 +25,8 @@ module Fluent::Plugin
 
     desc 'The command (program) to execute.'
     config_param :command, :string
+    desc 'Specify connect mode to executed process'
+    config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
 
     config_section :parse do
       config_set_default :@type, 'tsv'
@@ -72,9 +74,9 @@ module Fluent::Plugin
       super
 
       if @run_interval
-        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [:read], &method(:run))
+        child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
       else
-        child_process_execute(:exec_input, @command, immediate: true, mode: [:read], &method(:run))
+        child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
       end
     end
 

data/lib/fluent/plugin/output.rb

@@ -340,6 +340,7 @@ module Fluent
           buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
           @buffer = Plugin.new_buffer(buffer_type, parent: self)
           @buffer.configure(buffer_conf)
+          @buffer.enable_update_timekeys if @chunk_key_time
 
           @flush_at_shutdown = @buffer_config.flush_at_shutdown
           if @flush_at_shutdown.nil?

data/lib/fluent/plugin/parser_json.rb

@@ -71,8 +71,11 @@ module Fluent
       end
 
       def parse(text)
-        r = @load_proc.call(text)
-        time, record = convert_values(parse_time(r), r)
+        record = @load_proc.call(text)
+        time = parse_time(record)
+        if @execute_convert_values
+          time, record = convert_values(time, record)
+        end
         yield time, record
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil

data/lib/fluent/supervisor.rb

@@ -302,6 +302,7 @@ module Fluent
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
       ignore_repeated_log_interval = params['ignore_repeated_log_interval']
+      ignore_same_log_interval = params['ignore_same_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -309,7 +310,8 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
+                  ignore_same_log_interval: ignore_same_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -347,6 +349,7 @@ module Fluent
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
+        ignore_same_log_interval: ignore_same_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -441,10 +444,11 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
         $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
+        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -511,7 +515,8 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
+                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -635,7 +640,8 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: @system_config.ignore_same_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose, :ignore_repeated_log_interval,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -35,6 +35,7 @@ module Fluent
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.2'
+  VERSION = '1.11.3'
 
 end

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_in_exec.rb

@@ -240,4 +240,22 @@ EOC
       assert_equal [tag, time, record], event
     }
   end
+
+  test 'emit error message with read_with_stderr' do
+    d = create_driver %[
+      tag test
+      command ruby #{File.join(File.dirname(SCRIPT_PATH), 'foo_bar_baz_no_existence.rb')}
+      connect_mode read_with_stderr
+      <parse>
+        @type none
+      </parse>
+    ]
+    d.run(expect_records: 1, timeout: 10)
+
+    assert{ d.events.length > 0 }
+    d.events.each do |event|
+      assert_equal 'test', event[0]
+      assert_match /LoadError/, event[2]['message']
+    end
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -1403,6 +1403,9 @@ class TailInputTest < Test::Unit::TestCase
     end
 
     def test_tail_path_with_multiline_with_multiple_paths
+      if ENV["APPVEYOR"] && Fluent.windows?
+        omit "This testcase is unstable on AppVeyor."
+      end
       files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
       files.each { |file| File.open(file, "wb") { |f| } }
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -278,6 +278,7 @@ class HttpHelperTest < Test::Unit::TestCase
             driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: PORT, logger: NULL_LOGGER) do |s|
               s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
             end
+            omit "TLS connection should be aborted due to `Errno::ECONNABORTED`. Need to debug." if Fluent.windows?
 
             resp = secure_get("https://127.0.0.1:#{PORT}/example/hello", verify: false)
             assert_equal('200', resp.code)
@@ -293,6 +294,8 @@ class HttpHelperTest < Test::Unit::TestCase
           'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem'],
           'without passphrase' => [nil, 'cert.pem', 'cert-key.pem'])
         test 'load self-signed cert/key pair, verified from clients using cert files' do |(passphrase, cert, private_key)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
           cert_path = File.join(CERT_DIR, cert)
           private_key_path = File.join(CERT_DIR, private_key)
           opt = { 'insecure' => 'false', 'private_key_path' => private_key_path, 'cert_path' => cert_path }
@@ -315,6 +318,8 @@ class HttpHelperTest < Test::Unit::TestCase
           'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem', 'ca-cert-pass.pem'],
           'without passphrase' => [nil, 'cert.pem', 'cert-key.pem', 'ca-cert.pem'])
         test 'load cert by private CA cert file, verified from clients using CA cert file' do |(passphrase, cert, cert_key, ca_cert)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
           cert_path = File.join(CERT_CA_DIR, cert)
           private_key_path = File.join(CERT_CA_DIR, cert_key)
 

data/test/test_log.rb

@@ -411,6 +411,50 @@ class LogTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "ignore_same_log_interval" do
+    teardown do
+      Thread.current[:last_same_log] = nil
+    end
+
+    def test_same_message
+      message = "This is test"
+      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
+      log = Fluent::Log.new(logger, {ignore_same_log_interval: 5})
+
+      log.error message
+      10.times { |i|
+        Timecop.freeze(@timestamp + i + 1)
+        log.error message
+      }
+
+      expected = [
+        "2016-04-21 02:58:41 +0000 [error]: This is test\n",
+        "2016-04-21 02:58:47 +0000 [error]: This is test\n"
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+
+    def test_different_message
+      message = "This is test"
+      logger = ServerEngine::DaemonLogger.new(@log_device, {log_level: ServerEngine::DaemonLogger::INFO})
+      log = Fluent::Log.new(logger, {ignore_same_log_interval: 10})
+
+      log.error message
+      3.times { |i|
+        Timecop.freeze(@timestamp + i)
+        log.error message
+        log.error message
+        log.info "Hello! " + message
+      }
+
+      expected = [
+        "2016-04-21 02:58:41 +0000 [error]: This is test\n",
+        "2016-04-21 02:58:41 +0000 [info]: Hello! This is test\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
   def test_dup
     dl_opts = {}
     dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.11.2
+  version: 1.11.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-04 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack