fluentd 1.11.0 → 1.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 269cadf10ed4159e0c5279177cde46de14d563637b38ccadc71da090a098a22e
-  data.tar.gz: 9bf1dca5f46b1f3b245231c0e08b3b709add909840c3241bf29401952e02f27a
+  metadata.gz: 83bcc9640169d819af2a9336cc0bae773d4104b056467b6656115bba46565e02
+  data.tar.gz: a4e1c27ab6ba4454ae5aa8c2db25072503f10f21b158aecc686f2499e9416d48
 SHA512:
-  metadata.gz: 115e9f075e91136de66003ced2eda6076c1cff31c1e8edae5bb063f4c97420e4f832bcf6539a1ac4ee50bbe28d2050a6782840e88c951fa8fcbb52c66a889569
-  data.tar.gz: 25ebff981ebc41c4b6c0de4a6710350a07653be263019a138bb7be62d97dd6593bff67b7dd4d6ca4d37d0f84427b05d4c9b06f56cc9bc344fa46e0cfa82a7d68
+  metadata.gz: de9b224867b2b9dc5394889246e76b72449b1df6dfb8ddb982260ef0521d3a66d86d7c1cae319ddaa9c8cd00264be1c16539389c42da435c539534b537b567be
+  data.tar.gz: 4a077991b270c84aa8ab21e746ce75e06a5061ec70a2acd43fdac77127eb627d86241b18238f156a510decd56b8a311a54f51eaa723b69947cbd216c03accd9d

data/CHANGELOG.md

@@ -1,5 +1,96 @@
 # v1.11
 
+## Release v1.11.5 - 2020/11/06
+
+### Enhancement
+
+* formatter: Provide `newline` parameter to support `CRLF`
+  https://github.com/fluent/fluentd/pull/3152
+* out_http: adding support for intermediate certificates
+  https://github.com/fluent/fluentd/pull/3146
+
+### Bug fix
+
+* Fix a bug that windows service isn't stopped gracefuly
+  https://github.com/fluent/fluentd/pull/3156
+
+## Release v1.11.4 - 2020/10/13
+
+### Enhancement
+
+* inject: Support `unixtime_millis` in `time_type` parameter
+  https://github.com/fluent/fluentd/pull/3145
+
+### Bug fix
+
+* out_http: Fix broken data with `json_array true`
+  https://github.com/fluent/fluentd/pull/3144
+* output: Fix wrong logging issue for `${chunk_id}`
+  https://github.com/fluent/fluentd/pull/3134
+
+## Release v1.11.3 - 2020/09/30
+
+### Enhancement
+
+* in_exec: Add `connect_mode` parameter to read stderr
+  https://github.com/fluent/fluentd/pull/3108
+* parser_json: Improve the performance
+  https://github.com/fluent/fluentd/pull/3109
+* log: Add `ignore_same_log_interval` parameter
+  https://github.com/fluent/fluentd/pull/3119
+* Upgrade win32 gems
+  https://github.com/fluent/fluentd/pull/3100
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3094
+  https://github.com/fluent/fluentd/pull/3118
+
+### Bug fix
+
+* buffer: Fix calculation of timekey stats
+  https://github.com/fluent/fluentd/pull/3018
+* buffer: fix binmode usage for prevent gc
+  https://github.com/fluent/fluentd/pull/3138
+
+## Release v1.11.2 - 2020/08/04
+
+### Enhancement
+
+* `in_dummy` renamed to `in_sample`
+  https://github.com/fluent/fluentd/pull/3065
+* Allow regular expression in filter/match directive
+  https://github.com/fluent/fluentd/pull/3071
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3051
+
+### Bug fix
+
+* buffer: Fix log message for `chunk_limit_records` case
+  https://github.com/fluent/fluentd/pull/3079
+* buffer: Fix timekey optimization for non-windows platform
+  https://github.com/fluent/fluentd/pull/3092
+* cert: Raise an error for broken certificate file
+  https://github.com/fluent/fluentd/pull/3086
+* cert: Set TLS ciphers list correcty on older OpenSSL
+  https://github.com/fluent/fluentd/pull/3093
+
+## Release v1.11.1 - 2020/06/22
+
+### Enhancement
+
+* in_http: Add `dump_error_log` parameter
+  https://github.com/fluent/fluentd/pull/3035
+* in_http: Improve time field handling
+  https://github.com/fluent/fluentd/pull/3046
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3047
+
+### Bug fix
+
+* in_tail: Use actual path instead of based pattern for ignore list
+  https://github.com/fluent/fluentd/pull/3042
+* child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout
+  https://github.com/fluent/fluentd/pull/3044
+
 ## Release v1.11.0 - 2020/06/04
 
 ### New feature
@@ -10,7 +101,7 @@
 ### Enhancement
 
 * parser_syslog: Support any `time_format` for RFC3164 string parser
-  https://github.com/fluent/fluentd/pull/2988
+  https://github.com/fluent/fluentd/pull/3014
 * parser_syslog: Add new parser for RFC5424
   https://github.com/fluent/fluentd/pull/3015
 * Refactoring code

data/example/copy_roundrobin.conf

@@ -1,12 +1,12 @@
 <source>
-  @type  dummy
+  @type  sample
   @label @test
   tag    test.copy
   auto_increment_key id
 </source>
 
 <source>
-  @type dummy
+  @type sample
   @label @test
   tag    test.rr
   auto_increment_key id
@@ -36,4 +36,4 @@
       output_type ltsv
     </store>
   </match>
-</label>
+</label>

data/example/counter.conf

@@ -8,7 +8,7 @@
 </system>
 
 <source>
-  @type dummy
+  @type sample
   tag "test.data"
   auto_increment_key number
 </source>

data/example/filter_stdout.conf

@@ -1,6 +1,6 @@
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
 </source>
 
 <filter **>

data/example/{in_dummy_blocks.conf → in_sample_blocks.conf}

@@ -1,11 +1,11 @@
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
   rate 100
-  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
 </source>
 
-<match dummy>
+<match sample>
   @type null
   never_flush true
   <buffer>

data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf}

@@ -1,16 +1,16 @@
 <source>
-  @type dummy
+  @type sample
   @label @main
   tag "test.data"
   size 2
   rate 10
-  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
   auto_increment_key number
 </source>
 
 <label @main>
   <match test.data>
-    @type buffered_stdout
+    @type stdout
     <buffer>
       @type file
       path "#{Dir.pwd}/compressed_buffers"

data/example/logevents.conf

@@ -1,10 +1,10 @@
 <source>
-  @type dummy
-  @label @dummylog
+  @type sample
+  @label @samplelog
   tag "data"
-  dummy {"message":"yay"}
+  sample {"message":"yay"}
 </source>
-<label @dummylog>
+<label @samplelog>
   <match **>
     @type stdout
   </match>
@@ -22,4 +22,4 @@
   #     hostname_key "host"
   #   </inject>
   # </match>
-</label> 
+</label>

data/example/multi_filters.conf

@@ -1,7 +1,7 @@
 # This example is to measure optimized filter pipeline performance.
 
 <source>
-  @type dummy
+  @type sample
   tag   test
   size 1000
 </source>

data/example/out_exec_filter.conf

@@ -1,10 +1,10 @@
 <source>
-  @type dummy
+  @type sample
   @label @exec
   tag exec_input
   rate 10
   auto_increment_key num
-  dummy {"data":"mydata"}
+  sample {"data":"mydata"}
 </source>
 
 <label @exec>

data/example/out_forward.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_buf_file.conf

@@ -1,5 +1,5 @@
 <source>
-    @type dummy
+    @type sample
     tag test
 </source>
 

data/example/out_forward_client.conf

@@ -1,21 +1,21 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test3
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test4
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test5
 </source>
 

data/example/out_forward_heartbeat_none.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_sd.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_shared_key.conf

@@ -1,9 +1,9 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 

data/example/out_forward_tls.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_users.conf

@@ -1,13 +1,13 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test3
 </source>
 

data/example/out_null.conf

@@ -2,17 +2,17 @@
 # bundle exec bin/fluentd -c example/out_buffered_null.conf
 #   (+ --emit-error-log-interval 10)
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
   rate 500000000
-  dummy [
+  sample [
     {"message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
     {"message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
     {"message": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}
   ]
 </source>
 
-<match dummy.**>
+<match sample.**>
   @type null
   <buffer>
     flush_interval 60s

data/example/secondary_file.conf

@@ -3,7 +3,7 @@
 </system>
 
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/fluentd.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
-  gem.add_runtime_dependency("serverengine", [">= 2.0.4", "< 3.0.0"])
+  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -32,11 +32,11 @@ Gem::Specification.new do |gem|
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
-    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
-    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
-    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
-    gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
+    gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end
 
   gem.add_development_dependency("rake", ["~> 13.0"])

data/lib/fluent/command/fluentd.rb

@@ -185,6 +185,10 @@ if Fluent.windows?
     opts[:regwinsvcautostart] = s
   }
 
+  op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
+    opts[:regwinsvcdelaystart] = s
+  }
+
   op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
@@ -285,6 +289,13 @@ if winsvcinstmode = opts[:regwinsvc]
       dependencies: [""],
       display_name: opts[:winsvc_display_name]
     )
+
+    if opts[:regwinsvcdelaystart]
+      Service.configure(
+        service_name: opts[:winsvc_name],
+        delayed_start: true
+      )
+    end
   when 'u'
     if Service.status(opts[:winsvc_name]).current_state != 'stopped'
       begin

data/lib/fluent/log.rb

@@ -113,6 +113,7 @@ module Fluent
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
+      @ignore_same_log_interval = opts[:ignore_same_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -141,7 +142,8 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
-                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval,
+                             ignore_same_log_interval: @ignore_same_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -151,7 +153,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -428,6 +430,27 @@ module Fluent
       (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
     end
 
+    def ignore_same_log?(time, message)
+      cached_log = Thread.current[:last_same_log]
+      if cached_log.nil?
+        Thread.current[:last_same_log] = {message => time}
+        return false
+      end
+
+      prev_time = cached_log[message]
+      if prev_time
+        if (time - prev_time) <= @ignore_same_log_interval
+          true
+        else
+          cached_log[message] = time
+          false
+        end
+      else
+        cached_log[message] = time
+        false
+      end
+    end
+
     def suppress_stacktrace?(backtrace)
       cached_log = Thread.current[:last_repeated_stacktrace]
       return false if cached_log.nil?
@@ -507,7 +530,11 @@ module Fluent
         end
       }
 
-      if @ignore_repeated_log_interval
+      if @ignore_same_log_interval
+        if ignore_same_log?(time, message)
+          return nil, nil
+        end
+      elsif @ignore_repeated_log_interval
         if ignore_repeated_log?(:last_repeated_log, time, message)
           return nil, nil
         else
@@ -569,6 +596,9 @@ module Fluent
       if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
         @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
       end
+      if logger.instance_variable_defined?(:@ignore_same_log_interval)
+        @ignore_same_log_interval = logger.instance_variable_get(:@ignore_same_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format