fluentd 1.10.3 → 1.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a83d0f39bb55eb7343c7a19e126bd98ebf9ab907439ce95ef39546993eb9188
-  data.tar.gz: dac26a7c7a3db09b59ae5b96cf1e76f7dd7aa975a3e08474aa636f1950dfa4e8
+  metadata.gz: 9fac73611b9d1c9b470912f6de8294795e5e8809d1730fa9f692aae4777ffdc2
+  data.tar.gz: 88c9fe2ccf19fa1a1e82924072182ed7c0c688e70f25dcdff8245d386644faee
 SHA512:
-  metadata.gz: b39ee59f0649ce9f0558d335b8a0acbdfc371601bb1af9978cc97eb955fe4e3dfdb6ac7839583a146116479a61ff7d2afd3ce4eeccad5327273ea930aa9a73ff
-  data.tar.gz: ca26fc34d1327d39e97cbbaa3c782720971ae56273f9393656abbcc601584117b11d35c8db2d60e548ab6292e309cb27a056c3ad76fc644579f6ac5d4c2f46f0
+  metadata.gz: d8ee0bff2c2fb43bd8cbcf727de8cf31ab885ecd4e4eae299c2588559d665a4c309a8b575cedadd6534a51962934d6f643eda5baa6056759345814fb1ab084ee
+  data.tar.gz: 40c86faa9b747e65e32ffa004f5f1e8b5563fb81f5d686174da99866ad0c8789d91d465e78d5e4fff891359b05a71db4e204f2f91eed737c6cc5804999644ab7

data/CHANGELOG.md

@@ -1,5 +1,107 @@
+# v1.11
+
+## Release v1.11.3 - 2020/09/30
+
+### Enhancement
+
+* in_exec: Add `connect_mode` parameter to read stderr
+  https://github.com/fluent/fluentd/pull/3108
+* parser_json: Improve the performance
+  https://github.com/fluent/fluentd/pull/3109
+* log: Add `ignore_same_log_interval` parameter
+  https://github.com/fluent/fluentd/pull/3119
+* Upgrade win32 gems
+  https://github.com/fluent/fluentd/pull/3100
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3094
+  https://github.com/fluent/fluentd/pull/3118
+
+### Bug fix
+
+* buffer: Fix calculation of timekey stats
+  https://github.com/fluent/fluentd/pull/3018
+* buffer: fix binmode usage for prevent gc
+  https://github.com/fluent/fluentd/pull/3138
+
+## Release v1.11.2 - 2020/08/04
+
+### Enhancement
+
+* `in_dummy` renamed to `in_sample`
+  https://github.com/fluent/fluentd/pull/3065
+* Allow regular expression in filter/match directive
+  https://github.com/fluent/fluentd/pull/3071
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3051
+
+### Bug fix
+
+* buffer: Fix log message for `chunk_limit_records` case
+  https://github.com/fluent/fluentd/pull/3079
+* buffer: Fix timekey optimization for non-windows platform
+  https://github.com/fluent/fluentd/pull/3092
+* cert: Raise an error for broken certificate file
+  https://github.com/fluent/fluentd/pull/3086
+* cert: Set TLS ciphers list correcty on older OpenSSL
+  https://github.com/fluent/fluentd/pull/3093
+
+## Release v1.11.1 - 2020/06/22
+
+### Enhancement
+
+* in_http: Add `dump_error_log` parameter
+  https://github.com/fluent/fluentd/pull/3035
+* in_http: Improve time field handling
+  https://github.com/fluent/fluentd/pull/3046
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3047
+
+### Bug fix
+
+* in_tail: Use actual path instead of based pattern for ignore list
+  https://github.com/fluent/fluentd/pull/3042
+* child_process helper: Fix child process failure due to SIGPIPE if the command uses stdout
+  https://github.com/fluent/fluentd/pull/3044
+
+## Release v1.11.0 - 2020/06/04
+
+### New feature
+
+* in_unix: Use v1 API
+  https://github.com/fluent/fluentd/pull/2992
+
+### Enhancement
+
+* parser_syslog: Support any `time_format` for RFC3164 string parser
+  https://github.com/fluent/fluentd/pull/3014
+* parser_syslog: Add new parser for RFC5424
+  https://github.com/fluent/fluentd/pull/3015
+* Refactoring code
+  https://github.com/fluent/fluentd/pull/3019
+
+### Bug fix
+
+* in_gc_stat: Add `use_symbol_keys` parameter to emit string key record
+  https://github.com/fluent/fluentd/pull/3008
+
 # v1.10
 
+## Release v1.10.4 - 2020/05/12
+
+### Enhancement
+
+* out_http: Support single json array payload
+  https://github.com/fluent/fluentd/pull/2973
+* Refactoring
+  https://github.com/fluent/fluentd/pull/2988
+
+### Bug fix
+
+* supervisor: Call `File.umask(0)` for standalone worker
+  https://github.com/fluent/fluentd/pull/2987
+* out_forward: Fix ZeroDivisionError issue with `weight 0`
+  https://github.com/fluent/fluentd/pull/2989
+
 ## Release v1.10.3 - 2020/05/01
 
 ### Enhancement

data/CONTRIBUTING.md

@@ -24,7 +24,7 @@ If you find a bug of Fluentd or a mistake in the documentation, you can help us
 submitting an issue to Fluentd. Even better you can submit a Pull Request with a fix.
 
 * **Fluentd**: Use [fluentd](https://github.com/fluent/fluentd) repository. Fill issue template.
-* **Documentation**: Use [fluentd-docs](https://github.com/fluent/fluentd-docs-gitbook) repository.
+* **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
 
 If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
 And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.

data/example/copy_roundrobin.conf

@@ -1,12 +1,12 @@
 <source>
-  @type  dummy
+  @type  sample
   @label @test
   tag    test.copy
   auto_increment_key id
 </source>
 
 <source>
-  @type dummy
+  @type sample
   @label @test
   tag    test.rr
   auto_increment_key id
@@ -36,4 +36,4 @@
       output_type ltsv
     </store>
   </match>
-</label>
+</label>

data/example/counter.conf

@@ -8,7 +8,7 @@
 </system>
 
 <source>
-  @type dummy
+  @type sample
   tag "test.data"
   auto_increment_key number
 </source>

data/example/filter_stdout.conf

@@ -1,6 +1,6 @@
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
 </source>
 
 <filter **>

data/example/{in_dummy_blocks.conf → in_sample_blocks.conf}

@@ -1,11 +1,11 @@
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
   rate 100
-  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
 </source>
 
-<match dummy>
+<match sample>
   @type null
   never_flush true
   <buffer>

data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf}

@@ -1,16 +1,16 @@
 <source>
-  @type dummy
+  @type sample
   @label @main
   tag "test.data"
   size 2
   rate 10
-  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+  sample {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
   auto_increment_key number
 </source>
 
 <label @main>
   <match test.data>
-    @type buffered_stdout
+    @type stdout
     <buffer>
       @type file
       path "#{Dir.pwd}/compressed_buffers"

data/example/logevents.conf

@@ -1,10 +1,10 @@
 <source>
-  @type dummy
-  @label @dummylog
+  @type sample
+  @label @samplelog
   tag "data"
-  dummy {"message":"yay"}
+  sample {"message":"yay"}
 </source>
-<label @dummylog>
+<label @samplelog>
   <match **>
     @type stdout
   </match>
@@ -22,4 +22,4 @@
   #     hostname_key "host"
   #   </inject>
   # </match>
-</label> 
+</label>

data/example/multi_filters.conf

@@ -1,7 +1,7 @@
 # This example is to measure optimized filter pipeline performance.
 
 <source>
-  @type dummy
+  @type sample
   tag   test
   size 1000
 </source>

data/example/out_exec_filter.conf

@@ -1,10 +1,10 @@
 <source>
-  @type dummy
+  @type sample
   @label @exec
   tag exec_input
   rate 10
   auto_increment_key num
-  dummy {"data":"mydata"}
+  sample {"data":"mydata"}
 </source>
 
 <label @exec>

data/example/out_forward.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_buf_file.conf

@@ -1,5 +1,5 @@
 <source>
-    @type dummy
+    @type sample
     tag test
 </source>
 

data/example/out_forward_client.conf

@@ -1,21 +1,21 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test3
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test4
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test5
 </source>
 

data/example/out_forward_heartbeat_none.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_sd.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_shared_key.conf

@@ -1,9 +1,9 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 

data/example/out_forward_tls.conf

@@ -1,5 +1,5 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/example/out_forward_users.conf

@@ -1,13 +1,13 @@
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test2
 </source>
 <source>
-  @type dummy
+  @type sample
   tag test3
 </source>
 

data/example/out_null.conf

@@ -2,17 +2,17 @@
 # bundle exec bin/fluentd -c example/out_buffered_null.conf
 #   (+ --emit-error-log-interval 10)
 <source>
-  @type dummy
-  tag dummy
+  @type sample
+  tag sample
   rate 500000000
-  dummy [
+  sample [
     {"message": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
     {"message": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},
     {"message": "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}
   ]
 </source>
 
-<match dummy.**>
+<match sample.**>
   @type null
   <buffer>
     flush_interval 60s

data/example/secondary_file.conf

@@ -3,7 +3,7 @@
 </system>
 
 <source>
-  @type dummy
+  @type sample
   tag test
 </source>
 

data/fluentd.gemspec

@@ -32,11 +32,11 @@ Gem::Specification.new do |gem|
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
-    gem.add_runtime_dependency("win32-service", ["~> 0.8.3"])
-    gem.add_runtime_dependency("win32-ipc", ["~> 0.6.1"])
-    gem.add_runtime_dependency("win32-event", ["~> 0.6.1"])
-    gem.add_runtime_dependency("windows-pr", ["~> 1.2.5"])
-    gem.add_runtime_dependency("certstore_c", ["~> 0.1.2"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.1.5"])
+    gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
+    gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
+    gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
+    gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
   end
 
   gem.add_development_dependency("rake", ["~> 13.0"])

data/lib/fluent/command/fluentd.rb

@@ -185,6 +185,10 @@ if Fluent.windows?
     opts[:regwinsvcautostart] = s
   }
 
+  op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
+    opts[:regwinsvcdelaystart] = s
+  }
+
   op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
     opts[:fluentdopt] = s
   }
@@ -285,6 +289,13 @@ if winsvcinstmode = opts[:regwinsvc]
       dependencies: [""],
       display_name: opts[:winsvc_display_name]
     )
+
+    if opts[:regwinsvcdelaystart]
+      Service.configure(
+        service_name: opts[:winsvc_name],
+        delayed_start: true
+      )
+    end
   when 'u'
     if Service.status(opts[:winsvc_name]).current_state != 'stopped'
       begin

data/lib/fluent/log.rb

@@ -113,6 +113,7 @@ module Fluent
 
       @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       @ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
+      @ignore_same_log_interval = opts[:ignore_same_log_interval]
 
       @process_type = opts[:process_type] # :supervisor, :worker0, :workers Or :standalone
       @process_type ||= :standalone # to keep behavior of existing code
@@ -141,7 +142,8 @@ module Fluent
       dl_opts[:log_level] = @level - 1
       logger = ServerEngine::DaemonLogger.new(@out, dl_opts)
       clone = self.class.new(logger, suppress_repeated_stacktrace: @suppress_repeated_stacktrace, process_type: @process_type,
-                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval)
+                             worker_id: @worker_id, ignore_repeated_log_interval: @ignore_repeated_log_interval,
+                             ignore_same_log_interval: @ignore_same_log_interval)
       clone.format = @format
       clone.time_format = @time_format
       clone.log_event_enabled = @log_event_enabled
@@ -151,7 +153,7 @@ module Fluent
 
     attr_reader :format
     attr_reader :time_format
-    attr_accessor :log_event_enabled, :ignore_repeated_log_interval
+    attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
     attr_accessor :out
     attr_accessor :level
     attr_accessor :optional_header, :optional_attrs
@@ -428,6 +430,27 @@ module Fluent
       (cached_log.msg == message) && (time - cached_log.time <= @ignore_repeated_log_interval)
     end
 
+    def ignore_same_log?(time, message)
+      cached_log = Thread.current[:last_same_log]
+      if cached_log.nil?
+        Thread.current[:last_same_log] = {message => time}
+        return false
+      end
+
+      prev_time = cached_log[message]
+      if prev_time
+        if (time - prev_time) <= @ignore_same_log_interval
+          true
+        else
+          cached_log[message] = time
+          false
+        end
+      else
+        cached_log[message] = time
+        false
+      end
+    end
+
     def suppress_stacktrace?(backtrace)
       cached_log = Thread.current[:last_repeated_stacktrace]
       return false if cached_log.nil?
@@ -507,7 +530,11 @@ module Fluent
         end
       }
 
-      if @ignore_repeated_log_interval
+      if @ignore_same_log_interval
+        if ignore_same_log?(time, message)
+          return nil, nil
+        end
+      elsif @ignore_repeated_log_interval
         if ignore_repeated_log?(:last_repeated_log, time, message)
           return nil, nil
         else
@@ -569,6 +596,9 @@ module Fluent
       if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
         @ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
       end
+      if logger.instance_variable_defined?(:@ignore_same_log_interval)
+        @ignore_same_log_interval = logger.instance_variable_get(:@ignore_same_log_interval)
+      end
 
       self.format = @logger.format
       self.time_format = @logger.time_format