fluentd 1.13.3 → 1.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1603571f755a962fde001cbb33fd24ad8929633bdeb38832e61f45da2c0dfba5
-  data.tar.gz: 5c2d45299617e9e4424758c76eb53d78889387cedc28d1d8bac549402094b6af
+  metadata.gz: 9607315f05f94d8852e40219924b394e3744614f12b4109ea2c2c862a9c40e04
+  data.tar.gz: 12b6b47e9e1f7e43e112b6db097024dde27d692434550ea9807501010b2f51fc
 SHA512:
-  metadata.gz: 8dd4cc4947766345b5455fbae8043610d1c3d33e0e2ea9ea348a866722c099545301236d507d754e31b063017bb7daefdc8aef0e2a8078b67df83eb3c2489b2c
-  data.tar.gz: f0d126ed7704b519c2b7db5693de9685db7fd3ed1844f770a08e1e3351f3daf0f66e4b88c9435b3e456f40e38480c9af7652241f029a36b7a13184364a595251
+  metadata.gz: 0110ef8bb482a0b7d4193cb96712f8011dee152637ccf8140077861074db25d2efeaf784902475ca1b65acbca34997edfbf074bd3abcdb2364fa840055503d4f
+  data.tar.gz: 760b7f1254b51a072dcc6b44069b6857f365da60e61b210c985b1271858b828600e4eaa7ca8257b76ea8a9d3108953ddf6d80da37fe84e23d2f6ef0e17571b4f

data/.drone.yml

@@ -1,13 +1,13 @@
 kind: pipeline
-name: fluentd-test-arm64-2-6-3
+name: fluentd-test-arm64-3-0
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-2-6-3
-  image: arm64v8/ruby:2.6.3
+- name: fluentd-test-arm64-3-0
+  image: arm64v8/ruby:3.0
   commands:
   - apt update
   - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
   - bundle exec rake test
 ---
 kind: pipeline
-name: fluentd-test-arm64-latest
+name: fluentd-test-arm64-2-7
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-latest
-  image: arm64v8/ruby:latest
+- name: fluentd-test-arm64-2-7
+  image: arm64v8/ruby:2.7
   commands:
   - apt update
   - apt -y install libgmp3-dev

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -50,6 +50,7 @@ body:
       label: Your Configuration
       description: |
         Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
     validations:
       required: true
   - type: textarea

data/.github/workflows/windows-test.yaml

@@ -18,7 +18,7 @@ jobs:
           - windows-latest
         experimental: [false]
         include:
-          - ruby-version: '3.0.1'
+          - ruby-version: '3.0.2'
             os: windows-latest
             experimental: true
             # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
@@ -28,7 +28,7 @@ jobs:
             # * https://github.com/ruby/fiddle/issues/72
             # * https://bugs.ruby-lang.org/issues/17813
             # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.1/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
+            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
 
     name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
@@ -38,7 +38,7 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Add Fiddle 1.0.8
-        if: ${{ matrix.ruby-version == '3.0.1' }}
+        if: ${{ matrix.ruby-version == '3.0.2' }}
         run: gem install fiddle --version 1.0.8
       - name: Install dependencies
         run: ridk exec bundle install

data/CHANGELOG.md

@@ -1,3 +1,99 @@
+# v1.14.2
+
+## Release v1.14.2 - 2021/10/29
+
+IMPORTANT: This release contain the fix for CVE-2021-41186 -
+ReDoS vulnerability in `parser_apache2`.
+This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
+We recommend to upgrade Fluentd to v1.14.2 or use patched version of
+`parser_apache2` plugin.
+
+### Enhancement
+
+* fluent-cat: Add `--event-time` option to send specified event time for testing.
+
+### Bug fix
+
+* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
+  https://github.com/fluent/fluentd/pull/3524
+* Fixed ReDoS vulnerability in parser_apache2.
+  This vulnerability is caused by a certain pattern of a broken apache log.
+
+# v1.14.1
+
+## Release v1.14.1 - 2021/09/29
+
+### Enhancement
+
+* in_tail: Added file related metrics.
+  These metrics should be collected same as fluent-bit's in_tail.
+  https://github.com/fluent/fluentd/pull/3504
+* out_forward: Changed to use metrics mechanism for node statistics
+  https://github.com/fluent/fluentd/pull/3506
+
+### Bug fix
+
+* in_tail: Fixed a crash bug that it raise undefined method of eof? error.
+  This error may happen only when `read_bytes_limit_per_second` was specified.
+  https://github.com/fluent/fluentd/pull/3500
+* out_forward: Fixed a bug that node statistics information is not included correctly.
+  https://github.com/fluent/fluentd/pull/3503
+  https://github.com/fluent/fluentd/pull/3507
+* Fixed a error when using `@include` directive
+  It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
+  https://github.com/fluent/fluentd/pull/3517
+* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
+  It didn't work even if a user set it.
+  https://github.com/fluent/fluentd/pull/3515
+* Fixed not to output nanoseconds field of next retry time in warning log
+  Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
+  next_retry_seconds -> next_retry_time)
+  https://github.com/fluent/fluentd/pull/3518
+
+# v1.14.0
+
+## Release v1.14.0 - 2021/08/30
+
+### Enhancement
+
+* Added `enable_input_metrics`, `enable_size_metrics` system
+  configuration parameter
+  This feature might need to pay higher CPU cost, so input event metrics
+  features are disabled by default. These features are also enabled by
+  `--enable-input-metrics`,`--enable-size-metrics` command line
+  option.
+  https://github.com/fluent/fluentd/pull/3440
+* Added reserved word `@ROOT` for getting root router.
+  This is incompatible change. Do not use `@ROOT` for label name.
+  https://github.com/fluent/fluentd/pull/3358
+* in_syslog: Added `send_keepalive_packet` option
+  https://github.com/fluent/fluentd/pull/3474
+* in_http: Added `cors_allow_credentials` option.
+  This option tells browsers whether to expose the response to
+  frontend when the credentials mode is "include".
+  https://github.com/fluent/fluentd/pull/3481
+  https://github.com/fluent/fluentd/pull/3491
+
+### Bug fix
+
+* in_tail: Fixed a bug that deleted paths are not removed
+  from pos file by file compaction at start up
+  https://github.com/fluent/fluentd/pull/3467
+* in_tail: Revived a warning message of retrying unaccessible file
+  https://github.com/fluent/fluentd/pull/3478
+* TLSServer: Fixed a crash bug on logging peer host name errors
+  https://github.com/fluent/fluentd/pull/3483
+
+### Misc
+
+* Added metrics plugin mechanism
+  The implementations is changed to use metrics plugin.
+  In the future, 3rd party plugin will be able to handle these metrics.
+  https://github.com/fluent/fluentd/pull/3471
+  https://github.com/fluent/fluentd/pull/3473
+  https://github.com/fluent/fluentd/pull/3479
+  https://github.com/fluent/fluentd/pull/3484
+
 # v1.13.3
 
 ## Release v1.13.3 - 2021/07/27

data/fluentd.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile

data/lib/fluent/command/cat.rb

@@ -35,6 +35,7 @@ format = 'json'
 message_key = 'message'
 time_as_integer = false
 retry_limit = 5
+event_time = nil
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
   retry_limit = n
 }
 
+op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
+  event_time = v
+}
+
 singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -134,7 +139,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
     @tag = tag
     @connector = connector
     @socket = false
@@ -148,6 +153,7 @@ class Writer
     @retry_wait = 1
     @retry_limit = retry_limit
     @time_as_integer = time_as_integer
+    @event_time = event_time
 
     super()
   end
@@ -166,7 +172,11 @@ class Writer
       end
     end
 
-    time = Fluent::EventTime.now
+    time = if @event_time
+             Fluent::EventTime.parse(@event_time)
+           else
+             Fluent::EventTime.now
+           end
     time = time.to_i if @time_as_integer
     entry = if secondary_record?(record)
               # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
 w.start
 
 case format

data/lib/fluent/command/fluentd.rb

@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
   opts[:strict_config_value] = b
 }
 
+op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
+  opts[:enable_input_metrics] = b
+}
+
+op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
+  opts[:enable_size_metrics] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/compat/output.rb

@@ -310,7 +310,7 @@ module Fluent
       # original implementation of v0.12 BufferedOutput
       def emit(tag, es, chain, key="")
         # this method will not be used except for the case that plugin calls super
-        @emit_count += 1
+        @emit_count_metrics.inc
         data = format_stream(tag, es)
         if @buffer.emit(key, data, chain)
           submit_flush
@@ -337,14 +337,14 @@ module Fluent
       # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
       def handle_stream_simple(tag, es, enqueue: false)
         if @overrides_emit
-          current_emit_count = @emit_count
+          current_emit_count = @emit_count_metrics.get
           size = es.size
           key = data = nil
           begin
             emit(tag, es, NULL_OUTPUT_CHAIN)
             key, data = self.last_emit_via_buffer
           ensure
-            @emit_count = current_emit_count
+            @emit_count_metrics.set(current_emit_count)
             self.last_emit_via_buffer = nil
           end
           # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -363,7 +364,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -373,7 +375,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += size }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         [meta]
       end
 

data/lib/fluent/config/parser.rb

@@ -93,7 +93,7 @@ module Fluent
           basepath = '/'
           fname = path
           require 'open-uri'
-          open(uri) {|f|
+          URI.open(uri) {|f|
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
         end

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = open(uri) { |f| f.read }
+          data = URI.open(uri) { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/event_router.rb

@@ -47,6 +47,8 @@ module Fluent
       @match_cache = MatchCache.new
       @default_collector = default_collector
       @emit_error_handler = emit_error_handler
+      @metric_callbacks = {}
+      @caller_plugin_id = nil
     end
 
     attr_accessor :default_collector
@@ -83,6 +85,22 @@ module Fluent
       @match_rules << Rule.new(pattern, collector)
     end
 
+    def add_metric_callbacks(caller_plugin_id, callback)
+      @metric_callbacks[caller_plugin_id] = callback
+    end
+
+    def caller_plugin_id=(caller_plugin_id)
+      @caller_plugin_id = caller_plugin_id
+    end
+
+    def find_callback
+      if @caller_plugin_id
+        @metric_callbacks[@caller_plugin_id]
+      else
+        nil
+      end
+    end
+
     def emit(tag, time, record)
       unless record.nil?
         emit_stream(tag, OneEventStream.new(time, record))
@@ -95,6 +113,9 @@ module Fluent
 
     def emit_stream(tag, es)
       match(tag).emit_events(tag, es)
+      if callback = find_callback
+        callback.call(es)
+      end
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -175,7 +196,11 @@ module Fluent
           if optimizable?
             optimized_filter_stream(tag, es)
           else
-            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+            @filters.reduce(es) { |acc, filter|
+              filtered_es = filter.filter_stream(tag, acc)
+              filter.measure_metrics(filtered_es)
+              filtered_es
+            }
           end
         end
 
@@ -193,6 +218,7 @@ module Fluent
                   begin
                     filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record && filtered_time
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end
@@ -200,6 +226,7 @@ module Fluent
                   begin
                     filtered_record = filter.filter(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end

data/lib/fluent/plugin/bare_output.rb

@@ -23,37 +23,78 @@ require 'fluent/plugin_helper'
 module Fluent
   module Plugin
     class BareOutput < Base
+      include PluginHelper::Mixin # for metrics
+
       # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
       # This output plugin base class is only for meta-output plugins
       # which cannot be implemented on MultiOutput.
       # E.g,: forest, config-expander
 
+      helpers_internal :metrics
+
       include PluginId
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      attr_reader :num_errors, :emit_count, :emit_records
-
       def process(tag, es)
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'bare_output' => stats }
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/buf_file.rb

@@ -39,8 +39,8 @@ module Fluent
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-      config_param :file_permission, :string, default: nil # '0644'
-      config_param :dir_permission,  :string, default: nil # '0755'
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
 
       def initialize
         super