fluentd 1.13.3-x86-mingw32 → 1.14.3-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd9819bfeaa8fe8be243f6c4cf3af80a6ff93e47c7ff4c6bbe1523c5b8ff6c78
-  data.tar.gz: 5868dca822982fe4501beba7c2637f647334a8ff187b097aca2805be1c6eb16d
+  metadata.gz: 3ba01ca31fbaac62a1c0ea213d4f4dccd7c1a2a21be730a62f9b82007e8a67e3
+  data.tar.gz: 5b4dbb5aae91e85043e295f037fac15f877a084ee5c6289d09a0e8b4b7cf0c9c
 SHA512:
-  metadata.gz: e419985d7db6bfbeebdd9da123af3a109cc8fb502a6953fd09deb7d92289ed9dc1e6af6e26b74c2ec47e938d87b14d81627801a25a402bd571015e28987e27ca
-  data.tar.gz: 817231dbb280c9fd18fe09eb8fc2912d6de761be28cd5394b8b96b03435f5555903caea124d2d1edd98c0ba8a4c8460f85e9057127ef2fee1a8147d6e680b071
+  metadata.gz: ecf03095be6cc94747984eb828d607a9ddb70a43e88fa32b483ba38450c78ad13d309914d3cf8f5146a6d0d97e0fcc803178090217c861ad09c44e033f5a19c7
+  data.tar.gz: 902ef5dcd2289d72da71495877ec1b8337477f177b81b7c8002a2fa7beec11a2571957539044e16f2bc7c5a6a6be32dbd72e6867d2056a03ff41e216d5a2af22

data/.drone.yml

@@ -1,13 +1,13 @@
 kind: pipeline
-name: fluentd-test-arm64-2-6-3
+name: fluentd-test-arm64-3-0
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-2-6-3
-  image: arm64v8/ruby:2.6.3
+- name: fluentd-test-arm64-3-0
+  image: arm64v8/ruby:3.0
   commands:
   - apt update
   - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
   - bundle exec rake test
 ---
 kind: pipeline
-name: fluentd-test-arm64-latest
+name: fluentd-test-arm64-2-7
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-- name: fluentd-test-arm64-latest
-  image: arm64v8/ruby:latest
+- name: fluentd-test-arm64-2-7
+  image: arm64v8/ruby:2.7
   commands:
   - apt update
   - apt -y install libgmp3-dev

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -50,6 +50,7 @@ body:
       label: Your Configuration
       description: |
         Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+      render: apache
     validations:
       required: true
   - type: textarea

data/.github/workflows/windows-test.yaml

@@ -18,7 +18,7 @@ jobs:
           - windows-latest
         experimental: [false]
         include:
-          - ruby-version: '3.0.1'
+          - ruby-version: '3.0.2'
             os: windows-latest
             experimental: true
             # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
@@ -28,7 +28,7 @@ jobs:
             # * https://github.com/ruby/fiddle/issues/72
             # * https://bugs.ruby-lang.org/issues/17813
             # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.1/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
+            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
 
     name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
@@ -38,7 +38,7 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Add Fiddle 1.0.8
-        if: ${{ matrix.ruby-version == '3.0.1' }}
+        if: ${{ matrix.ruby-version == '3.0.2' }}
         run: gem install fiddle --version 1.0.8
       - name: Install dependencies
         run: ridk exec bundle install

data/CHANGELOG.md

@@ -1,3 +1,129 @@
+# v1.14.3
+
+## Release v1.14.3 - 2021/11/26
+
+### Enhancement
+
+* Changed to accept `http_parser.rb` 0.8.0.
+  `http_parser.rb` 0.8.0 is ready for Ractor.
+  https://github.com/fluent/fluentd/pull/3544
+
+### Bug fix
+
+* in_tail: Fixed a bug that no new logs are read when
+  `enable_stat_watcher true` and `enable_watch_timer false` is set.
+  https://github.com/fluent/fluentd/pull/3541
+* in_tail: Fixed a bug that the beginning and initial lines are lost
+  after startup when `read_from_head false` and path includes wildcard '*'.
+  https://github.com/fluent/fluentd/pull/3542
+* Fixed a bug that processing messages were lost when
+  BufferChunkOverflowError was thrown even though only a specific
+  message size exceeds chunk_limit_size.
+  https://github.com/fluent/fluentd/pull/3553
+  https://github.com/fluent/fluentd/pull/3562
+
+### Misc
+
+* Bump up required version of `win32-service` gem.
+  newer version is required to implement additional `fluent-ctl` commands.
+  https://github.com/fluent/fluentd/pull/3556
+
+# v1.14.2
+
+## Release v1.14.2 - 2021/10/29
+
+IMPORTANT: This release contain the fix for CVE-2021-41186 -
+ReDoS vulnerability in `parser_apache2`.
+This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
+We recommend to upgrade Fluentd to v1.14.2 or use patched version of
+`parser_apache2` plugin.
+
+### Enhancement
+
+* fluent-cat: Add `--event-time` option to send specified event time for testing.
+
+### Bug fix
+
+* Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
+  https://github.com/fluent/fluentd/pull/3524
+* Fixed ReDoS vulnerability in parser_apache2.
+  This vulnerability is caused by a certain pattern of a broken apache log.
+
+# v1.14.1
+
+## Release v1.14.1 - 2021/09/29
+
+### Enhancement
+
+* in_tail: Added file related metrics.
+  These metrics should be collected same as fluent-bit's in_tail.
+  https://github.com/fluent/fluentd/pull/3504
+* out_forward: Changed to use metrics mechanism for node statistics
+  https://github.com/fluent/fluentd/pull/3506
+
+### Bug fix
+
+* in_tail: Fixed a crash bug that it raise undefined method of eof? error.
+  This error may happen only when `read_bytes_limit_per_second` was specified.
+  https://github.com/fluent/fluentd/pull/3500
+* out_forward: Fixed a bug that node statistics information is not included correctly.
+  https://github.com/fluent/fluentd/pull/3503
+  https://github.com/fluent/fluentd/pull/3507
+* Fixed a error when using `@include` directive
+  It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
+  https://github.com/fluent/fluentd/pull/3517
+* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
+  It didn't work even if a user set it.
+  https://github.com/fluent/fluentd/pull/3515
+* Fixed not to output nanoseconds field of next retry time in warning log
+  Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
+  next_retry_seconds -> next_retry_time)
+  https://github.com/fluent/fluentd/pull/3518
+
+# v1.14.0
+
+## Release v1.14.0 - 2021/08/30
+
+### Enhancement
+
+* Added `enable_input_metrics`, `enable_size_metrics` system
+  configuration parameter
+  This feature might need to pay higher CPU cost, so input event metrics
+  features are disabled by default. These features are also enabled by
+  `--enable-input-metrics`,`--enable-size-metrics` command line
+  option.
+  https://github.com/fluent/fluentd/pull/3440
+* Added reserved word `@ROOT` for getting root router.
+  This is incompatible change. Do not use `@ROOT` for label name.
+  https://github.com/fluent/fluentd/pull/3358
+* in_syslog: Added `send_keepalive_packet` option
+  https://github.com/fluent/fluentd/pull/3474
+* in_http: Added `cors_allow_credentials` option.
+  This option tells browsers whether to expose the response to
+  frontend when the credentials mode is "include".
+  https://github.com/fluent/fluentd/pull/3481
+  https://github.com/fluent/fluentd/pull/3491
+
+### Bug fix
+
+* in_tail: Fixed a bug that deleted paths are not removed
+  from pos file by file compaction at start up
+  https://github.com/fluent/fluentd/pull/3467
+* in_tail: Revived a warning message of retrying unaccessible file
+  https://github.com/fluent/fluentd/pull/3478
+* TLSServer: Fixed a crash bug on logging peer host name errors
+  https://github.com/fluent/fluentd/pull/3483
+
+### Misc
+
+* Added metrics plugin mechanism
+  The implementations is changed to use metrics plugin.
+  In the future, 3rd party plugin will be able to handle these metrics.
+  https://github.com/fluent/fluentd/pull/3471
+  https://github.com/fluent/fluentd/pull/3473
+  https://github.com/fluent/fluentd/pull/3479
+  https://github.com/fluent/fluentd/pull/3484
+
 # v1.13.3
 
 ## Release v1.13.3 - 2021/07/27

data/README.md

@@ -88,6 +88,8 @@ You can run specified test via `TEST` environment variable:
 
 A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
 
+See [SECURITY](SECURITY.md) to contact us about vulnerability.
+
 ## Contributors:
 
 Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).

data/SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+## Supported Versions
+
+| Version   | Supported          |
+| -------   | ------------------ |
+| 1.14.x    | :white_check_mark: |
+| <= 1.13.x | :x:                |
+
+## Reporting a Vulnerability
+
+Please contact to current active maintainers. (in alphabetical order)
+
+* ashie@clear-code.com
+* fujimoto@clear-code.com
+* hatake@calyptia.com
+* hayashi@clear-code.com
+

data/fluentd.gemspec

@@ -23,11 +23,11 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
   gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
   gem.platform = fake_platform unless fake_platform.empty?
   if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
     gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
-    gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
+    gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
     gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
     gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
     gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])

data/lib/fluent/command/cat.rb

@@ -35,6 +35,7 @@ format = 'json'
 message_key = 'message'
 time_as_integer = false
 retry_limit = 5
+event_time = nil
 
 op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
   port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
   retry_limit = n
 }
 
+op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
+  event_time = v
+}
+
 singleton_class.module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -134,7 +139,7 @@ class Writer
     end
   end
 
-  def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
+  def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
     @tag = tag
     @connector = connector
     @socket = false
@@ -148,6 +153,7 @@ class Writer
     @retry_wait = 1
     @retry_limit = retry_limit
     @time_as_integer = time_as_integer
+    @event_time = event_time
 
     super()
   end
@@ -166,7 +172,11 @@ class Writer
       end
     end
 
-    time = Fluent::EventTime.now
+    time = if @event_time
+             Fluent::EventTime.parse(@event_time)
+           else
+             Fluent::EventTime.now
+           end
     time = time.to_i if @time_as_integer
     entry = if secondary_record?(record)
               # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
   }
 end
 
-w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
+w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
 w.start
 
 case format

data/lib/fluent/command/fluentd.rb

@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
   opts[:strict_config_value] = b
 }
 
+op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
+  opts[:enable_input_metrics] = b
+}
+
+op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
+  opts[:enable_size_metrics] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/compat/output.rb

@@ -310,7 +310,7 @@ module Fluent
       # original implementation of v0.12 BufferedOutput
       def emit(tag, es, chain, key="")
         # this method will not be used except for the case that plugin calls super
-        @emit_count += 1
+        @emit_count_metrics.inc
         data = format_stream(tag, es)
         if @buffer.emit(key, data, chain)
           submit_flush
@@ -337,14 +337,14 @@ module Fluent
       # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
       def handle_stream_simple(tag, es, enqueue: false)
         if @overrides_emit
-          current_emit_count = @emit_count
+          current_emit_count = @emit_count_metrics.get
           size = es.size
           key = data = nil
           begin
             emit(tag, es, NULL_OUTPUT_CHAIN)
             key, data = self.last_emit_via_buffer
           ensure
-            @emit_count = current_emit_count
+            @emit_count_metrics.set(current_emit_count)
             self.last_emit_via_buffer = nil
           end
           # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -363,7 +364,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -373,7 +375,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += size }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         [meta]
       end
 

data/lib/fluent/config/parser.rb

@@ -93,7 +93,7 @@ module Fluent
           basepath = '/'
           fname = path
           require 'open-uri'
-          open(uri) {|f|
+          URI.open(uri) {|f|
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
         end

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = open(uri) { |f| f.read }
+          data = URI.open(uri) { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/event_router.rb

@@ -47,6 +47,8 @@ module Fluent
       @match_cache = MatchCache.new
       @default_collector = default_collector
       @emit_error_handler = emit_error_handler
+      @metric_callbacks = {}
+      @caller_plugin_id = nil
     end
 
     attr_accessor :default_collector
@@ -83,6 +85,22 @@ module Fluent
       @match_rules << Rule.new(pattern, collector)
     end
 
+    def add_metric_callbacks(caller_plugin_id, callback)
+      @metric_callbacks[caller_plugin_id] = callback
+    end
+
+    def caller_plugin_id=(caller_plugin_id)
+      @caller_plugin_id = caller_plugin_id
+    end
+
+    def find_callback
+      if @caller_plugin_id
+        @metric_callbacks[@caller_plugin_id]
+      else
+        nil
+      end
+    end
+
     def emit(tag, time, record)
       unless record.nil?
         emit_stream(tag, OneEventStream.new(time, record))
@@ -95,6 +113,9 @@ module Fluent
 
     def emit_stream(tag, es)
       match(tag).emit_events(tag, es)
+      if callback = find_callback
+        callback.call(es)
+      end
     rescue => e
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
@@ -175,7 +196,11 @@ module Fluent
           if optimizable?
             optimized_filter_stream(tag, es)
           else
-            @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
+            @filters.reduce(es) { |acc, filter|
+              filtered_es = filter.filter_stream(tag, acc)
+              filter.measure_metrics(filtered_es)
+              filtered_es
+            }
           end
         end
 
@@ -193,6 +218,7 @@ module Fluent
                   begin
                     filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record && filtered_time
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end
@@ -200,6 +226,7 @@ module Fluent
                   begin
                     filtered_record = filter.filter(tag, filtered_time, filtered_record)
                     throw :break_loop unless filtered_record
+                    filter.measure_metrics(OneEventStream.new(time, record))
                   rescue => e
                     filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
                   end

data/lib/fluent/plugin/bare_output.rb

@@ -23,37 +23,78 @@ require 'fluent/plugin_helper'
 module Fluent
   module Plugin
     class BareOutput < Base
+      include PluginHelper::Mixin # for metrics
+
       # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
       # This output plugin base class is only for meta-output plugins
       # which cannot be implemented on MultiOutput.
       # E.g,: forest, config-expander
 
+      helpers_internal :metrics
+
       include PluginId
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      attr_reader :num_errors, :emit_count, :emit_records
-
       def process(tag, es)
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
       def initialize
         super
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+      end
+
+      def configure(conf)
+        super
+
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'bare_output' => stats }
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/buf_file.rb

@@ -39,8 +39,8 @@ module Fluent
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
 
-      config_param :file_permission, :string, default: nil # '0644'
-      config_param :dir_permission,  :string, default: nil # '0755'
+      config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
+      config_param :dir_permission,  :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
 
       def initialize
         super