fluentd 1.13.1 → 1.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2974ccbbcf3e5273d3561322c9786ca7ad4cdda66817f13e4ed73962f4762b7b
-  data.tar.gz: 5bd0cd30a89e325fd1cbb3868a3f13f4685c00ca557cd38dedf04c478c6dc249
+  metadata.gz: 7d57a00fbe9b7fdbbac7ce75dfd9a4cf2a963b2cc4e5288dd37caf967fc5572f
+  data.tar.gz: b447f9876816029af978af3a27efd579a51794b991a79b2a57dfea3a65b0d234
 SHA512:
-  metadata.gz: 4fba66b7426b7818c7e60c6120ce46de4396ce903474b7e3d4e5db0db3f52ed5b324c0d54d312267abcc110ce5010108cac5ceaddb503a2234a973c9f99f879b
-  data.tar.gz: d6d3771be3d7850e4bf265836cc2663c81f97f0c602e2e95457fc2d30ea8c8c7b67e9340ef5db276253c5d15f5538f2712614b9c8b9bfaee4b1e302761ca2052
+  metadata.gz: ae67626874b6294ec88fc20ebd61fd34e6b809199227624eb703e9a7010d4269b87a4d10d8cd02b689e82b6be175200ff7b7d1649bfd9b1df7e864834a6a16fd
+  data.tar.gz: cc3694e775abe4aba0223592f602f68be04367c5a0eb50e86129457d604c0bf8f557b5936e44fb550fafe849556687f84e75eba5dc44f2e806652c5694eb5804

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,69 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/windows-test.yaml

@@ -18,7 +18,7 @@ jobs:
           - windows-latest
         experimental: [false]
         include:
-          - ruby-version: '3.0.1'
+          - ruby-version: '3.0.2'
             os: windows-latest
             experimental: true
             # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
@@ -28,7 +28,7 @@ jobs:
             # * https://github.com/ruby/fiddle/issues/72
             # * https://bugs.ruby-lang.org/issues/17813
             # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.1/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
+            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
 
     name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
@@ -38,7 +38,7 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby-version }}
       - name: Add Fiddle 1.0.8
-        if: ${{ matrix.ruby-version == '3.0.1' }}
+        if: ${{ matrix.ruby-version == '3.0.2' }}
         run: gem install fiddle --version 1.0.8
       - name: Install dependencies
         run: ridk exec bundle install

data/CHANGELOG.md

@@ -1,3 +1,108 @@
+# v1.14.0
+
+## Release v1.14.0 - 2021/08/30
+
+### Enhancement
+
+* Added `enable_input_metrics`, `enable_size_metrics` system
+  configuration parameter
+  This feature might need to pay higher CPU cost, so input event metrics
+  features are disabled by default. These features are also enabled by
+  `--enable-input-metrics`,`--enable-size-metrics` command line
+  option.
+  https://github.com/fluent/fluentd/pull/3440
+* Added reserved word `@ROOT` for getting root router.
+  This is incompatible change. Do not use `@ROOT` for label name.
+  https://github.com/fluent/fluentd/pull/3358
+* in_syslog: Added `send_keepalive_packet` option
+  https://github.com/fluent/fluentd/pull/3474
+* in_http: Added `cors_allow_credentials` option.
+  This option tells browsers whether to expose the response to
+  frontend when the credentials mode is "include".
+  https://github.com/fluent/fluentd/pull/3481
+  https://github.com/fluent/fluentd/pull/3491
+
+### Bug fix
+
+* in_tail: Fixed a bug that deleted paths are not removed
+  from pos file by file compaction at start up
+  https://github.com/fluent/fluentd/pull/3467
+* in_tail: Revived a warning message of retrying unaccessible file
+  https://github.com/fluent/fluentd/pull/3478
+* TLSServer: Fixed a crash bug on logging peer host name errors
+  https://github.com/fluent/fluentd/pull/3483
+
+### Misc
+
+* Added metrics plugin mechanism
+  The implementations is changed to use metrics plugin.
+  In the future, 3rd party plugin will be able to handle these metrics.
+  https://github.com/fluent/fluentd/pull/3471
+  https://github.com/fluent/fluentd/pull/3473
+  https://github.com/fluent/fluentd/pull/3479
+  https://github.com/fluent/fluentd/pull/3484
+
+# v1.13.3
+
+## Release v1.13.3 - 2021/07/27
+
+### Bug fix
+
+* in_tail: Care DeletePending state on Windows
+  https://github.com/fluent/fluentd/pull/3457
+  https://github.com/fluent/fluentd/pull/3460
+* in_tail: Fix some pos_file bugs.
+  Avoid deleting pos_file entries unexpectedly when both
+  `pos_file_compaction_interval` and `follow_inode` are enabled.
+  Use `bytesize` instead of `size` for path length.
+  https://github.com/fluent/fluentd/pull/3459
+* in_tail: Fix detecting rotation twice on `follow_inode`.
+  https://github.com/fluent/fluentd/pull/3466
+
+### Misc
+
+* Remove needless spaces in a sample config file
+  https://github.com/fluent/fluentd/pull/3456
+
+### Enhancement
+
+# v1.13.2
+
+## Release v1.13.2 - 2021/07/12
+
+### Enhancement
+
+* fluent-plugin-generate: Storage plugin was supported.
+  https://github.com/fluent/fluentd/pull/3426
+* parser_json: Added support to customize configuration of oj options.
+  Use `FLUENT_OJ_OPTION_BIGDECIMAL_LOAD`, `FLUENT_OJ_OPTION_MAX_NESTING`,
+  `FLUENT_OJ_OPTION_MODE`, and `FLUENT_OJ_OPTION_USE_TO_JSON` environment
+  variable to configure it.
+  https://github.com/fluent/fluentd/pull/3315
+
+### Bug fix
+
+* binlog_reader: Fixed a crash bug by missing "fluent/env" dependency.
+  https://github.com/fluent/fluentd/pull/3443
+* Fixed a crash bug on outputting log at the early stage when parsing
+  config file.  This is a regression since v1.13.0. If you use invalid
+  '@' prefix parameter, remove it as a workaround.
+  https://github.com/fluent/fluentd/pull/3451
+* in_tail: Fixed a bug that when rotation is occurred, remaining lines
+  will be discarded if the throttling feature is enabled.
+  https://github.com/fluent/fluentd/pull/3390
+* fluent-plugin-generate: Fixed a crash bug during gemspec generation.
+  It was unexpectedly introduced by #3305, thus this bug was a
+  regression since 1.12.3.
+  https://github.com/fluent/fluentd/pull/3444
+
+### Misc
+
+* Fixed the runtime dependency version of http_parse.rb to 0.7.0.
+  It was fixed because false positive detection is occurred frequently
+  by security scanning tools.
+  https://github.com/fluent/fluentd/pull/3450
+
 # v1.13.1
 
 ## Release v1.13.1 - 2021/06/25

data/README.md

@@ -25,9 +25,9 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 
 Use Case | Description | Diagram
 -------- | ------------|:---------:
-Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/assets/img/recipes/elasticsearch-s3-fluentd.png" height="150"/>
+Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
 Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
-Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/assets/img/datasources/asynchronous_logging.png" height="150"/>
+Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
 
 ## Quick Start
 

data/example/v0_12_filter.conf

@@ -52,10 +52,10 @@
 </match>
 
 # Composing two filters. For all events with the tag foo.bar,
-# 
+#
 # 1. The first filter filters out all events that has the field "hello"
 # 2. Then, for those events, we downcase the value of the "name" field.
-# 
+#
 # - {"name":"SADA", "hello":100} gets filtered out
 # - {"name":"SADA"} becomes {"name":"sada"}
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"

data/fluentd.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])

data/lib/fluent/command/fluentd.rb

@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
   opts[:strict_config_value] = b
 }
 
+op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
+  opts[:enable_input_metrics] = b
+}
+
+op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
+  opts[:enable_size_metrics] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/command/plugin_generator.rb

@@ -27,7 +27,7 @@ class FluentPluginGenerator
   attr_reader :type, :name
   attr_reader :license_name
 
-  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
 
   def initialize(argv = ARGV)
     @argv = argv
@@ -164,8 +164,13 @@ BANNER
   end
 
   def locked_gem_version(gem_name)
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name(gem_name).version.version
+    end
   end
 
   def rake_version
@@ -177,8 +182,13 @@ BANNER
   end
 
   def bundler_version
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.bundler_version.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.bundler_version.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name("bundler").version.version
+    end
   end
 
   def class_name

data/lib/fluent/compat/output.rb

@@ -310,7 +310,7 @@ module Fluent
       # original implementation of v0.12 BufferedOutput
       def emit(tag, es, chain, key="")
         # this method will not be used except for the case that plugin calls super
-        @emit_count += 1
+        @emit_count_metrics.inc
         data = format_stream(tag, es)
         if @buffer.emit(key, data, chain)
           submit_flush
@@ -337,14 +337,14 @@ module Fluent
       # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
       def handle_stream_simple(tag, es, enqueue: false)
         if @overrides_emit
-          current_emit_count = @emit_count
+          current_emit_count = @emit_count_metrics.get
           size = es.size
           key = data = nil
           begin
             emit(tag, es, NULL_OUTPUT_CHAIN)
             key, data = self.last_emit_via_buffer
           ensure
-            @emit_count = current_emit_count
+            @emit_count_metrics.set(current_emit_count)
             self.last_emit_via_buffer = nil
           end
           # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -363,7 +364,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -373,7 +375,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += size }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         [meta]
       end
 

data/lib/fluent/config/types.rb

@@ -20,6 +20,10 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
     def self.size_value(str, opts = {}, name = nil)
       return nil if str.nil?
 
@@ -104,6 +108,16 @@ module Fluent
       Config.string_value(val, opts, name)
     }
 
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
     def self.enum_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
@@ -176,6 +190,7 @@ module Fluent
         when :bool then Config.bool_value(value, opts, name)
         when :time then Config.time_value(value, opts, name)
         when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end

data/lib/fluent/config/v1_parser.rb

@@ -37,6 +37,7 @@ module Fluent
         super(strscan, eval_context)
         @include_basepath = include_basepath
         @fname = fname
+        @logger = defined?($log) ? $log : nil
       end
 
       def parse!
@@ -99,7 +100,7 @@ module Fluent
 
           elsif root_element && skip(/(\@include|include)#{SPACING}/)
             if !prev_match.start_with?('@')
-              $log.warn "'include' is deprecated. Use '@include' instead"
+              @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
             end
             parse_include(attrs, elems)
 
@@ -123,7 +124,7 @@ module Fluent
                     parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
                   else
                     # TODO: This is for backward compatibility. It will throw an error in the future.
-                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
+                    @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
                   end
                 end