fluentd 1.13.0 → 1.14.0.rc

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20ece94d6aedf52ed5f7bdefedfda93056ef92bbc44a7b7a27c4b4e4660dc5e0
-  data.tar.gz: 76166c17b6018360db1d192c2a9aa8094c1fea780bc74f715264035f0485d601
+  metadata.gz: eab8e56529b8472ac765f6186687659e56081d0d2b21264c05f1e4550965eff6
+  data.tar.gz: f64f9a36befc145b9ae862f6eead5588670f5c3252ca821608ce2bc6d8649ed7
 SHA512:
-  metadata.gz: 214cbbef726516095e961786462e33b355fd77ecdcd278f5ada3a6f8d02489620ddbf84b9a7a4ba1de27b925f5455dda3c2b90c024bfa573afda05cc453d8e3a
-  data.tar.gz: 75de8aad92959bcb746be2c225c627e3f6590e2ffba38a91d0d0c540c7752bb2ebacc0623132a8ee613775ce63530a8083070fd008bb54646ecc37e4e2f6ca31
+  metadata.gz: 0a3d652b47ce320495511a39c41d6af65825a7455e1a67c7f2c93d7a77c13737acce3c86d79de0c0a95c6a053ba3e4bf3e0dfe19767327f4ea45f489321ef8a6
+  data.tar.gz: 0421fa9b68bf1317ff8e44f33908f7603f4298214daa4a0438e36586eca9829b2701144e2148a4a0ccbfeaaf87dd542be4227918eeab523dd3fdbcca0a08d839

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,69 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/linux-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['3.0', '2.7', '2.6', '2.5']
+        ruby-version: ['3.0', '2.7', '2.6']
         os: [ubuntu-latest]
         experimental: [false]
         include:

data/.github/workflows/windows-test.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7', '2.6', '2.5']
+        ruby-version: ['2.7', '2.6']
         os:
           - windows-latest
         experimental: [false]

data/.gitlab-ci.yml

@@ -13,16 +13,6 @@ stages:
   - build
   - tests
 
-2-5-8:
-  image: "ruby:2.5.8"
-  stage: build
-  extends: .install-template
-  script:
-    - bundle install --jobs=3 --retry=3
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-5-8"
-    paths:
-
 2-6-6:
   image: "ruby:2.6.6"
   stage: build
@@ -64,18 +54,6 @@ rubyhead:
     paths:
       - ./*
 
-2-5-8-test:
-  image: "ruby:2.5.8"
-  stage: tests
-  allow_failure: true
-  extends: .test-template
-  script:
-    - bundle exec rake test
-  cache:
-    key: "$CI_COMMIT_REF_SLUG 2-5-8"
-    paths:
-      - ./*
-
 2-6-6-test:
   image: "ruby:2.6.6"
   stage: tests

data/CHANGELOG.md

@@ -1,3 +1,90 @@
+# v1.13.3
+
+## Release v1.13.3 - 2021/07/27
+
+### Bug fix
+
+* in_tail: Care DeletePending state on Windows
+  https://github.com/fluent/fluentd/pull/3457
+  https://github.com/fluent/fluentd/pull/3460
+* in_tail: Fix some pos_file bugs.
+  Avoid deleting pos_file entries unexpectedly when both
+  `pos_file_compaction_interval` and `follow_inode` are enabled.
+  Use `bytesize` instead of `size` for path length.
+  https://github.com/fluent/fluentd/pull/3459
+* in_tail: Fix detecting rotation twice on `follow_inode`.
+  https://github.com/fluent/fluentd/pull/3466
+
+### Misc
+
+* Remove needless spaces in a sample config file
+  https://github.com/fluent/fluentd/pull/3456
+
+### Enhancement
+
+# v1.13.2
+
+## Release v1.13.2 - 2021/07/12
+
+### Enhancement
+
+* fluent-plugin-generate: Storage plugin was supported.
+  https://github.com/fluent/fluentd/pull/3426
+* parser_json: Added support to customize configuration of oj options.
+  Use `FLUENT_OJ_OPTION_BIGDECIMAL_LOAD`, `FLUENT_OJ_OPTION_MAX_NESTING`,
+  `FLUENT_OJ_OPTION_MODE`, and `FLUENT_OJ_OPTION_USE_TO_JSON` environment
+  variable to configure it.
+  https://github.com/fluent/fluentd/pull/3315
+
+### Bug fix
+
+* binlog_reader: Fixed a crash bug by missing "fluent/env" dependency.
+  https://github.com/fluent/fluentd/pull/3443
+* Fixed a crash bug on outputting log at the early stage when parsing
+  config file.  This is a regression since v1.13.0. If you use invalid
+  '@' prefix parameter, remove it as a workaround.
+  https://github.com/fluent/fluentd/pull/3451
+* in_tail: Fixed a bug that when rotation is occurred, remaining lines
+  will be discarded if the throttling feature is enabled.
+  https://github.com/fluent/fluentd/pull/3390
+* fluent-plugin-generate: Fixed a crash bug during gemspec generation.
+  It was unexpectedly introduced by #3305, thus this bug was a
+  regression since 1.12.3.
+  https://github.com/fluent/fluentd/pull/3444
+
+### Misc
+
+* Fixed the runtime dependency version of http_parse.rb to 0.7.0.
+  It was fixed because false positive detection is occurred frequently
+  by security scanning tools.
+  https://github.com/fluent/fluentd/pull/3450
+
+# v1.13.1
+
+## Release v1.13.1 - 2021/06/25
+
+### Bug fix
+
+* out_forward: Fixed a race condition on handshake
+  It's caused by using a same unpacker from multiple threads.
+  https://github.com/fluent/fluentd/pull/3405
+  https://github.com/fluent/fluentd/pull/3406
+* in_tail: Fixed to remove too much verbose debugging logs
+  It was unexpectedly introduced by #3185 log throttling feature.
+  https://github.com/fluent/fluentd/pull/3418
+* Fixed not to echo back the provides path as is on a 404 error
+  There was a potential cross-site scripting vector even though
+  it is quite difficult to exploit.
+  https://github.com/fluent/fluentd/pull/3427
+
+### Misc
+
+* Pretty print for Fluent::Config::Section has been supported
+  for debugging
+  https://github.com/fluent/fluentd/pull/3398
+* CI: Dropped to run CI for Ruby 2.5
+  https://github.com/fluent/fluentd/pull/3412
+
 # v1.13
 
 ## Release v1.13.0 - 2021/05/29

data/README.md

@@ -25,9 +25,9 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
 
 Use Case | Description | Diagram
 -------- | ------------|:---------:
-Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/assets/img/recipes/elasticsearch-s3-fluentd.png" height="150"/>
+Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
 Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
-Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/assets/img/datasources/asynchronous_logging.png" height="150"/>
+Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
 
 ## Quick Start
 

data/example/v0_12_filter.conf

@@ -52,10 +52,10 @@
 </match>
 
 # Composing two filters. For all events with the tag foo.bar,
-# 
+#
 # 1. The first filter filters out all events that has the field "hello"
 # 2. Then, for those events, we downcase the value of the "name" field.
-# 
+#
 # - {"name":"SADA", "hello":100} gets filtered out
 # - {"name":"SADA"} becomes {"name":"sada"}
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"

data/fluentd.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
-  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
+  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])

data/lib/fluent/command/fluentd.rb

@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
   opts[:strict_config_value] = b
 }
 
+op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
+  opts[:enable_input_metrics] = b
+}
+
+op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
+  opts[:enable_size_metrics] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/command/plugin_generator.rb

@@ -27,7 +27,7 @@ class FluentPluginGenerator
   attr_reader :type, :name
   attr_reader :license_name
 
-  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
 
   def initialize(argv = ARGV)
     @argv = argv
@@ -164,8 +164,13 @@ BANNER
   end
 
   def locked_gem_version(gem_name)
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name(gem_name).version.version
+    end
   end
 
   def rake_version
@@ -177,8 +182,13 @@ BANNER
   end
 
   def bundler_version
-    d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
-    d.locked_gems.bundler_version.version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.bundler_version.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name("bundler").version.version
+    end
   end
 
   def class_name

data/lib/fluent/compat/output.rb

@@ -310,7 +310,7 @@ module Fluent
       # original implementation of v0.12 BufferedOutput
       def emit(tag, es, chain, key="")
         # this method will not be used except for the case that plugin calls super
-        @emit_count += 1
+        @emit_count_metrics.inc
         data = format_stream(tag, es)
         if @buffer.emit(key, data, chain)
           submit_flush
@@ -337,14 +337,14 @@ module Fluent
       # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
       def handle_stream_simple(tag, es, enqueue: false)
         if @overrides_emit
-          current_emit_count = @emit_count
+          current_emit_count = @emit_count_metrics.get
           size = es.size
           key = data = nil
           begin
             emit(tag, es, NULL_OUTPUT_CHAIN)
             key, data = self.last_emit_via_buffer
           ensure
-            @emit_count = current_emit_count
+            @emit_count_metrics.set(current_emit_count)
             self.last_emit_via_buffer = nil
           end
           # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -363,7 +364,8 @@ module Fluent
           write_guard do
             @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
           end
-          @counter_mutex.synchronize{ @emit_records += size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
           return [meta]
         end
 
@@ -373,7 +375,8 @@ module Fluent
         write_guard do
           @buffer.write({meta => data}, enqueue: enqueue)
         end
-        @counter_mutex.synchronize{ @emit_records += size }
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         [meta]
       end
 

data/lib/fluent/config.rb

@@ -62,7 +62,7 @@ module Fluent
         Parser.parse(str, fname, basepath)
       when :ruby
         require 'fluent/config/dsl'
-        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file") if $log
         Config::DSL::Parser.parse(str, File.join(basepath, fname))
       else
         raise "[BUG] unknown configuration parser specification:'#{parser}'"

data/lib/fluent/config/section.rb

@@ -50,6 +50,11 @@ module Fluent
         "<Fluent::Config::Section #{@params.to_json}>"
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       def nil?
         false
       end

data/lib/fluent/config/types.rb

@@ -20,6 +20,10 @@ require 'fluent/config/error'
 
 module Fluent
   module Config
+    def self.reformatted_value(type, val, opts = {}, name = nil)
+      REFORMAT_VALUE.call(type, val, opts, name)
+    end
+
     def self.size_value(str, opts = {}, name = nil)
       return nil if str.nil?
 
@@ -104,6 +108,16 @@ module Fluent
       Config.string_value(val, opts, name)
     }
 
+    def self.symbol_value(val, opts = {}, name = nil)
+      return nil if val.nil? || val.empty?
+
+      val.delete_prefix(":").to_sym
+    end
+
+    SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
+      Config.symbol_value(val, opts, name)
+    }
+
     def self.enum_value(val, opts = {}, name = nil)
       return nil if val.nil?
 
@@ -176,6 +190,7 @@ module Fluent
         when :bool then Config.bool_value(value, opts, name)
         when :time then Config.time_value(value, opts, name)
         when :regexp then Config.regexp_value(value, opts, name)
+        when :symbol then Config.symbol_value(value, opts, name)
         else
           raise "unknown type in REFORMAT: #{type}"
         end