fluentd 1.12.1 → 1.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04be4b818522c222348ba309bccd34f03d0aa79bfc2fbd21f2ce738677d7a5e5
-  data.tar.gz: '078ce4c3003b74be7356367ff81d96d910a5df322ebe2319d96120cb1e2898b3'
+  metadata.gz: 9be763d670c8f331be578f85392cc428502a275a370e3a2b6520a08f22861da8
+  data.tar.gz: ffd8aaf0703bad823a2848b1f939d317bbe903ed6ae48196f4a84fc737db73fc
 SHA512:
-  metadata.gz: 77298246f6064ddf342765c37efb269236025566b33828cac9c88d69b6dabbbd2881c28ddae699d4620817a44a77310e15c4ac191a0ad51b6aeab51eb7442961
-  data.tar.gz: 318668fa9fde451292fd3b4284fc513124e6aa29218388a8a7cdfaaeb3943b3d3c4ffa45041cb0b2487da6573a9818f225c81b8769b7c025342bc5a6b5b05b53
+  metadata.gz: 7fad389c0066167a21d41c654c2ec74501b21e2dcf35d120e34fff7084b641a9468dcefcf75679ba8e5de5391e4bd58bbd60cbb310d31e3e01119a80408936a5
+  data.tar.gz: 8fbd2fbe2d88cb9c9cb35eabfea19706361225381775e08e0b5ee5adc8228b2e65f305d34410aa76d6cbbf773c8b3dd4a123da41f81e36fb294ef948b25ff98f

data/.deepsource.toml

@@ -0,0 +1,13 @@
+version = 1
+
+test_patterns = ["test/**/test_*.rb"]
+
+exclude_patterns = [
+    "bin/**",
+    "docs/**",
+    "example/**"
+]
+
+[[analyzers]]
+name = "ruby"
+enabled = true

data/.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Ask a Question
-    url: https://groups.google.com/forum/#!forum/fluentd
-    about: I have questions about Fluentd and plugins. Please ask and answer questions here
+    url: https://discuss.fluentd.org/
+    about: I have questions about Fluentd and plugins. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/workflows/linux-test.yaml

@@ -0,0 +1,36 @@
+name: Testing on Ubuntu
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['3.0', '2.7', '2.6', '2.5']
+        os: [ubuntu-latest]
+        experimental: [false]
+        include:
+          - ruby-version: head
+            os: ubuntu-latest
+            experimental: true
+
+    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install addons
+        run: sudo apt-get install libgmp3-dev libcap-ng-dev
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test

data/.github/workflows/macos-test.yaml

@@ -0,0 +1,30 @@
+name: Testing on macOS
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['head', '2.7']
+        os: [macos-latest]
+        experimental: [true]
+
+    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test

data/.github/workflows/{build.yaml → windows-test.yaml}

@@ -1,4 +1,4 @@
-name: build
+name: Testing on Windows
 
 on:
   push:
@@ -8,22 +8,23 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
 
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: [2.7, 2.6, 2.5]
+        ruby-version: ['3.0', '2.7', '2.6', '2.5']
+        os:
+          - windows-latest
 
+    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}
-      - name: Install addons
-        run: sudo apt-get install libgmp3-dev libcap-ng-dev
       - name: Install dependencies
-        run: bundle install
+        run: ridk exec bundle install
       - name: Run tests
         run: bundle exec rake test TESTOPTS=-v

data/CHANGELOG.md

@@ -1,5 +1,52 @@
 # v1.12
 
+## Release v1.12.2 - 2021/03/29
+
+### Enhancement
+
+* out_copy: Add ignore_if_prev_successes
+  https://github.com/fluent/fluentd/pull/3190
+  https://github.com/fluent/fluentd/pull/3287
+* Support multiple kind of timestamp format
+  https://github.com/fluent/fluentd/pull/3252
+* formatter_ltsv: suppress delimiters in output
+  https://github.com/fluent/fluentd/pull/1666
+  https://github.com/fluent/fluentd/pull/3288
+  https://github.com/fluent/fluentd/pull/3289
+
+### Bug fix
+
+* in_tail: Expect ENOENT during stat
+  https://github.com/fluent/fluentd/pull/3275
+* out_forward: Prevent transferring duplicate logs on restart
+  https://github.com/fluent/fluentd/pull/3267
+  https://github.com/fluent/fluentd/pull/3285
+* in_tail: Handle to send rotated logs when mv is used for rotating
+  https://github.com/fluent/fluentd/pull/3294
+* fluent-plugin-config-format: Fill an uninitialized instance variable
+  https://github.com/fluent/fluentd/pull/3297
+* Fix MessagePackEventStream issue with Enumerable methods
+  https://github.com/fluent/fluentd/pull/2116
+
+### Misc
+
+* Add webrick to support Ruby 3.0
+  https://github.com/fluent/fluentd/pull/3257
+* Suggest Discource instead of Google Groups
+  https://github.com/fluent/fluentd/pull/3261
+* Update MAINTAINERS.md
+  https://github.com/fluent/fluentd/pull/3282
+* Introduce DeepSource to check code quality
+  https://github.com/fluent/fluentd/pull/3286
+  https://github.com/fluent/fluentd/pull/3259
+  https://github.com/fluent/fluentd/pull/3291
+* Migrate to GitHub Actions and stabilize tests
+  https://github.com/fluent/fluentd/pull/3266
+  https://github.com/fluent/fluentd/pull/3268
+  https://github.com/fluent/fluentd/pull/3281
+  https://github.com/fluent/fluentd/pull/3283
+  https://github.com/fluent/fluentd/pull/3290
+
 ## Release v1.12.1 - 2021/02/18
 
 ### Enhancement

data/MAINTAINERS.md

@@ -1,8 +1,11 @@
 # Fluentd Maintainers
 
 - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
-- [Okkez](https://github.com/okkez), [Clearcode](https://www.clear-code.com/)
-- [Hiroshi Hatake](https://github.com/cosmo0920), [Clearcode](https://www.clear-code.com/)
+- [Okkez](https://github.com/okkez)
+- [Hiroshi Hatake](https://github.com/cosmo0920), [ClearCode](https://www.clear-code.com/)
 - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
 - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
 - [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)
+- [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
+- [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
+- [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)

data/fluentd.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
+  gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
 
   # build gem for a certain platform. see also Rakefile
   fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s

data/lib/fluent/command/bundler_injection.rb

@@ -40,6 +40,6 @@ else
     File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
   ] + ARGV
 
-  exec *cmdline
+  exec(*cmdline)
   exit! 127
 end

data/lib/fluent/command/cat.rb

@@ -101,7 +101,6 @@ rescue
   usage $!.to_s
 end
 
-require 'thread'
 require 'socket'
 require 'yajl'
 require 'msgpack'

data/lib/fluent/command/plugin_config_formatter.rb

@@ -44,6 +44,7 @@ class FluentPluginConfigFormatter
     @verbose = false
     @libs = []
     @plugin_dirs = []
+    @table = false
     @options = {}
 
     prepare_option_parser
@@ -162,7 +163,7 @@ class FluentPluginConfigFormatter
     else
       sections, params = base_section.partition {|_name, value| value[:section] }
     end
-    if @table and not params.empty?
+    if @table && (not params.empty?)
       dumped << "### Configuration\n\n"
       dumped << "|parameter|type|description|default|\n"
       dumped << "|---|---|---|---|\n"

data/lib/fluent/compat/parser.rb

@@ -244,10 +244,10 @@ module Fluent
         end
 
         def convert_value_to_nil(value)
-          if value and @null_empty_string
+          if value && @null_empty_string
             value = (value == '') ? nil : value
           end
-          if value and @null_value_pattern
+          if value && @null_value_pattern
             value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
           end
           value

data/lib/fluent/config/section.rb

@@ -179,7 +179,7 @@ module Fluent
             end
 
             if section_params[varname].nil?
-              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+              unless proxy.defaults.has_key?(varname) && proxy.defaults[varname].nil?
                 logger.error "config error in:\n#{conf}" if logger
                 raise ConfigError, "'#{name}' parameter is required but nil is specified"
               end

data/lib/fluent/config/types.rb

@@ -186,7 +186,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+                val.start_with?('{') ? JSON.parse(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
               else
                 val
               end
@@ -213,7 +213,7 @@ module Fluent
       return nil if val.nil?
 
       param = if val.is_a?(String)
-                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+                val.start_with?('[') ? JSON.parse(val) : val.strip.split(/\s*,\s*/)
               else
                 val
               end

data/lib/fluent/event.rb

@@ -254,19 +254,9 @@ module Fluent
     end
 
     def each(unpacker: nil, &block)
-      if @unpacked_times
-        @unpacked_times.each_with_index do |time, i|
-          block.call(time, @unpacked_records[i])
-        end
-      else
-        @unpacked_times = []
-        @unpacked_records = []
-        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker).feed_each(@data) do |time, record|
-          @unpacked_times << time
-          @unpacked_records << record
-          block.call(time, record)
-        end
-        @size = @unpacked_times.size
+      ensure_unpacked!(unpacker: unpacker)
+      @unpacked_times.each_with_index do |time, i|
+        block.call(time, @unpacked_records[i])
       end
       nil
     end

data/lib/fluent/load.rb

@@ -1,4 +1,3 @@
-require 'thread'
 require 'socket'
 require 'fcntl'
 require 'time'

data/lib/fluent/plugin/formatter_ltsv.rb

@@ -27,14 +27,14 @@ module Fluent
 
       config_param :delimiter, :string, default: "\t".freeze
       config_param :label_delimiter, :string, default: ":".freeze
+      config_param :replacement, :string, default: " ".freeze
       config_param :add_newline, :bool, default: true
 
-      # TODO: escaping for \t in values
       def format(tag, time, record)
         formatted = ""
         record.each do |label, value|
           formatted << @delimiter if formatted.length.nonzero?
-          formatted << "#{label}#{@label_delimiter}#{value}"
+          formatted << "#{label}#{@label_delimiter}#{value.to_s.gsub(@delimiter, @replacement)}"
         end
         formatted << @newline if @add_newline
         formatted

data/lib/fluent/plugin/in_http.rb

@@ -503,7 +503,7 @@ module Fluent::Plugin
         # For every incoming request, we check if we have some CORS
         # restrictions and allow listed origins through @cors_allow_origins.
         unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?('*') or include_cors_allow_origin
+          unless @cors_allow_origins.include?('*') || include_cors_allow_origin
             send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
             return
           end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -338,7 +338,7 @@ module Fluent::Plugin
         obj.merge!(pe.statistics['output'] || {})
       end
 
-      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] and pe.instance_variable_defined?(:@retry)
+      obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
 
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]

data/lib/fluent/plugin/in_tail.rb

@@ -313,11 +313,17 @@ module Fluent::Plugin
       (paths - excluded).select { |path|
         FileTest.exist?(path)
       }.each { |path|
-        target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
-        if @follow_inodes
-          hash[target_info.ino] = target_info
-        else
-          hash[target_info.path] = target_info
+        # Even we just checked for existence, there is a race condition here as
+        # of which stat() might fail with ENOENT. See #3224.
+        begin
+          target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
+          if @follow_inodes
+            hash[target_info.ino] = target_info
+          else
+            hash[target_info.path] = target_info
+          end
+        rescue Errno::ENOENT
+          $log.warn "expand_paths: stat() for #{path} failed with ENOENT. Skip file."
         end
       }
       hash
@@ -406,8 +412,15 @@ module Fluent::Plugin
           log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
           next
         end
-        target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
-        @tails[target_info] = tw
+
+        begin
+          target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
+          @tails[target_info] = tw
+        rescue Errno::ENOENT
+          $log.warn "stat() for #{target_info.path} failed with ENOENT. Drop tail watcher for now."
+          # explicitly detach and unwatch watcher `tw`.
+          stop_watchers(target_info, immediate: true, unwatched: true)
+        end
       }
     end
 
@@ -767,16 +780,22 @@ module Fluent::Plugin
           end
 
           if watcher_needs_update
-            # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-            # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-            if stat
-              target_info = TargetInfo.new(@path, stat.ino)
-              if @follow_inodes
-                # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+            if @follow_inodes
+              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
+              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
+              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              if stat
+                target_info = TargetInfo.new(@path, stat)
                 @update_watcher.call(target_info, @pe)
-              else
-                @update_watcher.call(target_info, swap_state(@pe))
               end
+            else
+              # Permit to handle if stat is nil (file not present).
+              # If a file is mv-ed and a new file is created during
+              # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
+              # and `#stop_watchers()` for the path because `target_paths_hash`
+              # always contains the path.
+              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
+              @update_watcher.call(target_info, swap_state(@pe))
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/out_copy.rb

@@ -27,20 +27,28 @@ module Fluent::Plugin
     desc 'Pass different record to each `store` plugin by specified method'
     config_param :copy_mode, :enum, list: [:no_copy, :shallow, :deep, :marshal], default: :no_copy
 
-    attr_reader :ignore_errors
+    attr_reader :ignore_errors, :ignore_if_prev_successes
 
     def initialize
       super
       @ignore_errors = []
+      @ignore_if_prev_successes = []
     end
 
     def configure(conf)
       super
 
       @copy_proc = gen_copy_proc
-      @stores.each { |store|
-        @ignore_errors << (store.arg == 'ignore_error')
+      @stores.each_with_index { |store, i|
+        if i == 0 && store.arg.include?('ignore_if_prev_success')
+          raise Fluent::ConfigError, "ignore_if_prev_success must specify 2nd or later <store> directives"
+        end
+        @ignore_errors << (store.arg.include?('ignore_error'))
+        @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+        log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
+      end
     end
 
     def multi_workers_ready?
@@ -55,10 +63,15 @@ module Fluent::Plugin
         }
         es = m
       end
-
+      success = Array.new(outputs.size)
       outputs.each_with_index do |output, i|
         begin
-          output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
+          if i > 0 && success[i - 1] && @ignore_if_prev_successes[i]
+            log.debug "ignore copy because prev_success in #{output.plugin_id}", index: i
+          else
+            output.emit_events(tag, @copy_proc ? @copy_proc.call(es) : es)
+            success[i] = true
+          end
         rescue => e
           if @ignore_errors[i]
             log.error "ignore emit error in #{output.plugin_id}", error: e