fluentd 1.9.3 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d67b7f6379705086047bb2d3adb34fc3c1dbb3c29b1dcd901b9cac62da28978f
-  data.tar.gz: ca8ee48ac0a2d435a87f189812e48c09d9d2947c06b6c2d3ce0e35a7e4c126e1
+  metadata.gz: 3df6cf76d3bcc80301a18f9f265ae75012356f13c39a7cf6b9ad7ccd1d598540
+  data.tar.gz: 94fef906686bc9af303a6745a44634d76edbe42c73edf5b6a7af7d4b8e08c664
 SHA512:
-  metadata.gz: ab9d1deb64f93efa57575b28d2f72eb8ad4ef1874a4d86d6cdf38110968d3b7ea18773e658589151d338d6cc6b24187468c2864fa65616f6dbeb483fbe7bf600
-  data.tar.gz: 147c88ed0e30a82f2b8e72cb6600737c9c44a83eecb1d974d5dc26634918bfb725e813ca9f65d56689067c979e1f488f29a2c899dafaa41ea9128ad552f285d6
+  metadata.gz: 9a5f3959f750f2c1568d8177f4ccc5f90048ffa34b34b723f71808fede41d1af0dfd5bec58606f1c910e8e1525986ff68a4ac284ad8034554a217c8895007da1
+  data.tar.gz: 84c375c61ba1b29e97dc99e61e48613e7972194c75929881ee04ad5fece0e866b8b591097a9093bd54c69cdbab963af3bce5319c9b64cc1727b86cfd29329a18

data/.github/workflows/issue-auto-closer.yml

@@ -0,0 +1,12 @@
+name: Autocloser
+on: [issues]
+jobs:
+  autoclose:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Autoclose issues that did not follow issue template
+      uses: roots/issue-closer-action@v1.1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template"
+        issue-pattern: "(.*Describe the bug.*)|(.*Is your feature request related to a problem.*)"

data/CHANGELOG.md

@@ -1,3 +1,51 @@
+# v1.10
+
+## Release v1.10.0 - 2020/03/24
+
+### New feature
+
+* sd plugin: Add RSV record plugin
+  https://github.com/fluent/fluentd/pull/2876
+
+### Enhancement
+
+* server: Add `cert_verifier` parameter for TLS transport
+  https://github.com/fluent/fluentd/pull/2888
+* parser_syslog: Support customized time format
+  https://github.com/fluent/fluentd/pull/2886
+* in_dummy: Delete `suspend` parameter
+  https://github.com/fluent/fluentd/pull/2897
+* Refactor code
+  https://github.com/fluent/fluentd/pull/2858
+  https://github.com/fluent/fluentd/pull/2862
+  https://github.com/fluent/fluentd/pull/2864
+  https://github.com/fluent/fluentd/pull/2869
+  https://github.com/fluent/fluentd/pull/2870
+  https://github.com/fluent/fluentd/pull/2874
+  https://github.com/fluent/fluentd/pull/2881
+  https://github.com/fluent/fluentd/pull/2885
+  https://github.com/fluent/fluentd/pull/2894
+  https://github.com/fluent/fluentd/pull/2896
+  https://github.com/fluent/fluentd/pull/2898
+  https://github.com/fluent/fluentd/pull/2899
+  https://github.com/fluent/fluentd/pull/2900
+  https://github.com/fluent/fluentd/pull/2901
+  https://github.com/fluent/fluentd/pull/2906
+
+### Bug fix
+
+* out_forward: windows: Permit to specify `linger_timeout`
+  https://github.com/fluent/fluentd/pull/2868
+* parser_syslog: Fix syslog format detection
+  https://github.com/fluent/fluentd/pull/2879
+* buffer: Fix `available_buffer_space_ratio` calculation
+  https://github.com/fluent/fluentd/pull/2882
+* tls: Support CRLF based X.509 certificates
+  https://github.com/fluent/fluentd/pull/2890
+* msgpack_factory mixin: Fix performance penalty for deprecation log
+  https://github.com/fluent/fluentd/pull/2903
+
+
 # v1.9
 
 ## Release v1.9.3 - 2020/03/05

data/lib/fluent/msgpack_factory.rb

@@ -23,25 +23,32 @@ module Fluent
 
     module Mixin
       def msgpack_factory
-        if $log
-          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.engine_factory')
         end
         MessagePackFactory.engine_factory
       end
 
       def msgpack_packer(*args)
-        if $log
-          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_packer')
         end
         MessagePackFactory.msgpack_packer(*args)
       end
 
       def msgpack_unpacker(*args)
-        if $log
-          $log.warn('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')
+        unless @deprecated_log_done
+          deprecated_log('Deprecated method: this method is going to be deleted. Use Fluent::MessagePackFactory.msgpack_unpacker')
         end
         MessagePackFactory.msgpack_unpacker(*args)
       end
+
+      def deprecated_log(str)
+        if $log
+          $log.warn(str)
+          @deprecated_log_done = true
+        end
+      end
     end
 
     def self.engine_factory(enable_time_support: false)

data/lib/fluent/plugin/buffer.rb

@@ -758,13 +758,13 @@ module Fluent
 
       def statistics
         stage_size, queue_size = @stage_size, @queue_size
-        buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size).round
+        buffer_space = 1.0 - ((stage_size + queue_size * 1.0) / @total_limit_size)
         stats = {
           'stage_length' => @stage.size,
           'stage_byte_size' => stage_size,
           'queue_length' => @queue.size,
           'queue_byte_size' => queue_size,
-          'available_buffer_space_ratios' => buffer_space * 100,
+          'available_buffer_space_ratios' => (buffer_space * 100).round(1),
           'total_queued_size' => stage_size + queue_size,
         }
 

data/lib/fluent/plugin/in_dummy.rb

@@ -37,7 +37,7 @@ module Fluent::Plugin
     desc "If specified, each generated event has an auto-incremented key field."
     config_param :auto_increment_key, :string, default: nil
     desc "The boolean to suspend-and-resume incremental value after restart"
-    config_param :suspend, :bool, default: false
+    config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
     desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
     config_param :dummy, default: [{"message"=>"dummy"}] do |val|
       begin

data/lib/fluent/plugin/in_tail.rb

@@ -319,7 +319,7 @@ module Fluent::Plugin
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+      tw = TailWatcher.new(path, pe, log, @read_from_head, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -555,6 +555,21 @@ module Fluent::Plugin
       es
     end
 
+    private
+
+    def io_handler(watcher, path)
+      TailWatcher::IOHandler.new(
+        watcher,
+        path: path,
+        log: log,
+        read_lines_limit: @read_lines_limit,
+        open_on_every_update: @open_on_every_update,
+        from_encoding: @from_encoding,
+        encoding: @encoding,
+        &method(:receive_lines)
+      )
+    end
+
     class StatWatcher < Coolio::StatWatcher
       def initialize(path, log, &callback)
         @callback = callback
@@ -586,22 +601,16 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(path, pe, log, read_from_head, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
+      def initialize(path, pe, log, read_from_head, update_watcher, line_buffer_timer_flusher, io_handler_build)
         @path = path
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
-        @read_lines_limit = read_lines_limit
-        @receive_lines = receive_lines
         @update_watcher = update_watcher
-
-        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
-        @io_handler = nil
         @log = log
-
+        @rotate_handler = RotateHandler.new(log, &method(:on_rotate))
         @line_buffer_timer_flusher = line_buffer_timer_flusher
-        @from_encoding = from_encoding
-        @encoding = encoding
-        @open_on_every_update = open_on_every_update
+        @io_handler = nil
+        @io_handler_build = io_handler_build
         @watchers = []
       end
 
@@ -709,9 +718,7 @@ module Fluent::Plugin
       end
 
       def io_handler
-        IOHandler.new(self, path: @path, log: @log, read_lines_limit: @read_lines_limit, open_on_every_update: @open_on_every_update, from_encoding: @from_encoding, encoding: @encoding) do |lines|
-          @receive_lines.call(lines, self)
-        end
+        @io_handler_build.call(self, @path)
       end
 
       def swap_state(pe)
@@ -801,6 +808,19 @@ module Fluent::Plugin
           @notify_mutex.synchronize { handle_notify }
         end
 
+        def close
+          if @io && !@io.closed?
+            @io.close
+            @io = nil
+          end
+        end
+
+        def opened?
+          !!@io
+        end
+
+        private
+
         def handle_notify
           with_io do |io|
             begin
@@ -822,7 +842,7 @@ module Fluent::Plugin
               end
 
               unless @lines.empty?
-                if @receive_lines.call(@lines)
+                if @receive_lines.call(@lines, @watcher)
                   @watcher.pe.update_pos(io.pos - @fifo.bytesize)
                   @lines.clear
                 else
@@ -833,17 +853,6 @@ module Fluent::Plugin
           end
         end
 
-        def close
-          if @io && !@io.closed?
-            @io.close
-            @io = nil
-          end
-        end
-
-        def opened?
-          !!@io
-        end
-
         def open
           io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)

data/lib/fluent/plugin/out_forward.rb

@@ -382,9 +382,7 @@ module Fluent::Plugin
           cert_logical_store_name: @tls_cert_logical_store_name,
           cert_use_enterprise_store: @tls_cert_use_enterprise_store,
 
-          # Enabling SO_LINGER causes data loss on Windows
-          # https://github.com/fluent/fluentd/issues/1968
-          linger_timeout: Fluent.windows? ? nil : @send_timeout,
+          linger_timeout: @send_timeout,
           send_timeout: @send_timeout,
           recv_timeout: @ack_response_timeout,
           connect_timeout: @connect_timeout,
@@ -510,7 +508,7 @@ module Fluent::Plugin
 
     class Node
       extend Forwardable
-      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby, :username, :password, :shared_key
+      def_delegators :@server, :discovery_id, :host, :port, :name, :weight, :standby
 
       # @param connection_manager [Fluent::Plugin::ForwardOutput::ConnectionManager]
       # @param ack_handler [Fluent::Plugin::ForwardOutput::AckHandler]
@@ -542,8 +540,8 @@ module Fluent::Plugin
           log: @log,
           hostname: sender.security && sender.security.self_hostname,
           shared_key: server.shared_key || (sender.security && sender.security.shared_key) || '',
-          password: server.password,
-          username: server.username,
+          password: server.password || '',
+          username: server.username || '',
         )
 
         @unpacker = Fluent::MessagePackFactory.msgpack_unpacker

data/lib/fluent/plugin/out_forward/handshake_protocol.rb

@@ -105,6 +105,10 @@ module Fluent::Plugin
           .hexdigest
         ping = ['PING', @hostname, @shared_key_salt, shared_key_hexdigest]
         if !ri.auth.empty?
+          if @username.nil? || @password.nil?
+            raise PingpongError, "username and password are required"
+          end
+
           password_hexdigest = Digest::SHA512.new.update(ri.auth).update(@username).update(@password).hexdigest
           ping.push(@username, password_hexdigest)
         else

data/lib/fluent/plugin/parser_syslog.rb

@@ -23,6 +23,7 @@ module Fluent
     class SyslogParser < Parser
       Plugin.register_parser('syslog', self)
 
+      # TODO: Remove them since these regexps are no longer needed. but keep them for compatibility for now
       # From existence TextParser pattern
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
@@ -36,7 +37,16 @@ module Fluent
       REGEXP_RFC5424_WITH_PRI = Regexp.new(<<~'EOS'.chomp % REGEXP_RFC5424, Regexp::MULTILINE)
         \A<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} %s\z
       EOS
-      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
+
+      REGEXP_DETECT_RFC5424 = /^\<[0-9]{1,3}\>[1-9]\d{0,2}/
+
+      RFC3164_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      RFC3164_CAPTURES = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
+      RFC3164_PRI_REGEXP = /^<(?<pri>[0-9]{1,3})>/
+
+      RFC5424_WITHOUT_TIME_AND_PRI_REGEXP = /(?<host>[!-~]{1,255}) (?<ident>[!-~]{1,48}) (?<pid>[!-~]{1,128}) (?<msgid>[!-~]{1,32}) (?<extradata>(?:\-|(?:\[.*?(?<!\\)\])+))(?: (?<message>.+))?\z/m
+      RFC5424_CAPTURES = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP.names.freeze
+      RFC5424_PRI_REGEXP = /^<(?<pri>\d{1,3})>\d\d{0,2}\s/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       desc 'If the incoming logs have priority prefix, e.g. <9>, set true'
@@ -53,6 +63,8 @@ module Fluent
       def initialize
         super
         @mutex = Mutex.new
+        @space_count = nil
+        @space_count_rfc5424 = nil
       end
 
       def configure(conf)
@@ -66,21 +78,21 @@ module Fluent
                   when :rfc3164
                     if @regexp_parser
                       class << self
-                        alias_method :parse, :parse_plain
+                        alias_method :parse, :parse_rfc3164_regex
                       end
                     else
                       class << self
                         alias_method :parse, :parse_rfc3164
                       end
                     end
-                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                    RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
                   when :rfc5424
                     class << self
-                      alias_method :parse, :parse_plain
+                      alias_method :parse, :parse_rfc5424_regex
                     end
                     @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
                     @support_rfc5424_without_subseconds = true
-                    @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
+                    RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
                   when :auto
                     class << self
                       alias_method :parse, :parse_auto
@@ -89,10 +101,14 @@ module Fluent
                     @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
                     nil
                   end
+
+        @space_count = @time_format.squeeze(' ').count(' ') + 1
+        @space_count_rfc5424 = @rfc5424_time_format.squeeze(' ').count(' ') + 1
         @time_parser = time_parser_create
         @time_parser_rfc5424_without_subseconds = time_parser_create(format: "%Y-%m-%dT%H:%M:%S%z")
       end
 
+      # this method is for tests
       def patterns
         {'format' => @regexp, 'time_format' => @time_format}
       end
@@ -103,51 +119,115 @@ module Fluent
 
       def parse_auto(text, &block)
         if REGEXP_DETECT_RFC5424.match(text)
-          @regexp = @with_priority ? REGEXP_RFC5424_WITH_PRI : REGEXP_RFC5424_NO_PRI
+          @regexp = RFC5424_WITHOUT_TIME_AND_PRI_REGEXP
           @time_parser = @time_parser_rfc5424
           @support_rfc5424_without_subseconds = true
-          parse_plain(text, &block)
+          parse_rfc5424_regex(text, &block)
         else
-          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @regexp = RFC3164_WITHOUT_TIME_AND_PRI_REGEXP
           @time_parser = @time_parser_rfc3164
           if @regexp_parser
-            parse_plain(text, &block)
+            parse_rfc3164_regex(text, &block)
           else
             parse_rfc3164(text, &block)
           end
         end
       end
 
-      def parse_plain(text, &block)
-        m = @regexp.match(text)
-        unless m
-          yield nil, nil
-          return
+      def parse_rfc3164_regex(text, &block)
+        idx = 0
+        record = {}
+
+        if @with_priority
+          if RFC3164_PRI_REGEXP.match?(text)
+            v = text.index('>')
+            record['pri'] = text[1..v].to_i # trim `<` and ``>
+            idx = v + 1
+          else
+            yield(nil, nil)
+            return
+          end
         end
 
-        time = nil
+        i = idx - 1
+        sq = false
+        @space_count.times do
+          while text[i + 1] == ' '.freeze
+            sq = true
+            i += 1
+          end
+
+          i = text.index(' '.freeze, i + 1)
+        end
+
+        time_str = sq ? text.slice(idx, i - idx).squeeze(' ') : text.slice(idx, i - idx)
+        time = @mutex.synchronize { @time_parser.parse(time_str) }
+        if @keep_time_key
+          record['time'] = time_str
+        end
+
+        parse_plain(time, text, i + 1, record, RFC3164_CAPTURES, &block)
+      end
+
+      def parse_rfc5424_regex(text, &block)
+        idx = 0
         record = {}
 
-        m.names.each { |name|
-          if value = m[name]
+        if @with_priority
+          if (m = RFC5424_PRI_REGEXP.match(text))
+            record['pri'] = m['pri']
+            idx = m.end(0)
+          else
+            yield(nil, nil)
+            return
+          end
+        end
+
+        i = idx - 1
+        sq = false
+        @space_count_rfc5424.times {
+          while text[i + 1] == ' '.freeze
+            sq = true
+            i += 1
+          end
+
+          i = text.index(' '.freeze, i + 1)
+        }
+
+        time_str = sq ? text.slice(idx, i - idx).squeeze(' '.freeze) : text.slice(idx, i - idx)
+        time = @mutex.synchronize do
+          begin
+            @time_parser.parse(time_str)
+          rescue Fluent::TimeParser::TimeParseError => e
+            if @support_rfc5424_without_subseconds
+              log.trace(e)
+              @time_parser_rfc5424_without_subseconds.parse(time_str)
+            else
+              raise
+            end
+          end
+        end
+
+        if @keep_time_key
+          record['time'] = time_str
+        end
+        parse_plain(time, text, i + 1, record, RFC5424_CAPTURES, &block)
+      end
+
+      # @param time [EventTime]
+      # @param idx [Integer] note: this argument is needed to avoid string creation
+      # @param record [Hash]
+      # @param capture_list [Array] for performance
+      def parse_plain(time, text, idx, record, capture_list, &block)
+        m = @regexp.match(text, idx)
+        if m.nil?
+          yield nil, nil
+          return
+        end
+
+        capture_list.each { |name|
+          if value = (m[name] rescue nil)
             case name
-            when "pri"
-              record['pri'] = value.to_i
-            when "time"
-              time = @mutex.synchronize do
-                time_str = value.squeeze(' ')
-                begin
-                  @time_parser.parse(time_str)
-                rescue Fluent::TimeParser::TimeParseError => e
-                  if @support_rfc5424_without_subseconds
-                    log.trace(e)
-                    @time_parser_rfc5424_without_subseconds.parse(time_str)
-                  else
-                    raise
-                  end
-                end
-              end
-              record[name] = value if @keep_time_key
             when "message"
               value.chomp!
               record[name] = value