fluentd 0.14.13-x64-mingw32 → 0.14.17-x64-mingw32

data/lib/fluent/config/element.rb

@@ -35,10 +35,13 @@ module Fluent
 
         # it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
         @logger = defined?($log) ? $log : nil
+
+        @target_worker_id = nil
       end
 
       attr_accessor :name, :arg, :unused, :v1_config, :corresponding_proxies, :unused_in
       attr_writer :elements
+      attr_reader :target_worker_id
 
       RESERVED_PARAMETERS_COMPAT = {
         '@type' => 'type',
@@ -213,6 +216,25 @@ module Fluent
         v.each_char { |c| result << LiteralParser.unescape_char(c) }
         result
       end
+
+      def set_target_worker_id(worker_id)
+        @target_worker_id = worker_id
+        @elements.each { |e|
+          e.set_target_worker_id(worker_id)
+        }
+      end
+
+      def for_every_workers?
+        @target_worker_id == nil
+      end
+
+      def for_this_worker?
+        @target_worker_id == Fluent::Engine.worker_id
+      end
+
+      def for_another_worker?
+        @target_worker_id != nil && @target_worker_id != Fluent::Engine.worker_id
+      end
     end
   end
 end

data/lib/fluent/config/literal_parser.rb

@@ -181,6 +181,8 @@ module Fluent
           "\f"
         when "b"
           "\b"
+        when "0"
+          "\0"
         when /[a-zA-Z0-9]/
           parse_error! "unexpected back-slash escape character '#{c}'"
         else  # symbols

data/lib/fluent/config/types.rb

@@ -64,7 +64,7 @@ module Fluent
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val.to_s.encode(Encoding::UTF_8) }
+    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -85,7 +85,7 @@ module Fluent
         value
       else
         case type
-        when :string  then value.to_s.encode(Encoding::UTF_8)
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
         when :integer then value.to_i
         when :float   then value.to_f
         when :size then Config.size_value(value)

data/lib/fluent/engine.rb

@@ -31,6 +31,7 @@ module Fluent
       @root_agent = nil
       @default_loop = nil
       @engine_stopped = false
+      @_worker_id = nil
 
       @log_event_router = nil
       @log_emit_thread = nil
@@ -42,6 +43,8 @@ module Fluent
       @suppress_config_dump = false
 
       @system_config = SystemConfig.new
+
+      @dry_run_mode = false
     end
 
     MAINLOOP_SLEEP_INTERVAL = 0.3
@@ -53,6 +56,8 @@ module Fluent
     attr_reader :matches, :sources
     attr_reader :system_config
 
+    attr_accessor :dry_run_mode
+
     def init(system_config)
       @system_config = system_config
 
@@ -97,12 +102,21 @@ module Fluent
                     else
                       "section <#{e.name}> is not used in <#{parent_name}>"
                     end
-          $log.warn :worker0, message
+          if e.for_every_workers?
+            $log.warn :worker0, message
+          elsif e.for_this_worker?
+            $log.warn message
+          end
           next
         end
         unless e.name == 'system'
           unless @without_source && e.name == 'source'
-            $log.warn :worker0, "parameter '#{key}' in #{e.to_s.strip} is not used."
+            message = "parameter '#{key}' in #{e.to_s.strip} is not used."
+            if e.for_every_workers?
+              $log.warn :worker0, message
+            elsif e.for_this_worker?
+              $log.warn message
+            end
           end
         end
       }
@@ -128,7 +142,7 @@ module Fluent
 
         unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
         unless unmatched_tags.empty?
-          $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
         end
       rescue ArgumentError # ArgumentError "#{label_name} label not found"
         # use default event router if <label @FLUENT_LOG> is missing in configuration
@@ -139,7 +153,7 @@ module Fluent
 
           unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
           unless unmatched_tags.empty?
-            $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+            $log.warn "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
           end
         end
       end
@@ -147,7 +161,7 @@ module Fluent
       $log.enable_event(true) if @log_event_router
 
       unless @suppress_config_dump
-        $log.info :worker0, "using configuration file: #{conf.to_s.rstrip}"
+        $log.info :supervisor, "using configuration file: #{conf.to_s.rstrip}"
       end
     end
 
@@ -200,10 +214,6 @@ module Fluent
     end
 
     def run
-      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
-      # so it's (almost) a single worker, worker_id=0
-      worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
-
       begin
         $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
         start
@@ -251,8 +261,15 @@ module Fluent
       @log_event_queue.push([tag, time, record])
     end
 
-    private
+    def worker_id
+      return @_worker_id if @_worker_id
+      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
+      # so it's (almost) a single worker, worker_id=0
+      @_worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+      @_worker_id
+    end
 
+    private
     def start
       @root_agent.start
     end

data/lib/fluent/env.rb

@@ -14,15 +14,15 @@
 #    limitations under the License.
 #
 
+require 'serverengine/utils'
+
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
-  IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
-  private_constant :IS_WINDOWS
 
   def self.windows?
-    IS_WINDOWS
+    ServerEngine.windows?
   end
 end

data/lib/fluent/log.rb

@@ -357,6 +357,9 @@ module Fluent
     def write(data)
       @out.write(data)
     end
+    # We need `#<<` method to use this logger class with other
+    # libraries such as aws-sdk
+    alias << write
 
     def flush
       @out.flush
@@ -469,7 +472,7 @@ module Fluent
     extend Forwardable
     def_delegators '@logger', :enable_color?, :enable_debug, :enable_event,
       :disable_events, :log_event_enabled, :log_event_enamed=, :time_format, :time_format=,
-      :event, :caller_line, :puts, :write, :flush, :reset, :out, :out=,
+      :event, :caller_line, :puts, :write, :<<, :flush, :reset, :out, :out=,
       :optional_header, :optional_header=, :optional_attrs, :optional_attrs=
   end
 

data/lib/fluent/plugin/base.rb

@@ -51,6 +51,9 @@ module Fluent
       end
 
       def configure(conf)
+        if conf.respond_to?(:for_this_worker?) && conf.for_this_worker?
+          system_config_override(workers: 1)
+        end
         super
         @_state ||= State.new(false, false, false, false, false, false, false, false, false)
         @_state.configure = true

data/lib/fluent/plugin/filter.rb

@@ -79,13 +79,13 @@ module Fluent
         return nil if implmented_methods.include?(:filter_stream)
         case
         when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
-          raise "BUG: Filter plugins MUST be implemented either `filter` or `filter_with_time`"
+          raise "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
         when implmented_methods.include?(:filter)
           false
         when implmented_methods.include?(:filter_with_time)
           true
         else
-          raise NotImplementedError, "BUG: Filter plugins MUST be implmented either `filter` or `filter_with_time`"
+          raise NotImplementedError, "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
         end
       end
     end

data/lib/fluent/plugin/filter_parser.rb

@@ -32,6 +32,7 @@ module Fluent::Plugin
     config_param :inject_key_prefix, :string, default: nil
     config_param :replace_invalid_sequence, :bool, default: false
     config_param :hash_value_field, :string, default: nil
+    config_param :emit_invalid_record_to_error, :bool, default: true
 
     attr_reader :parser
 
@@ -49,7 +50,9 @@ module Fluent::Plugin
     def filter_with_time(tag, time, record)
       raw_value = record[@key_name]
       if raw_value.nil?
-        router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        if @emit_invalid_record_to_error
+          router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        end
         if @reserve_data
           return time, handle_parsed(tag, record, time, {})
         else
@@ -67,7 +70,9 @@ module Fluent::Plugin
             r = handle_parsed(tag, record, t, values)
             return t, r
           else
-            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+            if @emit_invalid_record_to_error
+              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+            end
             if @reserve_data
               t = time
               r = handle_parsed(tag, record, time, {})
@@ -78,8 +83,11 @@ module Fluent::Plugin
           end
         end
       rescue Fluent::Plugin::Parser::ParserError => e
-        router.emit_error_event(tag, time, record, e)
-        return FAILED_RESULT
+        if @emit_invalid_record_to_error
+          raise e
+        else
+          return FAILED_RESULT
+        end
       rescue ArgumentError => e
         raise unless @replace_invalid_sequence
         raise unless e.message.index("invalid byte sequence in") == 0
@@ -87,8 +95,11 @@ module Fluent::Plugin
         raw_value = raw_value.scrub(REPLACE_CHAR)
         retry
       rescue => e
-        router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("parse failed #{e.message}"))
-        return FAILED_RESULT
+        if @emit_invalid_record_to_error
+          raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
+        else
+          return FAILED_RESULT
+        end
       end
     end
 

data/lib/fluent/plugin/in_forward.rb

@@ -211,7 +211,7 @@ module Fluent::Plugin
           options = on_message(msg, chunk_size, conn)
           if options && r = response(options)
             log.trace "sent response to fluent socket", address: conn.remote_addr, response: r
-            conn.on_write_complete{ conn.close } if @deny_keepalive
+            conn.on(:write_complete) { |c| c.close } if @deny_keepalive
             send_data.call(serializer, r)
           else
             if @deny_keepalive

data/lib/fluent/plugin/in_http.rb

@@ -319,6 +319,8 @@ module Fluent::Plugin
           when /Origin/i
             @origin  = v
           when /X-Forwarded-For/i
+            # For multiple X-Forwarded-For headers. Use first header value.
+            v = v.first if v.is_a?(Array)
             @remote_addr = v.split(",").first
           end
         }
@@ -373,6 +375,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
+        elsif @content_type =~ /^application\/msgpack/
+          params['msgpack'] = @body
         end
         path_info = uri.path
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -226,6 +226,11 @@ module Fluent::Plugin
       @first_warn = false
     end
 
+    def configure(conf)
+      super
+      @port += fluentd_worker_id
+    end
+
     def multi_workers_ready?
       true
     end
@@ -233,7 +238,7 @@ module Fluent::Plugin
     def start
       super
 
-      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
+      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
       @srv = WEBrick::HTTPServer.new({
           BindAddress: @bind,
           Port: @port,
@@ -273,8 +278,8 @@ module Fluent::Plugin
 
     MONITOR_INFO = {
       'output_plugin' => ->(){ is_a?(::Fluent::Plugin::Output) },
-      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil?; @buffer.queue.size },
-      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil?; @buffer.stage_size },
+      'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
+      'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size },
       'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
     }
 

data/lib/fluent/plugin/in_syslog.rb

@@ -85,15 +85,16 @@ module Fluent::Plugin
     config_param :source_hostname_key, :string, default: nil
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
-
     desc 'The field name of the priority.'
     config_param :priority_key, :string, default: nil
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
 
-    config_param :blocking_timeout, :time, default: 0.5
+    desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 2048
 
+    config_param :blocking_timeout, :time, default: 0.5
+
     config_section :parse do
       config_set_default :@type, DEFAULT_PARSER
       config_param :with_priority, :bool, default: true

data/lib/fluent/plugin/in_tail.rb

@@ -42,6 +42,7 @@ module Fluent::Plugin
       @tails = {}
       @pf_file = nil
       @pf = nil
+      @ignore_list = []
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -79,6 +80,10 @@ module Fluent::Plugin
     config_param :open_on_every_update, :bool, default: false
     desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
     config_param :limit_recently_modified, :time, default: nil
+    desc 'Enable the option to skip the refresh of watching list on startup.'
+    config_param :skip_refresh_on_startup, :bool, default: false
+    desc 'Ignore repeated permission error logs'
+    config_param :ignore_repeated_permission_error, :bool, default: false
 
     attr_reader :paths
 
@@ -160,7 +165,7 @@ module Fluent::Plugin
         @pf = PositionFile.parse(@pf_file)
       end
 
-      refresh_watchers
+      refresh_watchers unless @skip_refresh_on_startup
       timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
     end
 
@@ -187,14 +192,20 @@ module Fluent::Plugin
         path = date.strftime(path)
         if path.include?('*')
           paths += Dir.glob(path).select { |p|
-            if File.readable?(p) && !File.directory?(p)
+            is_file = !File.directory?(p)
+            if File.readable?(p) && is_file
               if @limit_recently_modified && File.mtime(p) < (date - @limit_recently_modified)
                 false
               else
                 true
               end
             else
-              log.warn "#{p} unreadable. It is excluded and would be examined next time."
+              if is_file
+                unless @ignore_list.include?(path)
+                  log.warn "#{p} unreadable. It is excluded and would be examined next time."
+                  @ignore_list << path if @ignore_repeated_permission_error
+                end
+              end
               false
             end
           }

data/lib/fluent/plugin/in_udp.rb

@@ -34,7 +34,10 @@ module Fluent::Plugin
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
 
-    config_param :body_size_limit, :size, default: 4096
+    desc "Deprecated parameter. Use message_length_limit instead"
+    config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
+    desc "The max bytes of message"
+    config_param :message_length_limit, :size, default: 4096
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -43,6 +46,7 @@ module Fluent::Plugin
       super
       @_event_loop_blocking_timeout = @blocking_timeout
       @source_hostname_key ||= @source_host_key if @source_host_key
+      @message_length_limit = @body_size_limit if @body_size_limit
 
       @parser = parser_create
     end
@@ -55,7 +59,7 @@ module Fluent::Plugin
       super
 
       log.info "listening udp socket", bind: @bind, port: @port
-      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @body_size_limit) do |data, sock|
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
         data.chomp!
         begin
           @parser.parse(data) do |time, record|