fluentd 0.14.13-x64-mingw32 → 0.14.17-x64-mingw32

data/lib/fluent/plugin/out_file.rb

@@ -105,6 +105,11 @@ module Fluent::Plugin
         buffer_conf['path'] = conf['path'] || '/tmp/dummy_path'
       end
 
+      if conf.has_key?('utc') || conf.has_key?('localtime')
+        param_name = conf.has_key?('utc') ? 'utc' : 'localtime'
+        log.warn "'#{param_name}' is deperecated for output plugin. This parameter is used for formatter plugin in compatibility layer. If you want to use same feature, use timekey_use_utc parameter in <buffer> directive instead"
+      end
+
       super
 
       @compress_method = SUPPORTED_COMPRESS_MAP[@compress]

data/lib/fluent/plugin/out_forward.rb

@@ -420,7 +420,7 @@ module Fluent::Plugin
     # return chunk id to be committed
     def read_ack_from_sock(sock, unpacker)
       begin
-        raw_data = sock.recv(@read_length)
+        raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
       rescue Errno::ECONNRESET
         raw_data = ""
       end
@@ -431,6 +431,7 @@ module Fluent::Plugin
       if raw_data.empty?
         log.warn "destination node closed the connection. regard it as unavailable.", host: info.node.host, port: info.node.port
         info.node.disable!
+        rollback_write(info.chunk_id)
         return nil
       else
         unpacker.feed(raw_data)
@@ -450,6 +451,8 @@ module Fluent::Plugin
       log.error "unexpected error while receiving ack message", error: e
       log.error_backtrace
     ensure
+      info.sock.close_write rescue nil
+      info.sock.close rescue nil
       @sock_ack_waiting_mutex.synchronize do
         @sock_ack_waiting.delete(info)
       end
@@ -492,7 +495,7 @@ module Fluent::Plugin
 
           readable_sockets.each do |sock|
             chunk_id = read_ack_from_sock(sock, unpacker)
-            commit_write(chunk_id)
+            commit_write(chunk_id) if chunk_id
           end
         rescue => e
           log.error "unexpected error while receiving ack", error: e

data/lib/fluent/plugin/output.rb

@@ -122,12 +122,17 @@ module Fluent
         raise NotImplementedError, "BUG: output plugins MUST implement this method"
       end
 
-      def formatted_to_msgpack_binary
+      def formatted_to_msgpack_binary?
         # To indicate custom format method (#format) returns msgpack binary or not.
         # If #format returns msgpack binary, override this method to return true.
         false
       end
 
+      # Compatibility for existing plugins
+      def formatted_to_msgpack_binary
+        formatted_to_msgpack_binary?
+      end
+
       def prefer_buffered_processing
         # override this method to return false only when all of these are true:
         #  * plugin has both implementation for buffered and non-buffered methods
@@ -589,13 +594,13 @@ module Fluent
           example = @argument[:example]
           timekey = @argument[:timekey]
           if !sec && timekey
-            raise Fluent::ConfigError, "Parameter '#{name}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
           end
           if sec && !timekey
-            raise Fluent::ConfigError, "Parameter '#{name}' has timestamp placeholders, but chunk key 'time' is not configured"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has timestamp placeholders, but chunk key 'time' is not configured"
           end
           if sec && timekey && timekey < sec
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
           end
         end
 
@@ -603,10 +608,10 @@ module Fluent
           parts = @argument[:parts]
           tagkey = @argument[:tagkey]
           if tagkey && parts.empty?
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have tag placeholder"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have tag placeholder"
           end
           if !tagkey && !parts.empty?
-            raise Fluent::ConfigError, "Parameter '#{@name}' has tag placeholders, but chunk key 'tag' is not configured"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has tag placeholders, but chunk key 'tag' is not configured"
           end
         end
 
@@ -615,11 +620,11 @@ module Fluent
           chunk_keys = @argument[:chunkkeys]
           if (chunk_keys - keys).size > 0
             not_specified = (chunk_keys - keys).sort
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
           end
           if (keys - chunk_keys).size > 0
             not_satisfied = (keys - chunk_keys).sort
-            raise Fluent::ConfigError, "Parameter '#{@name}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
           end
         end
       end

data/lib/fluent/plugin/parser_apache2.rb

@@ -21,7 +21,7 @@ module Fluent
     class Apache2Parser < Parser
       Plugin.register_parser('apache2', self)
 
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize

data/lib/fluent/plugin/parser_syslog.rb

@@ -27,9 +27,13 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
         super
@@ -39,7 +43,27 @@ module Fluent
       def configure(conf)
         super
 
-        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
+        @regexp = case @message_format
+                  when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
+                    REGEXP_RFC5424
+                  when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
+                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
+                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
+                    nil
+                  end
         @time_parser = time_parser_create
       end
 
@@ -48,6 +72,21 @@ module Fluent
       end
 
       def parse(text)
+        # This is overwritten in configure
+      end
+
+      def parse_auto(text, &block)
+        if REGEXP_DETECT_RFC5424.match(text)
+          @regexp = REGEXP_RFC5424
+          @time_parser = @time_parser_rfc5424
+        else
+          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @time_parser = @time_parser_rfc3164
+        end
+        parse_plain(text, &block)
+      end
+
+      def parse_plain(text, &block)
         m = @regexp.match(text)
         unless m
           yield nil, nil

data/lib/fluent/plugin_helper/cert_option.rb

@@ -46,12 +46,12 @@ module Fluent
         case
         when conf.cert_path
           raise Fluent::ConfigError, "private_key_path is required when cert_path is specified" unless conf.private_key_path
-          raise Fluent::ConfigError, "private_key_passphrase is required when cert_path is specified" unless conf.private_key_passphrase
+          log.warn "For security reason, setting private_key_passphrase is recommended when cert_path is specified" unless conf.private_key_passphrase
           cert_option_load(conf.cert_path, conf.private_key_path, conf.private_key_passphrase)
 
         when conf.ca_cert_path
           raise Fluent::ConfigError, "ca_private_key_path is required when ca_cert_path is specified" unless conf.ca_private_key_path
-          raise Fluent::ConfigError, "ca_private_key_passphrase is required when ca_cert_path is specified" unless conf.ca_private_key_passphrase
+          log.warn "For security reason, setting ca_private_key_passphrase is recommended when ca_cert_path is specified" unless conf.ca_private_key_passphrase
           generate_opts = cert_option_cert_generation_opts_from_conf(conf)
           cert_option_generate_server_pair_by_ca(
             conf.ca_cert_path,

data/lib/fluent/plugin_helper/compat_parameters.rb

@@ -55,7 +55,7 @@ module Fluent
         "keys" => "keys", # CSVParser, TSVParser (old ValuesParser)
         "time_key"    => "time_key",
         "time_format" => "time_format",
-        "localtim" => nil,
+        "localtime" => nil,
         "utc" => nil,
         "delimiter"   => "delimiter",
         "keep_time_key" => "keep_time_key",

data/lib/fluent/plugin_helper/storage.rb

@@ -30,7 +30,7 @@ module Fluent
       StorageState = Struct.new(:storage, :running)
 
       def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
-        if conf && !conf.arg.empty?
+        if conf && conf.respond_to?(:arg) && !conf.arg.empty?
           usage = conf.arg
         end
         if !usage.empty? && usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/

data/lib/fluent/root_agent.rb

@@ -64,11 +64,37 @@ module Fluent
     attr_reader :labels
 
     def configure(conf)
+      # initialize <worker> elements
+      conf.elements(name: 'worker').each do |e|
+        target_worker_id_str = e.arg
+        if target_worker_id_str.empty?
+          raise ConfigError, "Missing worker id on <worker> directive"
+        end
+
+        target_worker_id = target_worker_id_str.to_i
+        if target_worker_id < 0 || target_worker_id > (Fluent::Engine.system_config.workers - 1)
+          raise ConfigError, "worker id #{target_worker_id} specified by <worker> directive is not allowed. Available worker id is between 0 and #{(Fluent::Engine.system_config.workers - 1)}"
+        end
+
+        ## On dry_run mode, all worker sections have to be configured on supervisor (recognized as worker_id = 0).
+        target_worker_id = 0 if Fluent::Engine.dry_run_mode
+
+        e.elements.each do |elem|
+          unless ['source', 'match', 'filter', 'label'].include?(elem.name)
+            raise ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+          end
+          elem.set_target_worker_id(target_worker_id)
+        end
+        conf += e
+      end
+      conf.elements.delete_if{|e| e.name == 'worker'}
+
       error_label_config = nil
 
       # initialize <label> elements before configuring all plugins to avoid 'label not found' in input, filter and output.
       label_configs = {}
       conf.elements(name: 'label').each { |e|
+        next if e.for_another_worker?
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
 
@@ -90,6 +116,7 @@ module Fluent
         log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
       else
         conf.elements(name: 'source').each { |e|
+          next if e.for_another_worker?
           type = e['@type']
           raise ConfigError, "Missing '@type' parameter on <source> directive" unless type
           add_source(type, e)
@@ -136,8 +163,12 @@ module Fluent
     def start
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
-      end
-      lifecycle(desc: true) do |i|
+        # Input#start sometimes emits lots of evetns with in_tail/`read_from_head true` case
+        # and it causes deadlock for small buffer/queue output. To avoid such problem,
+        # buffer related output threads should be run before `Input#start`.
+        # This is why after_start should be called immediately after start call.
+        # This depends on `desc: true` because calling plugin order of `desc: true` is
+        # Output, Filter, Label, Output with Router, then Input.
         i.after_start unless i.after_started?
       end
     end
@@ -173,7 +204,7 @@ module Fluent
             log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
             instance.send(method) unless instance.send(checker)
           rescue Exception => e
-            log.warn "unexpected error while calling #{method} on #{kind} plugin", pluguin: instance.class, plugin_id: instance.plugin_id, error: e
+            log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
           end
         end
@@ -231,7 +262,8 @@ module Fluent
     end
 
     def add_source(type, conf)
-      log.info :worker0, "adding source", type: type
+      log_type = conf.for_this_worker? ? :default : :worker0
+      log.info log_type, "adding source", type: type
 
       input = Plugin.new_input(type)
       # <source> emits events to the top-level event router (RootAgent#event_router).

data/lib/fluent/supervisor.rb

@@ -61,6 +61,7 @@ module Fluent
 
     def after_run
       stop_rpc_server if @rpc_endpoint
+      Fluent::Supervisor.cleanup_resources
     end
 
     def run_rpc_server
@@ -383,10 +384,17 @@ module Fluent
         supervise: true,
         standalone_worker: false,
         signame: nil,
-        winsvcreg: nil,
       }
     end
 
+    def self.cleanup_resources
+      unless Fluent.windows?
+        if ENV.has_key?('SERVERENGINE_SOCKETMANAGER_PATH')
+          FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+        end
+      end
+    end
+
     def initialize(opt)
       @daemonize = opt[:daemonize]
       @supervise = opt[:supervise]
@@ -449,7 +457,7 @@ module Fluent
         end
       end
 
-      dry_run if @dry_run
+      dry_run_cmd if @dry_run
       supervise
     end
 
@@ -482,6 +490,10 @@ module Fluent
       read_config
       set_system_config
 
+      if @standalone_worker && @workers != 1
+        raise Fluent::ConfigError, "invalid number of workers (must be 1 or unspecified) with --no-supervisor: #{@workers}"
+      end
+
       install_main_process_signal_handlers
 
       # This is the only log messsage for @standalone_worker
@@ -493,6 +505,7 @@ module Fluent
         init_engine
         run_configure
         run_engine
+        self.class.cleanup_resources if @standalone_worker
         exit 0
       end
     end
@@ -505,17 +518,32 @@ module Fluent
       ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
     end
 
-    def dry_run
+    def dry_run_cmd
       $log.info "starting fluentd-#{Fluent::VERSION} as dry run mode"
-      change_privilege
-      init_engine
-      run_configure
+      @system_config.suppress_config_dump = true
+      dry_run
       exit 0
     rescue => e
       $log.error "dry run failed: #{e}"
       exit 1
     end
 
+    ## Set Engine's dry_run_mode true to override all target_id of worker sections
+    def dry_run
+      begin
+        Fluent::Engine.dry_run_mode = true
+        change_privilege
+        init_engine
+        run_configure
+      rescue Fluent::ConfigError => e
+        $log.error "config error", file: @config_path, error: e
+        $log.debug_backtrace
+        exit!(1)
+      ensure
+        Fluent::Engine.dry_run_mode = false
+      end
+    end
+
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
       $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
@@ -523,6 +551,9 @@ module Fluent
     end
 
     def supervise
+      # Make dumpable conf, which is set corresponding_proxies for all elements in all worker sections
+      dry_run
+
       Process.setproctitle("supervisor:#{@process_name}") if @process_name
       $log.info "starting fluentd-#{Fluent::VERSION}", pid: Process.pid
 

data/lib/fluent/system_config.rb

@@ -98,6 +98,13 @@ module Fluent
             next # doesn't exist in command line options
           when :emit_error_log_interval
             system.emit_error_log_interval = @suppress_interval if @suppress_interval
+          when :log_level
+            ll_value = instance_variable_get("@log_level")
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            if ll_value != Fluent::Log::LEVEL_INFO
+              system.log_level = ll_value
+            end
           else
             next unless instance_variable_defined?("@#{param}")
             supervisor_value = instance_variable_get("@#{param}")

data/lib/fluent/time.rb

@@ -208,6 +208,7 @@ module Fluent
       @parse = case
                when format_with_timezone && strptime then ->(v){ Fluent::EventTime.from_time(strptime.exec(v)) }
                when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
+               when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
                when strptime then ->(v){ t = strptime.exec(v);         Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
                when format   then ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
                else ->(v){ Fluent::EventTime.parse(v) }

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.13'
+  VERSION = '0.14.17'
 
 end

data/lib/fluent/winsvc.rb

@@ -14,10 +14,9 @@
 #    limitations under the License.
 #
 
-INTEVENTOBJ_NAME = "fluentdwinsvc"
-
 begin
 
+  require 'optparse'
   require 'windows/debug'
   require 'Windows/Library'
   require 'win32/daemon'
@@ -27,34 +26,50 @@ begin
   include Windows::Library
   include Windows::Debug
 
-  def read_fluentdopt
+  op = OptionParser.new
+  opts = {service_name: nil}
+  op.on('--service-name NAME', "The name of the Windows Service") {|name|
+    opts[:service_name] = name
+  }
+  op.parse(ARGV)
+  if opts[:service_name] == nil
+    raise "Error: No Windows Service name set. Use '--service-name'"
+  end 
+
+  def read_fluentdopt(service_name)
     require 'win32/Registry'
-    Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc") do |reg|
+    Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\#{service_name}") do |reg|
       reg.read("fluentdopt")[1] rescue ""
     end
   end
 
-  def service_main_start
+  def service_main_start(service_name)
     ruby_path = 0.chr * 260
     GetModuleFileName.call(0, ruby_path,260)
     ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
     rubybin_dir = ruby_path[0, ruby_path.rindex("/")]
-    opt = read_fluentdopt
-    Process.spawn(rubybin_dir + "/ruby.exe " + rubybin_dir + "/fluentd " + opt + " -x " + INTEVENTOBJ_NAME)
+    opt = read_fluentdopt(service_name)
+    Process.spawn("\"#{rubybin_dir}/ruby.exe\" \"#{rubybin_dir}/fluentd\" #{opt} -x #{service_name}")
   end
 
   class FluentdService < Daemon
     @pid = 0
+    @service_name = ''
 
+    def initialize(service_name)
+      @service_name = service_name
+    end
+    
     def service_main
-      @pid = service_main_start
+    
+      @pid = service_main_start(@service_name)
       while running?
         sleep 10
       end
     end
 
     def service_stop
-      ev = Win32::Event.open(INTEVENTOBJ_NAME)
+      ev = Win32::Event.open(@service_name)
       ev.set
       ev.close
       if @pid > 0
@@ -63,9 +78,8 @@ begin
     end
   end
 
-  FluentdService.mainloop
+  FluentdService.new(opts[:service_name]).mainloop
 
 rescue Exception => err
   raise
 end
-