fluentd 0.14.13 → 0.14.14.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 158d62963384d98d44438e5f3a8e31c97f3124e4
-  data.tar.gz: 59a98cfbf508e67e462849f8ada07a509255f8bf
+  metadata.gz: e1ebffe0eb082e2cf131d53ad1481cff9ef4648a
+  data.tar.gz: 85603a42d5d04d6b72d7d7a7829c9139d7a3bd67
 SHA512:
-  metadata.gz: bf8ab978620b5472988d8483828a9b6e5004e5028b86165b31628810c93d15f591b07fd24e0f6f9929b9390bd8de7ccb1035eeb6d389c8b862638a1325f8fc6a
-  data.tar.gz: 52257e481b34bddce80b2c7f7442328f704f0b11ee1c3495fa895dea4902040ff3eec6ad65c62a261dcb659c5f48bd3cd0455abe045c860259dad420d1b2f9e5
+  metadata.gz: 6ccaff3d1a6c1d0cf31760cac64e907951d319d624a2033870d070658e62368b137ed6e8aecde67909ede472b25b862c70f4845710b62c5aa30ec7c3560331cb
+  data.tar.gz: 722a6b02123610def0f9dc3e2d7f806148e88dca697d3c75ccf70fc36ad5db06ff391b702442952717f42e48e92f8ecc1bdf9f8be958fdf74def5d1ca2f324ff

data/README.md

@@ -26,6 +26,31 @@ Mobile/Web Application Logging | Fluentd can function as middleware to enable as
     $ fluentd -c conf/fluent.conf &
     $ echo '{"json":"message"}' | fluent-cat debug.test
 
+## Development
+
+### Prerequisites
+
+- Ruby 2.1 or later
+- git
+
+`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
+
+### Install dependent gems
+
+Use bundler:
+
+    $ gem install bundler
+    $ bundle install --path vendor/bundle
+
+### Run test
+
+    $ bundle exec rake test
+
+You can run specified test via `TEST` environment variable:
+
+    $ bundle exec rake test TEST=test/test_specified_path.rb
+    $ bundle exec rake test TEST=test/test_*.rb
+
 ## Fluentd UI: Admin GUI
 
 [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.

data/lib/fluent/config/types.rb

@@ -64,7 +64,7 @@ module Fluent
       end
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val.to_s.encode(Encoding::UTF_8) }
+    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -85,7 +85,7 @@ module Fluent
         value
       else
         case type
-        when :string  then value.to_s.encode(Encoding::UTF_8)
+        when :string  then value.to_s.force_encoding(Encoding::UTF_8)
         when :integer then value.to_i
         when :float   then value.to_f
         when :size then Config.size_value(value)

data/lib/fluent/env.rb

@@ -14,15 +14,15 @@
 #    limitations under the License.
 #
 
+require 'serverengine/utils'
+
 module Fluent
   DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
   DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
-  IS_WINDOWS = /mswin|mingw/ === RUBY_PLATFORM
-  private_constant :IS_WINDOWS
 
   def self.windows?
-    IS_WINDOWS
+    ServerEngine.windows?
   end
 end

data/lib/fluent/log.rb

@@ -357,6 +357,9 @@ module Fluent
     def write(data)
       @out.write(data)
     end
+    # We need `#<<` method to use this logger class with other
+    # libraries such as aws-sdk
+    alias << write
 
     def flush
       @out.flush

data/lib/fluent/plugin/filter_parser.rb

@@ -32,6 +32,7 @@ module Fluent::Plugin
     config_param :inject_key_prefix, :string, default: nil
     config_param :replace_invalid_sequence, :bool, default: false
     config_param :hash_value_field, :string, default: nil
+    config_param :emit_invalid_record_to_error, :bool, default: true
 
     attr_reader :parser
 
@@ -49,7 +50,9 @@ module Fluent::Plugin
     def filter_with_time(tag, time, record)
       raw_value = record[@key_name]
       if raw_value.nil?
-        router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        if @emit_invalid_record_to_error
+          router.emit_error_event(tag, time, record, ArgumentError.new("#{@key_name} does not exist"))
+        end
         if @reserve_data
           return time, handle_parsed(tag, record, time, {})
         else
@@ -67,7 +70,9 @@ module Fluent::Plugin
             r = handle_parsed(tag, record, t, values)
             return t, r
           else
-            router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+            if @emit_invalid_record_to_error
+              router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not match with data '#{raw_value}'"))
+            end
             if @reserve_data
               t = time
               r = handle_parsed(tag, record, time, {})
@@ -78,8 +83,11 @@ module Fluent::Plugin
           end
         end
       rescue Fluent::Plugin::Parser::ParserError => e
-        router.emit_error_event(tag, time, record, e)
-        return FAILED_RESULT
+        if @emit_invalid_record_to_error
+          raise e
+        else
+          return FAILED_RESULT
+        end
       rescue ArgumentError => e
         raise unless @replace_invalid_sequence
         raise unless e.message.index("invalid byte sequence in") == 0
@@ -87,8 +95,11 @@ module Fluent::Plugin
         raw_value = raw_value.scrub(REPLACE_CHAR)
         retry
       rescue => e
-        router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("parse failed #{e.message}"))
-        return FAILED_RESULT
+        if @emit_invalid_record_to_error
+          raise Fluent::Plugin::Parser::ParserError, "parse failed #{e.message}"
+        else
+          return FAILED_RESULT
+        end
       end
     end
 

data/lib/fluent/plugin/in_http.rb

@@ -373,6 +373,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
+        elsif @content_type =~ /^application\/msgpack/
+          params['msgpack'] = @body
         end
         path_info = uri.path
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -226,6 +226,11 @@ module Fluent::Plugin
       @first_warn = false
     end
 
+    def configure(conf)
+      super
+      @port += fluentd_worker_id
+    end
+
     def multi_workers_ready?
       true
     end
@@ -233,7 +238,7 @@ module Fluent::Plugin
     def start
       super
 
-      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins"
+      log.debug "listening monitoring http server on http://#{@bind}:#{@port}/api/plugins for worker#{fluentd_worker_id}"
       @srv = WEBrick::HTTPServer.new({
           BindAddress: @bind,
           Port: @port,

data/lib/fluent/plugin/in_tail.rb

@@ -79,6 +79,8 @@ module Fluent::Plugin
     config_param :open_on_every_update, :bool, default: false
     desc 'Limit the watching files that the modification time is within the specified time range (when use \'*\' in path).'
     config_param :limit_recently_modified, :time, default: nil
+    desc 'Enable the option to skip the refresh of watching list on startup.'
+    config_param :skip_refresh_on_startup, :bool, default: false
 
     attr_reader :paths
 
@@ -160,7 +162,7 @@ module Fluent::Plugin
         @pf = PositionFile.parse(@pf_file)
       end
 
-      refresh_watchers
+      refresh_watchers unless @skip_refresh_on_startup
       timer_execute(:in_tail_refresh_watchers, @refresh_interval, &method(:refresh_watchers))
     end
 

data/lib/fluent/plugin/output.rb

@@ -589,13 +589,13 @@ module Fluent
           example = @argument[:example]
           timekey = @argument[:timekey]
           if !sec && timekey
-            raise Fluent::ConfigError, "Parameter '#{name}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholders for timekey #{timekey.to_i}"
           end
           if sec && !timekey
-            raise Fluent::ConfigError, "Parameter '#{name}' has timestamp placeholders, but chunk key 'time' is not configured"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has timestamp placeholders, but chunk key 'time' is not configured"
           end
           if sec && timekey && timekey < sec
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have timestamp placeholder for #{title}('#{example}') for timekey #{timekey.to_i}"
           end
         end
 
@@ -603,10 +603,10 @@ module Fluent
           parts = @argument[:parts]
           tagkey = @argument[:tagkey]
           if tagkey && parts.empty?
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have tag placeholder"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have tag placeholder"
           end
           if !tagkey && !parts.empty?
-            raise Fluent::ConfigError, "Parameter '#{@name}' has tag placeholders, but chunk key 'tag' is not configured"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has tag placeholders, but chunk key 'tag' is not configured"
           end
         end
 
@@ -615,11 +615,11 @@ module Fluent
           chunk_keys = @argument[:chunkkeys]
           if (chunk_keys - keys).size > 0
             not_specified = (chunk_keys - keys).sort
-            raise Fluent::ConfigError, "Parameter '#{@name}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' doesn't have enough placeholders for keys #{not_specified.join(',')}"
           end
           if (keys - chunk_keys).size > 0
             not_satisfied = (keys - chunk_keys).sort
-            raise Fluent::ConfigError, "Parameter '#{@name}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
+            raise Fluent::ConfigError, "Parameter '#{name}: #{string}' has placeholders, but chunk keys doesn't have keys #{not_satisfied.join(',')}"
           end
         end
       end

data/lib/fluent/plugin/parser_apache2.rb

@@ -21,7 +21,7 @@ module Fluent
     class Apache2Parser < Parser
       Plugin.register_parser('apache2', self)
 
-      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>(?:[^\"]|\\.)*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>(?:[^\"]|\\.)*)" "(?<agent>(?:[^\"]|\\.)*)")?$/
       TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
 
       def initialize

data/lib/fluent/plugin/parser_syslog.rb

@@ -27,9 +27,13 @@ module Fluent
       REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_RFC5424 = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
+      REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/
 
       config_set_default :time_format, "%b %d %H:%M:%S"
       config_param :with_priority, :bool, default: false
+      config_param :message_format, :enum, list: [:rfc3164, :rfc5424, :auto], default: :rfc3164
+      config_param :rfc5424_time_format, :string, default: "%Y-%m-%dT%H:%M:%S.%L%z"
 
       def initialize
         super
@@ -39,7 +43,27 @@ module Fluent
       def configure(conf)
         super
 
-        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @time_parser_rfc3164 = @time_parser_rfc5424 = nil
+        @regexp = case @message_format
+                  when :rfc3164
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @with_priority ? REGEXP_WITH_PRI : REGEXP
+                  when :rfc5424
+                    class << self
+                      alias_method :parse, :parse_plain
+                    end
+                    @time_format = @rfc5424_time_format unless conf.has_key?('time_format')
+                    REGEXP_RFC5424
+                  when :auto
+                    class << self
+                      alias_method :parse, :parse_auto
+                    end
+                    @time_parser_rfc3164 = time_parser_create(format: @time_format)
+                    @time_parser_rfc5424 = time_parser_create(format: @rfc5424_time_format)
+                    nil
+                  end
         @time_parser = time_parser_create
       end
 
@@ -48,6 +72,21 @@ module Fluent
       end
 
       def parse(text)
+        # This is overwritten in configure
+      end
+
+      def parse_auto(text, &block)
+        if REGEXP_DETECT_RFC5424.match(text)
+          @regexp = REGEXP_RFC5424
+          @time_parser = @time_parser_rfc5424
+        else
+          @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+          @time_parser = @time_parser_rfc3164
+        end
+        parse_plain(text, &block)
+      end
+
+      def parse_plain(text, &block)
         m = @regexp.match(text)
         unless m
           yield nil, nil

data/lib/fluent/plugin_helper/storage.rb

@@ -30,7 +30,7 @@ module Fluent
       StorageState = Struct.new(:storage, :running)
 
       def storage_create(usage: '', type: nil, conf: nil, default_type: nil)
-        if conf && !conf.arg.empty?
+        if conf && conf.respond_to?(:arg) && !conf.arg.empty?
           usage = conf.arg
         end
         if !usage.empty? && usage !~ /^[a-zA-Z][-_.a-zA-Z0-9]*$/

data/lib/fluent/root_agent.rb

@@ -136,8 +136,12 @@ module Fluent
     def start
       lifecycle(desc: true) do |i| # instance
         i.start unless i.started?
-      end
-      lifecycle(desc: true) do |i|
+        # Input#start sometimes emits lots of evetns with in_tail/`read_from_head true` case
+        # and it causes deadlock for small buffer/queue output. To avoid such problem,
+        # buffer related output threads should be run before `Input#start`.
+        # This is why after_start should be called immediately after start call.
+        # This depends on `desc: true` because calling plugin order of `desc: true` is
+        # Output, Filter, Label, Output with Router, then Input.
         i.after_start unless i.after_started?
       end
     end
@@ -173,7 +177,7 @@ module Fluent
             log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
             instance.send(method) unless instance.send(checker)
           rescue Exception => e
-            log.warn "unexpected error while calling #{method} on #{kind} plugin", pluguin: instance.class, plugin_id: instance.plugin_id, error: e
+            log.warn "unexpected error while calling #{method} on #{kind} plugin", plugin: instance.class, plugin_id: instance.plugin_id, error: e
             log.warn_backtrace
           end
         end

data/lib/fluent/supervisor.rb

@@ -61,6 +61,7 @@ module Fluent
 
     def after_run
       stop_rpc_server if @rpc_endpoint
+      Fluent::Supervisor.cleanup_resources
     end
 
     def run_rpc_server
@@ -387,6 +388,14 @@ module Fluent
       }
     end
 
+    def self.cleanup_resources
+      unless Fluent.windows?
+        if ENV.has_key?('SERVERENGINE_SOCKETMANAGER_PATH')
+          FileUtils.rm_f(ENV['SERVERENGINE_SOCKETMANAGER_PATH'])
+        end
+      end
+    end
+
     def initialize(opt)
       @daemonize = opt[:daemonize]
       @supervise = opt[:supervise]
@@ -493,6 +502,7 @@ module Fluent
         init_engine
         run_configure
         run_engine
+        self.class.cleanup_resources if @standalone_worker
         exit 0
       end
     end

data/lib/fluent/system_config.rb

@@ -98,6 +98,13 @@ module Fluent
             next # doesn't exist in command line options
           when :emit_error_log_interval
             system.emit_error_log_interval = @suppress_interval if @suppress_interval
+          when :log_level
+            ll_value = instance_variable_get("@log_level")
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            if ll_value != Fluent::Log::LEVEL_INFO
+              system.log_level = ll_value
+            end
           else
             next unless instance_variable_defined?("@#{param}")
             supervisor_value = instance_variable_get("@#{param}")

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.13'
+  VERSION = '0.14.14.pre.1'
 
 end

data/test/config/test_system_config.rb

@@ -13,11 +13,13 @@ module Fluent::Config
   end
 
   class FakeSupervisor
+    attr_writer :log_level
+
     def initialize
       @workers = nil
       @root_dir = nil
       @log = FakeLoggerInitializer.new
-      @log_level = nil
+      @log_level = Fluent::Log::LEVEL_INFO
       @suppress_interval = nil
       @suppress_config_dump = nil
       @suppress_repeated_stacktrace = nil
@@ -55,7 +57,7 @@ module Fluent::Config
       assert_nil(sc.without_source)
       assert_equal(1, s.instance_variable_get(:@workers))
       assert_nil(s.instance_variable_get(:@root_dir))
-      assert_nil(s.instance_variable_get(:@log_level))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
       assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
       assert_nil(s.instance_variable_get(:@emit_error_log_interval))
       assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -99,7 +101,7 @@ module Fluent::Config
       sc.apply(s)
       assert_equal(1, s.instance_variable_get(:@workers))
       assert_nil(s.instance_variable_get(:@root_dir))
-      assert_nil(s.instance_variable_get(:@log_level))
+      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
       assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
       assert_nil(s.instance_variable_get(:@emit_error_log_interval))
       assert_nil(s.instance_variable_get(:@suppress_config_dump))
@@ -109,16 +111,43 @@ module Fluent::Config
       assert_nil(s.instance_variable_get(:@dir_permission))
     end
 
-    test 'log_level' do
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'info' => Fluent::Log::LEVEL_INFO,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is applied when log_level related command line option is not passed' do |level|
+      conf = parse_text(<<-EOS)
+        <system>
+          log_level #{Fluent::Log::LEVEL_TEXT[level]}
+        </system>
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
+      sc.apply(s)
+      assert_equal(level, s.instance_variable_get("@log").level)
+    end
+
+    # info is removed because info level can't be specified via command line
+    data('trace' => Fluent::Log::LEVEL_TRACE,
+         'debug' => Fluent::Log::LEVEL_DEBUG,
+         'warn' => Fluent::Log::LEVEL_WARN,
+         'error' => Fluent::Log::LEVEL_ERROR,
+         'fatal' => Fluent::Log::LEVEL_FATAL)
+    test 'log_level is ignored when log_level related command line option is passed' do |level|
       conf = parse_text(<<-EOS)
         <system>
-          log_level warn
+          log_level info
         </system>
       EOS
       s = FakeSupervisor.new
+      s.log_level = level
       sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
       sc.apply(s)
-      assert_equal(Fluent::Log::LEVEL_WARN, s.instance_variable_get("@log").level)
+      assert_equal(level, s.instance_variable_get("@log").level)
     end
 
     test 'process global overridable variables' do
@@ -130,6 +159,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
+      sc.attach(s)
       sc.apply(s)
       assert_equal(0655, s.instance_variable_get(:@file_permission))
       assert_equal(0765, s.instance_variable_get(:@dir_permission))

data/test/config/test_types.rb

@@ -69,6 +69,17 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal Encoding::UTF_8, Config::STRING_TYPE.call('test', {}).encoding
     end
 
+    data('latin' => 'Märch',
+         'ascii' => 'ascii',
+         'space' => '     ',
+         'number' => '1',
+         'Hiragana' => 'あいうえお')
+    test 'string w/ binary' do |str|
+      actual = Config::STRING_TYPE.call(str.b, {})
+      assert_equal str, actual
+      assert_equal Encoding::UTF_8, actual.encoding
+    end
+
     test 'enum' do
       assert_equal :val, Config::ENUM_TYPE.call('val', {list: [:val, :value, :v]})
       assert_equal :v, Config::ENUM_TYPE.call('v', {list: [:val, :value, :v]})
@@ -143,6 +154,14 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_raise(RuntimeError.new("unknown type in REFORMAT: foo")){ Config::HASH_TYPE.call("x:1,y:2", {value_type: :foo}) }
     end
 
+    data('latin' => ['3:Märch', {"3"=>"Märch"}],
+         'ascii' => ['ascii:ascii', {"ascii"=>"ascii"}],
+         'number' => ['number:1', {"number"=>"1"}],
+         'Hiragana' => ['hiragana:あいうえお', {"hiragana"=>"あいうえお"}])
+    test 'hash w/ binary' do |(target, expected)|
+      assert_equal(expected, Config::HASH_TYPE.call(target.b, { value_type: :string }))
+    end
+
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
       assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))

data/test/plugin/test_filter_parser.rb

@@ -662,4 +662,39 @@ class ParserFilterTest < Test::Unit::TestCase
       end
     end
   end
+
+  class EmitInvalidRecordToErrorTest < self
+    def test_pattern_is_mismached_with_emit_invalid_record_to_error
+      d = create_driver(CONFIG_UNMATCHED_PATTERN_LOG + "emit_invalid_record_to_error false")
+      flexmock(d.instance.router).should_receive(:emit_error_event).never
+      assert_nothing_raised {
+        d.run do
+          d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
+        end
+      }
+      assert_equal 0, d.filtered.length
+    end
+
+    def test_parser_error_with_emit_invalid_record_to_error
+      d = create_driver(CONFIG_INVALID_TIME_VALUE + "emit_invalid_record_to_error false")
+      flexmock(d.instance.router).should_receive(:emit_error_event).never
+      assert_nothing_raised {
+        d.run do
+          d.feed(@tag, Fluent::EventTime.now.to_i, {'data' => '{"time":[], "f1":"v1"}'})
+        end
+      }
+      assert_equal 0, d.filtered.length
+    end
+
+    def test_key_not_exist_with_emit_invalid_record_to_error
+      d = create_driver(CONFIG_NOT_IGNORE + "emit_invalid_record_to_error false")
+      flexmock(d.instance.router).should_receive(:emit_error_event).never
+      assert_nothing_raised {
+        d.run do
+          d.feed(@tag, Fluent::EventTime.now.to_i, {'foo' => 'bar'})
+        end
+      }
+      assert_equal 0, d.filtered.length
+    end
+  end
 end

data/test/plugin/test_in_http.rb

@@ -313,6 +313,28 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  def test_application_msgpack
+    d = create_driver
+    time = event_time("2011-01-02 13:14:15 UTC")
+    time_i = time.to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}?time=#{time_i.to_s}", record.to_msgpack, {"Content-Type"=>"application/msgpack"})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+  
   def test_msgpack
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")

data/test/plugin/test_in_monitor_agent.rb

@@ -204,10 +204,9 @@ EOC
     unless header.has_key?('Content-Type')
       header['Content-Type'] = 'application/octet-stream'
     end
-    res = Net::HTTP.start(url.host, url.port) {|http|
+    Net::HTTP.start(url.host, url.port) {|http|
       http.request(req)
     }
-    res.body
   end
 
   sub_test_case "servlets" do
@@ -268,7 +267,7 @@ plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tret
       expected_test_filter_response = "\
 plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
 
-      response = get("http://127.0.0.1:#{@port}/api/plugins")
+      response = get("http://127.0.0.1:#{@port}/api/plugins").body
       test_in = response.split("\n")[0]
       test_filter = response.split("\n")[3]
       assert_equal(expected_test_in_response, test_in)
@@ -306,7 +305,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       }
       expected_null_response.merge!("config" => {"@id" => "null", "@type" => "null"}) if with_config
       expected_null_response.merge!("retry" => {}) if with_retry
-      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json").body)
       test_in_response = response["plugins"][0]
       null_response = response["plugins"][5]
       assert_equal(expected_test_in_response, test_in_response)
@@ -343,7 +342,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       }
       expected_null_response.merge!("config" => {"@id" => "null", "@type" => "null"}) if with_config
       expected_null_response.merge!("retry" => {}) if with_retry
-      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json#{query_param}"))
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json#{query_param}").body)
       test_in_response = response["plugins"][0]
       null_response = response["plugins"][5]
       assert_equal(expected_test_in_response, test_in_response)
@@ -376,7 +375,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "type"            => "null",
         "instance_variables" => {"id" => "null", "num_errors" => 0}
       }
-      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json?with_config=no&with_retry=no&with_ivars=id,num_errors"))
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json?with_config=no&with_retry=no&with_ivars=id,num_errors").body)
       test_in_response = response["plugins"][0]
       null_response = response["plugins"][5]
       assert_equal(expected_test_in_response, test_in_response)
@@ -394,7 +393,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       expected_response_regex = /pid:\d+\tppid:\d+\tconfig_path:\/etc\/fluent\/fluent.conf\tpid_file:\tplugin_dirs:\[\"\/etc\/fluent\/plugin\"\]\tlog_path:/
 
       assert_match(expected_response_regex,
-                   get("http://127.0.0.1:#{@port}/api/config"))
+                   get("http://127.0.0.1:#{@port}/api/config").body)
     end
 
     test "/api/config.json" do
@@ -405,7 +404,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
   tag monitor
 ")
       d.instance.start
-      res = JSON.parse(get("http://127.0.0.1:#{@port}/api/config.json"))
+      res = JSON.parse(get("http://127.0.0.1:#{@port}/api/config.json").body)
       assert_equal("/etc/fluent/fluent.conf", res["config_path"])
       assert_nil(res["pid_file"])
       assert_equal(["/etc/fluent/plugin"], res["plugin_dirs"])
@@ -474,7 +473,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         output.submit_flush_once
         sleep 0.1 until output.buffer.queued?
       end
-      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json").body)
       test_out_fail_write_response = response["plugins"][1]
       # remove dynamic keys
       response_retry_count = test_out_fail_write_response.delete("retry_count")
@@ -486,4 +485,32 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert{ response_retry_count == response_retry["steps"] + 1 }
     end
   end
+
+  sub_test_case "check the port number of http server" do
+    test "on single worker environment" do
+      port = unused_port
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{port}
+")
+      d.instance.start
+      assert_equal("200", get("http://127.0.0.1:#{port}/api/plugins").code)
+    end
+
+    test "worker_id = 2 on multi worker environment" do
+      port = unused_port
+      Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+        d = Fluent::Test::Driver::Input.new(Fluent::Plugin::MonitorAgentInput)
+        d.instance.instance_eval{ @_fluentd_worker_id = 2 }
+        d.configure("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{port - 2}
+")
+        d.instance.start
+      end
+      assert_equal("200", get("http://127.0.0.1:#{port}/api/plugins").code)
+    end
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -1198,4 +1198,19 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal expected_files, plugin.expand_paths.sort
     end
   end
+
+  def test_skip_refresh_on_startup
+    FileUtils.touch("#{TMP_DIR}/tail.txt")
+    config = config_element('', '', {
+                              'format' => 'none',
+                              'refresh_interval' => 1,
+                              'skip_refresh_on_startup' => true
+                            })
+    d = create_driver(config)
+    d.run(shutdown: false) {}
+    assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
+    # detect a file at first execution of in_tail_refresh_watchers timer
+    waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
+    d.instance_shutdown
+  end
 end

data/test/plugin/test_output.rb

@@ -315,7 +315,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:time?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
           validators.first.validate!
         end
       end
@@ -325,7 +325,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:time?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 30") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
           validators.first.validate!
         end
       end
@@ -335,7 +335,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:tag?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
           validators.first.validate!
         end
       end
@@ -345,7 +345,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/file.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:tag?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
           validators.first.validate!
         end
       end
@@ -355,7 +355,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:keys?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys group,username") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
           validators.first.validate!
         end
       end
@@ -365,7 +365,7 @@ class OutputTest < Test::Unit::TestCase
         validators = @i.placeholder_validators(:path, "/my/path/file.log")
         assert_equal 1, validators.size
         assert_equal 1, validators.select(&:keys?).size
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys group,username") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
           validators.first.validate!
         end
       end
@@ -374,14 +374,14 @@ class OutputTest < Test::Unit::TestCase
     sub_test_case '#placeholder_validate!' do
       test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
           @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
         end
       end
 
       test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 180") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have timestamp placeholders for timekey 180") do
           @i.placeholder_validate!(:path, "/my/path/file.log")
         end
         assert_nothing_raised do
@@ -391,7 +391,7 @@ class OutputTest < Test::Unit::TestCase
 
       test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y%m%d_%H.log' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
           @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
         end
         assert_nothing_raised do
@@ -401,14 +401,14 @@ class OutputTest < Test::Unit::TestCase
 
       test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.${tag[2]}.log' has tag placeholders, but chunk key 'tag' is not configured") do
           @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
         end
       end
 
       test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
           @i.placeholder_validate!(:path, "/my/path/file.log")
         end
         assert_nothing_raised do
@@ -418,14 +418,14 @@ class OutputTest < Test::Unit::TestCase
 
       test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys service,username") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.${username}.log' has placeholders, but chunk keys doesn't have keys service,username") do
           @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
         end
       end
 
       test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service,username") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys service,username") do
           @i.placeholder_validate!(:path, "/my/path/file.log")
         end
         assert_nothing_raised do
@@ -435,10 +435,10 @@ class OutputTest < Test::Unit::TestCase
 
       test 'raise configuration error for a template and configuration with keys mismatch' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.${username}.log' doesn't have enough placeholders for keys service") do
           @i.placeholder_validate!(:path, "/my/path/file.${username}.log")
         end
-        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys username") do
+        assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${service}/file.log' doesn't have enough placeholders for keys username") do
           @i.placeholder_validate!(:path, "/my/path/${service}/file.log")
         end
         assert_nothing_raised do

data/test/plugin/test_parser_apache2.rb

@@ -35,4 +35,12 @@ class Apache2ParserTest < ::Test::Unit::TestCase
       assert_equal(@expected, record)
     }
   end
+
+  def test_parse_with_escape_sequence
+    @parser.instance.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
+      assert_equal('/\"', record['path'])
+      assert_equal('referer \\\ \"', record['referer'])
+      assert_equal('user agent \\\ \"', record['agent'])
+    }
+  end
 end

data/test/plugin/test_parser_syslog.rb

@@ -63,4 +63,180 @@ class SyslogParserTest < ::Test::Unit::TestCase
       assert_equal "Feb 28 00:00:12", record['time']
     end
   end
+
+  class TestRFC5424Regexp < self
+    def test_parse_with_rfc5424_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_message_without_time_format
+      @parser.configure(
+                        'message_format' => 'rfc5424',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'rfc5424',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+  end
+
+  class TestAutoRegexp < self
+    def test_auto_with_legacy_syslog_message
+      @parser.configure(
+                        'time_format' => '%b %d %M:%S:%H',
+                        'mseeage_format' => 'auto',
+                        )
+      text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
+      end
+    end
+
+    def test_auto_with_legacy_syslog_priority_message
+      @parser.configure(
+                        'time_format' => '%b %d %M:%S:%H',
+                        'with_priority' => true,
+                        'mseeage_format' => 'auto',
+                        )
+      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 6), record)
+      end
+    end
+
+    def test_parse_with_rfc5424_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'auto',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_rfc5424_structured_message
+      @parser.configure(
+                        'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'message_format' => 'auto',
+                        )
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_both_message_type
+      @parser.configure(
+        'time_format' => '%b %d %M:%S:%H',
+        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+        'message_format' => 'auto',
+      )
+      text = 'Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      text = 'Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected, record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+
+    def test_parse_with_both_message_type_and_priority
+      @parser.configure(
+                        'time_format' => '%b %d %M:%S:%H',
+                        'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
+                        'with_priority' => true,
+                        'message_format' => 'auto',
+                        )
+      text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 6), record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "11111", record["pid"]
+        assert_equal "ID24224", record["msgid"]
+        assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
+                     record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+      text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
+        assert_equal(@expected.merge('pri' => 16), record)
+      end
+      text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
+      @parser.instance.parse(text) do |time, record|
+        assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
+        assert_equal "-", record["pid"]
+        assert_equal "-", record["msgid"]
+        assert_equal "-", record["extradata"]
+        assert_equal "Hi, from Fluentd!", record["message"]
+      end
+    end
+  end
 end

data/test/plugin_helper/test_storage.rb

@@ -99,6 +99,15 @@ class StorageHelperTest < Test::Unit::TestCase
     assert_equal 0, d._storages.size
   end
 
+  test 'can be configured with hash' do
+    d = Dummy.new
+    d.configure(config_element())
+    conf = { '@type' => 'example' }
+    assert_nothing_raised do
+      d.storage_create(conf: conf)
+    end
+  end
+
   test 'can override default configuration parameters, but not overwrite whole definition' do
     d = Dummy.new
     d.configure(config_element())

data/test/test_log.rb

@@ -720,6 +720,10 @@ class PluginLoggerTest < Test::Unit::TestCase
       @log.write("log")
     end
 
+    def test_write_alias
+      assert(@log.respond_to?(:<<))
+    end
+
     def test_out
       assert_equal(@log.out, @logger.out)
       @log.out = Object.new

data/test/test_plugin_classes.rb

@@ -21,6 +21,28 @@ module FluentTest
     end
   end
 
+  class FluentTestGenInput < ::Fluent::Plugin::Input
+    ::Fluent::Plugin.register_input('test_in_gen', self)
+
+    attr_reader :started
+
+    config_param :num, :integer, default: 10000
+
+    def start
+      super
+      @started = true
+
+      @num.times { |i|
+        router.emit("test.evet", Fluent::EventTime.now, {'message' => 'Hello!', 'key' => "value#{i}", 'num' => i})
+      }
+    end
+
+    def shutdown
+      @started = false
+      super
+    end
+  end
+
   class FluentTestOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out', self)
 
@@ -112,6 +134,19 @@ module FluentTest
 
   class FluentTestBufferedOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_buffered', self)
+
+    attr_reader :started
+
+    def start
+      super
+      @started = true
+    end
+
+    def shutdown
+      @started = false
+      super
+    end
+
     def write(chunk)
       # drop everything
     end

data/test/test_root_agent.rb

@@ -174,10 +174,15 @@ EOC
   end
 
   sub_test_case 'start/shutdown' do
-    setup do
-      @ra = RootAgent.new(log: $log)
-      stub(Engine).root_agent { @ra }
-      @ra.configure(Config.parse(<<-EOC, "(test)", "(test_dir)", true))
+    def setup_root_agent(conf)
+      ra = RootAgent.new(log: $log)
+      stub(Engine).root_agent { ra }
+      ra.configure(Config.parse(conf, "(test)", "(test_dir)", true))
+      ra
+    end
+
+    test 'plugin status' do
+      ra = setup_root_agent(<<-EOC)
 <source>
   @type test_in
   @id test_in
@@ -191,19 +196,41 @@ EOC
   @id test_out
 </match>
 EOC
-      @ra
+      ra.start
+      assert_true ra.inputs.first.started
+      assert_true ra.filters.first.started
+      assert_true ra.outputs.first.started
+
+      ra.shutdown
+      assert_false ra.inputs.first.started
+      assert_false ra.filters.first.started
+      assert_false ra.outputs.first.started
     end
 
-    test 'plugin status' do
-      @ra.start
-      assert_true @ra.inputs.first.started
-      assert_true @ra.filters.first.started
-      assert_true @ra.outputs.first.started
+    test 'output plugin threads should run before input plugin is blocked with buffer full' do
+      ra = setup_root_agent(<<-EOC)
+<source>
+  @type test_in_gen
+  @id test_in_gen
+</source>
+<match **>
+  @type test_out_buffered
+  @id test_out_buffered
+  <buffer>
+    chunk_limit_size 1k
+    queue_limit_length 2
+    flush_thread_count 2
+    overflow_action block
+  </buffer>
+</match>
+EOC
+      waiting(5) { ra.start }
+      assert_true ra.inputs.first.started
+      assert_true ra.outputs.first.started
 
-      @ra.shutdown
-      assert_false @ra.inputs.first.started
-      assert_false @ra.filters.first.started
-      assert_false @ra.outputs.first.started
+      ra.shutdown
+      assert_false ra.inputs.first.started
+      assert_false ra.outputs.first.started
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.13
+  version: 0.14.14.pre.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-17 00:00:00.000000000 Z
+date: 2017-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -708,9 +708,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.2