fluentd 0.12.37 → 0.12.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f88c1cef594f4ca5773a92310fd32f0fbf728a89
-  data.tar.gz: e9fc666c618c2b9bf35d10c21466a0539bb45d10
+  metadata.gz: 8fc381ec2e8ffc7a3f6e196b597a324f3e6e553e
+  data.tar.gz: 2f1832c98553174c7fe96f87832898e669f985cb
 SHA512:
-  metadata.gz: 5652613563369d7f73bd290c5e550a9fd02bff462502bf9074a399a4fa2df89e3da69a4f63e34480751d3411938f57346aa1d1e049a26fb921b40609c7d7d311
-  data.tar.gz: 641ea08ba43f5f646a6444f91939c7f99fc95d5d59a562f59123e64f2d79bf120adb17e32e3b9dea3daa51bc432619db688567c74319b131743ad6c1b475606b
+  metadata.gz: cabc4f4b85ff50f0421b63f3ddceb2046a8354fac5097af1cbee43b32124e110ce8b12bf23fc55682c91266a63643ae7f38d142af146f903a4b049b162eb3073
+  data.tar.gz: bd375fc4feee2fa831c4bb03d1ed3118c3807434e250a0be635dadf5cac1654cdab666b069b1754d07d45c91c0207785ca083424968071c7baadb5d6080cb222

data/ChangeLog

@@ -1,5 +1,24 @@
 # v0.12
 
+## Release 0.12.38 - 2017/07/13
+
+### New features / Enhancement
+
+* in_syslog: Add Add allow_without_priority and default_priority parameters
+  https://github.com/fluent/fluentd/pull/1608
+* in_syslog: More characters are available in tag part of syslog format
+  https://github.com/fluent/fluentd/pull/1609
+* in_syslog: Add resolve_hostname and source_address_key parameters
+  https://github.com/fluent/fluentd/pull/1615
+* filter_grep: Support new configuration format by config_section
+  https://github.com/fluent/fluentd/pull/1627
+
+### Bug fixes
+
+* log: Capture more fluentd log events during shutdown
+  https://github.com/fluent/fluentd/pull/1618
+  https://github.com/fluent/fluentd/pull/1625
+
 ## Release 0.12.37 - 2017/06/21
 
 ### New features / Enhancement

data/lib/fluent/agent.rb

@@ -38,6 +38,11 @@ module Fluent
       @started_outputs = []
       @started_filters = []
 
+      @outputs_for_log_event = []
+      @filters_for_log_event = []
+      @started_outputs_for_log_event = []
+      @started_filters_for_log_event = []
+
       @log = Engine.log
       @event_router = EventRouter.new(NoMatchMatch.new(log), self)
       @error_collector = nil
@@ -70,16 +75,47 @@ module Fluent
       @outputs.each { |o|
         o.start
         @started_outputs << o
+        @started_outputs_for_log_event << o if @outputs_for_log_event.include?(o)
       }
 
       @filters.each { |f|
         f.start
         @started_filters << f
+        @started_filters_for_log_event << f if @filters_for_log_event.include?(f)
       }
     end
 
     def shutdown
-      @started_filters.map { |f|
+      (@started_filters - @started_filters_for_log_event).map { |f|
+        Thread.new do
+          begin
+            log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
+            f.shutdown
+          rescue => e
+            log.warn "unexpected error while shutting down filter plugins", plugin: f.class, plugin_id: f.plugin_id, error_class: e.class, error: e
+            log.warn_backtrace
+          end
+        end
+      }.each { |t| t.join }
+
+      # Output plugin as filter emits records at shutdown so emit problem still exist.
+      # This problem will be resolved after actual filter mechanizm.
+      (@started_outputs - @started_outputs_for_log_event).map { |o|
+        Thread.new do
+          begin
+            log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
+            o.shutdown
+          rescue => e
+            log.warn "unexpected error while shutting down output plugins", plugin: o.class, plugin_id: o.plugin_id, error_class: e.class, error: e
+            log.warn_backtrace
+          end
+        end
+      }.each { |t| t.join }
+
+      ## execute callback from Engine to flush log event queue before shutting down corresponding filters and outputs
+      yield if block_given?
+
+      @started_filters_for_log_event.map { |f|
         Thread.new do
           begin
             log.info "shutting down filter#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(f.class), plugin_id: f.plugin_id
@@ -93,7 +129,7 @@ module Fluent
 
       # Output plugin as filter emits records at shutdown so emit problem still exist.
       # This problem will be resolved after actual filter mechanizm.
-      @started_outputs.map { |o|
+      @started_outputs_for_log_event.map { |o|
         Thread.new do
           begin
             log.info "shutting down output#{@context.nil? ? '' : " in #{@context}"}", type: Plugin.lookup_name_from_class(o.class), plugin_id: o.plugin_id
@@ -134,6 +170,10 @@ module Fluent
       @outputs << output
       @event_router.add_rule(pattern, output)
 
+      if match_event_log_tag?(pattern)
+        @outputs_for_log_event << output
+      end
+
       output
     end
 
@@ -146,9 +186,17 @@ module Fluent
       @filters << filter
       @event_router.add_rule(pattern, filter)
 
+      if match_event_log_tag?(pattern)
+        @filters_for_log_event << filter
+      end
+
       filter
     end
 
+    def match_event_log_tag?(pattern)
+      EventRouter::Rule.new(pattern, nil).match?($log.tag)
+    end
+
     # For handling invalid record
     def emit_error_event(tag, time, record, error)
     end

data/lib/fluent/engine.rb

@@ -165,7 +165,6 @@ module Fluent
       $log.disable_events(Thread.current)
 
       while sleep(LOG_EMIT_INTERVAL)
-        break if @log_event_loop_stop
         next if @log_event_queue.empty?
 
         # NOTE: thead-safe of slice! depends on GVL
@@ -179,6 +178,7 @@ module Fluent
             $log.error "failed to emit fluentd's log event", tag: tag, event: record, error_class: e.class, error: e
           end
         }
+        break if @log_event_loop_stop
       end
     end
 
@@ -209,10 +209,13 @@ module Fluent
         $log.error_backtrace
       ensure
         $log.info "shutting down fluentd"
-        shutdown
         if @log_emit_thread
-          @log_event_loop_stop = true
-          @log_emit_thread.join
+          shutdown do
+            @log_event_loop_stop = true
+            @log_emit_thread.join
+          end
+        else
+          shutdown
         end
       end
     end
@@ -237,8 +240,8 @@ module Fluent
       @root_agent.start
     end
 
-    def shutdown
-      @root_agent.shutdown
+    def shutdown(&block)
+      @root_agent.shutdown(&block)
     end
   end
 

data/lib/fluent/parser.rb

@@ -537,9 +537,9 @@ module Fluent
 
     class SyslogParser < Parser
       # From existence TextParser pattern
-      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
       # From in_syslog default pattern
-      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
        REGEXP_RFC5424 = /\A^(?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_RFC5424_WITH_PRI = /\A^\<(?<pri>[0-9]{1,3})\>[1-9]\d{0,2} (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|[^ ])) (?<message>.+)$\z/
       REGEXP_DETECT_RFC5424 = /^\<.*\>[1-9]\d{0,2}/

data/lib/fluent/plugin/filter_grep.rb

@@ -28,32 +28,59 @@ module Fluent
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
+
+    config_section :regexp, param_name: :regexps, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
+
+    config_section :exclude, param_name: :excludes, multi: true do
+      desc "The field name to which the regular expression is applied."
+      config_param :key, :string
+      desc "The regular expression."
+      config_param :pattern do |value|
+        Regexp.compile(value)
+      end
+    end
 
     # for test
-    attr_reader :regexps
-    attr_reader :excludes
+    attr_reader :_regexps
+    attr_reader :_excludes
 
     def configure(conf)
       super
 
-      @regexps = {}
+      @_regexps = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
-        @regexps[key] = Regexp.compile(regexp)
+        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
+        @_regexps[key] = Regexp.compile(regexp)
       end
 
-      @excludes = {}
+      @_excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
-        @excludes[key] = Regexp.compile(exclude)
+        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
+        @_excludes[key] = Regexp.compile(exclude)
+      end
+
+      @regexps.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
+        @_regexps[e.key] = e.pattern
+      end
+      @excludes.each do |e|
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
+        @_excludes[e.key] = e.pattern
       end
     end
 
@@ -61,10 +88,10 @@ module Fluent
       result = nil
       begin
         catch(:break_loop) do
-          @regexps.each do |key, regexp|
+          @_regexps.each do |key, regexp|
             throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
           end
-          @excludes.each do |key, exclude|
+          @_excludes.each do |key, exclude|
             throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
           end
           result = record

data/lib/fluent/plugin/in_syslog.rb

@@ -93,6 +93,10 @@ module Fluent
     config_param :source_host_key, :string, default: 'source_host'.freeze, deprecated: "use source_hostname_key instead"
     desc "The field name of the client's hostname."
     config_param :source_hostname_key, :string, default: nil
+    desc "The field name of the client's source address."
+    config_param :source_address_key, :string, default: nil
+    desc 'Try to resolve hostname from IP addresses or not.'
+    config_param :resolve_hostname, :bool, default: nil
     desc 'The field name of the priority.'
     config_param :priority_key, :string, default: nil
     desc 'The field name of the facility.'
@@ -101,14 +105,28 @@ module Fluent
     config_param :message_length_limit, :size, default: 2048
     config_param :blocking_timeout, :time, default: 0.5
 
+    desc 'If true, accept syslog message without PRI part'
+    config_param :allow_without_priority, :bool, default: false
+    # 13 is the default value of rsyslog and syslog-ng
+    desc 'The default PRI value (0 - 191 are available)'
+    config_param :default_priority, :integer, default: 13
+
     def configure(conf)
       super
 
+      if @default_priority < 0 && @default_priority > 191
+        raise ConfigError, "default_priority must be 0 ~ 191"
+      end
+
+      if @allow_without_priority && conf['message_format'] == 'auto'
+        raise ConfigError, "message_format auto isn't allowed when allow_without_priority is true"
+      end
+
+      conf['with_priority'] = !@allow_without_priority
       if conf.has_key?('format')
         @parser = Plugin.new_parser(conf['format'])
         @parser.configure(conf)
       else
-        conf['with_priority'] = true
         @parser = TextParser::SyslogParser.new
         @parser.configure(conf)
         @use_default = true
@@ -117,15 +135,25 @@ module Fluent
       if @source_hostname_key.nil? && @include_source_host
         @source_hostname_key = @source_host_key
       end
+      if @source_hostname_key
+        if @resolve_hostname.nil?
+          @resolve_hostname = true
+        elsif !@resolve_hostname # user specifies "false" in configure
+          raise Fluent::ConfigError, "resolve_hostname must be true with source_hostname_key"
+        end
+      end
     end
 
     def start
-      callback = if @use_default
-                   method(:receive_data)
+      callback = if @allow_without_priority
+                   method(:receive_data_allow_without_priority)
                  else
-                   method(:receive_data_parser)
+                   if @use_default
+                     method(:receive_data_default)
+                   else
+                     method(:receive_data_with_format)
+                   end
                  end
-
       @loop = Coolio::Loop.new
       @handler = listen(callback)
       @loop.attach(@handler)
@@ -149,7 +177,16 @@ module Fluent
 
     private
 
-    def receive_data_parser(data, addr)
+    ## SyslogParser with PRI part
+    def receive_data_default(data, addr)
+      parse_text(data, addr)
+    rescue => e
+      log.error data.dump, error: e.to_s
+      log.error_backtrace
+    end
+
+    ## PRI part parser + custom parser without PRI part
+    def receive_data_with_format(data, addr)
       m = SYSLOG_REGEXP.match(data)
       unless m
         log.warn "invalid syslog message: #{data.dump}"
@@ -158,48 +195,50 @@ module Fluent
       pri = m[1].to_i
       text = m[2]
 
-      @parser.parse(text) { |time, record|
-        unless time && record
-          log.warn "pattern not match: #{text.inspect}"
-          return
-        end
-
-        facility = FACILITY_MAP[pri >> 3]
-        priority = PRIORITY_MAP[pri & 0b111]
+      parse_text(text, addr, pri)
+    rescue => e
+      log.error data.dump, error: e.to_s
+      log.error_backtrace
+    end
 
-        record[@priority_key] = priority if @priority_key
-        record[@facility_key] = facility if @facility_key
-        record[@source_hostname_key] = addr[2] if @source_hostname_key
+    ## PRI part parser + SyslogParser without PRI part | custom parser without PRI part
+    def receive_data_allow_without_priority(data, addr)
+      m = SYSLOG_REGEXP.match(data)
+      if m
+        pri = m[1].to_i
+        text = m[2]
+      else
+        pri = @default_priority
+        text = data
+      end
 
-        tag = "#{@tag}.#{facility}.#{priority}"
-        emit(tag, time, record)
-      }
+      parse_text(text, addr, pri)
     rescue => e
       log.error data.dump, error: e.to_s
       log.error_backtrace
     end
 
-    def receive_data(data, addr)
-      @parser.parse(data) { |time, record|
+    def parse_text(text, addr, pri = nil)
+      @parser.parse(text) { |time, record|
         unless time && record
-          log.warn "invalid syslog message", data: data
+          log.warn "pattern not match: #{text.inspect}"
           return
         end
 
-        pri = record.delete('pri'.freeze)
+        ## from receive_data_default
+        pri = record.delete('pri'.freeze) unless pri
+
         facility = FACILITY_MAP[pri >> 3]
         priority = PRIORITY_MAP[pri & 0b111]
 
         record[@priority_key] = priority if @priority_key
         record[@facility_key] = facility if @facility_key
         record[@source_hostname_key] = addr[2] if @source_hostname_key
+        record[@source_address_key] = addr[3] if @source_address_key
 
         tag = "#{@tag}.#{facility}.#{priority}"
         emit(tag, time, record)
       }
-    rescue => e
-      log.error data.dump, error: e.to_s
-      log.error_backtrace
     end
 
     private
@@ -209,10 +248,10 @@ module Fluent
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
+        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, @resolve_hostname)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback, !!@source_hostname_key)
+        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback, @resolve_hostname)
       end
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.37'
+  VERSION = '0.12.38'
 
 end

data/test/plugin/test_filter_grep.rb

@@ -22,12 +22,51 @@ class GrepFilterTest < Test::Unit::TestCase
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._regexps['message'])
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance._excludes['message'])
+    end
+
+    sub_test_case "duplicate key" do
+      test "flat" do
+        conf = %[
+          regexp1 message test
+          regexp2 message test2
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "section" do
+        conf = %[
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+          <regexp>
+            key message
+            pattern test2
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+      test "mix" do
+        conf = %[
+          regexp1 message test
+          <regexp>
+            key message
+            pattern test
+          </regexp>
+        ]
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
     end
   end
 
@@ -75,6 +114,38 @@ class GrepFilterTest < Test::Unit::TestCase
       end
     end
 
+    test 'regexps' do
+      conf = %[
+        <regexp>
+          key message
+          pattern WARN
+        </regexp>
+      ]
+      es = emit(conf, messages)
+      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      assert_block('only WARN logs') do
+        es.all? { |t, r|
+          !r['message'].include?('INFO')
+        }
+      end
+    end
+
+    test 'excludes' do
+      conf = %[
+        <exclude>
+          key message
+          pattern favicon
+        </exclude>
+      ]
+      es = emit(conf, messages)
+      assert_equal(3, es.instance_variable_get(:@record_array).size)
+      assert_block('remove favicon logs') do
+        es.all? { |t, r|
+          !r['message'].include?('favicon')
+        }
+      end
+    end
+
     sub_test_case 'with invalid sequence' do
       def messages
         [

data/test/plugin/test_in_syslog.rb

@@ -35,6 +35,74 @@ class SyslogInputTest < Test::Unit::TestCase
     }
   end
 
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    test 'resolve_hostname must be true with source_hostname_key' do
+      assert_raise(Fluent::ConfigError) {
+        create_driver(CONFIG + <<EOS)
+resolve_hostname false
+source_hostname_key hostname
+EOS
+      }
+    end
+
+    LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
+    LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
+    DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'source_hostname_key and source_address_key parameter feature should add record(s)' do |keys|
+      conf = CONFIG.dup
+      if keys.include?(:hostname)
+        conf << <<EOL
+source_hostname_key source_hostname
+EOL
+        end
+      if keys.include?(:address)
+        conf << <<EOL
+source_address_key source_address
+EOL
+      end
+      tests = create_test_case
+      d = create_driver(conf)
+
+      d.run do
+        u = UDPSocket.new
+        u.connect('127.0.0.1', PORT)
+        tests.each {|test|
+          u.send(test['msg'], 0)
+        }
+        sleep 1
+      end
+
+      d.emits.each { |tag, _time, record|
+        if keys.include?(:hostname)
+          assert_true record.has_key?('source_hostname')
+          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
+          unless keys.include?(:address)
+            assert_false record.has_key?('source_address')
+          end
+        end
+        if keys.include?(:address)
+          assert_true record.has_key?('source_address')
+          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
+          unless keys.include?(:hostname)
+            assert_false record.has_key?('source_hostname')
+          end
+        end
+      }
+    end
+
+    data('resolve_hostname' => 'resolve_hostname true',
+         'source_hostname_key' => 'source_hostname_key source_host')
+    def test_configure_reslove_hostname(param)
+      d = create_driver([CONFIG, param].join("\n"))
+      assert_true d.instance.resolve_hostname
+    end
+  end
+
   def test_time_format
     configs = {'127.0.0.1' => CONFIG}
     configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
@@ -213,6 +281,77 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests, {facility: facility})
   end
 
+  def test_allow_without_priority_with_default_format
+    d = create_driver([CONFIG, 'allow_without_priority true'].join("\n"))
+
+    tests = [
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.user.notice'},
+    ]
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_allow_without_priority_with_json_format
+    d = create_driver([CONFIG, %[
+      allow_without_priority true
+      format json
+    ]].join("\n"))
+
+    message = 'foo'
+    tests = [
+      {'msg' => '<6>' + {'message' => message}.to_json + "\n", 'expected' => message},
+      {'msg' => {'message' => message}.to_json + "\n", 'expected' => message, 'tag' => 'syslog.user.notice'},
+    ]
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests)
+  end
+
+  def test_default_priority
+    d = create_driver([CONFIG, %[
+      allow_without_priority true
+      default_priority 100
+    ]].join("\n"))
+
+    tests = [
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.ntp.warn'},
+    ]
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests)
+
+  end
+
   def create_test_case(large_message = false)
     # actual syslog message has "\n"
     if large_message
@@ -231,7 +370,7 @@ class SyslogInputTest < Test::Unit::TestCase
 
   def compare_test_result(events, tests, options = {})
     events.each_index { |i|
-      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
+      assert_equal((tests[i]['tag'] || 'syslog.kern.info'), events[i][0]) # <6> means kern.info
       assert_equal(tests[i]['expected'], events[i][2]['message'])
       assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
       assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]

data/test/test_parser.rb

@@ -374,6 +374,24 @@ module ParserTest
       end
     end
 
+    def test_parse_various_characters_for_tag
+      ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
+      @parser.configure({})
+      @parser.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(@expected.merge('ident' => ident), record)
+      }
+    end
+
+    def test_parse_various_characters_for_tag_with_priority
+      ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
+      @parser.configure({'with_priority' => true})
+      @parser.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
+        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
+      }
+    end
+
     class TestRFC5424Regexp < self
       def test_parse_with_rfc5424_message
         @parser.configure(

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.37
+  version: 0.12.38
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack