fluentd 0.12.33 → 0.12.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: daedf4abbe7d43acdd8df2b8ac75f4f93dc8f252
-  data.tar.gz: 99ea120dbf1257a4be97fc480a4f022b7cc638bf
+  metadata.gz: ffd21d1304c2f951d877baf533d64ddb647805de
+  data.tar.gz: 44faf3d4806bfc29b9cf2a1eab46b4fffd2c7b21
 SHA512:
-  metadata.gz: d9440abbc6e21242e9281fe3c9ca469fa12f9b47abe8115559c4702fdb5f763fa8576d986434ea76829da45b21a05edfead3217d8ab5cfc798f28de2dea0d982
-  data.tar.gz: dab81f6a5f102c884c0ab0ec6628934aeb5fff690e87ea24148b421fae70e8efab72425b59e8f558052fb8e7f6dd3a097a1268b9af794abd8e4f5e0397b67176
+  metadata.gz: e30eb92f16978286232710abdb1512144cfbe44c8b96b1acbe200530e9aa3b446e609c1d69b2b5b4ca82e3d3101f6755e0c090061713f90af6a0e024b736780e
+  data.tar.gz: f9bb8a3b4019a9de03ad06f6486f74de29abf5c49cf6a98eecc8720776ad8cf4aa01fd8a72c161c95092786bafec953cdda4d2a10d721cdb4de43abe9f14b819

data/ChangeLog

@@ -1,10 +1,24 @@
 # v0.12
 
+## Release 0.12.34 - 2017/03/25
+
+### New features / Enhancement
+
+* in_syslog/in_udp/in_tcp: Fix hostname lookup and unify parameter name to source_hostname_key
+  https://github.com/fluent/fluentd/pull/1515
+* in_http: Support application/msgpack header
+  https://github.com/fluent/fluentd/pull/1506
+
+### Bug fixes
+
+* engine: Label's plugins should be flushed when receive signal
+  https://github.com/fluent/fluentd/pull/1508
+
 ## Release 0.12.33 - 2017/03/09
 
 ### New features / Enhancement
 
-* in_syslog: Backport 'Support rfc5424 syslog format'
+* parser_syslog: Backport 'Support rfc5424 syslog format'
   https://github.com/fluent/fluentd/pull/1495
 * in_forward: Add source_address_key and fix source_hostname_key parameters
   https://github.com/fluent/fluentd/pull/1490

data/lib/fluent/plugin/in_http.rb

@@ -355,6 +355,8 @@ module Fluent
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif @content_type =~ /^application\/json/
           params['json'] = @body
+        elsif @content_type =~ /^application\/msgpack/
+          params['msgpack'] = @body
         end
         path_info = uri.path
 

data/lib/fluent/plugin/in_syslog.rb

@@ -88,15 +88,18 @@ module Fluent
       end
     end
     desc 'If true, add source host to event record.'
-    config_param :include_source_host, :bool, default: false
+    config_param :include_source_host, :bool, default: false, deprecated: "use source_hostname_key instead"
     desc 'Specify key of source host when include_source_host is true.'
-    config_param :source_host_key, :string, default: 'source_host'.freeze
+    config_param :source_host_key, :string, default: 'source_host'.freeze, deprecated: "use source_hostname_key instead"
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
     desc 'The field name of the priority.'
     config_param :priority_key, :string, default: nil
     desc 'The field name of the facility.'
     config_param :facility_key, :string, default: nil
-    config_param :blocking_timeout, :time, default: 0.5
+    desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 2048
+    config_param :blocking_timeout, :time, default: 0.5
 
     def configure(conf)
       super
@@ -110,6 +113,10 @@ module Fluent
         @parser.configure(conf)
         @use_default = true
       end
+
+      if @source_hostname_key.nil? && @include_source_host
+        @source_hostname_key = @source_host_key
+      end
     end
 
     def start
@@ -162,7 +169,7 @@ module Fluent
 
         record[@priority_key] = priority if @priority_key
         record[@facility_key] = facility if @facility_key
-        record[@source_host_key] = addr[2] if @include_source_host
+        record[@source_hostname_key] = addr[2] if @source_hostname_key
 
         tag = "#{@tag}.#{facility}.#{priority}"
         emit(tag, time, record)
@@ -185,7 +192,7 @@ module Fluent
 
         record[@priority_key] = priority if @priority_key
         record[@facility_key] = facility if @facility_key
-        record[@source_host_key] = addr[2] if @include_source_host
+        record[@source_hostname_key] = addr[2] if @source_hostname_key
 
         tag = "#{@tag}.#{facility}.#{priority}"
         emit(tag, time, record)
@@ -202,10 +209,10 @@ module Fluent
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
+        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
-        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback)
+        Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback, !!@source_hostname_key)
       end
     end
 

data/lib/fluent/plugin/in_tcp.rb

@@ -28,7 +28,7 @@ module Fluent
 
     def listen(callback)
       log.info "listening tcp socket on #{@bind}:#{@port}"
-      Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback)
+      Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback, !!@source_hostname_key)
     end
   end
 end

data/lib/fluent/plugin/in_udp.rb

@@ -21,13 +21,23 @@ module Fluent
     Plugin.register_input('udp', self)
 
     config_set_default :port, 5160
-    config_param :body_size_limit, :size, default: 4096
+
+    desc "Deprecated parameter. Use message_length_limit instead"
+    config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
+    desc "The max bytes of message"
+    config_param :message_length_limit, :size, default: 4096
+
+    def configure(conf)
+      super
+
+      @message_length_limit = @body_size_limit if @body_size_limit
+    end
 
     def listen(callback)
       log.info "listening udp socket on #{@bind}:#{@port}"
       @usock = SocketUtil.create_udp_socket(@bind)
       @usock.bind(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)
+      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
     end
   end
 end

data/lib/fluent/plugin/socket_util.rb

@@ -32,9 +32,10 @@ module Fluent
     module_function :create_udp_socket
 
     class UdpHandler < Coolio::IO
-      def initialize(io, log, body_size_limit, callback)
+      def initialize(io, log, body_size_limit, callback, resolve_hostname = false)
         super(io)
         @io = io
+        @io.do_not_reverse_lookup = !resolve_hostname
         @log = log
         @body_size_limit = body_size_limit
         @callback = callback
@@ -52,9 +53,10 @@ module Fluent
     class TcpHandler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
-      def initialize(io, log, delimiter, callback)
+      def initialize(io, log, delimiter, callback, resolve_hostname = false)
         super(io)
         if io.is_a?(TCPSocket)
+          io.do_not_reverse_lookup = resolve_hostname
           @addr = (io.peeraddr rescue PEERADDR_FAILED)
 
           opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
@@ -104,13 +106,16 @@ module Fluent
       config_param :port, :integer, default: 5150
       desc 'The bind address to listen to.'
       config_param :bind, :string, default: '0.0.0.0'
+      desc "Deprecated parameter. Use source_hostname_key instead"
+      config_param :source_host_key, :string, default: nil, deprecated: "Use source_hostname_key instead"
       desc "The field name of the client's hostname."
-      config_param :source_host_key, :string, default: nil
+      config_param :source_hostname_key, :string, default: nil
       config_param :blocking_timeout, :time, default: 0.5
 
       def configure(conf)
         super
 
+        @source_hostname_key = @source_host_key if @source_host_key
         @parser = Plugin.new_parser(@format)
         @parser.configure(conf)
       end
@@ -145,7 +150,7 @@ module Fluent
             return
           end
 
-          record[@source_host_key] = addr[3] if @source_host_key
+          record[@source_hostname_key] = addr[2] if @source_hostname_key
           router.emit(@tag, time, record)
         }
       rescue => e

data/lib/fluent/root_agent.rb

@@ -138,6 +138,11 @@ module Fluent
       super
     end
 
+    def flush!
+      super
+      @labels.each{ |name, label| label.flush! }
+    end
+
     def suppress_interval(interval_time)
       @suppress_emit_error_log_interval = interval_time
       @next_emit_error_log_time = Time.now.to_i

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.33'
+  VERSION = '0.12.34'
 
 end

data/test/plugin/test_in_http.rb

@@ -205,6 +205,22 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_application_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag,time,record|
+        res = post("/#{tag}?time=#{time.to_s}", record.to_msgpack, {"content-type"=>"application/msgpack; charset=utf-8"})
+        assert_equal "200", res.code
+      }
+    end
+  end
+  
   def test_msgpack
     d = create_driver
 

data/test/plugin/test_in_syslog.rb

@@ -155,15 +155,18 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests)
   end
 
-  def test_msg_size_with_include_source_host
-    d = create_driver([CONFIG, 'include_source_host'].join("\n"))
+  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
+  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
+
+  data('old parameter' => 'include_source_host',
+       'new parameter' => 'source_hostname_key source_host')
+  def test_msg_size_with_include_source_host(param)
+    d = create_driver([CONFIG, param].join("\n"))
     tests = create_test_case
 
-    host = nil
     d.run do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
-      host = u.peeraddr[2]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
@@ -171,7 +174,7 @@ class SyslogInputTest < Test::Unit::TestCase
     end
 
     assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {host: host})
+    compare_test_result(d.emits, tests, {host: LOCALHOST_HOSTNAME})
   end
 
   def test_msg_size_with_include_priority
@@ -235,4 +238,49 @@ class SyslogInputTest < Test::Unit::TestCase
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
+
+  class SyslogMessageFormatTest < self
+    def test_syslog_rfc5424_format
+      d = create_driver(CONFIG + 'message_format rfc5424')
+      tests = [
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
+      ]
+
+      run_tests(d, tests)
+      compare_test_result(d)
+    end
+
+    def test_syslog_auto_format
+      d = create_driver(CONFIG + 'message_format auto')
+      tests = [
+        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
+        '<6>Sep 11 00:00:00 localhost fluentd: Hi, from Fluentd!'
+      ]
+
+      run_tests(d, tests)
+      compare_test_result(d)
+    end
+
+    def run_tests(d, tests)
+      d.run do
+        u = Fluent::SocketUtil.create_udp_socket('127.0.0.1')
+        u.connect('127.0.0.1', PORT)
+        tests.each {|test|
+          u.send(test, 0)
+        }
+        sleep 1
+      end
+    end
+
+    def compare_test_result(d)
+      emits = d.emits
+      assert_equal 2, emits.size
+      emits.each_index {|i|
+        record =  emits[i][2]
+        assert_equal('fluentd', record['ident'])
+        assert_equal('Hi, from Fluentd!', record['message'])
+      }
+    end
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -32,10 +32,19 @@ class UdpInputTest < Test::Unit::TestCase
       d = create_driver(v)
       assert_equal PORT, d.instance.port
       assert_equal k, d.instance.bind
-      assert_equal 4096, d.instance.body_size_limit
+      assert_equal 4096, d.instance.message_length_limit
     }
   end
 
+  data(
+    'message_length_limit' => 'message_length_limit 2048',
+    'body_size_limit' => 'body_size_limit 2048'
+  )
+  test 'message_length_limit/body_size_limit compatibility' do |param|
+    d = create_driver(CONFIG + param)
+    assert_equal 2048, d.instance.message_length_limit
+  end
+
   def test_time_format
     configs = {'127.0.0.1' => CONFIG}
     configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.33
+  version: 0.12.34
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-09 00:00:00.000000000 Z
+date: 2017-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack