fluentd 0.12.27 → 0.12.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6f3ea036e68e5a0a46afbe518b05590fec7addf
-  data.tar.gz: 1e73ba803fdbceec16d3e7d39f80744b696d271b
+  metadata.gz: 648289c24c5bb2d91083867a5e1bfb4199988800
+  data.tar.gz: 3415cae010e8d511e7b9ff3ec42cbc1bf2a57729
 SHA512:
-  metadata.gz: 8ea885c9bbd62870782c04c8da7e3a29e78fa28c2ecc944d57454cc271238dc11b7d0a9de56c77aaa922871151eba2520133399c48826711d6a2b6d3154c4910
-  data.tar.gz: 189f245592e68f78d61cda44c7253d7e5ec90a01369adf90e05a463a2a2461e6f26f838635d939b626f317af864d9c13e421db3a3190b6f2d59efd86c9a11567
+  metadata.gz: 89a9aab0f91f45d2eee9f6ed210a2788cbbddeb5ca357224f8d357e361ef960b8eaef45ed4f6fcd3b3772306129363698fc734c3fdff76b889a754b02bec4fe8
+  data.tar.gz: 5ca47b0c94fd74e29e4f81c7a7f21952c8aa058de62ea6ceb7fe9f4676483ef4e6fb0b7473ce1d018dedf18dec591e7508c5383ee8a6037ad8b495ad1554a8cb

data/ChangeLog

@@ -1,5 +1,17 @@
 # v0.12
 
+## Release 0.12.28 - 2016/08/14
+
+### New features / Enhancement
+
+* in_forward: Add source_hostname_key parameter
+  https://github.com/fluent/fluentd/pull/807
+
+### Bug fixes
+
+* parser: Ensure strict mode is used for Oj parser
+  https://github.com/fluent/fluentd/pull/1147
+
 ## Release 0.12.27 - 2016/08/03
 
 ### New features / Enhancement

data/lib/fluent/parser.rb

@@ -260,7 +260,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {bigdecimal_load: :float}
+          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError

data/lib/fluent/plugin/in_forward.rb

@@ -47,6 +47,8 @@ module Fluent
     config_param :chunk_size_limit, :size, default: nil
     desc 'Skip an event if incoming event is invalid.'
     config_param :skip_invalid_event, :bool, default: false
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
 
     def configure(conf)
       super
@@ -133,7 +135,7 @@ module Fluent
     #   3: object record
     #   4: object option (optional)
     # }
-    def on_message(msg, chunk_size, source)
+    def on_message(msg, chunk_size, peeraddr)
       if msg.nil?
         # for future TCP heartbeat_request
         return
@@ -141,7 +143,7 @@ module Fluent
 
       # TODO: raise an exception if broken chunk is generated by recoverable situation
       unless msg.is_a?(Array)
-        log.warn "incoming chunk is broken:", source: source, msg: msg
+        log.warn "incoming chunk is broken:", source: source_message(peeraddr), msg: msg
         return
       end
 
@@ -149,23 +151,24 @@ module Fluent
       entries = msg[1]
 
       if @chunk_size_limit && (chunk_size > @chunk_size_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source, limit: @chunk_size_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_limit', dropped:", tag: tag, source: source_message(peeraddr), limit: @chunk_size_limit, size: chunk_size
         return
       elsif @chunk_size_warn_limit && (chunk_size > @chunk_size_warn_limit)
-        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source, limit: @chunk_size_warn_limit, size: chunk_size
+        log.warn "Input chunk size is larger than 'chunk_size_warn_limit':", tag: tag, source: source_message(peeraddr), limit: @chunk_size_warn_limit, size: chunk_size
       end
 
       if entries.class == String
         # PackedForward
         es = MessagePackEventStream.new(entries)
-        es = check_and_skip_invalid_event(tag, es, source) if @skip_invalid_event
+        es = check_and_skip_invalid_event(tag, es, peeraddr) if @skip_invalid_event
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
       elsif entries.class == Array
         # Forward
         es = if @skip_invalid_event
-               check_and_skip_invalid_event(tag, entries, source)
+               check_and_skip_invalid_event(tag, entries, peeraddr)
              else
                es = MultiEventStream.new
                entries.each { |e|
@@ -177,6 +180,7 @@ module Fluent
                }
                es
              end
+        es = add_source_host(es, peeraddr[2]) if @source_hostname_key
         router.emit_stream(tag, es)
         option = msg[2]
 
@@ -185,11 +189,12 @@ module Fluent
         time = msg[1]
         record = msg[2]
         if @skip_invalid_event && invalid_event?(tag, time, record)
-          log.warn "got invalid event and drop it:", source: source, tag: tag, time: time, record: record
+          log.warn "got invalid event and drop it:", source: source_message(peeraddr), tag: tag, time: time, record: record
           return msg[3] # retry never succeeded so return ack and drop incoming event.
         end
         return if record.nil?
         time = Engine.now if time == 0
+        record[@source_hostname_key] = peeraddr[2] if @source_hostname_key
         router.emit(tag, time, record)
         option = msg[3]
       end
@@ -202,11 +207,11 @@ module Fluent
       !(time.is_a?(Integer) && record.is_a?(Hash) && tag.is_a?(String))
     end
 
-    def check_and_skip_invalid_event(tag, es, source)
+    def check_and_skip_invalid_event(tag, es, peeraddr)
       new_es = MultiEventStream.new
       es.each { |time, record|
         if invalid_event?(tag, time, record)
-          log.warn "skip invalid event:", source: source, tag: tag, time: time, record: record
+          log.warn "skip invalid event:", source: source_message(peeraddr), tag: tag, time: time, record: record
           next
         end
         new_es.add(time, record)
@@ -214,16 +219,29 @@ module Fluent
       new_es
     end
 
+    def add_source_host(es, host)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        record[@source_hostname_key] = host
+        new_es.add(time, record)
+      }
+      new_es
+    end
+
+    def source_message(peeraddr)
+      _, port, host, addr = peeraddr
+      "host: #{host}, addr: #{addr}, port: #{port}"
+    end
+
     class Handler < Coolio::Socket
       PEERADDR_FAILED = ["?", "?", "name resolusion failed", "?"]
 
       def initialize(io, linger_timeout, log, on_message)
         super(io)
 
+        @peeraddr = nil
         if io.is_a?(TCPSocket) # for unix domain socket support in the future
-          proto, port, host, addr = ( io.peeraddr rescue PEERADDR_FAILED )
-          @source = "host: #{host}, addr: #{addr}, port: #{port}"
-
+          @peeraddr = (io.peeraddr rescue PEERADDR_FAILED)
           opt = [1, linger_timeout].pack('I!I!')  # { int l_onoff; int l_linger; }
           io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
         end
@@ -252,7 +270,7 @@ module Fluent
           @serializer = :to_json.to_proc
           @y = Yajl::Parser.new
           @y.on_parse_complete = lambda { |obj|
-            option = @on_message.call(obj, @chunk_counter, @source)
+            option = @on_message.call(obj, @chunk_counter, @peeraddr)
             respond option if option
             @chunk_counter = 0
           }
@@ -280,7 +298,7 @@ module Fluent
       def on_read_msgpack(data)
         @chunk_counter += data.bytesize
         @u.feed_each(data) do |obj|
-          option = @on_message.call(obj, @chunk_counter, @source)
+          option = @on_message.call(obj, @chunk_counter, @peeraddr)
           respond option if option
           @chunk_counter = 0
         end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.27'
+  VERSION = '0.12.28'
 
 end

data/test/plugin/test_in_forward.rb

@@ -17,6 +17,7 @@ class ForwardInputTest < Test::Unit::TestCase
     port #{PORT}
     bind 127.0.0.1
   ]
+  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
 
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
@@ -197,7 +198,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -226,7 +227,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -253,7 +254,7 @@ class ForwardInputTest < Test::Unit::TestCase
     # d.run => send_data
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
       end
     end
 
@@ -275,7 +276,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     # d.run => send_data
     d.run do
-      d.instance.send(:on_message, data, 1000000000, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
+      d.instance.send(:on_message, data, 1000000000, PEERADDR)
     end
 
     # check emitted data
@@ -391,7 +392,6 @@ class ForwardInputTest < Test::Unit::TestCase
 
     assert_equal events, d.emits
     assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
-
   end
 
   def test_not_respond_to_message_not_requiring_ack
@@ -498,5 +498,53 @@ class ForwardInputTest < Test::Unit::TestCase
     @responses << res if try_to_receive_response
   end
 
+  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
+  test 'message protocol with source_hostname_key' do
+    execute_test { |events|
+      events.each { |tag, time, record|
+        send_data [tag, time, record].to_msgpack
+      }
+    }
+  end
+
+  test 'forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ['tag1', entries].to_msgpack
+    }
+  end
+
+  test 'packed forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = ''
+      events.each { |tag, time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    }
+  end
+
+  def execute_test(&block)
+    d = create_driver(CONFIG + 'source_hostname_key source')
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      block.call(events)
+    end
+
+    d.emits.each { |tag, _time, record|
+      assert_true record.has_key?('source')
+    }
+  end
+
   # TODO heartbeat
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.27
+  version: 0.12.28
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-03 00:00:00.000000000 Z
+date: 2016-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack