fluentd 0.12.18 → 0.12.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07240444ef56a97335d1f42c0153a8be4059e77a
-  data.tar.gz: 31cec22258753e9e6e51780b6a8d33e1194879c2
+  metadata.gz: 251e0a23a2109a9b8434de8be77676e4acb84b0f
+  data.tar.gz: 495d9a6ca6baa5266da3c04f11f5b65336252674
 SHA512:
-  metadata.gz: 404559093a12caba1b073e0106287975f9e3dd3ea4ae8be191e95a71abf8d49b13c81c53539830bc2672e0a4741d5088cae800b6906fe3789a7849a8f19997b0
-  data.tar.gz: d8d4411e79d2211526ca7ce69d47263c18b7d8b47606be018a926b2dbf44a25195ffc35fa4aadb23a77067e390d7f4dbe2d395b9d81e912ad25aca085683e63b
+  metadata.gz: 9799adf3dbb5078f797aedf12275283b153e8201c1ab6fa4310bf80e9460911f3077499def817a82f2709cbc4a9fda82be1f88b7b0149b17d45b98cf4f8c237a
+  data.tar.gz: a1ef697d0b1ae85a414b9a3f65aaef5b108fea491c13edf0f9c17ab965d2dfd2606b276d81762c5d59f185aaba25bb74424b40d04048608dcb6b937d69411c2d

data/ChangeLog

@@ -1,5 +1,21 @@
 # v0.12
 
+## Release 0.12.19 - 2015/12/21
+
+### New features / Enhancement
+
+* input: Standardize logging level for source configs
+  https://github.com/fluent/fluentd/pull/743
+* buffer: Add buffer_queue_full_action parameter
+  https://github.com/fluent/fluentd/pull/745
+* parser/formatter: Use 'oj' when oj is installed in json parser/formatter
+  https://github.com/fluent/fluentd/pull/748
+
+### Bug fixes
+
+* fluent-cat: fix the regression of --msgpack option
+  https://github.com/fluent/fluentd/pull/747
+
 ## Release 0.12.18 - 2015/12/09
 
 ### New features / Enhancement

data/fluentd.gemspec

@@ -37,4 +37,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("timecop", [">= 0.3.0"])
   gem.add_development_dependency("test-unit", ["~> 3.1.4"])
   gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("oj", ["~> 2.14"])
 end

data/lib/fluent/buffer.rb

@@ -143,6 +143,8 @@ module Fluent
     config_param :buffer_chunk_limit, :size, :default => 8*1024*1024
     desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, :default => 256
+    desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
+    config_param :buffer_queue_full_action, :enum, :list => [:exception, :block], :default => :exception
 
     alias chunk_limit buffer_chunk_limit
     alias chunk_limit= buffer_chunk_limit=
@@ -151,6 +153,10 @@ module Fluent
 
     def configure(conf)
       super
+
+      if @buffer_queue_full_action == :block
+        $log.warn "'block' action stops input process until the buffer full is resolved. Check your pipeline this action is fit or not"
+      end
     end
 
     def start
@@ -179,16 +185,28 @@ module Fluent
       key = key.to_s
 
       synchronize do
-        # chunk unique id is generated in #new_chunk
-        chunk = (@map[key] ||= new_chunk(key))
+        begin
+          # chunk unique id is generated in #new_chunk
+          chunk = (@map[key] ||= new_chunk(key))
 
-        if storable?(chunk, data)
-          chain.next
-          chunk << data
-          return false
+          if storable?(chunk, data)
+            chain.next
+            chunk << data
+            return false
 
-        elsif @queue.size >= @buffer_queue_limit
-          raise BufferQueueLimitError, "queue size exceeds limit"
+          elsif @queue.size >= @buffer_queue_limit
+            raise BufferQueueLimitError, "queue size exceeds limit"
+          end
+        rescue BufferQueueLimitError => e
+          case @buffer_queue_full_action
+          when :exception
+            raise e
+          when :block
+            # This is rough implementation. New Buffer API should improve this routine by using wait/signal
+            $log.debug "buffer queue is full. Wait 1 second to re-emit events"
+            sleep 1
+            retry
+          end
         end
 
         if data.bytesize > @buffer_chunk_limit

data/lib/fluent/command/cat.rb

@@ -288,6 +288,8 @@ when 'json'
   end
 
 when 'msgpack'
+  require 'fluent/engine'
+
   begin
     u = Fluent::Engine.msgpack_factory.unpacker($stdin)
     u.each {|record|

data/lib/fluent/engine.rb

@@ -17,6 +17,7 @@
 module Fluent
   require 'fluent/event_router'
   require 'fluent/root_agent'
+  require 'fluent/time'
 
   class EngineClass
     class DummyMessagePackFactory

data/lib/fluent/formatter.rb

@@ -131,8 +131,23 @@ module Fluent
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
+      config_param :json_parser, :string, :default => 'oj'
+
+      def configure(conf)
+        super
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = {:mode => :compat}
+          @dump_proc = Oj.method(:dump)
+        rescue LoadError
+          @dump_proc = Yajl.method(:dump)
+        end
+      end
+
       def format_record(record)
-        "#{Yajl.dump(record)}\n"
+        "#{@dump_proc.call(record)}\n"
       end
     end
 

data/lib/fluent/parser.rb

@@ -237,6 +237,7 @@ module Fluent
     class JSONParser < Parser
       config_param :time_key, :string, :default => 'time'
       config_param :time_format, :string, :default => nil
+      config_param :json_parser, :string, :default => 'oj'
 
       def configure(conf)
         super
@@ -245,10 +246,20 @@ module Fluent
           @time_parser = TimeParser.new(@time_format)
           @mutex = Mutex.new
         end
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          @load_proc = Oj.method(:load)
+          @error_class = Oj::ParseError
+        rescue LoadError
+          @load_proc = Yajl.method(:load)
+          @error_class = Yajl::ParseError
+        end
       end
 
       def parse(text)
-        record = Yajl.load(text)
+        record = @load_proc.call(text)
 
         value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
         if value
@@ -274,7 +285,7 @@ module Fluent
         else
           return time, record
         end
-      rescue Yajl::ParseError
+      rescue @error_class
         if block_given?
           yield nil, nil
         else

data/lib/fluent/plugin/in_stream.rb

@@ -179,7 +179,7 @@ module Fluent
         File.unlink(@path)
       end
       FileUtils.mkdir_p File.dirname(@path)
-      log.debug "listening fluent socket on #{@path}"
+      log.info "listening fluent socket on #{@path}"
       s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
       s.listen(@backlog) unless @backlog.nil?
       s

data/lib/fluent/plugin/in_syslog.rb

@@ -173,7 +173,7 @@ module Fluent
     private
 
     def listen(callback)
-      log.debug "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
+      log.info "listening syslog socket on #{@bind}:#{@port} with #{@protocol_type}"
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)

data/lib/fluent/plugin/in_tcp.rb

@@ -25,7 +25,7 @@ module Fluent
     config_param :delimiter, :string, :default => "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
 
     def listen(callback)
-      log.debug "listening tcp socket on #{@bind}:#{@port}"
+      log.info "listening tcp socket on #{@bind}:#{@port}"
       Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, @delimiter, callback)
     end
   end

data/lib/fluent/plugin/in_udp.rb

@@ -24,7 +24,7 @@ module Fluent
     config_param :body_size_limit, :size, :default => 4096
 
     def listen(callback)
-      log.debug "listening udp socket on #{@bind}:#{@port}"
+      log.info "listening udp socket on #{@bind}:#{@port}"
       @usock = SocketUtil.create_udp_socket(@bind)
       @usock.bind(@bind, @port)
       SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)

data/lib/fluent/test/formatter_test.rb

@@ -16,6 +16,8 @@
 
 module Fluent
   module Test
+    require 'fluent/config'
+
     class FormatterTestDriver
       def initialize(klass_or_str, proc=nil, &block)
         if klass_or_str.is_a?(Class)
@@ -41,8 +43,7 @@ module Fluent
         when Fluent::Config::Element
           @config = conf
         when String
-          io = StringIO.new(conf)
-          @config = Config::Parser.parse(io, 'fluent.conf')
+          @config = Config.parse(conf, 'fluent.conf')
         when Hash
           @config = Config::Element.new('ROOT', '', conf, [])
         else

data/lib/fluent/test/parser_test.rb

@@ -16,6 +16,8 @@
 
 module Fluent
   module Test
+    require 'fluent/config'
+
     class ParserTestDriver
       def initialize(klass_or_str, format=nil, conf={}, &block)
         if klass_or_str.is_a?(Class)
@@ -47,8 +49,7 @@ module Fluent
         when Fluent::Config::Element
           @config = conf
         when String
-          io = StringIO.new(conf)
-          @config = Config::Parser.parse(io, 'fluent.conf')
+          @config = Config.parse(conf, 'fluent.conf')
         when Hash
           @config = Config::Element.new('ROOT', '', conf, [])
         else

data/lib/fluent/time.rb

@@ -0,0 +1 @@
+# placeholder for v0.14 time module

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.18'
+  VERSION = '0.12.19'
 
 end

data/test/plugin/test_out_stdout.rb

@@ -30,14 +30,20 @@ class StdoutOutputTest < Test::Unit::TestCase
     end
   end
 
-  def test_emit_json
-    d = create_driver(CONFIG + "\noutput_type json")
+  data('oj' => 'oj', 'yajl' => 'yajl')
+  def test_emit_json(data)
+    d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
     time = Time.now
     out = capture_log { d.emit({'test' => 'test'}, time) }
     assert_equal "#{time.localtime} test: {\"test\":\"test\"}\n", out
 
-    # NOTE: Float::NAN is not jsonable
-    assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
+    if data == 'yajl'
+      # NOTE: Float::NAN is not jsonable
+      assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
+    else
+      out = capture_log { d.emit({'test' => Float::NAN}, time) }
+      assert_equal "#{time.localtime} test: {\"test\":NaN}\n", out
+    end
   end
 
   def test_emit_hash

data/test/test_buffer.rb

@@ -4,6 +4,7 @@ require 'fluent/buffer'
 
 require 'stringio'
 require 'msgpack'
+require 'timeout'
 
 module FluentBufferTest
   class BufferTest < Test::Unit::TestCase
@@ -526,5 +527,80 @@ module FluentBufferTest
 
       assert chunks.reduce(true){|a,b| a && b.purged }
     end
+
+    sub_test_case 'buffer_queue_full_action' do
+      def dummy_buffer(action)
+        db = DummyBuffer.new
+        db.configure(
+          "buffer_chunk_limit" => 1024,
+          "buffer_queue_limit" => 1,
+          "buffer_queue_full_action" => action
+        )
+        db.start
+        db
+      end
+
+      def data
+        @data ||= "a" * 1024
+      end
+
+      def chain
+        @chain ||= DummyChain.new
+      end
+
+      def test_emit_with_exception
+        db = dummy_buffer(:exception)
+
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+
+        assert_raise(Fluent::BufferQueueLimitError) {
+          assert db.emit('key', data, chain)
+        }
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
+      def test_emit_with_block
+        db = dummy_buffer(:block)
+
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+
+        begin
+          # with block, emit events to full queue causes sleep loop
+          timeout(1) {
+            assert db.emit('key', data, chain)
+          }
+          flunk("timeout must happen")
+        rescue Timeout::Error => e
+        end
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
+      def pop_chunk(db)
+        out = DummyOutput.new
+        c1 = DummyChunk.new('k1', 1)
+
+        pop_return_value = nil
+        c1.synchronize do
+          pop_return_value = Thread.new {
+            db.pop(out)
+          }.value
+        end
+        pop_return_value
+      end
+    end
   end
 end

data/test/test_formatter.rb

@@ -130,15 +130,17 @@ module FormatterTest
       @time = Engine.now
     end
 
-    def test_format
-      @formatter.configure({})
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format(data)
+      @formatter.configure('json_parser' => data)
       formatted = @formatter.format(tag, @time, record)
 
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
 
-    def test_format_with_include_tag
-      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_tag(data)
+      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record
@@ -146,8 +148,9 @@ module FormatterTest
       assert_equal("#{Yajl.dump(r)}\n", formatted)
     end
 
-    def test_format_with_include_time
-      @formatter.configure('include_time_key' => 'true', 'localtime' => '')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_time(data)
+      @formatter.configure('include_time_key' => 'true', 'localtime' => '', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record
@@ -155,8 +158,9 @@ module FormatterTest
       assert_equal("#{Yajl.dump(r)}\n", formatted)
     end
 
-    def test_format_with_include_time_as_number
-      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_with_include_time_as_number(data)
+      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch', 'json_parser' => data)
       formatted = @formatter.format(tag, @time, record.dup)
 
       r = record

data/test/test_parser.rb

@@ -1,6 +1,7 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/parser'
+require 'fluent/plugin/string_util'
 
 module ParserTest
   include Fluent
@@ -372,7 +373,9 @@ module ParserTest
       @parser = TextParser::JSONParser.new
     end
 
-    def test_parse
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse(data)
+      @parser.configure('json_parser' => data)
       @parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
         assert_equal({
@@ -383,9 +386,11 @@ module ParserTest
       }
     end
 
-    def test_parse_without_time
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_without_time(data)
       time_at_start = Time.now.to_i
 
+      @parser.configure('json_parser' => data)
       @parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert time && time >= time_at_start, "parser puts current time without time input"
         assert_equal({
@@ -397,7 +402,7 @@ module ParserTest
 
       parser = TextParser::JSONParser.new
       parser.estimate_current_event = false
-      parser.configure({})
+      parser.configure('json_parser' => data)
       parser.parse('{"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
         assert_equal({
           'host'   => '192.168.0.1',
@@ -408,17 +413,21 @@ module ParserTest
       }
     end
 
-    def test_parse_with_invalid_time
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_invalid_time(data)
+      @parser.configure('json_parser' => data)
       assert_raise Fluent::ParserError do
         @parser.parse('{"time":[],"k":"v"}') { |time, record| }
       end
     end
 
-    def test_parse_with_keep_time_key
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_keep_time_key(data)
       parser = TextParser::JSONParser.new
       parser.configure(
-        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
-        'keep_time_key'=>'true',
+        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key' => 'true',
+        'json_parser' => data
       )
       text = "28/Feb/2013:12:00:00 +0900"
       parser.parse("{\"time\":\"#{text}\"}") do |time, record|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.18
+  version: 0.12.19
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-09 00:00:00.000000000 Z
+date: 2015-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -266,6 +266,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
 description: Fluentd is an open source data collector designed to scale and simplify
   log management. It can collect, process and ship many kinds of data in near real-time.
 email:
@@ -384,6 +398,7 @@ files:
 - lib/fluent/test/input_test.rb
 - lib/fluent/test/output_test.rb
 - lib/fluent/test/parser_test.rb
+- lib/fluent/time.rb
 - lib/fluent/timezone.rb
 - lib/fluent/version.rb
 - test/config/assertions.rb