fluentd 0.12.23 → 0.12.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ddb93865f4a907d91de1d6518b0eecff806c20d
-  data.tar.gz: cc1925a4af249b8b599adfa0fb2095a39107fd85
+  metadata.gz: ab6393d8dfc67b70656190c768de2acb86e8b453
+  data.tar.gz: 70f4e1abb3b9d3dda769e4e9ded10dff755602c8
 SHA512:
-  metadata.gz: 9dec5f61c9e67be7ca905cae26026f5097cc741fdf2ffc382db9dbfe1e32d64b78b13a30e256bc654c1be6c25543546353ffe910af0c43c1a903a68a6c421d8b
-  data.tar.gz: 29a02b240b0798f44412d1112f3586a21801636b9e1b91d505c3d7c9fc1051f1ea9f5c2e38faf307512a74743e3282acca361dbe302922b5b150bf0204a1acf3
+  metadata.gz: 222628da1465e330bed1f54024845fc9cebd54e80b2c457fb61c8ec46ac8f429a5bd92d43530fb7bc1b34376a00c363ad6b39dedcfd13520a554cafc6d7d7468
+  data.tar.gz: 00061a3007fd77070568dd0fb59460cf059fe13bdb9d1ce90823e89ce9f91a121a2adec4961d4978987c3198468229edc0bbd88fc0d0c3b68a67d02afc39d8d8

data/.travis.yml

@@ -1,7 +1,6 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1
   - 2.2.3
@@ -32,3 +31,4 @@ matrix:
     - rvm: ruby-head
     - rvm: rbx-2
     - rvm: 2.3.0
+    - rvm: 1.9.3

data/ChangeLog

@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.24 - 2016/05/20
+
+### New features / Enhancement
+
+* buffer: Add drop_oldest_chunk to buffer_queue_full_action parameter
+  https://github.com/fluent/fluentd/pull/934
+* in_tail: Add "encoding" parameter
+  https://github.com/fluent/fluentd/pull/889
+* in_tail: Add "path_key" patameter
+  https://github.com/fluent/fluentd/pull/951
+* in_monitor_agent: Add emit_config parameter to emit plugin configuration together
+  https://github.com/fluent/fluentd/pull/963
+* config: Support multiline string in " quoted string
+  https://github.com/fluent/fluentd/pull/929
+
+### Bug fixes
+
+* in_http: Add 'Access-Control-Allow-Origin' to response header
+  https://github.com/fluent/fluentd/pull/882
+* parser: Fix logger on Fluent::StringUtil to use $log
+  https://github.com/fluent/fluentd/pull/926
+* out_forward: Add missing error class definition
+  https://github.com/fluent/fluentd/pull/953
+
 ## Release 0.12.23 - 2016/05/03
 
 ### Bug fixes

data/fluentd.gemspec

@@ -27,7 +27,9 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0.0"])
   gem.add_runtime_dependency("tzinfo-data", [">= 1.0.0"])
-  gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
+  unless "".respond_to?(:scrub)
+    gem.add_runtime_dependency("string-scrub", [">= 0.1.1"])
+  end
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
   gem.add_development_dependency("flexmock", ["~> 1.3.3"])

data/lib/fluent/buffer.rb

@@ -150,7 +150,7 @@ module Fluent
     desc 'The length limit of the chunk queue.'
     config_param :buffer_queue_limit, :integer, default: 256
     desc 'The action when the size of buffer queue exceeds the buffer_queue_limit.'
-    config_param :buffer_queue_full_action, :enum, list: [:exception, :block], default: :exception
+    config_param :buffer_queue_full_action, :enum, list: [:exception, :block, :drop_oldest_chunk], default: :exception
 
     alias chunk_limit buffer_chunk_limit
     alias chunk_limit= buffer_chunk_limit=
@@ -212,6 +212,9 @@ module Fluent
             $log.debug "buffer queue is full. Wait 1 second to re-emit events"
             sleep 1
             retry
+          when :drop_oldest_chunk
+            $log.debug "buffer queue is full. Dropping oldest chunk"
+            pop(nil)
           end
         end
 
@@ -326,7 +329,7 @@ module Fluent
       begin
         # #push(key) does not push empty chunks into queue.
         # so this check is nonsense...
-        if !chunk.empty?
+        if !chunk.empty? && !out.nil?
           write_chunk(chunk, out)
         end
 

data/lib/fluent/config/basic_parser.rb

@@ -28,6 +28,7 @@ module Fluent
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
       ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
+      LINE_END_WITHOUT_SPACING_AND_COMMENT = /(?:\z|[\r\n])/
 
       module ClassMethods
         def symbol(string)
@@ -71,6 +72,10 @@ module Fluent
         @ss[0]
       end
 
+      def check(pattern)
+        @ss.check(pattern)
+      end
+
       def line_end
         skip(LINE_END)
       end

data/lib/fluent/config/literal_parser.rb

@@ -81,6 +81,11 @@ module Fluent
         while true
           if skip(/\"/)
             return string.join
+          elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+            if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
+              string << s
+            end
+            skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
           elsif s = scan(/\\./)
             string << eval_escape_char(s[1,1])
           elsif skip(/\#\{/)

data/lib/fluent/plugin/in_http.rb

@@ -362,6 +362,7 @@ module Fluent
         code, header, body = *@callback.call(path_info, params)
         body = body.to_s
 
+        header['Access-Control-Allow-Origin'] = @origin if !@cors_allow_origins.nil? && @cors_allow_origins.include?(@origin)
         if @keep_alive
           header['Connection'] = 'Keep-Alive'
           send_response(code, header, body)

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -32,6 +32,7 @@ module Fluent
     config_param :port, :integer, default: 24220
     config_param :tag, :string, default: nil
     config_param :emit_interval, :time, default: 60
+    config_param :emit_config, :bool, default: false
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -253,7 +254,7 @@ module Fluent
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {with_config: false}
+        opts = {with_config: @emit_config}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now

data/lib/fluent/plugin/in_tail.rb

@@ -50,6 +50,16 @@ module Fluent
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'The encoding of the input.'
+    config_param :encoding, default: nil do |encoding_name|
+      begin
+        Encoding.find(encoding_name)
+      rescue ArgumentError => e
+        raise ConfigError, e.message
+      end
+    end
+    desc 'Add the log path being tailed to records. Specify the field name to be used.'
+    config_param :path_key, :string, default: nil
 
     attr_reader :paths
 
@@ -222,6 +232,7 @@ module Fluent
     def flush_buffer(tw)
       if lb = tw.line_buffer
         lb.chomp!
+        lb.force_encoding(@encoding) if @encoding
         @parser.parse(lb) { |time, record|
           if time && record
             tag = if @tag_prefix || @tag_suffix
@@ -229,6 +240,7 @@ module Fluent
                   else
                     @tag
                   end
+            record[@path_key] ||= tw.path unless @path_key.nil?
             router.emit(tag, time, record)
           else
             log.warn "got incomplete line at shutdown from #{tw.path}: #{lb.inspect}"
@@ -266,11 +278,13 @@ module Fluent
       return true
     end
 
-    def convert_line_to_event(line, es)
+    def convert_line_to_event(line, es, tail_watcher)
       begin
         line.chomp!  # remove \n
+        line.force_encoding(@encoding) if @encoding
         @parser.parse(line) { |time, record|
           if time && record
+            record[@path_key] ||= tail_watcher.path unless @path_key.nil?
             es.add(time, record)
           else
             log.warn "pattern not match: #{line.inspect}"
@@ -285,7 +299,7 @@ module Fluent
     def parse_singleline(lines, tail_watcher)
       es = MultiEventStream.new
       lines.each { |line|
-        convert_line_to_event(line, es)
+        convert_line_to_event(line, es, tail_watcher)
       }
       es
     end
@@ -298,7 +312,7 @@ module Fluent
         lines.each { |line|
           if @parser.firstline?(line)
             if lb
-              convert_line_to_event(lb, es)
+              convert_line_to_event(lb, es, tail_watcher)
             end
             lb = line
           else
@@ -315,7 +329,7 @@ module Fluent
           lb << line
           @parser.parse(lb) { |time, record|
             if time && record
-              convert_line_to_event(lb, es)
+              convert_line_to_event(lb, es, tail_watcher)
               lb = ''
             end
           }

data/lib/fluent/plugin/out_forward.rb

@@ -30,6 +30,9 @@ module Fluent
   class ForwardOutputResponseError < ForwardOutputError
   end
 
+  class ForwardOutputConnectionClosedError < ForwardOutputError
+  end
+
   class ForwardOutputACKTimeoutError < ForwardOutputResponseError
   end
 

data/lib/fluent/plugin/string_util.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require "string-scrub" unless "".respond_to?(:scrub)
+
 module Fluent
   module StringUtil
     def match_regexp(regexp, string)
@@ -21,7 +23,7 @@ module Fluent
         return regexp.match(string)
       rescue ArgumentError => e
         raise e unless e.message.index("invalid byte sequence in".freeze).zero?
-        log.info "invalid byte sequence is replaced in `#{string}`"
+        $log.info "invalid byte sequence is replaced in `#{string}`"
         string = string.scrub('?')
         retry
       end

data/lib/fluent/registry.rb

@@ -40,7 +40,7 @@ module Fluent
       if value = @map[type]
         return value
       end
-      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluentd-plugin' to find plugins"  # TODO error class
+      raise ConfigError, "Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"  # TODO error class
     end
 
     def search(type)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.23'
+  VERSION = '0.12.24'
 
 end

data/test/config/test_config_parser.rb

@@ -5,6 +5,7 @@ require "fluent/config/error"
 require "fluent/config/basic_parser"
 require "fluent/config/literal_parser"
 require "fluent/config/v1_parser"
+require 'fluent/config/parser'
 
 module Fluent::Config
   module V1TestHelper
@@ -115,6 +116,7 @@ module Fluent::Config
         test "requires escaping double quote" do
           assert_text_parsed_as(e('ROOT', '', {"k1" => '"'}), '  k1 "\\""')
           assert_parse_error('  k1 """')
+          assert_parse_error('  k1 ""\'')
         end
 
         test "removes backslash in front of a normal character" do
@@ -124,6 +126,39 @@ module Fluent::Config
         test "accepts escape characters" do
           assert_text_parsed_as(e('ROOT', '', {"k1" => "\n"}), '  k1 "\\n"')
         end
+
+        test "support multiline string" do
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2]
+            }),
+            %[k1      "line1
+                       line2"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1                       line2]
+            }),
+            %[k1      "line1\\
+                       line2"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2
+                       line3]
+            }),
+            %[k1      "line1
+                       line2
+                       line3"]
+          )
+          assert_text_parsed_as(e('ROOT', '',
+            {"k1" => %[line1
+                       line2                       line3]
+            }),
+            %[k1      "line1
+                       line2\\
+                       line3"]
+          )
+        end
       end
 
       sub_test_case 'single quoted string' do

data/test/config/test_literal_parser.rb

@@ -109,6 +109,8 @@ module Fluent::Config
       test('"\\0"') { assert_parse_error('"\\0"') }  # unknown escaped character
       test('"\\1"') { assert_parse_error('"\\1"') }  # unknown escaped character
       test('"t') { assert_parse_error('"t') }  # non-terminated quoted character
+      test("\"t\nt\"") { assert_text_parsed_as("t\nt", "\"t\nt\"" ) } # multiline string
+      test("\"t\\\nt\"") { assert_text_parsed_as("tt", "\"t\\\nt\"" ) } # multiline string
       test('t"') { assert_text_parsed_as('t"', 't"') }
       test('"."') { assert_text_parsed_as('.', '"."') }
       test('"*"') { assert_text_parsed_as('*', '"*"') }

data/test/plugin/test_filter_stdout.rb

@@ -1,7 +1,7 @@
 require_relative '../helper'
 require 'fluent/plugin/filter_stdout'
 require 'timecop'
-require 'flexmock'
+require 'flexmock/test_unit'
 
 class StdoutFilterTest < Test::Unit::TestCase
   include Fluent
@@ -13,6 +13,8 @@ class StdoutFilterTest < Test::Unit::TestCase
   end
 
   def teardown
+    super # FlexMock::TestCase requires this
+    # http://flexmock.rubyforge.org/FlexMock/TestCase.html
     Timecop.return
   end
 
@@ -101,12 +103,12 @@ class StdoutFilterTest < Test::Unit::TestCase
 
   # Capture the log output of the block given
   def capture_log(d, &block)
-    tmp = d.instance.log
-    d.instance.log = StringIO.new
+    tmp = d.instance.log.out
+    d.instance.log.out = StringIO.new
     yield
-    return d.instance.log.string
+    return d.instance.log.out.string
   ensure
-    d.instance.log = tmp
+    d.instance.log.out = tmp
   end
 end
 

data/test/plugin/test_in_http.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_http'
 require 'net/http'
 
 class HttpInputTest < Test::Unit::TestCase
@@ -269,6 +270,79 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_cors_allowed
+    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    assert_equal ["http://foo.com"], d.instance.cors_allow_origins
+
+    test_in_http_cros_allowed = nil
+    acao = nil
+
+    begin
+      d.run do
+        Net::HTTP.start("127.0.0.1", PORT) do |http|
+          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://foo.com"})
+          res = http.request(req)
+
+          acao = res["Access-Control-Allow-Origin"]
+        end
+      end
+      test_in_http_cros_allowed = true
+    rescue
+      test_in_http_cros_allowed = false
+    end
+
+    assert_equal true, test_in_http_cros_allowed
+    assert_equal "http://foo.com", acao
+  end
+
+  def test_cors_disallowed
+    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    assert_equal ["http://foo.com"], d.instance.cors_allow_origins
+
+    test_in_http_cros_disallowed = nil
+    response_code = nil
+
+    begin
+      d.run do
+        Net::HTTP.start("127.0.0.1", PORT) do |http|
+          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://bar.com"})
+          res = http.request(req)
+
+          response_code = res.code
+        end
+      end
+      test_in_http_cros_disallowed = true
+    rescue
+      test_in_http_cros_disallowed = false
+    end
+
+    assert_equal true, test_in_http_cros_disallowed
+    assert_equal "403", response_code
+  end
+
+  $test_in_http_connection_object_ids = []
+  $test_in_http_content_types = []
+  $test_in_http_content_types_flag = false
+  module ContentTypeHook
+    def on_headers_complete(headers)
+      super
+      if $test_in_http_content_types_flag
+        $test_in_http_content_types << self.content_type
+      end
+    end
+
+    def on_message_begin
+      super
+      if $test_in_http_content_types_flag
+        $test_in_http_connection_object_ids << @io_handler.object_id
+      end
+    end
+  end
+
+  class Fluent::HttpInput::Handler
+    prepend ContentTypeHook
+  end
+
   def test_if_content_type_is_initialized_properly
     # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.
     # Especially when in Keep-Alive and the second request has no 'Content-Type'.

data/test/plugin/test_in_tail.rb

@@ -1,7 +1,7 @@
 require_relative '../helper'
 require 'fluent/test'
 require 'net/http'
-require 'flexmock'
+require 'flexmock/test_unit'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -51,6 +51,17 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal 1000, d.instance.read_lines_limit
   end
 
+  def test_configure_encoding
+    # valid encoding
+    d = create_driver(SINGLE_LINE_CONFIG + 'encoding utf-8')
+    assert_equal Encoding::UTF_8, d.instance.encoding
+
+    # invalid encoding
+    assert_raise(Fluent::ConfigError) do
+      create_driver(SINGLE_LINE_CONFIG + 'encoding no-such-encoding')
+    end
+  end
+
   # TODO: Should using more better approach instead of sleep wait
 
   def test_emit
@@ -300,6 +311,27 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal({"message" => "tab	"}, emits[5][2])
   end
 
+  data(
+    'default encoding' => ['', Encoding::ASCII_8BIT],
+    'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
+  def test_encoding(data)
+    encoding_config, encoding = data
+
+    d = create_driver(SINGLE_LINE_CONFIG + CONFIG_READ_FROM_HEAD + encoding_config)
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(encoding, emits[0][2]['message'].encoding)
+  end
+
   # multiline mode test
 
   def test_multiline
@@ -377,6 +409,33 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  data(
+    'default encoding' => ['', Encoding::ASCII_8BIT],
+    'explicit encoding config' => ['encoding utf-8', Encoding::UTF_8])
+  def test_multiline_encoding_of_flushed_record(data)
+    encoding_config, encoding = data
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      multiline_flush_interval 2s
+      read_from_head true
+      #{encoding_config}
+    ]
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f|
+        f.puts "s test"
+      }
+
+      sleep 4
+      emits = d.emits
+      assert_equal(1, emits.length)
+      assert_equal(encoding, emits[0][2]['message1'].encoding)
+    end
+  end
+
   def test_multiline_with_multiple_formats
     File.open("#{TMP_DIR}/tail.txt", "w") { |f| }
 
@@ -680,4 +739,119 @@ class TailInputTest < Test::Unit::TestCase
       d.emits
     end
   end
+
+  sub_test_case "tail_path" do
+    def test_tail_path_with_singleline
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(%[path_key path] + SINGLE_LINE_CONFIG)
+
+      d.run do
+        sleep 1
+
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert_equal(true, emits.length > 0)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_with_firstline
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver %[
+        path_key path
+        format multiline
+        format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+        format_firstline /^[s]/
+      ]
+      d.run do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+          f.puts "f test1"
+          f.puts "s test2"
+          f.puts "f test3"
+          f.puts "f test4"
+          f.puts "s test5"
+          f.puts "s test6"
+          f.puts "f test7"
+          f.puts "s test8"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length == 4)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_without_firstline
+      File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+      d = create_driver %[
+        path_key path
+        format multiline
+        format1 /(?<var1>foo \\d)\\n/
+        format2 /(?<var2>bar \\d)\\n/
+        format3 /(?<var3>baz \\d)/
+      ]
+      d.run do
+        File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+          f.puts "foo 1"
+          f.puts "bar 1"
+          f.puts "baz 1"
+        }
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length > 0)
+      emits.each do |emit|
+        assert_equal("#{TMP_DIR}/tail.txt", emit[2]["path"])
+      end
+    end
+
+    def test_tail_path_with_multiline_with_multiple_paths
+      files = ["#{TMP_DIR}/tail1.txt", "#{TMP_DIR}/tail2.txt"]
+      files.each { |file| File.open(file, "wb") { |f| } }
+
+      d = create_driver(%[
+        path #{files[0]},#{files[1]}
+        path_key path
+        tag t1
+        format multiline
+        format1 /^[s|f] (?<message>.*)/
+        format_firstline /^[s]/
+      ], false)
+      d.run do
+        files.each do |file|
+          File.open(file, 'ab') { |f|
+            f.puts "f #{file} line should be ignored"
+            f.puts "s test1"
+            f.puts "f test2"
+            f.puts "f test3"
+            f.puts "s test4"
+          }
+        end
+        sleep 1
+      end
+
+      emits = d.emits
+      assert(emits.length == 4)
+      assert_equal(files, [emits[0][2]["path"], emits[1][2]["path"]].sort)
+      # "test4" events are here because these events are flushed at shutdown phase
+      assert_equal(files, [emits[2][2]["path"], emits[3][2]["path"]].sort)
+    end
+  end
 end

data/test/plugin/test_string_util.rb

@@ -0,0 +1,26 @@
+require_relative '../helper'
+require 'fluent/plugin/string_util'
+
+class StringUtilTest < Test::Unit::TestCase
+  def setup
+    @null_value_pattern = Regexp.new("^(-|null|NULL)$")
+  end
+
+  sub_test_case 'valid string' do
+    test 'null string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "null").to_s, "null"
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "NULL").to_s, "NULL"
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "-").to_s, "-"
+    end
+
+    test 'normal string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "fluentd"), nil
+    end
+  end
+
+  sub_test_case 'invalid string' do
+    test 'normal string' do
+      assert_equal Fluent::StringUtil.match_regexp(@null_value_pattern, "\xff"), nil
+    end
+  end
+end

data/test/test_buffer.rb

@@ -589,6 +589,24 @@ module FluentBufferTest
         assert db.emit('key', data, chain)
       end
 
+      def test_emit_with_drop_oldest_chunk
+        db = dummy_buffer(:drop_oldest_chunk)
+      
+        assert !db.emit('key', data, chain)
+        assert db.emit('key', data, chain)
+        
+        # oldest (and only) chunk in queue is dropped before newer data is enqueued 
+        assert db.emit('key', data, chain)
+        
+        assert db.queue.size == 1
+
+        assert !pop_chunk(db)
+        assert db.queue.size == 0
+
+        # queue is now empty so can emit data again
+        assert db.emit('key', data, chain)
+      end
+
       def pop_chunk(db)
         out = DummyOutput.new
         c1 = DummyChunk.new('k1', 1)

data/test/test_log.rb

@@ -0,0 +1,340 @@
+require_relative 'helper'
+require 'fluent/engine'
+require 'fluent/log'
+
+class LogTest < Test::Unit::TestCase
+  def setup
+    @log_device = Fluent::Test::DummyLogDevice.new
+    @timestamp = Time.parse("2016-04-21 11:58:41 +0900")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    stub(Time).now { @timestamp }
+  end
+
+  def teardown
+    Thread.current[:last_repeated_stacktrace] = nil
+  end
+
+  sub_test_case "log level" do
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 1],
+      info: [Fluent::Log::LEVEL_INFO, 2],
+      warn: [Fluent::Log::LEVEL_WARN, 3],
+      error: [Fluent::Log::LEVEL_ERROR, 4],
+      fatal: [Fluent::Log::LEVEL_FATAL, 5],
+    )
+    def test_output(data)
+      log_level, start = data
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace "trace log"
+      log.debug "debug log"
+      log.info "info log"
+      log.warn "warn log"
+      log.error "error log"
+      log.fatal "fatal log"
+      expected = [
+        "#{@timestamp_str} [trace]: trace log\n",
+        "#{@timestamp_str} [debug]: debug log\n",
+        "#{@timestamp_str} [info]: info log\n",
+        "#{@timestamp_str} [warn]: warn log\n",
+        "#{@timestamp_str} [error]: error log\n",
+        "#{@timestamp_str} [fatal]: fatal log\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 1],
+      info: [Fluent::Log::LEVEL_INFO, 2],
+      warn: [Fluent::Log::LEVEL_WARN, 3],
+      error: [Fluent::Log::LEVEL_ERROR, 4],
+      fatal: [Fluent::Log::LEVEL_FATAL, 5],
+    )
+    def test_output_with_block(data)
+      log_level, start = data
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace { "trace log" }
+      log.debug { "debug log" }
+      log.info { "info log" }
+      log.warn { "warn log" }
+      log.error { "error log" }
+      log.fatal { "fatal log" }
+      expected = [
+        "#{@timestamp_str} [trace]: trace log\n",
+        "#{@timestamp_str} [debug]: debug log\n",
+        "#{@timestamp_str} [info]: info log\n",
+        "#{@timestamp_str} [warn]: warn log\n",
+        "#{@timestamp_str} [error]: error log\n",
+        "#{@timestamp_str} [fatal]: fatal log\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, { trace: true, debug: true, info: true, warn: true, error: true, fatal: true }],
+      debug: [Fluent::Log::LEVEL_DEBUG, { trace: false, debug: true, info: true, warn: true, error: true, fatal: true }],
+      info: [Fluent::Log::LEVEL_INFO, { trace: false, debug: false, info: true, warn: true, error: true, fatal: true }],
+      warn: [Fluent::Log::LEVEL_WARN, { trace: false, debug: false, info: false, warn: true, error: true, fatal: true }],
+      error: [Fluent::Log::LEVEL_ERROR, { trace: false, debug: false, info: false, warn: false, error: true, fatal: true }],
+      fatal: [Fluent::Log::LEVEL_FATAL, { trace: false, debug: false, info: false, warn: false, error: false, fatal: true }],
+    )
+    def test_execute_block(data)
+      log_level, expected = data
+      log = Fluent::Log.new(@log_device, log_level)
+      block_called = {
+        trace: false,
+        debug: false,
+        info: false,
+        warn: false,
+        error: false,
+        fatal: false,
+      }
+      log.trace { block_called[:trace] = true }
+      log.debug { block_called[:debug] = true }
+      log.info { block_called[:info] = true }
+      log.warn { block_called[:warn] = true }
+      log.error { block_called[:error] = true }
+      log.fatal { block_called[:fatal] = true }
+      assert_equal(expected, block_called)
+    end
+
+    data(
+      trace: [Fluent::Log::LEVEL_TRACE, 0],
+      debug: [Fluent::Log::LEVEL_DEBUG, 3],
+      info: [Fluent::Log::LEVEL_INFO, 6],
+      warn: [Fluent::Log::LEVEL_WARN, 9],
+      error: [Fluent::Log::LEVEL_ERROR, 12],
+      fatal: [Fluent::Log::LEVEL_FATAL, 15],
+    )
+    def test_backtrace(data)
+      log_level, start = data
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, log_level)
+      log.trace_backtrace(backtrace)
+      log.debug_backtrace(backtrace)
+      log.info_backtrace(backtrace)
+      log.warn_backtrace(backtrace)
+      log.error_backtrace(backtrace)
+      log.fatal_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [debug]: line 1\n",
+        "  #{@timestamp_str} [debug]: line 2\n",
+        "  #{@timestamp_str} [debug]: line 3\n",
+        "  #{@timestamp_str} [info]: line 1\n",
+        "  #{@timestamp_str} [info]: line 2\n",
+        "  #{@timestamp_str} [info]: line 3\n",
+        "  #{@timestamp_str} [warn]: line 1\n",
+        "  #{@timestamp_str} [warn]: line 2\n",
+        "  #{@timestamp_str} [warn]: line 3\n",
+        "  #{@timestamp_str} [error]: line 1\n",
+        "  #{@timestamp_str} [error]: line 2\n",
+        "  #{@timestamp_str} [error]: line 3\n",
+        "  #{@timestamp_str} [fatal]: line 1\n",
+        "  #{@timestamp_str} [fatal]: line 2\n",
+        "  #{@timestamp_str} [fatal]: line 3\n"
+      ][start..-1]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
+  sub_test_case "suppress repeated backtrace" do
+    def test_same_log_level
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE, suppress_repeated_stacktrace: true)
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace + ["line 4"])
+      log.trace_backtrace(backtrace)
+      log.trace_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: line 4\n",
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [trace]: suppressed same stacktrace\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+
+    def test_different_log_level
+      backtrace = ["line 1", "line 2", "line 3"]
+      log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE, suppress_repeated_stacktrace: true)
+      log.trace_backtrace(backtrace)
+      log.debug_backtrace(backtrace)
+      log.info_backtrace(backtrace)
+      log.warn_backtrace(backtrace)
+      log.error_backtrace(backtrace)
+      log.fatal_backtrace(backtrace)
+      expected = [
+        "  #{@timestamp_str} [trace]: line 1\n",
+        "  #{@timestamp_str} [trace]: line 2\n",
+        "  #{@timestamp_str} [trace]: line 3\n",
+        "  #{@timestamp_str} [debug]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [info]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [warn]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [error]: suppressed same stacktrace\n",
+        "  #{@timestamp_str} [fatal]: suppressed same stacktrace\n",
+      ]
+      assert_equal(expected, log.out.logs)
+    end
+  end
+
+  def test_dup
+    log1 = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+    log2 = log1.dup
+    log1.level = Fluent::Log::LEVEL_DEBUG
+    original_tag = log1.tag
+    log1.tag = "changed"
+    assert_equal(Fluent::Log::LEVEL_DEBUG, log1.level)
+    assert_equal(Fluent::Log::LEVEL_TRACE, log2.level)
+    assert_equal("changed", log1.tag)
+    assert_equal(original_tag, log2.tag)
+  end
+
+  def test_disable_events
+    log = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+    engine = log.instance_variable_get("@engine")
+    mock(engine).push_log_event(anything, anything, anything).once
+    log.trace "trace log"
+    log.disable_events(Thread.current)
+    log.trace "trace log"
+  end
+end
+
+class PluginLoggerTest < Test::Unit::TestCase
+  def setup
+    @log_device = Fluent::Test::DummyLogDevice.new
+    @timestamp = Time.parse("2016-04-21 11:58:41 +0900")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    stub(Time).now { @timestamp }
+    @logger = Fluent::Log.new(@log_device, Fluent::Log::LEVEL_TRACE)
+  end
+
+  def teardown
+    Thread.current[:last_repeated_stacktrace] = nil
+  end
+
+  def test_initialize
+    log = Fluent::PluginLogger.new(@logger)
+    logger = log.instance_variable_get("@logger")
+    assert_equal(logger, @logger)
+  end
+
+  def test_level
+    log = Fluent::PluginLogger.new(@logger)
+    assert_equal(log.level, @logger.level)
+    log.level = "fatal"
+    assert_equal(Fluent::Log::LEVEL_FATAL, log.level)
+    assert_equal(Fluent::Log::LEVEL_TRACE, @logger.level)
+  end
+
+  def test_enable_color
+    log = Fluent::PluginLogger.new(@logger)
+    log.enable_color(true)
+    assert_equal(true, log.enable_color?)
+    assert_equal(true, @logger.enable_color?)
+    log.enable_color(false)
+    assert_equal(false, log.enable_color?)
+    assert_equal(false, @logger.enable_color?)
+    log.enable_color
+    assert_equal(true, log.enable_color?)
+    assert_equal(true, @logger.enable_color?)
+  end
+
+  sub_test_case "delegators" do
+    def setup
+      super
+      @log = Fluent::PluginLogger.new(@logger)
+    end
+
+    def test_enable_debug
+      mock(@logger).enable_debug
+      @log.enable_debug
+    end
+
+    def test_enable_event
+      mock(@logger).enable_event
+      @log.enable_event
+    end
+
+    def test_disable_events
+      mock(@logger).disable_events(Thread.current)
+      @log.disable_events(Thread.current)
+    end
+
+    def test_tag
+      assert_equal(@log.tag, @logger.tag)
+      @log.tag = "dummy"
+      assert_equal(@log.tag, @logger.tag)
+    end
+
+    def test_time_format
+      assert_equal(@log.time_format, @logger.time_format)
+      @log.time_format = "time_format"
+      assert_equal(@log.time_format, @logger.time_format)
+    end
+
+    def test_event
+      mock(@logger).event(Fluent::Log::LEVEL_TRACE, { key: "value" })
+      @log.event(Fluent::Log::LEVEL_TRACE, { key: "value" })
+    end
+
+    def test_caller_line
+      mock(@logger).caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)
+      @log.caller_line(Time.now, 1, Fluent::Log::LEVEL_TRACE)
+    end
+
+    def test_puts
+      mock(@logger).puts("log")
+      @log.puts("log")
+    end
+
+    def test_write
+      mock(@logger).write("log")
+      @log.write("log")
+    end
+
+    def test_out
+      assert_equal(@log.out, @logger.out)
+      @log.out = Object.new
+      assert_equal(@log.out, @logger.out)
+    end
+  end
+end
+
+class PluginLoggerMixinTest < Test::Unit::TestCase
+  class DummyPlugin < Fluent::Input
+  end
+
+  def create_driver(conf)
+    Fluent::Test::TestDriver.new(DummyPlugin).configure(conf)
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def test_default_log
+    plugin = DummyPlugin.new
+    log = plugin.log
+    assert_equal($log, log)
+  end
+
+  def test_log_level
+    d = create_driver(%[log_level fatal])
+    log = d.instance.log
+    assert_not_equal($log.level, log.level)
+    assert_equal(Fluent::Log::LEVEL_FATAL, log.level)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.23
+  version: 0.12.24
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-04 00:00:00.000000000 Z
+date: 2016-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -140,26 +140,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: string-scrub
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.3
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 0.0.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.3
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: 0.0.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -451,6 +431,7 @@ files:
 - test/plugin/test_out_roundrobin.rb
 - test/plugin/test_out_stdout.rb
 - test/plugin/test_out_stream.rb
+- test/plugin/test_string_util.rb
 - test/scripts/exec_script.rb
 - test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/out_test.rb
@@ -463,6 +444,7 @@ files:
 - test/test_filter.rb
 - test/test_formatter.rb
 - test/test_input.rb
+- test/test_log.rb
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_output.rb
@@ -538,6 +520,7 @@ test_files:
 - test/plugin/test_out_roundrobin.rb
 - test/plugin/test_out_stdout.rb
 - test/plugin/test_out_stream.rb
+- test/plugin/test_string_util.rb
 - test/scripts/exec_script.rb
 - test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/out_test.rb
@@ -550,6 +533,7 @@ test_files:
 - test/test_filter.rb
 - test/test_formatter.rb
 - test/test_input.rb
+- test/test_log.rb
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_output.rb