fluentd 0.12.31 → 0.12.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0aa9f8f4d0ebedebdba427af533a3b6b67eb7bcd
-  data.tar.gz: f69ae9d212c29b4c68303460aa10a9c21aa2080a
+  metadata.gz: 29eef51ecd637ecb6cf5cc7d41bc31527513afff
+  data.tar.gz: 2077e5585e911081e2f76225ff97995aaa1b5100
 SHA512:
-  metadata.gz: 4a89d78f39a8f3d5ced5460274db3e0c8ac7923fe68e9dc0d119225f912aa30a37a067f5d7c403f156f12491d9bd3440eeb82913cf7e3092ce49199b9c7ddb4b
-  data.tar.gz: a3009318c51d0f8dcd774521c67b0935c6d619d5cd8b506e126cd3bf2e55c24c39c8150fca88cfb0eb25eaa0fae4fbf778f482d6fd26e52ea03d990c49c6fddb
+  metadata.gz: 416d42c9f2d15131fbe0b01c91a11b5cb22f3e54d052ebc948304e4f9a02ca1445f5c88882e9af01778e553fcb708c83b24a42b3442582a9e1ee8e6ad1ebf9e0
+  data.tar.gz: fee4ebcca6c1ee8eb452043552bc720fb8850641414bbf4586d9d668a93a9230c1ed6283220255d4150b7ae135af4010590082af962871bfae4ca8d0c37c2325

data/.travis.yml

@@ -3,10 +3,10 @@ language: ruby
 rvm:
   - 2.0.0
   - 2.1
-  - 2.2.3
-  - 2.3.0
+  - 2.2.6
+  - 2.3.3
+  - 2.4.0
   - ruby-head
-  - rbx-2
 
 os:
   - linux
@@ -15,7 +15,6 @@ os:
 branches:
   only:
     - master
-    - v0.10
     - v0.12
     - v0.14
 
@@ -29,6 +28,6 @@ sudo: false
 matrix:
   allow_failures:
     - rvm: ruby-head
-    - rvm: rbx-2
-    - rvm: 2.3.0
-    - rvm: 1.9.3
+  exclude:
+    - rvm: 2.2.6
+      os: osx

data/ChangeLog

@@ -1,5 +1,26 @@
 # v0.12
 
+## Release 0.12.32 - 2017/02/02
+
+### New features / Enhancement
+
+* formatter: Port add_newline parameter to v012
+  https://github.com/fluent/fluentd/pull/1447
+* record_transformer: remove_keys processing should be last
+  https://github.com/fluent/fluentd/pull/1433
+* in_tail: Capture unmatched lines
+  https://github.com/fluent/fluentd/pull/1421
+* in_monitor_agent: Port #1393 to v0.12 for with_ivars query paretemer
+  https://github.com/fluent/fluentd/pull/1402
+* in_monitor_agent: Port #1387 to v0.12 for retry field
+  https://github.com/fluent/fluentd/pull/1387
+* Allow msgpack-ruby v1
+
+### Bug fixes
+
+* in_exec: Ensure to sleep for prevending fork bomb
+  https://github.com/fluent/fluentd/pull/1348
+
 ## Release 0.12.31 - 2016/12/14
 
 ### New features / Enhancement

data/fluentd.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 0.6.0"])
+  gem.add_runtime_dependency("msgpack", [">= 0.5.11", "< 2"])
   gem.add_runtime_dependency("json", [">= 1.4.3"])
   gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
   gem.add_runtime_dependency("cool.io", [">= 1.2.2", "< 2.0.0"])

data/lib/fluent/formatter.rb

@@ -135,6 +135,7 @@ module Fluent
       include StructuredFormatMixin
 
       config_param :json_parser, :string, default: 'oj'
+      config_param :add_newline, :bool, default: true
 
       def configure(conf)
         super
@@ -147,19 +148,34 @@ module Fluent
         rescue LoadError
           @dump_proc = Yajl.method(:dump)
         end
+
+        # format json is used on various highload environment, so re-define method to skip if check
+        unless @add_newline
+          define_singleton_method(:format, method(:format_without_nl))
+        end
       end
 
       def format_record(record)
         "#{@dump_proc.call(record)}\n"
       end
+
+      def format_without_nl(tag, time, record)
+        @dump_proc.call(record)
+      end
     end
 
     class HashFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
 
+      config_param :add_newline, :bool, default: true
+
       def format_record(record)
-        "#{record.to_s}\n"
+        if @add_newline
+          "#{record.to_s}\n"
+        else
+          record.to_s
+        end
       end
     end
 
@@ -177,6 +193,7 @@ module Fluent
 
       config_param :delimiter, :string, default: "\t"
       config_param :label_delimiter, :string, default: ":"
+      config_param :add_newline, :bool, default: true
 
       def format(tag, time, record)
         filter_record(tag, time, record)
@@ -184,7 +201,7 @@ module Fluent
           result << @delimiter if result.length.nonzero?
           result << "#{pair.first}#{@label_delimiter}#{pair.last}"
         }
-        formatted << "\n"
+        formatted << "\n".freeze if @add_newline
         formatted
       end
     end
@@ -197,6 +214,7 @@ module Fluent
       end
       config_param :force_quotes, :bool, default: true
       config_param :fields, :array, value_type: :string
+      config_param :add_newline, :bool, default: true
 
       def initialize
         super
@@ -217,7 +235,9 @@ module Fluent
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, @generate_opts)
+        line = CSV.generate_line(row, @generate_opts)
+        line.chomp! unless @add_newline
+        line
       end
     end
 

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -95,14 +95,13 @@ module Fluent
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        placeholder_values.merge!({
-          'time'     => @placeholder_expander.time_value(time),
-          'record'   => record,
-        })
+        placeholder_values['time'] = @placeholder_expander.time_value(time)
+        placeholder_values['record'] = record
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
           time = new_record[@renew_time_key].to_i
         end
+        @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
         new_es.add(time, new_record)
       end
       new_es
@@ -131,7 +130,6 @@ module Fluent
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
-      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end

data/lib/fluent/plugin/in_exec.rb

@@ -140,10 +140,11 @@ module Fluent
           io = IO.popen(@command, "r")
           @parser.call(io)
           Process.waitpid(io.pid)
-          sleep @run_interval
         rescue
           log.error "exec failed to run or shutdown child process", error: $!.to_s, error_class: $!.class.to_s
           log.warn_backtrace $!.backtrace
+        ensure
+          sleep @run_interval
         end
       end
     end

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -34,6 +34,7 @@ module Fluent
     config_param :emit_interval, :time, default: 60
     config_param :emit_config, :bool, default: false
     config_param :include_config, :bool, default: true
+    config_param :include_retry, :bool, default: true
 
     class MonitorServlet < WEBrick::HTTPServlet::AbstractServlet
       def initialize(server, agent)
@@ -77,16 +78,24 @@ module Fluent
 
         # if ?debug=1 is set, set :with_debug_info for get_monitor_info
         # and :pretty_json for render_json_error
-        opts = {with_config: @agent.include_config}
+        opts = {with_config: @agent.include_config, with_retry: @agent.include_retry}
         if s = qs['debug'] and s[0]
           opts[:with_debug_info] = true
           opts[:pretty_json] = true
         end
 
+        if ivars = (qs['with_ivars'] || []).first
+          opts[:ivars] = ivars.split(',')
+        end
+
         if with_config = get_search_parameter(qs, 'with_config'.freeze)
           opts[:with_config] = Fluent::Config.bool_value(with_config)
         end
 
+        if with_retry = get_search_parameter(qs, 'with_retry'.freeze)
+          opts[:with_retry] = Fluent::Config.bool_value(with_retry)
+        end
+
         if tag = get_search_parameter(qs, 'tag'.freeze)
           # ?tag= to search an output plugin by match pattern
           if obj = @agent.plugin_info_by_tag(tag, opts)
@@ -259,7 +268,7 @@ module Fluent
         log.debug "tag parameter is specified. Emit plugins info to '#{@tag}'"
 
         @loop = Coolio::Loop.new
-        opts = {with_config: @emit_config}
+        opts = {with_config: @emit_config, with_retry: false}
         timer = TimerWatcher.new(@emit_interval, log) {
           es = MultiEventStream.new
           now = Engine.now
@@ -385,6 +394,7 @@ module Fluent
 
     # TODO: use %i() after drop ruby v1.9.3 support.
     IGNORE_ATTRIBUTES = %W(@config_root_section @config @masked_config).map(&:to_sym)
+    EMPTY_RESULT = {}
 
     # get monitor info from the plugin `pe` and return a hash object
     def get_monitor_info(pe, opts={})
@@ -404,6 +414,13 @@ module Fluent
         end
       }
 
+      if opts[:with_retry]
+        num_errors = pe.instance_variable_get(:@num_errors)
+        if num_errors
+          obj['retry'] = num_errors.zero? ? EMPTY_RESULT : get_retry_info(pe, num_errors)
+        end
+      end
+
       # include all instance variables if :with_debug_info is set
       if opts[:with_debug_info]
         iv = {}
@@ -415,11 +432,25 @@ module Fluent
           }
         end
         obj['instance_variables'] = iv
+      elsif ivars = opts[:ivars]
+        iv = {}
+        ivars.each {|name|
+          iname = "@#{name}"
+          iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
+        }
+        obj['instance_variables'] = iv
       end
 
       obj
     end
 
+    def get_retry_info(pe, num_errors)
+      retry_variables = {}
+      retry_variables['steps'] = num_errors
+      retry_variables['next_time'] = Time.at(pe.instance_variable_get('@next_retry_time'.freeze))
+      retry_variables
+    end
+
     def plugin_category(pe)
       case pe
       when Fluent::Input

data/lib/fluent/plugin/in_tail.rb

@@ -48,6 +48,8 @@ module Fluent
     config_param :read_lines_limit, :integer, default: 1000
     desc 'The interval of flushing the buffer for multiline format'
     config_param :multiline_flush_interval, :time, default: nil
+    desc 'Enable the option to emit unmatched lines.'
+    config_param :emit_unmatched_lines, :bool, default: false
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
     desc 'The encoding after conversion of the input.'
@@ -315,6 +317,11 @@ module Fluent
             record[@path_key] ||= tail_watcher.path unless @path_key.nil?
             es.add(time, record)
           else
+            if @emit_unmatched_lines
+              record = {'unmatched_line' => line}
+              record[@path_key] ||= tail_watcher.path unless @path_key.nil?
+              es.add(::Fluent::Engine.now, record)
+            end
             log.warn "pattern not match: #{line.inspect}"
           end
         }
@@ -345,6 +352,9 @@ module Fluent
             lb = line
           else
             if lb.nil?
+              if @emit_unmatched_lines
+                convert_line_to_event(line, es, tail_watcher)
+              end
               log.warn "got incomplete line before first line from #{tail_watcher.path}: #{line.inspect}"
             else
               lb << line

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.31'
+  VERSION = '0.12.32'
 
 end

data/test/plugin/test_filter_record_transformer.rb

@@ -99,6 +99,25 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       es = emit(config, msgs)
       es.each_with_index do |(time, record), i|
         assert_equal(times[i].to_i, time)
+        assert_true(record.has_key?('message'))
+      end
+    end
+
+    test 'renew_time_key and remove_keys' do
+      config = %[
+                 renew_time_key event_time_key
+                 remove_keys event_time_key
+                 auto_typecast true
+                 <record>
+                   event_time_key ${record["message"]}
+                 </record>
+               ]
+      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
+      msgs = times.map { |t| t.to_f.to_s }
+      filtered = emit(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
+        assert_equal(times[i].to_i, time)
+        assert_false(_record.has_key?('event_time_key'))
       end
     end
 

data/test/plugin/test_in_exec.rb

@@ -53,6 +53,17 @@ class ExecInputTest < Test::Unit::TestCase
     ]
   end
 
+  def invalid_json_config
+    # For counting command execution, redirect stderr to file
+    %[
+      command ruby #{@script} #{@test_time} 4
+      format json
+      tag_key tag
+      time_key time
+      run_interval 0.5
+    ]
+  end
+
   def test_configure
     d = create_driver
     assert_equal 'tsv', d.instance.format
@@ -130,4 +141,16 @@ class ExecInputTest < Test::Unit::TestCase
     assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
     assert_equal @test_time, emits[0][1]
   end
+
+  def test_emit_with_invalid_script
+    d = create_driver invalid_json_config
+
+    d.run do
+      sleep 2
+    end
+
+    assert_equal true, d.emits.empty?
+    logs = d.instance.log.logs
+    assert_equal true, logs.count { |line| line =~ /exec failed to run or shutdown child process/ }.between?(1, 4)
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -97,7 +97,6 @@ class TailInputTest < Test::Unit::TestCase
     }
 
     d = create_driver
-
     d.run do
       sleep 1
 
@@ -115,6 +114,34 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(1, d.emit_streams.size)
   end
 
+  def test_emit_with_emit_unmatched_lines_true
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver(%[
+      format /^(?<message>test.*)/
+      emit_unmatched_lines true
+    ])
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "test line 1"
+        f.puts "test line 2"
+        f.puts "bad line 1"
+        f.puts "test line 3"
+      }
+
+      sleep 1
+    end
+
+    events = d.emits
+    assert_equal(4, events.length)
+    assert_equal({"message" => "test line 1"}, events[0][2])
+    assert_equal({"message" => "test line 2"}, events[1][2])
+    assert_equal({"unmatched_line" => "bad line 1"}, events[2][2])
+    assert_equal({"message" => "test line 3"}, events[3][2])
+  end
+
   data('1' => [1, 2], '10' => [10, 1])
   def test_emit_with_read_lines_limit(data)
     limit, num_emits = data
@@ -419,6 +446,44 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal({"message1" => "test8"}, emits[3][2])
   end
 
+  def test_multiline_with_emit_unmatched_lines_true
+    File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
+
+    d = create_driver %[
+      format multiline
+      format1 /^s (?<message1>[^\\n]+)(\\nf (?<message2>[^\\n]+))?(\\nf (?<message3>.*))?/
+      format_firstline /^[s]/
+      emit_unmatched_lines true
+    ]
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "ab") { |f|
+        f.puts "f test1"
+        f.puts "s test2"
+        f.puts "f test3"
+        f.puts "f test4"
+        f.puts "s test5"
+        f.puts "s test6"
+        f.puts "f test7"
+        f.puts "s test8"
+      }
+      sleep 1
+
+      events = d.emits
+      assert_equal(4, events.length)
+      assert_equal({"unmatched_line" => "f test1"}, events[0][2])
+      assert_equal({"message1" => "test2", "message2" => "test3", "message3" => "test4"}, events[1][2])
+      assert_equal({"message1" => "test5"}, events[2][2])
+      assert_equal({"message1" => "test6", "message2" => "test7"}, events[3][2])
+
+      sleep 3
+      assert_equal(4, d.emits.length)
+    end
+
+    emits = d.emits
+    assert_equal(5, emits.length)
+    assert_equal({"message1" => "test8"}, emits[4][2])
+  end
+
   def test_multiline_with_flush_interval
     File.open("#{TMP_DIR}/tail.txt", "wb") { |f| }
 

data/test/scripts/exec_script.rb

@@ -17,6 +17,10 @@ def gen_raw_string(time)
   "#{time} hello"
 end
 
+def gen_invalid_json(time)
+  %({"tag": "tag1", "time": #{time}, "k1": "ok", })
+end
+
 time = ARGV.first
 time = Integer(time) rescue time
 
@@ -29,4 +33,6 @@ when 2
   print gen_msgpack(time)
 when 3
   print gen_raw_string(time)
+when 4
+  print gen_invalid_json(time)
 end

data/test/test_formatter.rb

@@ -142,6 +142,14 @@ module FormatterTest
       assert_equal("#{Yajl.dump(record)}\n", formatted)
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_format_without_nl(data)
+      @formatter.configure('json_parser' => data, 'add_newline' => false)
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal(Yajl.dump(record), formatted)
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_format_with_symbolic_record(data)
       @formatter.configure('json_parser' => data)
@@ -252,6 +260,13 @@ module FormatterTest
       assert_equal("message:awesome\n", formatted)
     end
 
+    def test_format_without_nl
+      @formatter.configure('add_newline' => false)
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message:awesome", formatted)
+    end
+
     def test_format_with_tag
       @formatter.configure('include_tag_key' => 'true')
       formatted = @formatter.format(tag, @time, record)
@@ -319,6 +334,15 @@ module FormatterTest
       assert_equal("\"awesome\",\"awesome2\"\n", formatted)
     end
 
+    def test_format_without_nl
+      @formatter.configure('fields' => 'message,message2', 'add_newline' => false)
+      formatted = @formatter.format(tag, @time, {
+        'message' => 'awesome',
+        'message2' => 'awesome2'
+      })
+      assert_equal("\"awesome\",\"awesome2\"", formatted)
+    end
+
     def test_format_with_tag
       @formatter.configure(
         'fields' => 'tag,message,message2',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.31
+  version: 0.12.32
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-14 00:00:00.000000000 Z
+date: 2017-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -19,7 +19,7 @@ dependencies:
         version: 0.5.11
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 0.5.11
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -495,7 +495,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector