fluentd 0.12.40 → 0.12.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 524485c147140fc4851fdaeab2ac208ae866363a
-  data.tar.gz: ba754459fd69e611c6930c0ac6307e47474d2503
+  metadata.gz: 0ec31bae71a398df42247c0d91a55f5575fe1623
+  data.tar.gz: 20c905e78d4d66f7c69c94be932dac503644e7f0
 SHA512:
-  metadata.gz: 722511c8f2827a6a68d4500ffe49ad761886738caed8a4823737b5d162b4cbefaeed4ecce216b144db07f9392df9b2bf4ef40cb1d24a7e750187a58ca6c57af8
-  data.tar.gz: 11eb85ece55f56692e2ecaea32eb881b9abf9d34c5b6d76358833eb5936a395e0dd080daaea9fa990c95e1b3f4bdf5bd5fbfd1aebb0ff2aac35a29bcd4f43449
+  metadata.gz: de3317a3a936c3432cadd2f39fda59918ec509eb19f0dd99da0d806a9dacedff35aa52af816b6a05a01e5ba207f27a2510cfc3e9738566226a96f3513ead97e3
+  data.tar.gz: ed3ea856639b975f67679fe2e7a6015cf6072d2d8edee017e5765424713dc3c85ef1660d56dc792d98ba970583f9b7d6f9e6db5f876f5cc7356a814ec02bc2df

data/{ChangeLog → CHANGELOG.md}

@@ -1,5 +1,23 @@
 # v0.12
 
+## Release 0.12.41 - 2017/11/15
+
+### New features / Enhancements
+
+* in_udp: Add remove_newline parameter
+  https://github.com/fluent/fluentd/pull/1748
+
+### Bug fixes
+
+* out_forward: Fix elapsed time miscalculation in tcp heartbeat
+  https://github.com/fluent/fluentd/pull/1738
+* in_tail: Skip setup failed watcher to avoid resource leak and log bloat
+  https://github.com/fluent/fluentd/pull/1752
+* agent: Add error location to emit error logs
+  https://github.com/fluent/fluentd/pull/1746
+* filter_parser: Fix dumpped result for avoiding escape sequence injection
+  https://github.com/fluent/fluentd/pull/1733
+
 ## Release 0.12.40 - 2017/08/25
 
 ### Bug fixes

data/lib/fluent/plugin/filter_parser.rb

@@ -53,7 +53,7 @@ class Fluent::ParserFilter < Fluent::Filter
             if @emit_invalid_record_to_error
               router.emit_error_event(tag, time, record, ::Fluent::ParserError.new("pattern not match with data '#{raw_value}'"))
             else
-              log.warn "pattern not match with data '#{raw_value}'" unless @suppress_parse_error_log
+              log.warn "pattern not match with data '#{raw_value.dump}'" unless @suppress_parse_error_log
             end
             if @reserve_data
               t = time

data/lib/fluent/plugin/in_tail.rb

@@ -24,6 +24,16 @@ module Fluent
   class NewTailInput < Input
     Plugin.register_input('tail', self)
 
+    class WatcherSetupError < StandardError
+      def initialize(msg)
+        @message = msg
+      end
+
+      def to_s
+        @message
+      end
+    end
+
     def initialize
       super
       @paths = []
@@ -65,6 +75,8 @@ module Fluent
     config_param :skip_refresh_on_startup, :bool, default: false
     desc 'Ignore repeated permission error logs'
     config_param :ignore_repeated_permission_error, :bool, default: false
+    # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
+    config_param :blocking_timeout, :time, default: 0.5
 
     attr_reader :paths
 
@@ -207,6 +219,9 @@ module Fluent
       tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher,  &method(:receive_lines))
       tw.attach(@loop)
       tw
+    rescue => e
+      tw.close if tw
+      raise e
     end
 
     def start_watchers(paths)
@@ -223,7 +238,13 @@ module Fluent
           end
         end
 
-        @tails[path] = setup_watcher(path, pe)
+        begin
+          tw = setup_watcher(path, pe)
+        rescue WatcherSetupError => e
+          log.warn "Skip #{path} because unexpected setup error happens: #{e}"
+          next
+        end
+        @tails[path] = tw
       }
     end
 
@@ -292,7 +313,7 @@ module Fluent
     end
 
     def run
-      @loop.run
+      @loop.run(@blocking_timeout)
     rescue
       log.error "unexpected error", error: $!.to_s
       log.error_backtrace
@@ -436,8 +457,8 @@ module Fluent
       end
 
       def detach
-        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger.attached?
+        @timer_trigger.detach if @enable_watch_timer && @timer_trigger && @timer_trigger.attached?
+        @stat_trigger.detach if @stat_trigger && @stat_trigger.attached?
       end
 
       def close(close_io = true)
@@ -485,7 +506,12 @@ module Fluent
               pos = @read_from_head ? 0 : fsize
               @pe.update(inode, pos)
             end
-            io.seek(pos)
+
+            begin
+              io.seek(pos)
+            rescue RangeError
+              raise WatcherSetupError, "seek error with #{@path}: file_position = #{pos.to_s(16)}"
+            end
 
             @io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
           else
@@ -690,6 +716,9 @@ module Fluent
             @fsize = fsize
           end
 
+        rescue WatcherSetupError => e
+          io.close if io
+          raise e
         rescue
           @log.error $!.to_s
           @log.error_backtrace

data/lib/fluent/plugin/in_udp.rb

@@ -26,6 +26,8 @@ module Fluent
     config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
     desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 4096
+    desc "Remove newline from the end of incoming payload"
+    config_param :remove_newline, :bool, default: true
 
     def configure(conf)
       super
@@ -37,7 +39,7 @@ module Fluent
       log.info "listening udp socket on #{@bind}:#{@port}"
       @usock = SocketUtil.create_udp_socket(@bind)
       @usock.bind(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key)
+      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key, @remove_newline)
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -394,9 +394,6 @@ module Fluent
     def on_timer
       return if @finished
       @nodes.each {|n|
-        if n.tick
-          rebuild_weight_array
-        end
         begin
           #log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
           if @heartbeat_type == :tcp
@@ -408,6 +405,9 @@ module Fluent
           # TODO log
           log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", error: $!.to_s
         end
+        if n.tick
+          rebuild_weight_array
+        end
       }
     end
 

data/lib/fluent/plugin/socket_util.rb

@@ -32,18 +32,19 @@ module Fluent
     module_function :create_udp_socket
 
     class UdpHandler < Coolio::IO
-      def initialize(io, log, body_size_limit, callback, resolve_hostname = false)
+      def initialize(io, log, body_size_limit, callback, resolve_hostname = false, remove_newline = true)
         super(io)
         @io = io
         @io.do_not_reverse_lookup = !resolve_hostname
         @log = log
         @body_size_limit = body_size_limit
+        @remove_newline = remove_newline
         @callback = callback
       end
 
       def on_readable
         msg, addr = @io.recvfrom_nonblock(@body_size_limit)
-        msg.chomp!
+        msg.chomp! if @remove_newline
         @callback.call(msg, addr)
       rescue => e
         @log.error "unexpected error", error: e, error_class: e.class

data/lib/fluent/root_agent.rb

@@ -177,7 +177,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time}
+      error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -189,7 +189,7 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {error_class: error.class, error: error.to_s, tag: tag}
+      error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
@@ -218,7 +218,7 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {error_class: error.class, error: error.to_s, tag: tag, time: time, record: record}
+        error_info = {error_class: error.class, error: error.to_s, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.40'
+  VERSION = '0.12.41'
 
 end

data/test/plugin/test_in_udp.rb

@@ -104,6 +104,26 @@ class UdpInputTest < Test::Unit::TestCase
     end
   }
 
+  test "remove_newline" do
+    tests = [{'msg' => "test1\n", 'expected' => "test1\n"},
+             {'msg' => "test2\n", 'expected' => "test2\n"}]
+
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      remove_newline false
+    !)
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each { |test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
   def compare_test_result(emits, tests)
     assert_equal(2, emits.size)
     emits.each_index {|i|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.40
+  version: 0.12.41
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-25 00:00:00.000000000 Z
+date: 2017-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -301,9 +301,9 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - AUTHORS
+- CHANGELOG.md
 - CONTRIBUTING.md
 - COPYING
-- ChangeLog
 - Gemfile
 - README.md
 - Rakefile
@@ -496,7 +496,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector