fluentd 0.12.41 → 0.12.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ec31bae71a398df42247c0d91a55f5575fe1623
-  data.tar.gz: 20c905e78d4d66f7c69c94be932dac503644e7f0
+  metadata.gz: dfec195ed913b2df7110db9f1daeff69618e0fe4
+  data.tar.gz: e04cae36275617e52cbfc29bcd00ad90e521d632
 SHA512:
-  metadata.gz: de3317a3a936c3432cadd2f39fda59918ec509eb19f0dd99da0d806a9dacedff35aa52af816b6a05a01e5ba207f27a2510cfc3e9738566226a96f3513ead97e3
-  data.tar.gz: ed3ea856639b975f67679fe2e7a6015cf6072d2d8edee017e5765424713dc3c85ef1660d56dc792d98ba970583f9b7d6f9e6db5f876f5cc7356a814ec02bc2df
+  metadata.gz: de52fc5d1eab2008728c18c7b04ea60f7c1a23d8341ea78d91fbe076d90e5c2dd4e85dc1e0519bb74691063b9e24ef4caf373738a5dddba032363268141b2504
+  data.tar.gz: 84a2f281792b3aa5c29ee53af28cb93de3bb24c3ca04af3a00d454a6313a1d5bda730956253e47310b5638f3b722e3eb1a2020d607a21e183c171b5e687cfd77

data/.travis.yml

@@ -1,11 +1,10 @@
 language: ruby
 
 rvm:
-  - 2.0.0
   - 2.1
   - 2.2.6
   - 2.3.3
-  - 2.4.0
+  - 2.4.2
   - ruby-head
 
 os:

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # v0.12
 
+## Release 0.12.42 - 2017/12/14
+
+### New features / Enhancements
+
+* Disable tracepoint feature to omit unnecessary insts
+  https://github.com/fluent/fluentd/pull/1764
+* in_udp: Add receive_buffer_size parameter
+  https://github.com/fluent/fluentd/pull/1788
+
+### Bug fixes
+
+* in_tail: Ensure close temporaly opened IO object
+  https://github.com/fluent/fluentd/pull/1756
+* in_tail: Explicitly update position to 0 when the file is truncated
+  https://github.com/fluent/fluentd/pull/1791
+* in_tail: Properly handle moved back and truncated case
+  https://github.com/fluent/fluentd/pull/1793
+* out_forward: Rebuild weight array to apply server setting properly
+  https://github.com/fluent/fluentd/pull/1784
+
 ## Release 0.12.41 - 2017/11/15
 
 ### New features / Enhancements

data/lib/fluent/plugin/in_tail.rb

@@ -490,8 +490,16 @@ module Fluent
               # assuming following situation:
               #   a) file was once renamed and backed, or
               #   b) symlink or hardlink to the same file is recreated
-              # in either case, seek to the saved position
-              pos = @pe.read_pos
+              # in either case of a and b, seek to the saved position
+              #   c) file was once renamed, truncated and then backed
+              # in this case, consider it truncated
+              last_pos = @pe.read_pos
+              if stat.size < last_pos
+                pos = 0
+                @pe.update(inode, pos)
+              else
+                pos = last_pos
+              end
             elsif last_inode != 0
               # this is FilePositionEntry and fluentd once started.
               # read data from the head of the rotated file.
@@ -526,7 +534,7 @@ module Fluent
             stat = io.stat
             inode = stat.ino
             if inode == @pe.read_inode # truncated
-              @pe.update_pos(stat.size)
+              @pe.update_pos(0)
               io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
               @io_handler.close
               @io_handler = io_handler
@@ -535,6 +543,7 @@ module Fluent
               io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
               @io_handler = io_handler
             else # file is rotated and new file found
+              io.close unless io.closed?
               detach
               @update_watcher.call(@path, swap_state(@pe))
             end

data/lib/fluent/plugin/in_udp.rb

@@ -28,6 +28,8 @@ module Fluent
     config_param :message_length_limit, :size, default: 4096
     desc "Remove newline from the end of incoming payload"
     config_param :remove_newline, :bool, default: true
+    desc "The max size of socket receive buffer. SO_RCVBUF"
+    config_param :receive_buffer_size, :size, default: nil
 
     def configure(conf)
       super
@@ -39,7 +41,7 @@ module Fluent
       log.info "listening udp socket on #{@bind}:#{@port}"
       @usock = SocketUtil.create_udp_socket(@bind)
       @usock.bind(@bind, @port)
-      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key, @remove_newline)
+      SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback, !!@source_hostname_key, @remove_newline, @receive_buffer_size)
     end
   end
 end

data/lib/fluent/plugin/out_forward.rb

@@ -263,6 +263,7 @@ module Fluent
 
       r = Random.new(@rand_seed)
       weight_array.sort_by! { r.rand }
+      p weight_array.map { |e| e.name }
 
       @weight_array = weight_array
     end
@@ -397,13 +398,17 @@ module Fluent
         begin
           #log.trace "sending heartbeat #{n.host}:#{n.port} on #{@heartbeat_type}"
           if @heartbeat_type == :tcp
-            send_heartbeat_tcp(n)
+            if send_heartbeat_tcp(n)
+              rebuild_weight_array
+            end
           else
             @usock.send "\0", 0, Socket.pack_sockaddr_in(n.port, n.resolved_host)
           end
-        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED
+        rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED => e
           # TODO log
-          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", error: $!.to_s
+          log.debug "failed to send heartbeat packet to #{n.host}:#{n.port}", error: e.to_s
+        rescue => e
+          log.debug "unexpected error happen during heartbeat to #{n.host}:#{n.port}", error: e.to_s
         end
         if n.tick
           rebuild_weight_array

data/lib/fluent/plugin/socket_util.rb

@@ -32,8 +32,11 @@ module Fluent
     module_function :create_udp_socket
 
     class UdpHandler < Coolio::IO
-      def initialize(io, log, body_size_limit, callback, resolve_hostname = false, remove_newline = true)
+      def initialize(io, log, body_size_limit, callback, resolve_hostname = false, remove_newline = true, receive_buffer_size = nil)
         super(io)
+        if io.is_a?(UDPSocket) && receive_buffer_size
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF, receive_buffer_size)
+        end
         @io = io
         @io.do_not_reverse_lookup = !resolve_hostname
         @log = log

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.41'
+  VERSION = '0.12.42'
 
 end

data/test/plugin/test_in_tail.rb

@@ -315,6 +315,54 @@ class TailInputTest < Test::Unit::TestCase
     file.close
   end
 
+  def test_truncate_file
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      sleep 1
+      File.truncate("#{TMP_DIR}/tail.txt", 6)
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(3, emits.length)
+    assert_equal({"message" => "test3"}, emits[0][2])
+    assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal({"message" => "test1"}, emits[2][2])
+  end
+
+  def test_move_truncate_move_back
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver
+    d.run do
+      sleep 1
+      FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
+      sleep 1
+      File.truncate("#{TMP_DIR}/tail2.txt", 6)
+      sleep 1
+      FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(1, emits.length)
+    assert_equal({"message" => "test1"}, emits[0][2])
+  end
+
   def test_lf
     File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
 

data/test/plugin/test_in_udp.rb

@@ -33,6 +33,7 @@ class UdpInputTest < Test::Unit::TestCase
       assert_equal PORT, d.instance.port
       assert_equal k, d.instance.bind
       assert_equal 4096, d.instance.message_length_limit
+      assert_equal nil, d.instance.receive_buffer_size
     }
   end
 
@@ -124,6 +125,39 @@ class UdpInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests)
   end
 
+  test "receive_buffer_size" do
+    # don't check exact value because it depends on platform and condition
+
+    # check if default socket and in_udp's one without receive_buffer_size have same size buffer
+    d0 = create_driver(BASE_CONFIG + %!
+      format none
+    !)
+    d0.run do
+      begin
+        sock = d0.instance.instance_variable_get(:@handler).instance_variable_get(:@io)
+        default_sock = UDPSocket.new
+        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+
+    # check if default socket and in_udp's one with receive_buffer_size have different size buffer
+    d1 = create_driver(BASE_CONFIG + %!
+      format none
+      receive_buffer_size 1001
+    !)
+    d1.run do
+      sock = d1.instance.instance_variable_get(:@handler).instance_variable_get(:@io)
+      begin
+        default_sock = UDPSocket.new
+        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+  end
+
   def compare_test_result(emits, tests)
     assert_equal(2, emits.size)
     emits.each_index {|i|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.41
+  version: 0.12.42
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2017-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack