fluentd 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88ca4ec7c1507eb95cad4a86d039a850bfe12424
-  data.tar.gz: b45cd0bea4a3e2091026ab0c0de8da58b2dc7de8
+  metadata.gz: 1585c5b08cc563404db3e0901bf745c79b2d4e9d
+  data.tar.gz: e010df6ece76b3b516d91798d295b37fcdaf6fab
 SHA512:
-  metadata.gz: 96e8b778f0ce8f7c3c7057a703842499a8b3d492298b54a858b91f8910f68a41618310a1f71f81c7c8628a8c74934a566c55ef31e866aee1c793e02f669a9ddf
-  data.tar.gz: 71131b5600b865b21170ef5a347b74c61ea872146ebfd34985466629c8379ecf6255d5abfc8ebf95a437a862ba285e1b4b8ca6d407fefe3c4703d88f688627f0
+  metadata.gz: dee5732e3915f8e03f697cb5d0176cf0eef2c490d0136d8b1f8cc53c2028abd269a01ae8dee13a335a36adb132b30ca39238d80968b0eb336996f626eb965530
+  data.tar.gz: 68dcb7db0688e8963d0de610371eb9f19d2cade0f8ddf631584a795ac66c27a1cfbd3f50c618178ef04cd17c3a14378fa8f4d22b0714acc2bab871903b9e5f64

data/CHANGELOG.md

@@ -1,7 +1,28 @@
 # v1.0
 
+## Release v1.0.1 - 2017/12/14
+
+### New features / Enhancements
+
+* in_udp: Add receive_buffer_size parameter
+  https://github.com/fluent/fluentd/pull/1788
+* in_tail: Add enable_stat_watcher option to disable inotify events
+  https://github.com/fluent/fluentd/pull/1775
+* Relax strptime gem version
+
+### Bug fixes
+
+* in_tail: Properly handle moved back and truncated case
+  https://github.com/fluent/fluentd/pull/1793
+* out_forward: Rebuild weight array to apply server setting properly
+  https://github.com/fluent/fluentd/pull/1784
+* fluent-plugin-config-formatter: Use v1.0 for URL
+  https://github.com/fluent/fluentd/pull/1781
+
 ## Release v1.0.0 - 2017/12/6
 
+See [CNCF announcment](https://www.cncf.io/blog/2017/12/06/fluentd-v1-0/) :)
+
 ### New features / Enhancements
 
 * out_copy: Support ignore_error argument in `<store>`

data/fluentd.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
   gem.add_runtime_dependency("tzinfo", ["~> 1.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
-  gem.add_runtime_dependency("strptime", ["~> 0.1.7"])
+  gem.add_runtime_dependency("strptime", ["~> 0.1"])
   gem.add_runtime_dependency("ruby_dig", ["~> 0.0.2"])
 
   # build gem for a certain platform. see also Rakefile

data/lib/fluent/command/plugin_config_formatter.rb

@@ -31,7 +31,7 @@ class FluentPluginConfigFormatter
     "buffer", "parser", "formatter", "storage"
   ]
 
-  DOCS_BASE_URL = "https://docs.fluentd.org/v0.14/articles/"
+  DOCS_BASE_URL = "https://docs.fluentd.org/v1.0/articles/"
 
   def initialize(argv = ARGV)
     @argv = argv

data/lib/fluent/plugin/in_tail.rb

@@ -80,6 +80,8 @@ module Fluent::Plugin
     config_param :emit_unmatched_lines, :bool, default: false
     desc 'Enable the additional watch timer.'
     config_param :enable_watch_timer, :bool, default: true
+    desc 'Enable the stat watcher based on inotify.'
+    config_param :enable_stat_watcher, :bool, default: true
     desc 'The encoding after conversion of the input.'
     config_param :encoding, :string, default: nil
     desc 'The encoding of the input.'
@@ -115,6 +117,10 @@ module Fluent::Plugin
 
       super
 
+      if !@enable_watch_timer && !@enable_stat_watcher
+        raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
+      end
+
       @paths = @path.split(',').map {|path| path.strip }
       if @paths.empty?
         raise Fluent::ConfigError, "tail: 'path' parameter is required on tail input"
@@ -254,10 +260,10 @@ module Fluent::Plugin
 
     def setup_watcher(path, pe)
       line_buffer_timer_flusher = (@multiline_mode && @multiline_flush_interval) ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @enable_watch_timer, @enable_stat_watcher, @read_lines_limit, method(:update_watcher), line_buffer_timer_flusher, @from_encoding, @encoding, open_on_every_update, &method(:receive_lines))
       tw.attach do |watcher|
         watcher.timer_trigger = timer_execute(:in_tail_timer_trigger, 1, &watcher.method(:on_notify)) if watcher.enable_watch_timer
-        event_loop_attach(watcher.stat_trigger)
+        event_loop_attach(watcher.stat_trigger) if watcher.enable_stat_watcher
       end
       tw
     rescue => e
@@ -457,17 +463,18 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, enable_watch_timer, enable_stat_watcher, read_lines_limit, update_watcher, line_buffer_timer_flusher, from_encoding, encoding, open_on_every_update, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
         @enable_watch_timer = enable_watch_timer
+        @enable_stat_watcher = enable_stat_watcher
         @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
-        @stat_trigger = StatWatcher.new(self, &method(:on_notify))
+        @stat_trigger = @enable_stat_watcher ? StatWatcher.new(self, &method(:on_notify)) : nil
         @timer_trigger = nil
 
         @rotate_handler = RotateHandler.new(self, &method(:on_rotate))
@@ -483,7 +490,7 @@ module Fluent::Plugin
       attr_reader :path
       attr_reader :log, :pe, :read_lines_limit, :open_on_every_update
       attr_reader :from_encoding, :encoding
-      attr_reader :stat_trigger, :enable_watch_timer
+      attr_reader :stat_trigger, :enable_watch_timer, :enable_stat_watcher
       attr_accessor :timer_trigger
       attr_accessor :line_buffer, :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
@@ -503,7 +510,7 @@ module Fluent::Plugin
 
       def detach
         @timer_trigger.detach if @enable_watch_timer && @timer_trigger && @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger && @stat_trigger.attached?
+        @stat_trigger.detach if @enable_stat_watcher && @stat_trigger && @stat_trigger.attached?
         @io_handler.on_notify if @io_handler
       end
 
@@ -540,7 +547,10 @@ module Fluent::Plugin
               # assuming following situation:
               #   a) file was once renamed and backed, or
               #   b) symlink or hardlink to the same file is recreated
-              # in either case, seek to the saved position
+              # in either case of a and b, seek to the saved position
+              #   c) file was once renamed, truncated and then backed
+              # in this case, consider it truncated
+              @pe.update(inode, 0) if fsize < @pe.read_pos
             elsif last_inode != 0
               # this is FilePositionEntry and fluentd once started.
               # read data from the head of the rotated file.

data/lib/fluent/plugin/in_udp.rb

@@ -40,6 +40,8 @@ module Fluent::Plugin
     config_param :message_length_limit, :size, default: 4096
     desc "Remove newline from the end of incoming payload"
     config_param :remove_newline, :bool, default: true
+    desc "The max size of socket receive buffer. SO_RCVBUF"
+    config_param :receive_buffer_size, :size, default: nil
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -61,7 +63,7 @@ module Fluent::Plugin
       super
 
       log.info "listening udp socket", bind: @bind, port: @port
-      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
         data.chomp! if @remove_newline
         begin
           @parser.parse(data) do |time, record|

data/lib/fluent/plugin/out_forward.rb

@@ -398,7 +398,9 @@ module Fluent::Plugin
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
-          n.send_heartbeat
+          if n.send_heartbeat
+            rebuild_weight_array
+          end
         rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
           log.debug "failed to send heartbeat packet", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
         rescue => e
@@ -665,11 +667,13 @@ module Fluent::Plugin
             ## don't send any data to not cause a compatibility problem
             # sock.write FORWARD_TCP_HEARTBEAT_DATA
 
-            # successful tcp connection establishment is considered as valid heartbeat
+            # successful tcp connection establishment is considered as valid heartbeat.
+            # When heartbeat is succeeded after detached, return true. It rebuilds weight array.
             heartbeat(true)
           end
         when :udp
           @usock.send "\0", 0, Socket.pack_sockaddr_in(@port, resolved_host)
+          nil
         when :none # :none doesn't use this class
           raise "BUG: heartbeat_type none must not use Node"
         else

data/lib/fluent/plugin_helper/socket_option.rb

@@ -24,7 +24,7 @@ module Fluent
       FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
       FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
 
-      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
+      def socket_option_validate!(protocol, resolve_name: nil, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
         unless resolve_name.nil?
           if protocol != :tcp && protocol != :udp && protocol != :tls
             raise ArgumentError, "BUG: resolve_name in available for tcp/udp/tls"
@@ -37,7 +37,7 @@ module Fluent
         end
       end
 
-      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil)
+      def socket_option_set(sock, resolve_name: nil, nonblock: false, linger_timeout: nil, recv_timeout: nil, send_timeout: nil, receive_buffer_size: nil)
         unless resolve_name.nil?
           sock.do_not_reverse_lookup = !resolve_name
         end
@@ -56,6 +56,9 @@ module Fluent
           optval = [send_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)
           socket_option_set_one(sock, :SO_SNDTIMEO, optval)
         end
+        if receive_buffer_size
+          socket_option_set_one(sock, :SO_RCVBUF, receive_buffer_size.to_i)
+        end
         sock
       end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 
 end

data/test/command/test_plugin_config_formatter.rb

@@ -179,10 +179,10 @@ TEXT
       expected = <<TEXT
 ## Plugin helpers
 
-* [inject](https://docs.fluentd.org/v0.14/articles/api-plugin-helper-inject)
-* [compat_parameters](https://docs.fluentd.org/v0.14/articles/api-plugin-helper-compat_parameters)
+* [inject](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-compat_parameters)
 
-* See also: [Input Plugin Overview](https://docs.fluentd.org/v0.14/articles/input-plugin-overview)
+* See also: [Input Plugin Overview](https://docs.fluentd.org/v1.0/articles/input-plugin-overview)
 
 ## TestFluentPluginConfigFormatter::SimpleInput
 
@@ -202,10 +202,10 @@ TEXT
       expected = <<TEXT
 ## Plugin helpers
 
-* [inject](https://docs.fluentd.org/v0.14/articles/api-plugin-helper-inject)
-* [compat_parameters](https://docs.fluentd.org/v0.14/articles/api-plugin-helper-compat_parameters)
+* [inject](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-inject)
+* [compat_parameters](https://docs.fluentd.org/v1.0/articles/api-plugin-helper-compat_parameters)
 
-* See also: [Output Plugin Overview](https://docs.fluentd.org/v0.14/articles/output-plugin-overview)
+* See also: [Output Plugin Overview](https://docs.fluentd.org/v1.0/articles/output-plugin-overview)
 
 ## TestFluentPluginConfigFormatter::ComplexOutput
 

data/test/plugin/test_in_tail.rb

@@ -41,6 +41,7 @@ class TailInputTest < Test::Unit::TestCase
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
   CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
+  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
   CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
   PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
@@ -83,6 +84,12 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    test "both enable_watch_timer and enable_stat_watcher are false" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
+      end
+    end
+
     sub_test_case "encoding" do
       test "valid" do
         conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
@@ -256,6 +263,30 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test3"}, events[0][2])
       assert_equal({"message" => "test4"}, events[1][2])
     end
+
+    data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
+         parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
+    def test_emit_with_disable_stat_watcher(data)
+      config = data
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(config)
+
+      d.run(expect_emits: 1) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+      end
+
+      events = d.events
+      assert(events.length > 0)
+      assert_equal({"message" => "test3"}, events[0][2])
+      assert_equal({"message" => "test4"}, events[1][2])
+    end
   end
 
   class TestWithSystem < self
@@ -440,6 +471,67 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_truncate_file
+    config = SINGLE_LINE_CONFIG
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(config)
+
+    d.run(expect_emits: 2) do
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+        f.puts "test3\ntest4"
+        f.flush
+      }
+      sleep 1
+      File.truncate("#{TMP_DIR}/tail.txt", 6)
+    end
+
+    events = d.events
+    assert_equal(3, events.length)
+    assert_equal({"message" => "test3"}, events[0][2])
+    assert_equal({"message" => "test4"}, events[1][2])
+    assert_equal({"message" => "test1"}, events[2][2])
+    assert(events[0][1].is_a?(Fluent::EventTime))
+    assert(events[1][1].is_a?(Fluent::EventTime))
+    assert(events[2][1].is_a?(Fluent::EventTime))
+    assert_equal(2, d.emit_count)
+  end
+
+  def test_move_truncate_move_back
+    config = SINGLE_LINE_CONFIG
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(config)
+
+    d.run(expect_emits: 1) do
+      if Fluent.windows?
+        FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
+      else
+        FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
+      end
+      sleep(1)
+      File.truncate("#{TMP_DIR}/tail2.txt", 6)
+      sleep(1)
+      if Fluent.windows?
+        FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt", force: true)
+      else
+        FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
+      end
+    end
+
+    events = d.events
+    assert_equal(1, events.length)
+    assert_equal({"message" => "test1"}, events[0][2])
+    assert(events[0][1].is_a?(Fluent::EventTime))
+    assert_equal(1, d.emit_count)
+  end
+
   def test_lf
     File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
 
@@ -926,7 +1018,7 @@ class TailInputTest < Test::Unit::TestCase
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -944,7 +1036,7 @@ class TailInputTest < Test::Unit::TestCase
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

data/test/plugin/test_in_udp.rb

@@ -55,6 +55,7 @@ class UdpInputTest < Test::Unit::TestCase
     assert_equal PORT, d.instance.port
     assert_equal bind, d.instance.bind
     assert_equal 4096, d.instance.message_length_limit
+    assert_equal nil, d.instance.receive_buffer_size
   end
 
   data(
@@ -172,4 +173,37 @@ class UdpInputTest < Test::Unit::TestCase
       assert_equal expected_record, d.events[i][2]
     end
   end
+
+  test 'receive_buffer_size' do
+    # doesn't check exact value because it depends on platform and condition
+
+    # check if default socket and in_udp's one without receive_buffer_size have same size buffer
+    d0 = create_driver(BASE_CONFIG + %!
+      format none
+    !)
+    d0.run do
+      sock = d0.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+
+    # check if default socket and in_udp's one with receive_buffer_size have different size buffer
+    d1 = create_driver(BASE_CONFIG + %!
+      format none
+      receive_buffer_size 1001
+    !)
+    d1.run do
+      sock = d1.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-06 00:00:00.000000000 Z
+date: 2017-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -152,14 +152,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.7
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: ruby_dig
   requirement: !ruby/object:Gem::Requirement