fluentd 1.9.3-x64-mingw32 → 1.10.0-x64-mingw32

data/lib/fluent/plugin/sd_file.rb

@@ -85,6 +85,7 @@ module Fluent
             fetch_server_info
           rescue => e
             @log.error("sd_file: #{e}")
+            return
           end
 
         if s.nil?

data/lib/fluent/plugin/sd_srv.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'resolv'
+
+require 'fluent/plugin_helper'
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class SrvServiceDiscovery < ServiceDiscovery
+      include PluginHelper::Mixin
+
+      Plugin.register_sd('srv', self)
+
+      helpers :timer
+
+      desc 'Service without underscore in RFC2782'
+      config_param :service, :string
+      desc 'Proto without underscore in RFC2782'
+      config_param :proto, :string, default: 'tcp'
+      desc 'Name in RFC2782'
+      config_param :hostname, :string
+      desc 'hostname of DNS server to request the SRV record'
+      config_param :dns_server_host, :string, default: nil
+      desc 'interval of requesting to DNS server'
+      config_param :interval, :integer, default: 60
+      desc "resolve hostname to IP addr of SRV's Target"
+      config_param :dns_lookup, :bool, default: true
+      desc 'The shared key per server'
+      config_param :shared_key, :string, default: nil, secret: true
+      desc 'The username for authentication'
+      config_param :username, :string, default: ''
+      desc 'The password for authentication'
+      config_param :password, :string, default: '', secret: true
+
+      def initialize
+        super
+        @target = nil
+      end
+
+      def configure(conf)
+        super
+
+        @target = "_#{@service}._#{@proto}.#{@hostname}"
+        @dns_resolve =
+          if @dns_server_host.nil?
+            Resolv::DNS.new
+          elsif @dns_server_host.include?(':') # e.g. 127.0.0.1:8600
+            host, port = @dns_server_host.split(':', 2)
+            Resolv::DNS.new(nameserver_port: [[host, port.to_i]])
+          else
+            Resolv::DNS.new(nameserver: @dns_server_host)
+          end
+
+        @services = fetch_srv_record
+      end
+
+      def start(queue)
+        timer_execute(:"sd_srv_record_#{@target}", @interval) do
+          refresh_srv_records(queue)
+        end
+
+        super()
+      end
+
+      private
+
+      def refresh_srv_records(queue)
+        s = begin
+              fetch_srv_record
+            rescue => e
+              @log.error("sd_srv: #{e}")
+              return
+            end
+
+        if s.nil? || s.empty?
+          return
+        end
+
+        diff = []
+        join = s - @services
+        # Need service_in first to guarantee that server exist at least one all time.
+        join.each do |j|
+          diff << ServiceDiscovery.service_in_msg(j)
+        end
+
+        drain = @services - s
+        drain.each do |d|
+          diff << ServiceDiscovery.service_out_msg(d)
+        end
+
+        @services = s
+
+        diff.each do |a|
+          queue.push(a)
+        end
+      end
+
+      def fetch_srv_record
+        adders = @dns_resolve.getresources(@target, Resolv::DNS::Resource::IN::SRV)
+
+        services = []
+
+        adders.each do |addr|
+          host = @dns_lookup ? dns_lookup!(addr.target) : addr.target
+          services << [
+            addr.priority,
+            Service.new(:srv, host.to_s, addr.port.to_i, addr.target.to_s, addr.weight, false, @username, @password, @shared_key)
+          ]
+        end
+
+        services.sort_by(&:first).flat_map { |s| s[1] }
+      end
+
+      def dns_lookup!(host)
+        # may need to cache the result
+        @dns_resolve.getaddress(host) # get first result for now
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -36,7 +36,7 @@ module Fluent
         end
 
         if conf.client_cert_auth
-            ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
         end
 
         ctx.ca_file = conf.ca_path
@@ -45,6 +45,16 @@ module Fluent
         if extra && !extra.empty?
           ctx.extra_chain_cert = extra
         end
+        if conf.cert_verifier
+          sandbox = Class.new
+          ctx.verify_callback = if File.exist?(conf.cert_verifier)
+                                  verifier = File.read(conf.cert_verifier)
+                                  sandbox.instance_eval(verifier, File.basename(conf.cert_verifier))
+                                else
+                                  sandbox.instance_eval(conf.cert_verifier)
+                                end
+        end
+
         Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
 
         ctx
@@ -171,9 +181,12 @@ module Fluent
 
       def cert_option_certificates_from_file(path)
         data = File.read(path)
-        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n?', Regexp::MULTILINE)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
         list = []
         data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        if list.length == 0
+          log.warn "cert_path does not contain a valid certificate"
+        end
         list
       end
     end

data/lib/fluent/plugin_helper/server.rb

@@ -243,7 +243,7 @@ module Fluent
         :protocol, :version, :min_version, :max_version, :ciphers, :insecure,
         :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
         :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
-        :generate_private_key_length,
+        :cert_verifier, :generate_private_key_length,
         :generate_cert_country, :generate_cert_state, :generate_cert_state,
         :generate_cert_locality, :generate_cert_common_name,
         :generate_cert_expiration, :generate_cert_digest,
@@ -281,6 +281,8 @@ module Fluent
           config_param :ca_private_key_path, :string, default: nil
           config_param :ca_private_key_passphrase, :string, default: nil, secret: true
 
+          config_param :cert_verifier, :string, default: nil
+
           # Options for generating certs by private CA certs or self-signed
           config_param :generate_private_key_length, :integer, default: 2048
           config_param :generate_cert_country, :string, default: 'US'

data/lib/fluent/plugin_helper/socket_option.rb

@@ -21,6 +21,8 @@ require 'fcntl'
 module Fluent
   module PluginHelper
     module SocketOption
+      # ref: https://docs.microsoft.com/en-us/windows/win32/api/winsock/ns-winsock-linger
+      FORMAT_STRUCT_LINGER_WINDOWS  = 'S!S!' # { u_short l_onoff; u_short l_linger; }
       FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
       FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
 
@@ -49,9 +51,25 @@ module Fluent
         if nonblock
           sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
         end
-        if linger_timeout
+        if Fluent.windows?
+          # To prevent closing socket forcibly on Windows,
+          # this options shouldn't be set up when linger_timeout equals to 0 (including nil).
+          # This unintended behavior always ocurrs on Windows when linger_timeout.to_i == 0.
+          # This unintented behavior causes "Errno::ECONNRESET: An existing connection was forcibly
+          # closed by the remote host." on Windows.
+          if linger_timeout.to_i > 0
+            if linger_timeout >= 2**16
+              log.warn "maximum linger_timeout is 65535(2^16 - 1). Set to 65535 forcibly."
+              linger_timeout = 2**16 - 1
+            end
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER_WINDOWS)
+            socket_option_set_one(sock, :SO_LINGER, optval)
+          end
+        else
+          if linger_timeout
           optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
           socket_option_set_one(sock, :SO_LINGER, optval)
+          end
         end
         if recv_timeout
           optval = [recv_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.9.3'
+  VERSION = '1.10.0'
 
 end

data/test/command/test_fluentd.rb

@@ -84,6 +84,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     null_stream.close rescue nil
   end
 
+  def eager_read(io)
+    buf = +''
+
+    loop do
+      b = io.read_nonblock(1024, nil, exception: false)
+      if b == :wait_readable || b.nil?
+        return buf
+      end
+      buf << b
+    end
+  end
+
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
     matched = false
     assert_error_msg = "matched correctly"
@@ -97,7 +109,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               next unless readables
               break if readables.first.eof?
 
-              buf = readables.first.readpartial(1024)
+              buf = eager_read(readables.first)
               # puts buf
               stdio_buf << buf
               lines = stdio_buf.split("\n")
@@ -150,7 +162,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               next unless readables
               next if readables.first.eof?
 
-              stdio_buf << readables.first.readpartial(1024)
+              stdio_buf << eager_read(readables.first)
               lines = stdio_buf.split("\n")
               if lines.any?{|line| line.include?("fluentd worker is now running") }
                 running = true

data/test/helper.rb

@@ -49,8 +49,8 @@ require 'fluent/plugin_helper'
 require 'fluent/msgpack_factory'
 require 'fluent/time'
 require 'serverengine'
-require 'helpers/fuzzy_assert'
-require 'helpers/process_extenstion'
+require_relative 'helpers/fuzzy_assert'
+require_relative 'helpers/process_extenstion'
 
 module Fluent
   module Plugin

data/test/plugin/in_tail/test_fifo.rb

@@ -0,0 +1,121 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+
+class IntailFIFO < Test::Unit::TestCase
+  sub_test_case '#read_line' do
+    test 'returns lines spliting per `\n`' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'concant line when line is separated' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3 + 'test').force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+    end
+
+    test 'returns lines which convert encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::UTF_8)
+      text = ("test\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::UTF_8, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    test 'reads lines as from_encoding' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+      fifo << text
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+      assert_equal ["test\n", "test\n", "test\n"], lines
+    end
+
+    sub_test_case 'when it includes multi byte chars' do
+      test 'handles it as ascii_8bit' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::ASCII_8BIT)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["てすと\n", "てすと\n", "てすと\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+
+      test 'replaces character with ? when convert error happens' do
+        fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::UTF_8, Encoding::ASCII_8BIT)
+        text = ("てすと\n" * 3).force_encoding(Encoding::UTF_8)
+        fifo << text
+        lines = []
+        fifo.read_lines(lines)
+        assert_equal Encoding::ASCII_8BIT, lines[0].encoding
+        assert_equal ["???\n", "???\n", "???\n"].map { |e| e.force_encoding(Encoding::ASCII_8BIT) }, lines
+      end
+    end
+
+    test 'reutrns nothing when buffer is empty' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+
+      text = "test\n" * 3
+      fifo << text
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal [], lines
+    end
+  end
+
+  sub_test_case '#<<' do
+    test 'does not make any change about encoding to an argument' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = ("test\n" * 3).force_encoding(Encoding::UTF_8)
+
+      assert_equal Encoding::UTF_8, text.encoding
+      fifo << text
+      assert_equal Encoding::UTF_8, text.encoding
+    end
+  end
+
+  sub_test_case '#bytesize' do
+    test 'reutrns buffer size' do
+      fifo = Fluent::Plugin::TailInput::TailWatcher::FIFO.new(Encoding::ASCII_8BIT, Encoding::ASCII_8BIT)
+      text = "test\n" * 3 + 'test'
+      fifo << text
+
+      assert_equal text.bytesize, fifo.bytesize
+      lines = []
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n"], lines
+
+      assert_equal 'test'.bytesize, fifo.bytesize
+      fifo << "2\n"
+      fifo.read_lines(lines)
+      assert_equal ["test\n", "test\n", "test\n", "test2\n"], lines
+
+      assert_equal 0, fifo.bytesize
+    end
+  end
+end

data/test/plugin/in_tail/test_io_handler.rb

@@ -0,0 +1,132 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/in_tail'
+require 'tempfile'
+
+class IntailIOHandlerTest < Test::Unit::TestCase
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
+  end
+
+  test '#on_notify load file content and passed it to receive_lines method' do
+    text = "this line is test\ntest line is test\n"
+    @file.write(text)
+    @file.close
+
+    watcher = 'watcher'
+
+    update_pos = 0
+
+    stub(watcher).pe do
+      pe = 'position_file'
+      stub(pe).read_pos { 0 }
+      stub(pe).update_pos { |val| update_pos = val }
+      pe
+    end
+
+    returned_lines = ''
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: false) do |lines, _watcher|
+      returned_lines << lines.join
+      true
+    end
+
+    r.on_notify
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+
+    r.on_notify
+
+    assert_equal text.bytesize, update_pos
+    assert_equal text, returned_lines
+  end
+
+  sub_test_case 'when open_on_every_update is true and read_pos returns always 0' do
+    test 'open new IO and change pos to 0 and read it' do
+      text = "this line is test\ntest line is test\n"
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = ''
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, log: $log, open_on_every_update: true) do |lines, _watcher|
+        returned_lines << lines.join
+        true
+      end
+
+      r.on_notify
+      assert_equal text.bytesize, update_pos
+      assert_equal text, returned_lines
+
+      r.on_notify
+      assert_equal text * 2, returned_lines
+    end
+  end
+
+  sub_test_case 'when limit is 5' do
+    test 'call receive_lines once when short line(less than 8192)' do
+      text = "line\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal 8, returned_lines[0].size
+    end
+
+    test 'call receive_lines some times when long line(more than 8192)' do
+      t = 'line' * (8192 / 8)
+      text = "#{t}\n" * 8
+      @file.write(text)
+      @file.close
+
+      update_pos = 0
+
+      watcher = 'watcher'
+      stub(watcher).pe do
+        pe = 'position_file'
+        stub(pe).read_pos { 0 }
+        stub(pe).update_pos { |val| update_pos = val }
+        pe
+      end
+
+      returned_lines = []
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, log: $log, open_on_every_update: false) do |lines, _watcher|
+        returned_lines << lines.dup
+        true
+      end
+
+      r.on_notify
+      assert_equal 5, returned_lines[0].size
+      assert_equal 3, returned_lines[1].size
+    end
+  end
+end