fluentd 1.6.3 → 1.7.1

data/test/helper.rb

@@ -50,6 +50,7 @@ require 'fluent/msgpack_factory'
 require 'fluent/time'
 require 'serverengine'
 require 'helpers/fuzzy_assert'
+require 'helpers/process_extenstion'
 
 module Fluent
   module Plugin

data/test/helpers/process_extenstion.rb

@@ -0,0 +1,33 @@
+require 'timecop'
+
+module Process
+  class << self
+    alias_method :clock_gettime_original, :clock_gettime
+
+    def clock_gettime(clock_id, unit = :float_second)
+      # now only support CLOCK_REALTIME
+      if Process::CLOCK_REALTIME == clock_id
+        t = Time.now
+
+        case unit
+        when :float_second
+          t.to_i + t.nsec / 1_000_000_000.0
+        when :float_millisecond
+          t.to_i * 1_000 + t.nsec / 1_000_000.0
+        when :float_microsecond
+          t.to_i * 1_000_000 + t.nsec / 1_000.0
+        when :second
+          t.to_i
+        when :millisecond
+          t.to_i * 1000 + t.nsec / 1_000_000
+        when :microsecond
+          t.to_i * 1_000_000 + t.nsec / 1_000
+        when :nanosecond
+          t.to_i * 1_000_000_000 + t.nsec
+        end
+      else
+        Process.clock_gettime_original(clock_id, unit)
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_ack_handler.rb

@@ -0,0 +1,101 @@
+require_relative '../../helper'
+require 'fluent/test/driver/output'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/ack_handler'
+
+class AckHandlerTest < Test::Unit::TestCase
+  data(
+    'chunk_id is matched' => [MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') }), Fluent::Plugin::ForwardOutput::AckHandler::Result::SUCCESS],
+    'chunk_id is not matched' => [MessagePack.pack({ 'ack' => 'unmatched' }), Fluent::Plugin::ForwardOutput::AckHandler::Result::CHUNKID_UNMATCHED],
+    'chunk_id is empty' => ['', Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED],
+  )
+  test 'returns chunk_id, node, sock and result status' do |args|
+    receved, state = args
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything) { |_| receved } # IO does not have recv
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_equal chunk_id, a1
+      assert_equal node, a2
+      assert_equal r, a3
+      assert_equal state, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+
+  test 'returns nil if raise an error' do
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything) { |_| raise 'unexpected error' } # IO does not have recv
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_nil a1
+      assert_nil a2
+      assert_nil a3
+      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+
+  test 'when ack is expired' do
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 0, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      mock(r).recv(anything).never
+      ack.enqueue(r)
+
+      a1 = a2 = a3 = a4 = nil
+      ack_handler.collect_response(1) do |cid, n, s, ret|
+        # This block is rescued by ack_handler so it needs to invoke assetion outside of this block
+        a1 = cid; a2 = n; a3 = s; a4 = ret
+      end
+
+      assert_equal chunk_id, a1
+      assert_equal node, a2
+      assert_equal r, a3
+      assert_equal Fluent::Plugin::ForwardOutput::AckHandler::Result::FAILED, a4
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
+end

data/test/plugin/out_forward/test_connection_manager.rb

@@ -0,0 +1,145 @@
+require_relative '../../helper'
+require 'fluent/test/driver/output'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/connection_manager'
+require 'fluent/plugin/out_forward/socket_cache'
+
+class ConnectionManager < Test::Unit::TestCase
+  sub_test_case '#connect' do
+    sub_test_case 'when socket_cache is nil' do
+      test 'creates socket and does not close when block is not given' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :established)
+      end
+
+      test 'creates socket and calls close when block is given' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            sock = 'sock'
+            mock(sock).close.once
+            mock(sock).close_write.once
+            sock
+          },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+
+      test 'when secure is true, state is helo' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: true,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :helo)
+      end
+
+      test 'when passed ack' do
+        sock = 'sock'
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            mock(sock).close.never
+            mock(sock).close_write.never
+            sock
+          },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        ack = mock('ack').enqueue(sock).once.subject
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+    end
+
+    sub_test_case 'when socket_cache exists' do
+      test 'calls connect_keepalive' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').never
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
+
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :established)
+      end
+
+      test 'calls connect_keepalive and closes socket with block' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').once
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock); sock },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
+
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+
+      test 'does not call dec_ref when ack is passed' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').never
+        sock = 'sock'
+        ack = stub('ack').enqueue(sock).once.subject
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            mock(sock).close.never
+            mock(sock).close_write.never
+            sock
+          },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: ack).once
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_handshake_protocol.rb

@@ -0,0 +1,103 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/handshake_protocol'
+require 'fluent/plugin/out_forward/connection_manager'
+
+class HandshakeProtocolTest < Test::Unit::TestCase
+  sub_test_case '#invok when helo state' do
+    test 'sends PING message and change state to pingpong' do
+      hostname = 'hostname'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', {}])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], '')
+        assert_equal(ping[5], '')
+      end
+    end
+
+    test 'returns PING message with username if auth exists' do
+      hostname = 'hostname'
+      username = 'username'
+      pass = 'pass'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: pass, username: username)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', { 'auth' => 'auth' }])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], username)
+        assert_not_equal(ping[5], pass) # should be hashed
+      end
+    end
+
+    data(
+      lack_of_elem: ['HELO'],
+      wrong_message: ['HELLO!', {}],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      assert_raise(Fluent::Plugin::ForwardOutput::HeloError) do
+        handshake.invoke(sock, ri, msg)
+      end
+
+      assert_equal(ri.state, :helo)
+    end
+  end
+
+  sub_test_case '#invok when pingpong state' do
+    test 'sends PING message and change state to pingpong' do
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      handshake.invoke(
+        '',
+        ri,
+        # 40a3.... = Digest::SHA512.new.update('ce1897b0d3dbd76b90d7fb96010dcac3').update('client_hostname').update('').update('shared_key').hexdigest
+        ['PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f']
+      )
+      assert_equal(ri.state, :established)
+    end
+
+    data(
+      lack_of_elem: ['PONG', true, '', 'client_hostname'],
+      wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      error_by_server: ['PONG', false, 'error', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      same_hostname_as_server: ['PONG', true, '', 'hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError) do
+        handshake.invoke('', ri, msg)
+      end
+
+      assert_equal(ri.state, :pingpong)
+    end
+  end
+end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -0,0 +1,60 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward/load_balancer'
+
+class LoadBalancerTest < Test::Unit::TestCase
+  sub_test_case 'select_healthy_node' do
+    test 'select healty node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+    end
+
+    test 'call like round robin' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        end
+      end
+    end
+
+    test 'raise an error if all node are unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
+  end
+end