fluentd 1.6.3 → 1.7.0.rc1

data/test/plugin/out_forward/test_connection_manager.rb

@@ -0,0 +1,145 @@
+require_relative '../../helper'
+require 'fluent/test/driver/output'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/connection_manager'
+require 'fluent/plugin/out_forward/socket_cache'
+
+class ConnectionManager < Test::Unit::TestCase
+  sub_test_case '#connect' do
+    sub_test_case 'when socket_cache is nil' do
+      test 'creates socket and does not close when block is not given' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :established)
+      end
+
+      test 'creates socket and calls close when block is given' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            sock = 'sock'
+            mock(sock).close.once
+            mock(sock).close_write.once
+            sock
+          },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+
+      test 'when secure is true, state is helo' do
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: true,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :helo)
+      end
+
+      test 'when passed ack' do
+        sock = 'sock'
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            mock(sock).close.never
+            mock(sock).close_write.never
+            sock
+          },
+          socket_cache: nil,
+        )
+
+        mock.proxy(cm).connect_keepalive(anything).never
+        ack = mock('ack').enqueue(sock).once.subject
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+    end
+
+    sub_test_case 'when socket_cache exists' do
+      test 'calls connect_keepalive' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').never
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock).close.never; sock },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
+
+        sock, ri = cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil)
+        assert_equal(sock, 'sock')
+        assert_equal(ri.state, :established)
+      end
+
+      test 'calls connect_keepalive and closes socket with block' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').once
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) { sock = 'sock'; mock(sock); sock },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: nil).once
+
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: nil) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+
+      test 'does not call dec_ref when ack is passed' do
+        cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+        mock(cache).checkin('sock').never
+        sock = 'sock'
+        ack = stub('ack').enqueue(sock).once.subject
+
+        cm = Fluent::Plugin::ForwardOutput::ConnectionManager.new(
+          log: $log,
+          secure: false,
+          connection_factory: -> (_, _, _) {
+            mock(sock).close.never
+            mock(sock).close_write.never
+            sock
+          },
+          socket_cache: cache,
+        )
+
+        mock.proxy(cm).connect_keepalive(host: 'host', port: 1234, hostname: 'hostname', ack: ack).once
+        cm.connect(host: 'host', port: 1234, hostname: 'hostname', ack: ack) do |sock, ri|
+          assert_equal(sock, 'sock')
+          assert_equal(ri.state, :established)
+        end
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_handshake_protocol.rb

@@ -0,0 +1,103 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/handshake_protocol'
+require 'fluent/plugin/out_forward/connection_manager'
+
+class HandshakeProtocolTest < Test::Unit::TestCase
+  sub_test_case '#invok when helo state' do
+    test 'sends PING message and change state to pingpong' do
+      hostname = 'hostname'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', {}])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], '')
+        assert_equal(ping[5], '')
+      end
+    end
+
+    test 'returns PING message with username if auth exists' do
+      hostname = 'hostname'
+      username = 'username'
+      pass = 'pass'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: pass, username: username)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', { 'auth' => 'auth' }])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], username)
+        assert_not_equal(ping[5], pass) # should be hashed
+      end
+    end
+
+    data(
+      lack_of_elem: ['HELO'],
+      wrong_message: ['HELLO!', {}],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      assert_raise(Fluent::Plugin::ForwardOutput::HeloError) do
+        handshake.invoke(sock, ri, msg)
+      end
+
+      assert_equal(ri.state, :helo)
+    end
+  end
+
+  sub_test_case '#invok when pingpong state' do
+    test 'sends PING message and change state to pingpong' do
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      handshake.invoke(
+        '',
+        ri,
+        # 40a3.... = Digest::SHA512.new.update('ce1897b0d3dbd76b90d7fb96010dcac3').update('client_hostname').update('').update('shared_key').hexdigest
+        ['PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f']
+      )
+      assert_equal(ri.state, :established)
+    end
+
+    data(
+      lack_of_elem: ['PONG', true, '', 'client_hostname'],
+      wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      error_by_server: ['PONG', false, 'error', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      same_hostname_as_server: ['PONG', true, '', 'hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError) do
+        handshake.invoke('', ri, msg)
+      end
+
+      assert_equal(ri.state, :pingpong)
+    end
+  end
+end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -0,0 +1,60 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward/load_balancer'
+
+class LoadBalancerTest < Test::Unit::TestCase
+  sub_test_case 'select_healthy_node' do
+    test 'select healty node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+    end
+
+    test 'call like round robin' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        end
+      end
+    end
+
+    test 'raise an error if all node are unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -0,0 +1,139 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/out_forward/socket_cache'
+require 'timecop'
+
+class SocketCacheTest < Test::Unit::TestCase
+  sub_test_case 'checkout_or' do
+    test 'when gived key does not exist' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.open { 'socket' }.subject
+      assert_equal('socket', c.checkout_or('key') { sock.open })
+    end
+
+    test 'when given key exists' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      socket = 'socket'
+      assert_equal(socket, c.checkout_or('key') { socket })
+      c.checkin(socket)
+
+      sock = dont_allow(mock!).open
+      assert_equal(socket, c.checkout_or('key') { sock.open })
+    end
+
+    test 'when given key exists but used by other' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      assert_equal(@sock, c.checkout_or('key') { @sock })
+
+      new_sock = 'new sock'
+      sock = mock!.open { new_sock }.subject
+      assert_equal(new_sock, c.checkout_or('key') { sock.open })
+    end
+
+    test "when given key's value was expired" do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(0, $log)
+      assert_equal(@sock, c.checkout_or('key') { @sock })
+
+      new_sock = 'new sock'
+      sock = mock!.open { new_sock }.subject
+      assert_equal(new_sock, c.checkout_or('key') { sock.open })
+    end
+  end
+
+  sub_test_case 'checkin' do
+    test 'when value exists' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      socket = 'socket'
+      c.checkout_or('key') { socket }
+      c.checkin(socket)
+
+      assert_equal(socket, c.instance_variable_get(:@available_sockets)['key'].first.sock)
+      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+    end
+
+    test 'when value does not exist' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      c.checkout_or('key') { 'sock' }
+      c.checkin('other sock')
+
+      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+    end
+  end
+
+  test 'revoke' do
+    c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+    socket = 'socket'
+    c.checkout_or('key') { socket }
+    c.revoke(socket)
+
+    assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)
+    assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+    assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+
+    sock = mock!.open { 1 }.subject
+    assert_equal(1, c.checkout_or('key') { sock.open })
+  end
+
+  sub_test_case 'clear' do
+    test 'when value is in available_sockets' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      m = mock!.close { 'closed' }.subject
+      m2 = mock!.close { 'closed' }.subject
+      m3 = mock!.close { 'closed' }.subject
+      c.checkout_or('key') { m }
+      c.revoke(m)
+      c.checkout_or('key') { m2 }
+      c.checkin(m2)
+      c.checkout_or('key2') { m3 }
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)
+
+      c.clear
+      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(0, c.instance_variable_get(:@inactive_sockets).size)
+    end
+  end
+
+  sub_test_case 'purge_obsolete_socks' do
+    test 'delete key in inactive_socks' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.close { 'closed' }.subject
+      c.checkout_or('key') { sock }
+      c.revoke(sock)
+      assert_false(c.instance_variable_get(:@inactive_sockets).empty?)
+
+      c.purge_obsolete_socks
+      assert_true(c.instance_variable_get(:@inactive_sockets).empty?)
+    end
+
+    test 'move key from available_sockets to inactive_sockets' do
+      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
+
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.close { 'closed' }.subject
+      sock2 = dont_allow(mock!).close
+      stub(sock).inspect
+      stub(sock2).inspect
+
+      c.checkout_or('key') { sock }
+      c.checkin(sock)
+
+      # wait timeout
+      Timecop.freeze(Time.parse('2016-04-13 14:20:00 +0900'))
+      c.checkout_or('key') { sock2 }
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
+
+      c.purge_obsolete_socks
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
+    end
+  end
+end