fluentd222 1.16.2-x86_64-linux

data/test/plugin/out_forward/test_handshake_protocol.rb

@@ -0,0 +1,112 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward'
+require 'fluent/plugin/out_forward/handshake_protocol'
+require 'fluent/plugin/out_forward/connection_manager'
+
+class HandshakeProtocolTest < Test::Unit::TestCase
+  sub_test_case '#invok when helo state' do
+    test 'sends PING message and change state to pingpong' do
+      hostname = 'hostname'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', {}])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], '')
+        assert_equal(ping[5], '')
+      end
+    end
+
+    test 'returns PING message with username if auth exists' do
+      hostname = 'hostname'
+      username = 'username'
+      pass = 'pass'
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: hostname, shared_key: 'shared_key', password: pass, username: username)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      handshake.invoke(sock, ri, ['HELO', { 'auth' => 'auth' }])
+
+      assert_equal(ri.state, :pingpong)
+      Fluent::MessagePackFactory.msgpack_unpacker.feed_each(sock.string) do |ping|
+        assert_equal(ping.size, 6)
+        assert_equal(ping[0], 'PING')
+        assert_equal(ping[1], hostname)
+        assert(ping[2].is_a?(String)) # content is hashed value
+        assert(ping[3].is_a?(String)) # content is hashed value
+        assert_equal(ping[4], username)
+        assert_not_equal(ping[5], pass) # should be hashed
+      end
+    end
+
+    data(
+      lack_of_elem: ['HELO'],
+      wrong_message: ['HELLO!', {}],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      sock = StringIO.new('')
+      assert_raise(Fluent::Plugin::ForwardOutput::HeloError) do
+        handshake.invoke(sock, ri, msg)
+      end
+
+      assert_equal(ri.state, :helo)
+    end
+  end
+
+  sub_test_case '#invok when pingpong state' do
+    test 'sends PING message and change state to pingpong' do
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      handshake.invoke(
+        '',
+        ri,
+        # 40a3.... = Digest::SHA512.new.update('ce1897b0d3dbd76b90d7fb96010dcac3').update('client_hostname').update('').update('shared_key').hexdigest
+        ['PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f']
+      )
+      assert_equal(ri.state, :established)
+    end
+
+    test 'raises an error when password and username are nil if auth exists' do
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: nil, username: nil)
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:helo)
+
+      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError.new('username and password are required')) do
+        handshake.invoke('', ri, ['HELO', { 'auth' => 'auth' }])
+      end
+    end
+
+    data(
+      lack_of_elem: ['PONG', true, '', 'client_hostname'],
+      wrong_message: ['WRONG_PONG', true, '', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      error_by_server: ['PONG', false, 'error', 'client_hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      same_hostname_as_server: ['PONG', true, '', 'hostname', '40a3c5943cc6256e0c5dcf176e97db3826b0909698c330dc8e53d15af63efb47e030d113130255dd6e7ced5176d2999cc2e02a44852d45152503af317b73b33f'],
+      wrong_key: ['PONG', true, '', 'hostname', 'wrong_key'],
+    )
+    test 'raises an error when message is' do |msg|
+      handshake = Fluent::Plugin::ForwardOutput::HandshakeProtocol.new(log: $log, hostname: 'hostname', shared_key: 'shared_key', password: '', username: '')
+      handshake.instance_variable_set(:@shared_key_salt, 'ce1897b0d3dbd76b90d7fb96010dcac3') # to fix salt
+
+      ri = Fluent::Plugin::ForwardOutput::ConnectionManager::RequestInfo.new(:pingpong, '', '')
+      assert_raise(Fluent::Plugin::ForwardOutput::PingpongError) do
+        handshake.invoke('', ri, msg)
+      end
+
+      assert_equal(ri.state, :pingpong)
+    end
+  end
+end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -0,0 +1,106 @@
+require_relative '../../helper'
+require 'flexmock/test_unit'
+
+require 'fluent/plugin/out_forward/load_balancer'
+
+class LoadBalancerTest < Test::Unit::TestCase
+  sub_test_case 'select_healthy_node' do
+    test 'select healthy node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+
+      lb.select_healthy_node do |node|
+        assert_equal(node, n2)
+      end
+    end
+
+    test 'call like round robin' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+
+      lb.rebuild_weight_array([n1, n2])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        end
+      end
+    end
+
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
+    test 'raise an error if all node are unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
+  end
+end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -0,0 +1,174 @@
+require_relative '../../helper'
+
+require 'fluent/plugin/out_forward/socket_cache'
+require 'timecop'
+
+class SocketCacheTest < Test::Unit::TestCase
+  sub_test_case 'checkout_or' do
+    test 'when given key does not exist' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.open { 'socket' }.subject
+      assert_equal('socket', c.checkout_or('key') { sock.open })
+    end
+
+    test 'when given key exists' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      socket = 'socket'
+      assert_equal(socket, c.checkout_or('key') { socket })
+      c.checkin(socket)
+
+      sock = dont_allow(mock!).open
+      assert_equal(socket, c.checkout_or('key') { sock.open })
+    end
+
+    test 'when given key exists but used by other' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      assert_equal('sock', c.checkout_or('key') { 'sock' })
+
+      new_sock = 'new sock'
+      sock = mock!.open { new_sock }.subject
+      assert_equal(new_sock, c.checkout_or('key') { sock.open })
+    end
+
+    test "when given key's value was expired" do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(0, $log)
+      assert_equal('sock', c.checkout_or('key') { 'sock' })
+
+      new_sock = 'new sock'
+      sock = mock!.open { new_sock }.subject
+      assert_equal(new_sock, c.checkout_or('key') { sock.open })
+    end
+
+    test 'reuse same hash object after calling purge_obsolete_socks' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      c.checkout_or('key') { 'socket' }
+      c.purge_obsolete_socks
+
+      assert_nothing_raised(NoMethodError) do
+        c.checkout_or('key') { 'new socket' }
+      end
+    end
+  end
+
+  sub_test_case 'checkin' do
+    test 'when value exists' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      socket = 'socket'
+      c.checkout_or('key') { socket }
+      c.checkin(socket)
+
+      assert_equal(socket, c.instance_variable_get(:@available_sockets)['key'].first.sock)
+      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+    end
+
+    test 'when value does not exist' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      c.checkout_or('key') { 'sock' }
+      c.checkin('other sock')
+
+      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+    end
+  end
+
+  test 'revoke' do
+    c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+    socket = 'socket'
+    c.checkout_or('key') { socket }
+    c.revoke(socket)
+
+    assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)
+    assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+    assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+
+    sock = mock!.open { 1 }.subject
+    assert_equal(1, c.checkout_or('key') { sock.open })
+  end
+
+  sub_test_case 'clear' do
+    test 'when value is in available_sockets' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      m = mock!.close { 'closed' }.subject
+      m2 = mock!.close { 'closed' }.subject
+      m3 = mock!.close { 'closed' }.subject
+      c.checkout_or('key') { m }
+      c.revoke(m)
+      c.checkout_or('key') { m2 }
+      c.checkin(m2)
+      c.checkout_or('key2') { m3 }
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(1, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(1, c.instance_variable_get(:@inactive_sockets).size)
+
+      c.clear
+      assert_equal(0, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(0, c.instance_variable_get(:@available_sockets)['key'].size)
+      assert_equal(0, c.instance_variable_get(:@inactive_sockets).size)
+    end
+  end
+
+  sub_test_case 'purge_obsolete_socks' do
+    def teardown
+      Timecop.return
+    end
+
+    test 'delete key in inactive_socks' do
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.close { 'closed' }.subject
+      c.checkout_or('key') { sock }
+      c.revoke(sock)
+      assert_false(c.instance_variable_get(:@inactive_sockets).empty?)
+
+      c.purge_obsolete_socks
+      assert_true(c.instance_variable_get(:@inactive_sockets).empty?)
+    end
+
+    test 'move key from available_sockets to inactive_sockets' do
+      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
+
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = mock!.close { 'closed' }.subject
+      sock2 = dont_allow(mock!).close
+      stub(sock).inspect
+      stub(sock2).inspect
+
+      c.checkout_or('key') { sock }
+      c.checkin(sock)
+
+      # wait timeout
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
+      c.checkout_or('key') { sock2 }
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
+
+      c.purge_obsolete_socks
+
+      assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
+      assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
+    end
+
+    test 'should not purge just after checkin and purge after timeout' do
+      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
+
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = dont_allow(mock!).close
+      stub(sock).inspect
+      c.checkout_or('key') { sock }
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
+      c.checkin(sock)
+
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900'))
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(0, c.instance_variable_get(:@available_sockets).size)
+    end
+  end
+end

data/test/plugin/test_bare_output.rb

@@ -0,0 +1,131 @@
+require_relative '../helper'
+require 'fluent/plugin/bare_output'
+require 'fluent/event'
+
+module FluentPluginBareOutputTest
+  class DummyPlugin < Fluent::Plugin::BareOutput
+    attr_reader :store
+    def initialize
+      super
+      @store = []
+    end
+    def process(tag, es)
+      es.each do |time, record|
+        @store << [tag, time, record]
+      end
+    end
+  end
+end
+
+class BareOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @p = FluentPluginBareOutputTest::DummyPlugin.new
+  end
+
+  test 'has healthy lifecycle' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'has plugin_id automatically generated' do
+    assert @p.respond_to?(:plugin_id_configured?)
+    assert @p.respond_to?(:plugin_id)
+
+    @p.configure(config_element())
+
+    assert !@p.plugin_id_configured?
+    assert @p.plugin_id
+    assert{ @p.plugin_id != 'mytest' }
+  end
+
+  test 'has plugin_id manually configured' do
+    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+    assert @p.plugin_id_configured?
+    assert_equal 'mytest', @p.plugin_id
+  end
+
+  test 'has plugin logger' do
+    assert @p.respond_to?(:log)
+    assert @p.log
+
+    # default logger
+    original_logger = @p.log
+
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    assert(@p.log.object_id != original_logger.object_id)
+    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+  end
+
+  test 'can load plugin helpers' do
+    assert_nothing_raised do
+      class FluentPluginBareOutputTest::DummyPlugin2 < Fluent::Plugin::BareOutput
+        helpers :storage
+      end
+    end
+  end
+
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.num_errors
+    assert_equal 0, @p.emit_count
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
+  test 'can get input event stream to write' do
+    @p.configure(config_element('ROOT'))
+    @p.start
+
+    es1 = Fluent::OneEventStream.new(event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'})
+    es2 = Fluent::ArrayEventStream.new([
+        [event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
+        [event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
+      ])
+    @p.emit_events('mytest1', es1)
+    @p.emit_events('mytest2', es2)
+
+    all_events = [
+      ['mytest1', event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}],
+      ['mytest2', event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
+      ['mytest2', event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
+    ]
+
+    assert_equal all_events, @p.store
+  end
+end