fluentd 0.12.43 → 0.14.0

data/test/plugin/test_in_forward.rb

@@ -7,6 +7,18 @@ require 'fluent/env'
 require 'fluent/plugin/in_forward'
 
 class ForwardInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
   def setup
     Fluent::Test.setup
     @responses = []  # for testing responses after sending data
@@ -17,7 +29,6 @@ class ForwardInputTest < Test::Unit::TestCase
     port #{PORT}
     bind 127.0.0.1
   ]
-  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
 
   def create_driver(conf=CONFIG)
     Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
@@ -41,15 +52,15 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_time
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
     Fluent::Engine.now = time
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data [tag, 0, record].to_msgpack
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
       }
     end
   end
@@ -57,14 +68,29 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_message
     d = create_driver
 
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
+      }
+    end
+  end
+
+  def test_message_with_time_as_integer
+    d = create_driver
+
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
       }
     end
   end
@@ -72,19 +98,19 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_message_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag2", time, {"a" => 2}
 
     d.run do
-      entries = d.expected_emits.map { |tag, time, record| [tag, time, record] }
+      entries = d.expected_emits.map {|tag, _time, record| [tag, _time, record] }
       # These entries are skipped
       entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
 
-      entries.each { |tag, time, record|
+      entries.each {|tag, _time, record|
         # Without ack, logs are sometimes not saved to logs during test.
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s, true
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s, true
       }
     end
 
@@ -94,30 +120,47 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_forward
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag1", time, {"a"=>2}
 
     d.run do
       entries = []
-      d.expected_emits.each {|tag,time,record|
+      d.expected_emits.each {|tag, _time,record|
         entries << [time, record]
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
   end
 
+  def test_forward_with_time_as_integer
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = []
+      d.expected_emits.each {|_tag, _time, record|
+        entries << [_time, record]
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
   def test_forward_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag1", time, {"a" => 2}
 
     d.run do
-      entries = d.expected_emits.map { |tag, time, record| [time, record] }
+      entries = d.expected_emits.map {|_tag, _time, record| [_time, record] }
       # These entries are skipped
       entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
 
@@ -130,6 +173,23 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_packed_forward
     d = create_driver
 
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = ''
+      d.expected_emits.each {|_tag, _time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
+  def test_packed_forward_with_time_as_integer
+    d = create_driver
+
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
     d.expect_emit "tag1", time, {"a"=>1}
@@ -137,8 +197,8 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       entries = ''
-      d.expected_emits.each {|tag,time,record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      d.expected_emits.each {|_tag, _time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
@@ -147,19 +207,19 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_packed_forward_with_skip_invalid_event
     d = create_driver(CONFIG + "skip_invalid_event true")
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a" => 1}
     d.expect_emit "tag1", time, {"a" => 2}
 
     d.run do
-      entries = d.expected_emits.map { |tag ,time, record| [time, record] }
+      entries = d.expected_emits.map {|_tag , _time, record| [_time, record] }
       # These entries are skipped
       entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
 
       packed_entries = ''
-      entries.each { |time, record|
-        Fluent::Engine.msgpack_factory.packer(packed_entries).write([time, record]).flush
+      entries.each {|_time, record|
+        Fluent::Engine.msgpack_factory.packer(packed_entries).write([_time, record]).flush
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
     end
@@ -176,8 +236,8 @@ class ForwardInputTest < Test::Unit::TestCase
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data [tag, time, record].to_json
+      d.expected_emits.each {|tag, _time, record|
+        send_data [tag, _time, record].to_json
       }
     end
   end
@@ -188,7 +248,7 @@ class ForwardInputTest < Test::Unit::TestCase
       chunk_size_limit 32M
     ])
 
-    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
     # generate over 16M chunk
     str = "X" * 1024 * 1024
@@ -198,7 +258,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
 
@@ -206,7 +266,7 @@ class ForwardInputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 16, emits.size
     assert emits.map(&:first).all?{|t| t == "test.tag" }
-    assert_equal (0...16).to_a, emits.map{|tag, t, record| t - time }
+    assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
 
     # check log
     assert d.instance.log.logs.select{|line|
@@ -219,7 +279,7 @@ class ForwardInputTest < Test::Unit::TestCase
     d = create_driver(CONFIG + %[
       chunk_size_warn_limit 16M
     ])
-    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
     # generate over 16M chunk
     str = "X" * 1024 * 1024
@@ -227,7 +287,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
 
@@ -244,7 +304,7 @@ class ForwardInputTest < Test::Unit::TestCase
       chunk_size_limit 32M
     ])
 
-    time = Time.parse("2014-04-25 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
 
     # generate over 32M chunk
     str = "X" * 1024 * 1024
@@ -254,7 +314,7 @@ class ForwardInputTest < Test::Unit::TestCase
     # d.run => send_data
     d.run do
       Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        d.instance.send(:on_message, obj, chunk.size, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
       end
     end
 
@@ -276,7 +336,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     # d.run => send_data
     d.run do
-      d.instance.send(:on_message, data, 1000000000, PEERADDR)
+      d.instance.send(:on_message, data, 1000000000, "host: 127.0.0.1, addr: 127.0.0.1, port: 0000")
     end
 
     # check emitted data
@@ -292,7 +352,7 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_respond_to_message_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -303,10 +363,10 @@ class ForwardInputTest < Test::Unit::TestCase
     expected_acks = []
 
     d.run do
-      events.each {|tag,time,record|
+      events.each {|tag, _time, record|
         op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
         expected_acks << op['chunk']
-        send_data [tag, time, record, op].to_msgpack, true
+        send_data [tag, _time, record, op].to_msgpack, true
       }
     end
 
@@ -318,7 +378,7 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_respond_to_forward_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -330,7 +390,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       entries = []
-      events.each {|tag,time,record|
+      events.each {|_tag, _time, record|
         entries << [time, record]
       }
       op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
@@ -345,7 +405,7 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_respond_to_packed_forward_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -357,8 +417,8 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       entries = ''
-      events.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+      events.each {|_tag, _time, record|
+        [_time, record].to_msgpack(entries)
       }
       op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
       expected_acks << op['chunk']
@@ -383,21 +443,22 @@ class ForwardInputTest < Test::Unit::TestCase
     expected_acks = []
 
     d.run do
-      events.each {|tag,time,record|
+      events.each {|tag, _time, record|
         op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
         expected_acks << op['chunk']
-        send_data [tag, time, record, op].to_json, true
+        send_data [tag, _time, record, op].to_json, true
       }
     end
 
     assert_equal events, d.emits
     assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
+
   end
 
   def test_not_respond_to_message_not_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -406,8 +467,8 @@ class ForwardInputTest < Test::Unit::TestCase
     d.expected_emits_length = events.length
 
     d.run do
-      events.each {|tag,time,record|
-        send_data [tag, time, record].to_msgpack, true
+      events.each {|tag, _time, record|
+        send_data [tag, _time, record].to_msgpack, true
       }
     end
 
@@ -418,7 +479,7 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_not_respond_to_forward_not_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -428,8 +489,8 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       entries = []
-      events.each {|tag,time,record|
-        entries << [time, record]
+      events.each {|_tag, _time, record|
+        entries << [_time, record]
       }
       send_data ["tag1", entries].to_msgpack, true
     end
@@ -441,7 +502,7 @@ class ForwardInputTest < Test::Unit::TestCase
   def test_not_respond_to_packed_forward_not_requiring_ack
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     events = [
       ["tag1", time, {"a"=>1}],
@@ -451,8 +512,8 @@ class ForwardInputTest < Test::Unit::TestCase
 
     d.run do
       entries = ''
-      events.each {|tag,time,record|
-        [time, record].to_msgpack(entries)
+      events.each {|_tag, _time, record|
+        [_time, record].to_msgpack(entries)
       }
       send_data ["tag1", entries].to_msgpack, true
     end
@@ -473,8 +534,8 @@ class ForwardInputTest < Test::Unit::TestCase
     d.expected_emits_length = events.length
 
     d.run do
-      events.each {|tag,time,record|
-        send_data [tag, time, record].to_json, true
+      events.each {|tag, _time, record|
+        send_data [tag, _time, record].to_json, true
       }
     end
 
@@ -498,105 +559,5 @@ class ForwardInputTest < Test::Unit::TestCase
     @responses << res if try_to_receive_response
   end
 
-  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
-  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
-  DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
-
-  sub_test_case 'source_hostname_key and source_address_key features' do
-    test 'resolve_hostname must be true with source_hostname_key' do
-      assert_raise(Fluent::ConfigError) {
-        create_driver(CONFIG + <<EOS)
-resolve_hostname false
-source_hostname_key hostname
-EOS
-      }
-    end
-    data(
-      both: [:hostname, :address],
-      hostname: [:hostname],
-      address: [:address],
-    )
-    test 'message protocol' do |keys|
-      execute_test(*keys) { |events|
-        events.each { |tag, time, record|
-          send_data [tag, time, record].to_msgpack
-        }
-      }
-    end
-
-    data(
-      both: [:hostname, :address],
-      hostname: [:hostname],
-      address: [:address],
-    )
-    test 'forward protocol' do |keys|
-      execute_test(*keys) { |events|
-        entries = []
-        events.each {|tag,time,record|
-          entries << [time, record]
-        }
-        send_data ['tag1', entries].to_msgpack
-      }
-    end
-
-    data(
-      both: [:hostname, :address],
-      hostname: [:hostname],
-      address: [:address],
-    )
-    test 'packed forward protocol' do |keys|
-      execute_test(*keys) { |events|
-        entries = ''
-        events.each { |tag, time, record|
-          Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
-        }
-        send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
-      }
-    end
-
-    def execute_test(*keys, &block)
-      conf = CONFIG.dup
-      if keys.include?(:hostname)
-        conf << <<EOL
-source_hostname_key source_hostname
-EOL
-      end
-      if keys.include?(:address)
-        conf << <<EOL
-source_address_key source_address
-EOL
-      end
-      d = create_driver(conf)
-
-      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-      events = [
-        ["tag1", time, {"a"=>1}],
-        ["tag1", time, {"a"=>2}]
-      ]
-      d.expected_emits_length = events.length
-
-      d.run do
-        block.call(events)
-      end
-
-      d.emits.each { |tag, _time, record|
-        if keys.include?(:hostname)
-          assert_true record.has_key?('source_hostname')
-          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
-          unless keys.include?(:address)
-            assert_false record.has_key?('source_address')
-          end
-        end
-        if keys.include?(:address)
-          assert_true record.has_key?('source_address')
-          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
-          unless keys.include?(:hostname)
-            assert_false record.has_key?('source_hostname')
-          end
-        end
-      }
-    end
-  end
-
   # TODO heartbeat
 end