fluentd 0.12.0.pre.1 → 0.12.0.pre.2

data/test/plugin/test_in_exec.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 require 'net/http'
 

data/test/plugin/test_in_forward.rb

@@ -1,9 +1,11 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
+require 'base64'
 
 class ForwardInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @responses = []  # for testing responses after sending data
   end
 
   PORT = unused_port
@@ -198,13 +200,214 @@ class ForwardInputTest < Test::Unit::TestCase
     }.size == 1, "large chunk warning is not logged"
   end
 
-  def send_data(data)
+  def test_respond_to_message_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      events.each {|tag,time,record|
+        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data [tag, time, record, op].to_msgpack, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
+  def test_respond_to_forward_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+      expected_acks << op['chunk']
+      send_data ["tag1", entries, op].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  def test_respond_to_packed_forward_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      entries = ''
+      events.each {|tag,time,record|
+        [time, record].to_msgpack(entries)
+      }
+      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+      expected_acks << op['chunk']
+      send_data ["tag1", entries, op].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  def test_respond_to_message_json_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      events.each {|tag,time,record|
+        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data [tag, time, record, op].to_json, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
+
+  end
+
+  def test_not_respond_to_message_not_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      events.each {|tag,time,record|
+        send_data [tag, time, record].to_msgpack, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil, nil], @responses
+  end
+
+  def test_not_respond_to_forward_not_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ["tag1", entries].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil], @responses
+  end
+
+  def test_not_respond_to_packed_forward_not_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      entries = ''
+      events.each {|tag,time,record|
+        [time, record].to_msgpack(entries)
+      }
+      send_data ["tag1", entries].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil], @responses
+  end
+
+  def test_not_respond_to_message_json_not_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      events.each {|tag,time,record|
+        send_data [tag, time, record].to_json, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil, nil], @responses
+  end
+
+  def send_data(data, try_to_receive_response=false, response_timeout=1)
     io = connect
     begin
       io.write data
+      if try_to_receive_response
+        if IO.select([io], nil, nil, response_timeout)
+          res = io.recv(1024)
+        end
+        # timeout means no response, so push nil to @responses
+      end
     ensure
       io.close
     end
+    @responses << res if try_to_receive_response
   end
 
   # TODO heartbeat

data/test/plugin/test_in_gc_stat.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 
 class GCStatInputTest < Test::Unit::TestCase

data/test/plugin/test_in_http.rb

@@ -1,5 +1,5 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 require 'net/http'
 
 class HttpInputTest < Test::Unit::TestCase
@@ -157,7 +157,7 @@ class HttpInputTest < Test::Unit::TestCase
 
   def test_with_regexp
     d = create_driver(CONFIG + %[
-      format /^(?<field_1>\\\\d+):(?<field_2>\\\\w+)$/
+      format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
       types field_1:integer
     ])
 
@@ -199,6 +199,62 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_if_content_type_is_initialized_properly
+    # This test is to check if Fluent::HttpInput::Handler's @content_type is initialized properly.
+    # Especially when in Keep-Alive and the second request has no 'Content-Type'.
+    #
+    # Actually, in the current implementation of in_http, we can't test it directly.
+    # So we replace Fluent::HttpInput::Handler temporally with the extended Handler
+    # in order to collect @content_type(s) per request.
+    # Finally, we check those collected @content_type(s).
+
+    # Save the original Handler
+    orig_handler = Fluent::HttpInput::Handler
+
+    begin
+      # Create the extended Handler which can store @content_type per request
+      ext_handler = Class.new(Fluent::HttpInput::Handler) do
+          @@content_types = []
+
+          def self.content_types
+            @@content_types
+          end
+
+          def on_message_complete
+            @@content_types << @content_type
+            super
+          end
+        end
+
+      # Replace the original Handler temporally with the extended one
+      Fluent::HttpInput.module_eval do
+        remove_const(:Handler) if const_defined?(:Handler)
+        const_set(:Handler, ext_handler)
+      end
+
+      d = create_driver
+
+      d.run do
+        # Send two requests the second one has no Content-Type in Keep-Alive
+        Net::HTTP.start("127.0.0.1", PORT) do |http|
+          req = Net::HTTP::Post.new("/foodb/bartbl", {"connection" => "keepalive", "content-type" => "application/json"})
+          res = http.request(req)
+
+          req = Net::HTTP::Get.new("/foodb/bartbl", {"connection" => "keepalive"})
+          res = http.request(req)
+        end
+
+        assert_equal(['application/json', ''], ext_handler.content_types)
+      end
+    ensure
+      # Revert the original Handler
+      Fluent::HttpInput.module_eval do
+        remove_const(:Handler) if const_defined?(:Handler)
+        const_set(:Handler, orig_handler)
+      end
+    end
+  end
+
   def post(path, params, header = {})
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Post.new(path, header)

data/test/plugin/test_in_object_space.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 
 class ObjectSpaceInputTest < Test::Unit::TestCase

data/test/plugin/test_in_status.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 
 class StatusInputTest < Test::Unit::TestCase

data/test/plugin/test_in_stream.rb

@@ -1,5 +1,5 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 
 module StreamInputTest
   def setup

data/test/plugin/test_in_syslog.rb

@@ -1,5 +1,5 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 
 class SyslogInputTest < Test::Unit::TestCase
   def setup

data/test/plugin/test_in_tail.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 require 'net/http'
 require 'flexmock'

data/test/plugin/test_in_tcp.rb

@@ -1,5 +1,5 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 
 class TcpInputTest < Test::Unit::TestCase
   def setup

data/test/plugin/test_in_udp.rb

@@ -1,5 +1,5 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 
 class UdpInputTest < Test::Unit::TestCase
   def setup

data/test/plugin/test_out_copy.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 
 class CopyOutputTest < Test::Unit::TestCase

data/test/plugin/test_out_exec.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 require 'fileutils'
 

data/test/plugin/test_out_exec_filter.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 require 'fileutils'
 

data/test/plugin/test_out_file.rb

@@ -1,3 +1,4 @@
+require 'helper'
 require 'fluent/test'
 require 'fileutils'
 require 'time'
@@ -63,6 +64,41 @@ class FileOutputTest < Test::Unit::TestCase
     d.run
   end
 
+  def test_timezone_1
+    d = create_driver %[
+      path #{TMP_DIR}/out_file_test
+      timezone Asia/Taipei
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit({"a"=>1}, time)
+    d.expect_format %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n]
+    d.run
+  end
+
+  def test_timezone_2
+    d = create_driver %[
+      path #{TMP_DIR}/out_file_test
+      timezone -03:30
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.emit({"a"=>1}, time)
+    d.expect_format %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}\n]
+    d.run
+  end
+
+  def test_timezone_invalid
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[
+        path #{TMP_DIR}/out_file_test
+        timezone Invalid/Invalid
+      ]
+    end
+  end
+
   def check_gzipped_result(path, expect)
     # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
     # Following code from https://www.ruby-forum.com/topic/971591#979520

data/test/plugin/test_out_forward.rb

@@ -1,26 +1,45 @@
-require 'fluent/test'
 require 'helper'
+require 'fluent/test'
 
 class ForwardOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
+  TARGET_HOST = '127.0.0.1'
+  TARGET_PORT = 13999
   CONFIG = %[
     send_timeout 51
     <server>
       name test
-      host 127.0.0.1
-      port 13999
+      host #{TARGET_HOST}
+      port #{TARGET_PORT}
     </server>
   ]
 
+  TARGET_CONFIG = %[
+    port #{TARGET_PORT}
+    bind #{TARGET_HOST}
+  ]
+
   def create_driver(conf=CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::ForwardOutput) do
-      def write(chunk)
-        chunk.read
+    Fluent::Test::OutputTestDriver.new(Fluent::ForwardOutput) {
+      attr_reader :responses, :exceptions
+
+      def initialize
+        super
+        @responses = []
+        @exceptions = []
       end
-    end.configure(conf)
+
+      def send_data(node, tag, chunk)
+        # Original #send_data returns nil when it does not wait for responses or when on response timeout.
+        @responses << super(node, tag, chunk)
+      rescue => e
+        @exceptions << e
+        raise e
+      end
+    }.configure(conf)
   end
 
   def test_configure
@@ -52,5 +71,257 @@ class ForwardOutputTest < Test::Unit::TestCase
     node.tick
     assert_equal node.available, false
   end
-end
 
+  def test_wait_response_timeout_config
+    d = create_driver(CONFIG)
+    assert_equal false, d.instance.extend_internal_protocol
+    assert_equal false, d.instance.require_ack_response
+    assert_equal 190, d.instance.ack_response_timeout
+
+    d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    assert d.instance.extend_internal_protocol
+    assert d.instance.require_ack_response
+    assert_equal 2, d.instance.ack_response_timeout
+  end
+
+  def test_send_to_a_node_supporting_responses
+    target_input_driver = create_target_input_driver(true)
+
+    d = create_driver(CONFIG + %[flush_interval 1s])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
+    assert_empty d.instance.exceptions
+  end
+
+  def test_send_to_a_node_not_supporting_responses
+    target_input_driver = create_target_input_driver
+
+    d = create_driver(CONFIG + %[flush_interval 1s])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
+    assert_empty d.instance.exceptions
+  end
+
+  def test_require_a_node_supporting_responses_to_respond_with_ack
+    target_input_driver = create_target_input_driver(true)
+
+    d = create_driver(CONFIG + %[
+      flush_interval 1s
+      require_ack_response true
+      ack_response_timeout 1s
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal 1, d.instance.responses.length
+    assert d.instance.responses[0].has_key?('ack')
+    assert_empty d.instance.exceptions
+  end
+
+  def test_require_a_node_not_supporting_responses_to_respond_with_ack
+    target_input_driver = create_target_input_driver
+
+    d = create_driver(CONFIG + %[
+      flush_interval 1s
+      require_ack_response true
+      ack_response_timeout 1s
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    node = d.instance.nodes.first
+    assert_equal false, node.available # node is regarded as unavailable when timeout
+
+    assert_empty d.instance.responses # send_data() raises exception, so response is missing
+    assert_equal 1, d.instance.exceptions.size
+  end
+
+  def create_target_input_driver(do_respond=false, conf=TARGET_CONFIG)
+    require 'fluent/plugin/in_forward'
+
+    DummyEngineDriver.new(Fluent::ForwardInput) {
+      handler_class = Class.new(Fluent::ForwardInput::Handler) { |klass|
+        attr_reader :chunk_counter # for checking if received data is successfully deserialized
+
+        def initialize(sock, log, on_message)
+          @sock = sock
+          @log = log
+          @chunk_counter = 0
+          @on_message = on_message
+        end
+
+        if do_respond
+          def write(data)
+            @sock.write data
+          rescue => e
+            @sock.close
+          end
+        else
+          def write(data)
+            # do nothing
+          end
+        end
+
+        def close
+          @sock.close
+        end
+      }
+
+      define_method(:start) do
+        @thread = Thread.new do
+          Socket.tcp_server_loop(@host, @port) do |sock, client_addrinfo|
+            begin
+              handler = handler_class.new(sock, @log, method(:on_message))
+              loop do
+                raw_data = sock.recv(1024)
+                handler.on_read(raw_data)
+                # chunk_counter is reset to zero only after all the data have been received and successfully deserialized.
+                break if handler.chunk_counter == 0
+              end
+              sleep  # wait for connection to be closed by client
+            ensure
+              sock.close
+            end
+          end
+        end
+      end
+
+      def shutdown
+        @thread.kill
+        @thread.join
+      end
+    }.configure(conf).inject_router()
+  end
+
+  class DummyEngineDriver < Fluent::Test::TestDriver
+    def initialize(klass, &block)
+      super(klass, &block)
+      @engine = DummyEngineClass.new
+      @klass = klass
+      # To avoid accessing Fluent::Engine, set Engine as a plugin's class constant (Fluent::SomePlugin::Engine).
+      # But this makes it impossible to run tests concurrently by threading in a process.
+      @klass.const_set(:Engine, @engine)
+    end
+
+    def inject_router
+      @instance.router = @engine
+      self
+    end
+
+    def run(&block)
+      super(&block)
+      @klass.class_eval do
+        remove_const(:Engine)
+      end
+    end
+
+    def emits
+      all = []
+      @engine.emit_streams.each {|tag,events|
+        events.each {|time,record|
+          all << [tag, time, record]
+        }
+      }
+      all
+    end
+
+    class DummyEngineClass
+      attr_reader :emit_streams
+
+      def initialize
+        @emit_streams ||= []
+      end
+
+      def emit_stream(tag, es)
+        @emit_streams << [tag, es.to_a]
+      end
+    end
+  end
+end