fluentd 0.14.0 → 0.14.1

data/test/helper.rb

@@ -42,6 +42,7 @@ require 'fileutils'
 require 'fluent/config/element'
 require 'fluent/log'
 require 'fluent/test'
+require 'fluent/test/helpers'
 require 'fluent/plugin/base'
 require 'fluent/log'
 require 'fluent/plugin_id'
@@ -67,30 +68,7 @@ unless defined?(Test::Unit::AssertionFailedError)
   end
 end
 
-def config_element(name = 'test', argument = '', params = {}, elements = [])
-  Fluent::Config::Element.new(name, argument, params, elements)
-end
-
-def event_time(str=nil)
-  if str
-    Fluent::EventTime.parse(str)
-  else
-    Fluent::EventTime.now
-  end
-end
-
-def msgpack(type)
-  case type
-  when :factory
-    Fluent::MessagePackFactory.factory
-  when :packer
-    Fluent::MessagePackFactory.packer
-  when :unpacker
-    Fluent::MessagePackFactory.unpacker
-  else
-    raise ArgumentError, "unknown msgpack object type '#{type}'"
-  end
-end
+include Fluent::Test::Helpers
 
 def unused_port(num = 1)
   ports = []

data/test/plugin/test_in_gc_stat.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_gc_stat'
 
 class GCStatInputTest < Test::Unit::TestCase
@@ -13,7 +13,7 @@ class GCStatInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::GCStatInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::GCStatInput).configure(conf)
   end
 
   def test_configure
@@ -31,9 +31,9 @@ class GCStatInputTest < Test::Unit::TestCase
       sleep 2
     end
 
-    emits = d.emits
-    assert(emits.length > 0)
-    assert_equal(stat, emits[0][2])
-    assert(emits[0][1].is_a?(Fluent::EventTime))
+    events = d.events
+    assert(events.length > 0)
+    assert_equal(stat, events[0][2])
+    assert(events[0][1].is_a?(Fluent::EventTime))
   end
 end

data/test/plugin/test_in_http.rb

@@ -147,6 +147,39 @@ class HttpInputTest < Test::Unit::TestCase
 
   end
 
+  def test_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+        assert_equal "200", res.code
+      }
+    end
+
+  end
+
+  def test_multi_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+      assert_equal "200", res.code
+    end
+
+  end
+
   def test_multi_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
 
@@ -302,50 +335,34 @@ class HttpInputTest < Test::Unit::TestCase
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_allowed = nil
-    acao = nil
+    time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
 
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://foo.com", "Content-Type" => "application/octet-stream"})
-          res = http.request(req)
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>1}
 
-          acao = res["Access-Control-Allow-Origin"]
-        end
-      end
-      test_in_http_cros_allowed = true
-    rescue
-      test_in_http_cros_allowed = false
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"Origin"=>"http://foo.com"})
+        assert_equal "200", res.code
+        assert_equal "http://foo.com", res["Access-Control-Allow-Origin"]
+      }
     end
-
-    assert_equal true, test_in_http_cros_allowed
-    assert_equal "http://foo.com", acao
   end
 
   def test_cors_disallowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_disallowed = nil
-    response_code = nil
+    time = Fluent::EventTime.new(Time.parse("2011-01-02 13:14:15 UTC").to_i)
 
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://bar.com", "Content-Type" => "application/octet-stream"})
-          res = http.request(req)
+    d.expected_emits_length = 0
+    d.run do
+      res = post("/tag1", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
 
-          response_code = res.code
-        end
-      end
-      test_in_http_cros_disallowed = true
-    rescue
-      test_in_http_cros_disallowed = false
+      res = post("/tag2", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
     end
-
-    assert_equal true, test_in_http_cros_disallowed
-    assert_equal "403", response_code
   end
 
   $test_in_http_connection_object_ids = []

data/test/plugin/{test_in_stream.rb → test_in_unix.rb}

@@ -1,6 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
-require 'fluent/plugin/in_stream'
+require 'fluent/plugin/in_unix'
 
 module StreamInputTest
   def setup

data/test/plugin/test_out_buffered_stdout.rb

@@ -0,0 +1,108 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_buffered_stdout'
+
+class BufferedStdoutOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedStdoutOutput).configure(conf)
+  end
+
+  test 'default configure' do
+    d = create_driver
+    assert_equal 'json', d.instance.output_type
+    assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
+    assert d.instance.buffer_config.flush_at_shutdown
+    assert_equal ['tag'], d.instance.buffer_config.chunk_keys
+    assert d.instance.chunk_key_tag
+    assert !d.instance.chunk_key_time
+    assert_equal [], d.instance.chunk_keys
+  end
+
+  test 'configure with output_type' do
+    d = create_driver(CONFIG + "\noutput_type json")
+    assert_equal 'json', d.instance.output_type
+
+    d = create_driver(CONFIG + "\noutput_type hash")
+    assert_equal 'hash', d.instance.output_type
+
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(CONFIG + "\noutput_type foo")
+    end
+  end
+
+  sub_test_case "emit json" do
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    test '#write(synchronous)' do |data|
+      d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
+      time = event_time()
+
+      out = capture_log do
+        d.run(default_tag: 'test', flush: true) do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+      assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+    end
+
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    test '#try_write(asynchronous)' do |data|
+      d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
+      time = event_time()
+      d.instance.delayed = true
+
+      out = capture_log do
+        d.run(default_tag: 'test', flush: true, shutdown: false) do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+
+      assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+    end
+  end
+
+  sub_test_case 'emit hash' do
+    test '#write(synchronous)' do
+      d = create_driver(CONFIG + "\noutput_type hash")
+      time = event_time()
+
+      out = capture_log do
+        d.run(default_tag: 'test', flush: true) do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+
+      assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
+    end
+
+    test '#try_write(asynchronous)' do
+      d = create_driver(CONFIG + "\noutput_type hash")
+      time = event_time()
+      d.instance.delayed = true
+
+      out = capture_log do
+        d.run(default_tag: 'test', flush: true, shutdown: false) do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+
+      assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
+    end
+  end
+
+  # Capture the log output of the block given
+  def capture_log(&block)
+    tmp = $log
+    $log = StringIO.new
+    yield
+    return $log.string
+  ensure
+    $log = tmp
+  end
+end

data/test/plugin/test_out_copy.rb

@@ -1,12 +1,14 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/multi_output'
 require 'fluent/plugin/out_copy'
+require 'fluent/event'
 
 class CopyOutputTest < Test::Unit::TestCase
   class << self
     def startup
       $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
       require 'fluent/plugin/out_test'
+      require 'fluent/plugin/out_test2'
     end
 
     def shutdown
@@ -20,21 +22,21 @@ class CopyOutputTest < Test::Unit::TestCase
 
   CONFIG = %[
     <store>
-      type test
+      @type test
       name c0
     </store>
     <store>
-      type test
+      @type test2
       name c1
     </store>
     <store>
-      type test
+      @type test
       name c2
     </store>
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput).configure(conf)
+    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(conf)
   end
 
   def test_configure
@@ -42,158 +44,117 @@ class CopyOutputTest < Test::Unit::TestCase
 
     outputs = d.instance.outputs
     assert_equal 3, outputs.size
-    assert_equal Fluent::TestOutput, outputs[0].class
-    assert_equal Fluent::TestOutput, outputs[1].class
-    assert_equal Fluent::TestOutput, outputs[2].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
     assert_equal "c0", outputs[0].name
     assert_equal "c1", outputs[1].name
     assert_equal "c2", outputs[2].name
   end
 
-  def test_emit
+  def test_feed_events
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
+    assert !d.instance.outputs[0].has_router?
+    assert_not_nil d.instance.outputs[1].router
+    assert !d.instance.outputs[2].has_router?
 
-    d.instance.outputs.each {|o|
-      assert_equal [
-          [time, {"a"=>1}],
-          [time, {"a"=>2}],
-        ], o.events
-    }
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
+    end
 
     d.instance.outputs.each {|o|
-      assert_not_nil o.router
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
     }
   end
 
-  def test_msgpack_es_emit_bug
-    d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
-
-    outputs = %w(p1 p2).map do |pname|
-      p = Fluent::Plugin.new_output('test')
-      p.configure('name' => pname)
-      p.define_singleton_method(:emit) do |tag, es, chain|
-        es.each do |time, record|
-          super(tag, [[time, record]], chain)
-        end
-      end
-      p
-    end
-
-    d.instance.instance_eval { @outputs = outputs }
+  def test_msgpack_unpacker_cache_bug_for_msgpack_event_stream
+    d = create_driver
 
-    es = if defined?(MessagePack::Packer)
-           time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-           packer = Fluent::Engine.msgpack_factory.packer
-           packer.pack([time, {"a" => 1}])
-           packer.pack([time, {"a" => 2}])
-           Fluent::MessagePackEventStream.new(packer.to_s)
-         else
-           events = "#{[time, {"a" => 1}].to_msgpack}#{[time, {"a" => 2}].to_msgpack}"
-           Fluent::MessagePackEventStream.new(events)
-         end
+    time = event_time("2011-01-02 13:14:15 UTC")
+    source = Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"a" => 2}] ])
+    es = Fluent::MessagePackEventStream.new(source.to_msgpack_stream)
 
-    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
 
     d.instance.outputs.each { |o|
-      assert_equal [
-        [time, {"a"=>1}],
-        [time, {"a"=>2}],
-      ], o.events
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
     }
   end
 
-  def create_event_test_driver(is_deep_copy = false)
-    deep_copy_config = %[
-deep_copy #{is_deep_copy}
-    <store>
-      type test
-      name c0
-    </store>
-    <store>
-      type test
-      name c1
-    </store>
-    <store>
-      type test
-      name c2
-    </store>
-]
-
-    output1 = Fluent::Plugin.new_output('test')
-    output1.configure('name' => 'output1')
-    output1.define_singleton_method(:emit) do |tag, es, chain|
+  def create_event_test_driver(does_deep_copy = false)
+    config = %[
+      deep_copy #{does_deep_copy}
+      <store>
+        @type test
+        name output1
+      </store>
+      <store>
+        @type test
+        name output2
+      </store>
+    ]
+
+    d = Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(config)
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
       es.each do |time, record|
         record['foo'] = 'bar'
-        super(tag, [[time, record]], chain)
-      end
-    end
-
-    output2 = Fluent::Plugin.new_output('test')
-    output2.configure('name' => 'output2')
-    output2.define_singleton_method(:emit) do |tag, es, chain|
-      es.each do |time, record|
-        super(tag, [[time, record]], chain)
       end
+      super(tag, es)
     end
-
-    outputs = [output1, output2]
-
-    d = Fluent::Test::OutputTestDriver.new(Fluent::CopyOutput)
-    d = d.configure(deep_copy_config)
-    d.instance.instance_eval { @outputs = outputs }
     d
   end
 
-  def test_one_event
-    time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-
-    d = create_event_test_driver(false)
-    es = Fluent::OneEventStream.new(time, {"a" => 1})
-    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
-
-    assert_equal [
-      [[time, {"a"=>1, "foo"=>"bar"}]],
-      [[time, {"a"=>1, "foo"=>"bar"}]]
-    ], d.instance.outputs.map{ |o| o.events }
+  time = event_time("2013-05-26 06:37:22 UTC")
+  mes0 = Fluent::MultiEventStream.new
+  mes0.add(time, {"a" => 1})
+  mes0.add(time, {"b" => 1})
+  mes1 = Fluent::MultiEventStream.new
+  mes1.add(time, {"a" => 1})
+  mes1.add(time, {"b" => 1})
+
+  data(
+    "OneEventStream without deep_copy"   => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
+    "OneEventStream with deep_copy"      => [true,  Fluent::OneEventStream.new(time, {"a" => 1})],
+    "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "ArrayEventStream with deep_copy"    => [true,  Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "MultiEventStream without deep_copy" => [false, mes0],
+    "MultiEventStream with deep_copy"    => [true,  mes1],
+  )
+  def test_deep_copy_controls_shallow_or_deep_copied(data)
+    does_deep_copy, es = data
+
+    d = create_event_test_driver(does_deep_copy)
+
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
 
-    d = create_event_test_driver(true)
-    es = Fluent::OneEventStream.new(time, {"a" => 1})
-    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
+    events = d.instance.outputs.map(&:events)
 
-    assert_equal [
-      [[time, {"a"=>1, "foo"=>"bar"}]],
-      [[time, {"a"=>1}]]
-    ], d.instance.outputs.map{ |o| o.events }
-  end
+    if does_deep_copy
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
 
-  def test_multi_event
-    time = Time.parse("2013-05-26 06:37:22 UTC").to_i
-
-    d = create_event_test_driver(false)
-    es = Fluent::MultiEventStream.new
-    es.add(time, {"a" => 1})
-    es.add(time, {"b" => 2})
-    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
-
-    assert_equal [
-      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
-      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]]
-    ], d.instance.outputs.map{ |o| o.events }
-
-    d = create_event_test_driver(true)
-    es = Fluent::MultiEventStream.new
-    es.add(time, {"a" => 1})
-    es.add(time, {"b" => 2})
-    d.instance.emit('test', es, Fluent::NullOutputChain.instance)
-
-    assert_equal [
-      [[time, {"a"=>1, "foo"=>"bar"}], [time, {"b"=>2, "foo"=>"bar"}]],
-      [[time, {"a"=>1}], [time, {"b"=>2}]]
-    ], d.instance.outputs.map{ |o| o.events }
+        assert{ record0.object_id != record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert !record1.has_key?("foo")
+      end
+    else
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
+
+        assert{ record0.object_id == record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert_equal "bar", record1["foo"]
+      end
+    end
   end
 end