fluentd 0.14.4-x86-mingw32

data/test/plugin/test_output_as_standard.rb

@@ -0,0 +1,374 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/msgpack_factory'
+require 'fluent/event'
+
+require 'json'
+require 'time'
+require 'timeout'
+
+require 'flexmock/test_unit'
+
+module FluentPluginStandardBufferedOutputTest
+  class DummyBareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+  class DummyAsyncOutput < DummyBareOutput
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+  class DummyAsyncStandardOutput < DummyBareOutput
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+end
+
+class StandardBufferedOutputTest < Test::Unit::TestCase
+  def create_output(type=:full)
+    case type
+    when :bare     then FluentPluginStandardBufferedOutputTest::DummyBareOutput.new
+    when :buffered then FluentPluginStandardBufferedOutputTest::DummyAsyncOutput.new
+    when :standard then FluentPluginStandardBufferedOutputTest::DummyAsyncStandardOutput.new
+    else
+      raise ArgumentError, "unknown type: #{type}"
+    end
+  end
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+  def test_event_stream
+    es = Fluent::MultiEventStream.new
+    es.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+    es.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+    es.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+    es.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+    es.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+    es.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+    es
+  end
+
+  setup do
+    @i = nil
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  sub_test_case 'standard buffered without any chunk keys' do
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
+      @i = create_output(:standard)
+      @i.configure(config_element())
+      @i.start
+      @i.after_start
+
+      m = create_metadata()
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"}))
+      @i.start
+      @i.after_start
+
+      m = create_metadata()
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'standard buffered with tag chunk key' do
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m = create_metadata(tag: "mytag.test")
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with predefined msgpack format, but time will be int if time_as_integer specified' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{"time_as_integer"=>"true"},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m = create_metadata(tag: "mytag.test")
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es}, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'standard buffered with time chunk key' do
+    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
+      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
+      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
+
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es3 = Fluent::MultiEventStream.new
+      es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({
+          m1 => es1,
+          m2 => es2,
+          m3 => es3,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false)
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+
+    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format, but time will be int if time_as_integer specified' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
+      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
+      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
+
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es3 = Fluent::MultiEventStream.new
+      es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).with({
+          m1 => es1,
+          m2 => es2,
+          m3 => es3,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false)
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'standard buffered with variable chunk keys' do
+    test '#execute_chunking calls @buffer.write(bulk: true) with predefined msgpack format' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(variables: {key: "my value", name: "moris1"})
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      m2 = create_metadata(variables: {key: "my value", name: "moris2"})
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).with({
+          m1 => es1,
+          m2 => es2,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, enqueue: false).once
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+
+    test '#execute_chunking calls @buffer.write(bulk: true) in times of # of variable variations with predefined msgpack format, but time will be int if time_as_integer specified' do
+      @i = create_output(:standard)
+      @i.configure(config_element('ROOT','',{"time_as_integer" => "true"},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(variables: {key: "my value", name: "moris1"})
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      m2 = create_metadata(variables: {key: "my value", name: "moris2"})
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).with({
+          m1 => es1,
+          m2 => es2,
+        }, format: Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, enqueue: false).once
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'custom format buffered without any chunk keys' do
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
+      @i = create_output(:buffered)
+      @i.register(:format){|tag, time, record| [time, record].to_json }
+      @i.configure(config_element())
+      @i.start
+      @i.after_start
+
+      m = create_metadata()
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json }}, format: nil, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'custom format buffered with tag chunk key' do
+    test '#execute_chunking calls @buffer.write(bulk: true) just once with customized format' do
+      @i = create_output(:buffered)
+      @i.register(:format){|tag, time, record| [time, record].to_json }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','tag',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m = create_metadata(tag: "mytag.test")
+      es = test_event_stream
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).once.with({m => es.map{|t,r| [t,r].to_json}}, format: nil, enqueue: false)
+
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+  sub_test_case 'custom format buffered with time chunk key' do
+    test '#execute_chunking calls @buffer.write with customized format' do
+      @i = create_output(:buffered)
+      @i.register(:format){|tag, time, record| [time, record].to_json }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','time',{"timekey" => "60",'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(timekey: Time.parse('2016-04-21 17:19:00 -0700').to_i)
+      m2 = create_metadata(timekey: Time.parse('2016-04-21 17:20:00 -0700').to_i)
+      m3 = create_metadata(timekey: Time.parse('2016-04-21 17:21:00 -0700').to_i)
+
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es2.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      es3 = Fluent::MultiEventStream.new
+      es3.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).with({
+          m1 => es1.map{|t,r| [t,r].to_json },
+          m2 => es2.map{|t,r| [t,r].to_json },
+          m3 => es3.map{|t,r| [t,r].to_json },
+        }, enqueue: false).once
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+
+  sub_test_case 'custom format buffered with variable chunk keys' do
+    test '#execute_chunking calls @buffer.write in times of # of variable variations with customized format' do
+      @i = create_output(:buffered)
+      @i.register(:format){|tag, time, record| [time, record].to_json }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer','key,name',{'flush_thread_burst_interval' => 0.01})]))
+      @i.start
+      @i.after_start
+
+      m1 = create_metadata(variables: {key: "my value", name: "moris1"})
+      es1 = Fluent::MultiEventStream.new
+      es1.add(event_time('2016-04-21 17:19:00 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:19:25 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:01 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:20:13 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+      es1.add(event_time('2016-04-21 17:21:32 -0700'), {"key" => "my value", "name" => "moris1", "message" => "hello!"})
+
+      m2 = create_metadata(variables: {key: "my value", name: "moris2"})
+      es2 = Fluent::MultiEventStream.new
+      es2.add(event_time('2016-04-21 17:19:13 -0700'), {"key" => "my value", "name" => "moris2", "message" => "hello!"})
+
+      buffer_mock = flexmock(@i.buffer)
+      buffer_mock.should_receive(:write).with({
+          m1 => es1.map{|t,r| [t,r].to_json },
+          m2 => es2.map{|t,r| [t,r].to_json },
+        }, enqueue: false).once
+
+      es = test_event_stream
+      @i.execute_chunking("mytag.test", es)
+    end
+  end
+end

data/test/plugin/test_owned_by.rb

@@ -0,0 +1,35 @@
+require_relative '../helper'
+require 'fluent/plugin/base'
+require 'fluent/plugin/input'
+require 'fluent/plugin/owned_by_mixin'
+
+module OwnedByMixinTestEnv
+  class DummyParent < Fluent::Plugin::Input
+    Fluent::Plugin.register_input('dummy_parent', self)
+  end
+  class DummyChild < Fluent::Plugin::Base
+    include Fluent::Plugin::OwnedByMixin
+    Fluent::Plugin.register_parser('dummy_child', self)
+  end
+end
+
+class OwnedByMixinTest < Test::Unit::TestCase
+  sub_test_case 'Owned plugins' do
+    setup do
+      Fluent::Test.setup
+    end
+
+    test 'inherits plugin id and logger from parent' do
+      parent = Fluent::Plugin.new_input('dummy_parent')
+      parent.configure(config_element('ROOT', '', {'@id' => 'my_parent_id', '@log_level' => 'trace'}))
+      child = Fluent::Plugin.new_parser('dummy_child', parent: parent)
+
+      assert_equal parent.object_id, child.owner.object_id
+
+      assert child.instance_eval{ @_plugin_id_configured }
+      assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
+
+      assert_equal Fluent::Log::LEVEL_TRACE, child.log.level
+    end
+  end
+end

data/test/plugin/test_parser_apache.rb

@@ -0,0 +1,42 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_apache'
+
+class ApacheParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = {})
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::ApacheParser.new).configure(conf)
+  end
+
+  data('parse' => :parse, 'call' => :call)
+  def test_call(method_name)
+    d = create_driver
+    m = d.instance.method(method_name)
+    m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
+      assert_equal(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      assert_equal({
+                     'user'    => '-',
+                     'method'  => 'GET',
+                     'code'    => '200',
+                     'size'    => '777',
+                     'host'    => '192.168.0.1',
+                     'path'    => '/'
+                   }, record)
+    }
+  end
+
+  def test_parse_with_keep_time_key
+    conf = {
+      'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+      'keep_time_key' => 'true',
+    }
+    d = create_driver(conf)
+    text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
+    d.instance.parse(text) do |_time, record|
+      assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
+    end
+  end
+end