fluentd 0.12.43 → 0.14.0

data/test/plugin/test_buffer_memory_chunk.rb

@@ -0,0 +1,265 @@
+require_relative '../helper'
+require 'fluent/plugin/buffer/memory_chunk'
+
+require 'json'
+
+class BufferMemoryChunkTest < Test::Unit::TestCase
+  setup do
+    @c = Fluent::Plugin::Buffer::MemoryChunk.new(Object.new)
+  end
+
+  test 'has blank chunk initially' do
+    assert @c.empty?
+    assert_equal '', @c.instance_eval{ @chunk }
+    assert_equal 0, @c.instance_eval{ @chunk_bytes }
+    assert_equal 0, @c.instance_eval{ @adding_bytes }
+    assert_equal 0, @c.instance_eval{ @adding_size }
+  end
+
+  test 'can #append, #commit and #read it' do
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+
+    content = @c.read
+    ds = content.split("\n").select{|d| !d.empty? }
+
+    assert_equal 2, ds.size
+    assert_equal d1, JSON.parse(ds[0])
+    assert_equal d2, JSON.parse(ds[1])
+
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+    @c.commit
+
+    content = @c.read
+    ds = content.split("\n").select{|d| !d.empty? }
+
+    assert_equal 4, ds.size
+    assert_equal d1, JSON.parse(ds[0])
+    assert_equal d2, JSON.parse(ds[1])
+    assert_equal d3, JSON.parse(ds[2])
+    assert_equal d4, JSON.parse(ds[3])
+  end
+
+  test 'can #concat, #commit and #read it' do
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"].join
+    @c.concat(data, 2)
+    @c.commit
+
+    content = @c.read
+    ds = content.split("\n").select{|d| !d.empty? }
+
+    assert_equal 2, ds.size
+    assert_equal d1, JSON.parse(ds[0])
+    assert_equal d2, JSON.parse(ds[1])
+
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
+    @c.commit
+
+    content = @c.read
+    ds = content.split("\n").select{|d| !d.empty? }
+
+    assert_equal 4, ds.size
+    assert_equal d1, JSON.parse(ds[0])
+    assert_equal d2, JSON.parse(ds[1])
+    assert_equal d3, JSON.parse(ds[2])
+    assert_equal d4, JSON.parse(ds[3])
+  end
+
+  test 'has its contents in binary (ascii-8bit)' do
+    data1 = "aaa bbb ccc".force_encoding('utf-8')
+    @c.append([data1])
+    @c.commit
+    assert_equal Encoding::ASCII_8BIT, @c.instance_eval{ @chunk.encoding }
+
+    content = @c.read
+    assert_equal Encoding::ASCII_8BIT, content.encoding
+  end
+
+  test 'has #bytesize and #size' do
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    @c.commit
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    first_bytesize = @c.bytesize
+
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+
+    assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 4, @c.size
+
+    @c.commit
+
+    assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 4, @c.size
+  end
+
+  test 'can #rollback to revert non-committed data' do
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    @c.rollback
+
+    assert @c.empty?
+
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    first_bytesize = @c.bytesize
+
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+
+    assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 4, @c.size
+
+    @c.rollback
+
+    assert_equal first_bytesize, @c.bytesize
+    assert_equal 2, @c.size
+  end
+
+  test 'can #rollback to revert non-committed data from #concat' do
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"].join
+    @c.concat(data, 2)
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    @c.rollback
+
+    assert @c.empty?
+
+    assert @c.empty?
+
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+
+    assert_equal (d1.to_json + "\n" + d2.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 2, @c.size
+
+    first_bytesize = @c.bytesize
+
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.concat([d3.to_json + "\n", d4.to_json + "\n"].join, 2)
+
+    assert_equal first_bytesize + (d3.to_json + "\n" + d4.to_json + "\n").bytesize, @c.bytesize
+    assert_equal 4, @c.size
+
+    @c.rollback
+
+    assert_equal first_bytesize, @c.bytesize
+    assert_equal 2, @c.size
+  end
+
+  test 'does nothing for #close' do
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+    @c.commit
+
+    content = @c.read
+
+    @c.close
+
+    assert_equal content, @c.read
+  end
+
+  test 'deletes all data by #purge' do
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+    @c.commit
+
+    @c.purge
+
+    assert @c.empty?
+    assert_equal 0, @c.bytesize
+    assert_equal 0, @c.size
+    assert_equal '', @c.read
+  end
+
+  test 'can #open its contents as io' do
+    d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
+    d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}
+    data = [d1.to_json + "\n", d2.to_json + "\n"]
+    @c.append(data)
+    @c.commit
+    d3 = {"f1" => 'x', "f2" => 'y', "f3" => 'z'}
+    d4 = {"f1" => 'a', "f2" => 'b', "f3" => 'c'}
+    @c.append([d3.to_json + "\n", d4.to_json + "\n"])
+    @c.commit
+
+    lines = []
+    @c.open do |io|
+      assert io
+      io.readlines.each do |l|
+        lines << l
+      end
+    end
+
+    assert_equal d1.to_json + "\n", lines[0]
+    assert_equal d2.to_json + "\n", lines[1]
+    assert_equal d3.to_json + "\n", lines[2]
+    assert_equal d4.to_json + "\n", lines[3]
+  end
+end

data/test/plugin/test_filter.rb

@@ -0,0 +1,255 @@
+require_relative '../helper'
+require 'fluent/plugin/filter'
+require 'fluent/event'
+require 'flexmock/test_unit'
+
+module FluentPluginFilterTest
+  class DummyPlugin < Fluent::Plugin::Filter
+  end
+  class NumDoublePlugin < Fluent::Plugin::Filter
+    def filter(tag, time, record)
+      r = record.dup
+      r["num"] = r["num"].to_i * 2
+      r
+    end
+  end
+  class IgnoreForNumPlugin < Fluent::Plugin::Filter
+    def filter(tag, time, record)
+      if record["num"].is_a? Numeric
+        nil
+      else
+        record
+      end
+    end
+  end
+  class RaiseForNumPlugin < Fluent::Plugin::Filter
+    def filter(tag, time, record)
+      if record["num"].is_a? Numeric
+        raise "Value of num is Number!"
+      end
+      record
+    end
+  end
+end
+
+class FilterPluginTest < Test::Unit::TestCase
+  DummyRouter = Struct.new(:emits) do
+    def emit_error_event(tag, time, record, error)
+      self.emits << [tag, time, record, error]
+    end
+  end
+
+  teardown do
+    if @p
+      @p.stop unless @p.stopped?
+      @p.before_shutdown unless @p.before_shutdown?
+      @p.shutdown unless @p.shutdown?
+      @p.after_shutdown unless @p.after_shutdown?
+      @p.close unless @p.closed?
+      @p.terminate unless @p.terminated?
+    end
+  end
+
+  sub_test_case 'for basic dummy plugin' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::DummyPlugin.new
+    end
+
+    test 'has healthy lifecycle' do
+      assert !@p.configured?
+      @p.configure(config_element())
+      assert @p.configured?
+
+      assert !@p.started?
+      @p.start
+      assert @p.start
+
+      assert !@p.stopped?
+      @p.stop
+      assert @p.stopped?
+
+      assert !@p.before_shutdown?
+      @p.before_shutdown
+      assert @p.before_shutdown?
+
+      assert !@p.shutdown?
+      @p.shutdown
+      assert @p.shutdown?
+
+      assert !@p.after_shutdown?
+      @p.after_shutdown
+      assert @p.after_shutdown?
+
+      assert !@p.closed?
+      @p.close
+      assert @p.closed?
+
+      assert !@p.terminated?
+      @p.terminate
+      assert @p.terminated?
+    end
+
+    test 'has plugin_id automatically generated' do
+      assert @p.respond_to?(:plugin_id_configured?)
+      assert @p.respond_to?(:plugin_id)
+
+      @p.configure(config_element())
+
+      assert !@p.plugin_id_configured?
+      assert @p.plugin_id
+      assert{ @p.plugin_id != 'mytest' }
+    end
+
+    test 'has plugin_id manually configured' do
+      @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+      assert @p.plugin_id_configured?
+      assert_equal 'mytest', @p.plugin_id
+    end
+
+    test 'has plugin logger' do
+      assert @p.respond_to?(:log)
+      assert @p.log
+
+      # default logger
+      original_logger = @p.log
+
+      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+      assert{ @p.log.object_id != original_logger.object_id }
+      assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+    end
+
+    test 'can load plugin helpers' do
+      assert_nothing_raised do
+        class FluentPluginFilterTest::DummyPlugin2 < Fluent::Plugin::Filter
+          helpers :storage
+        end
+      end
+    end
+
+    test 'plugin does not define #filter raises error' do
+      es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      assert_raise NotImplementedError do
+        @p.filter_stream('testing', es)
+      end
+    end
+  end
+
+  sub_test_case 'normal filter plugin' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::NumDoublePlugin.new
+    end
+
+    test 'filters events correctly' do
+      test_es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => "2", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      es = @p.filter_stream('testing', test_es)
+      assert es.is_a? Fluent::EventStream
+
+      ary = []
+      es.each do |time, r|
+        ary << [time, r]
+      end
+
+      assert_equal 3, ary.size
+
+      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]
+      assert_equal "Hello filters!", ary[0][1]["message"]
+      assert_equal 2, ary[0][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:03 -0700'), ary[1][0]
+      assert_equal 4, ary[1][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[2][0]
+      assert_equal 6, ary[2][1]["num"]
+    end
+  end
+
+  sub_test_case 'filter plugin returns nil for some records' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::IgnoreForNumPlugin.new
+    end
+
+    test 'filter_stream ignores records which #filter return nil' do
+      test_es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Ignored, yay!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      es = @p.filter_stream('testing', test_es)
+      assert es.is_a? Fluent::EventStream
+
+      ary = []
+      es.each do |time, r|
+        ary << [time, r]
+      end
+
+      assert_equal 2, ary.size
+
+      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]
+      assert_equal "Hello filters!", ary[0][1]["message"]
+      assert_equal "1", ary[0][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]
+      assert_equal "3", ary[1][1]["num"]
+    end
+  end
+
+  sub_test_case 'filter plugin raises error' do
+    setup do
+      Fluent::Test.setup
+      @p = FluentPluginFilterTest::RaiseForNumPlugin.new
+    end
+
+    test 'has router and can emit events to error streams' do
+      assert @p.has_router?
+
+      @p.configure(config_element())
+      assert @p.router
+
+      @p.router = DummyRouter.new([])
+
+      test_es = [
+        [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],
+        [event_time('2016-04-19 13:01:03 -0700'), {"num" => 2, "message" => "Hello error router!"}],
+        [event_time('2016-04-19 13:01:05 -0700'), {"num" => "3", "message" => "Hello filters!"}],
+      ]
+      es = @p.filter_stream('testing', test_es)
+      assert es.is_a? Fluent::EventStream
+
+      ary = []
+      es.each do |time, r|
+        ary << [time, r]
+      end
+
+      assert_equal 2, ary.size
+
+      assert_equal event_time('2016-04-19 13:01:00 -0700'), ary[0][0]
+      assert_equal "Hello filters!", ary[0][1]["message"]
+      assert_equal "1", ary[0][1]["num"]
+
+      assert_equal event_time('2016-04-19 13:01:05 -0700'), ary[1][0]
+      assert_equal "3", ary[1][1]["num"]
+
+      assert_equal 1, @p.router.emits.size
+
+      error_emits = @p.router.emits
+
+      assert_equal "testing", error_emits[0][0]
+      assert_equal event_time('2016-04-19 13:01:03 -0700'), error_emits[0][1]
+      assert_equal({"num" => 2, "message" => "Hello error router!"}, error_emits[0][2])
+      assert{ error_emits[0][3].is_a? RuntimeError }
+      assert_equal "Value of num is Number!", error_emits[0][3].message
+    end
+  end
+end