fluentd222 1.16.2-x86_64-linux

data/test/plugin/test_in_unix.rb

@@ -0,0 +1,181 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_unix'
+
+class UnixInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @d = nil
+  end
+
+  def teardown
+    @d.instance_shutdown if @d
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
+  CONFIG = %[
+    path #{TMP_DIR}/unix
+    backlog 1000
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::UnixInput).configure(conf)
+  end
+
+  def packer(*args)
+    Fluent::MessagePackFactory.msgpack_packer(*args)
+  end
+
+  def unpacker
+    Fluent::MessagePackFactory.msgpack_unpacker
+  end
+
+  def send_data(data)
+    io = UNIXSocket.new("#{TMP_DIR}/unix")
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  def test_configure
+    @d = create_driver
+    assert_equal "#{TMP_DIR}/unix", @d.instance.path
+    assert_equal 1000, @d.instance.backlog
+  end
+
+  def test_time
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", 0, {"a" => 1}],
+      ["tag2", nil, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
+      }
+    end
+
+    @d.events.each_with_index { |e, i|
+      orig = records[i]
+      assert_equal(orig[0], e[0])
+      assert_true(time <= e[1])
+      assert_equal(orig[2], e[2])
+    }
+  end
+
+  def test_message
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
+      }
+    end
+
+    assert_equal(records, @d.events)
+  end
+
+  def test_forward
+    @d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}]
+    ]
+
+    @d.run(expect_records: records.length, timeout: 20) do
+      entries = []
+      records.each {|tag, _time, record|
+        entries << [_time, record]
+      }
+      send_data packer.write(["tag1", entries]).to_s
+    end
+    assert_equal(records, @d.events)
+  end
+
+  def test_packed_forward
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 20) do
+      entries = ''
+      records.each {|_tag, _time, record|
+        packer(entries).write([_time, record]).flush
+      }
+      send_data packer.write(["tag1", entries]).to_s
+    end
+    assert_equal(records, @d.events)
+  end
+
+  def test_message_json
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      tag, _time, record = records[0]
+      send_data [tag, _time.to_i, record].to_json
+      tag, _time, record = records[1]
+      send_data [tag, _time.to_f, record].to_json
+    end
+
+    assert_equal(records, @d.events)
+  end
+
+  def test_message_with_tag
+    @d = create_driver(CONFIG + "tag new_tag")
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
+      }
+    end
+
+    @d.events.each { |event|
+      assert_equal("new_tag", event[0])
+    }
+  end
+
+  data('string chunk' => 'broken string',
+       'integer chunk' => 10)
+  def test_broken_message(data)
+    @d = create_driver
+    @d.run(shutdown: false, timeout: 5) do
+      @d.instance.__send__(:on_message, data)
+    end
+
+    assert_equal 0, @d.events.size
+
+    logs = @d.instance.log.logs
+    assert_equal 1, logs.count { |line|
+      line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
+    }, "should not accept broken chunk"
+  end
+end unless Fluent.windows?

data/test/plugin/test_input.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/plugin/input'
+require 'flexmock/test_unit'
+
+module FluentPluginInputTest
+  class DummyPlugin < Fluent::Plugin::Input
+  end
+end
+
+class InputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @p = FluentPluginInputTest::DummyPlugin.new
+  end
+
+  test 'has healthy lifecycle' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'has plugin_id automatically generated' do
+    assert @p.respond_to?(:plugin_id_configured?)
+    assert @p.respond_to?(:plugin_id)
+
+    @p.configure(config_element())
+
+    assert !@p.plugin_id_configured?
+    assert @p.plugin_id
+    assert{ @p.plugin_id != 'mytest' }
+  end
+
+  test 'has plugin_id manually configured' do
+    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+    assert @p.plugin_id_configured?
+    assert_equal 'mytest', @p.plugin_id
+  end
+
+  test 'has plugin logger' do
+    assert @p.respond_to?(:log)
+    assert @p.log
+
+    # default logger
+    original_logger = @p.log
+
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    assert(@p.log.object_id != original_logger.object_id)
+    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+  end
+
+  test 'can load plugin helpers' do
+    assert_nothing_raised do
+      class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input
+        helpers :storage
+      end
+    end
+  end
+
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
+  test 'are not available with multi workers configuration in default' do
+    assert_false @p.multi_workers_ready?
+  end
+
+  test 'has router and can emit into it' do
+    assert @p.has_router?
+
+    @p.configure(config_element())
+    assert @p.router
+
+    DummyRouter = Struct.new(:emits) do
+      def emit(tag, es)
+        self.emits << [tag, es]
+      end
+    end
+    @p.router = DummyRouter.new([])
+    @p.router.emit('mytag', [])
+    @p.router.emit('mytag.testing', ['it is not es, but no problem for tests'])
+
+    assert_equal ['mytag', []], @p.router.emits[0]
+    assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1]
+  end
+
+  test 'has router for specified label if configured' do
+    @p.configure(config_element())
+    original_router = @p.router
+
+    router_mock = flexmock('mytest')
+    router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock'])
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).once.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    @p.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    assert{ @p.router.object_id != original_router.object_id }
+
+    @p.router.emit('mytag.testing', ['for mock'])
+  end
+end

data/test/plugin/test_metadata.rb

@@ -0,0 +1,89 @@
+require_relative '../helper'
+require 'fluent/plugin/buffer'
+
+class BufferMetadataTest < Test::Unit::TestCase
+
+  def meta(timekey=nil, tag=nil, variables=nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'about metadata' do
+    test 'comparison of variables should be stable' do
+      m = meta(nil, nil, nil)
+      # different sets of keys
+      assert_equal(-1, m.cmp_variables({}, {a: 1}))
+      assert_equal(1, m.cmp_variables({a: 1}, {}))
+      assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))
+      # same set of keys
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))
+      assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys
+      assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))
+    end
+
+    test 'comparison of metadata should be stable' do
+      n = Time.now.to_i
+
+      assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))
+      assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))
+      assert_equal(0, meta(nil, "t1", nil) <=> meta(nil, "t1", nil))
+      assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))
+      assert_equal(0, meta(nil, nil, {a: "1"}) <=> meta(nil, nil, {a: "1"}))
+      assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))
+      assert_equal(0, meta(n, "t1", {}) <=> meta(n, "t1", {}))
+      assert_equal(0, meta(n, "t1", {a: "x", b: 10}) <=> meta(n, "t1", {a: "x", b: 10}))
+
+      # timekey is 1st comparison key
+      assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))
+      assert_equal(1, meta(n + 1, "a", nil) <=> meta(n - 1, "b", nil))
+      assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))
+
+      # tag is 2nd
+      assert_equal(-1, meta(nil, "a", {}) <=> meta(nil, "b", {}))
+      assert_equal(-1, meta(n, "a", {}) <=> meta(n, "b", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {a: 1}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 10, b: 1}))
+
+      # variables is the last
+      assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(-1, meta(n, "t", {}) <=> meta(n, "t", {a: 1}))
+      assert_equal(1, meta(n, "t", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 1, b: 1}))
+      assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(1, meta(n, "t", {b: 1}) <=> meta(n, "t", {a: 1}))
+    end
+
+    test 'metadata can be sorted' do
+      n = Time.now.to_i
+      m0 = meta(nil, nil, nil)
+      m1 = meta(n - 1, nil, nil)
+      m2 = meta(n - 1, "a", nil)
+      m3 = meta(n - 1, "a", {a: 1})
+      m4 = meta(n - 1, "a", {a: 100})
+      m5 = meta(n - 1, "a", {a: 100, b: 1})
+      m6 = meta(n - 1, "aa", nil)
+      m7 = meta(n - 1, "aa", {a: 1})
+      m8 = meta(n - 1, "b", nil)
+      m9 = meta(n, nil, nil)
+      m10 = meta(n + 1, nil, {a: 1})
+      expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze
+      ary = expected.dup
+      100.times do
+        assert_equal expected, ary.shuffle.sort
+      end
+    end
+  end
+end

data/test/plugin/test_metrics.rb

@@ -0,0 +1,294 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+
+class BareMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('bare', self)
+
+  private
+
+  # Just override for tests.
+  def has_methods_for_counter?
+    false
+  end
+end
+
+class BasicCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :set, :set_counter
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set_counter(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class BasicGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub(value)
+    @data -= value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :dec, :dec_gauge
+      alias_method :set, :set_gauge
+      alias_method :sub, :sub_gauge
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec_gauge
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub_gauge(value)
+    @data -= value
+  end
+  def set_gauge(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class StorageTest < Test::Unit::TestCase
+  sub_test_case 'BareMetrics' do
+    setup do
+      @m = BareMetrics.new
+      @m.configure(config_element())
+    end
+
+    test 'is configured with plugin information and system config' do
+      m = BareMetrics.new
+      m.configure(config_element('metrics', '', {}))
+
+      assert_false m.use_gauge_metric
+      assert_false m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test 'all bare operations are not defined yet' do
+      assert_raise NotImplementedError do
+        @m.get
+      end
+      assert_raise NotImplementedError do
+        @m.inc
+      end
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+      assert_raise NotImplementedError do
+        @m.add(10)
+      end
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+      assert_raise NotImplementedError do
+        @m.set(123)
+      end
+    end
+  end
+
+  sub_test_case 'BasicCounterMetric' do
+    setup do
+      @m = BasicCounterMetrics.new
+      @m.configure(config_element('metrics', '', {'@id' => '1'}))
+    end
+
+    test 'all basic counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'AliasedCounterMetric' do
+    setup do
+      @m = AliasedCounterMetrics.new
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'BasicGaugeMetric' do
+    setup do
+      @m = BasicGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all basic gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+
+  sub_test_case 'AliasedGaugeMetric' do
+    setup do
+      @m = AliasedGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+end