fluentd-hubspot 0.14.14.1

data/test/plugin/test_multi_output.rb

@@ -0,0 +1,180 @@
+require_relative '../helper'
+require 'fluent/plugin/multi_output'
+require 'fluent/event'
+
+require 'json'
+require 'time'
+require 'timeout'
+
+module FluentPluginMultiOutputTest
+  class DummyMultiOutput < Fluent::Plugin::MultiOutput
+    attr_reader :events
+    def initialize
+      super
+      @events = []
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      es.each do |time, record|
+        @events << [tag, time, record]
+      end
+    end
+  end
+  class DummyCompatMultiOutput < Fluent::Plugin::MultiOutput
+    def initialize
+      super
+      @compat = true
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      # ...
+    end
+  end
+
+  class Dummy1Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_1', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_2', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy3Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_3', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy4Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_4', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+end
+
+class MultiOutputTest < Test::Unit::TestCase
+  def create_output(type=:multi)
+    case type
+    when :compat_multi
+      FluentPluginMultiOutputTest::DummyCompatMultiOutput.new
+    else
+      FluentPluginMultiOutputTest::DummyMultiOutput.new
+    end
+  end
+
+  sub_test_case 'basic multi output plugin' do
+    setup do
+      Fluent::Test.setup
+      @i = create_output()
+    end
+
+    teardown do
+      @i.log.out.reset
+    end
+
+    test '#configure raises error if <store> sections are missing' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' }, [])
+      assert_raise Fluent::ConfigError do
+        @i.configure(conf)
+      end
+    end
+
+    test '#configure initialize child plugins and call these #configure' do
+      assert_equal [], @i.outputs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      assert_equal 4, @i.outputs.size
+
+      assert @i.outputs[0].is_a? FluentPluginMultiOutputTest::Dummy1Output
+      assert @i.outputs[0].configured
+
+      assert @i.outputs[1].is_a? FluentPluginMultiOutputTest::Dummy2Output
+      assert @i.outputs[1].configured
+
+      assert @i.outputs[2].is_a? FluentPluginMultiOutputTest::Dummy3Output
+      assert @i.outputs[2].configured
+
+      assert @i.outputs[3].is_a? FluentPluginMultiOutputTest::Dummy4Output
+      assert @i.outputs[3].configured
+    end
+
+    test '#configure warns if "type" is used in <store> sections instead of "@type"' do
+      assert_equal [], @i.log.out.logs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      assert_equal 4, @i.outputs.size
+
+      logs = @i.log.out.logs
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+    end
+
+    test '#emit_events calls #process always' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      @i.start
+
+      assert @i.events.empty?
+
+      @i.emit_events(
+        'test.tag',
+        Fluent::ArrayEventStream.new(
+          [
+            [event_time(), {"message" => "multi test 1"}],
+            [event_time(), {"message" => "multi test 1"}],
+          ]
+        )
+      )
+
+      assert_equal 2, @i.events.size
+    end
+  end
+end

data/test/plugin/test_out_copy.rb

@@ -0,0 +1,160 @@
+require_relative '../helper'
+require 'fluent/test/driver/multi_output'
+require 'fluent/plugin/out_copy'
+require 'fluent/event'
+
+class CopyOutputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
+      require 'fluent/plugin/out_test'
+      require 'fluent/plugin/out_test2'
+    end
+
+    def shutdown
+      $LOAD_PATH.shift
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    <store>
+      @type test
+      name c0
+    </store>
+    <store>
+      @type test2
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    outputs = d.instance.outputs
+    assert_equal 3, outputs.size
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+    assert_equal "c0", outputs[0].name
+    assert_equal "c1", outputs[1].name
+    assert_equal "c2", outputs[2].name
+  end
+
+  def test_feed_events
+    d = create_driver
+
+    assert !d.instance.outputs[0].has_router?
+    assert_not_nil d.instance.outputs[1].router
+    assert !d.instance.outputs[2].has_router?
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
+    end
+
+    d.instance.outputs.each {|o|
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
+    }
+  end
+
+  def test_msgpack_unpacker_cache_bug_for_msgpack_event_stream
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    source = Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"a" => 2}] ])
+    es = Fluent::MessagePackEventStream.new(source.to_msgpack_stream)
+
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
+
+    d.instance.outputs.each { |o|
+      assert_equal [ [time, {"a"=>1}], [time, {"a"=>2}] ], o.events
+    }
+  end
+
+  def create_event_test_driver(does_deep_copy = false)
+    config = %[
+      deep_copy #{does_deep_copy}
+      <store>
+        @type test
+        name output1
+      </store>
+      <store>
+        @type test
+        name output2
+      </store>
+    ]
+
+    d = Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::CopyOutput).configure(config)
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      es.each do |time, record|
+        record['foo'] = 'bar'
+      end
+      super(tag, es)
+    end
+    d
+  end
+
+  time = event_time("2013-05-26 06:37:22 UTC")
+  mes0 = Fluent::MultiEventStream.new
+  mes0.add(time, {"a" => 1})
+  mes0.add(time, {"b" => 1})
+  mes1 = Fluent::MultiEventStream.new
+  mes1.add(time, {"a" => 1})
+  mes1.add(time, {"b" => 1})
+
+  data(
+    "OneEventStream without deep_copy"   => [false, Fluent::OneEventStream.new(time, {"a" => 1})],
+    "OneEventStream with deep_copy"      => [true,  Fluent::OneEventStream.new(time, {"a" => 1})],
+    "ArrayEventStream without deep_copy" => [false, Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "ArrayEventStream with deep_copy"    => [true,  Fluent::ArrayEventStream.new([ [time, {"a" => 1}], [time, {"b" => 2}] ])],
+    "MultiEventStream without deep_copy" => [false, mes0],
+    "MultiEventStream with deep_copy"    => [true,  mes1],
+  )
+  def test_deep_copy_controls_shallow_or_deep_copied(data)
+    does_deep_copy, es = data
+
+    d = create_event_test_driver(does_deep_copy)
+
+    d.run(default_tag: 'test') do
+      d.feed(es)
+    end
+
+    events = d.instance.outputs.map(&:events)
+
+    if does_deep_copy
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
+
+        assert{ record0.object_id != record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert !record1.has_key?("foo")
+      end
+    else
+      events[0].each_with_index do |entry0, i|
+        record0 = entry0.last
+        record1 = events[1][i].last
+
+        assert{ record0.object_id == record1.object_id }
+        assert_equal "bar", record0["foo"]
+        assert_equal "bar", record1["foo"]
+      end
+    end
+  end
+end
+

data/test/plugin/test_out_exec.rb

@@ -0,0 +1,310 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_exec'
+require 'fileutils'
+
+class ExecOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR, secure: true)
+    if File.exist?(TMP_DIR)
+      # ensure files are closed for Windows, on which deleted files
+      # are still visible from filesystem
+      GC.start(full_mark: true, immediate_sweep: true)
+      FileUtils.remove_entry_secure(TMP_DIR)
+    end
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_exec#{ENV['TEST_ENV_NUMBER']}"
+
+  def create_driver(config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
+  end
+
+  def create_test_data
+    time = event_time("2011-01-02 13:14:15.123")
+    records = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
+    return time, records
+  end
+
+  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      keys ["k1", "kx"]
+    </format>
+  ]
+
+  test 'configure in default' do
+    d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+    assert{ d.instance.formatter.is_a? Fluent::Plugin::TSVFormatter }
+    assert_equal ["k1", "kx"], d.instance.formatter.keys
+    assert_nil d.instance.inject_config
+  end
+
+  TSV_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+  ]
+  TSV_CONFIG_WITH_SUBSEC = %[
+    command cat >#{TMP_DIR}/out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S.%3N
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+  ]
+  TSV_CONFIG_WITH_BUFFER = TSV_CONFIG + %[
+    <buffer time>
+      @type memory
+      timekey 3600
+      flush_thread_count 5
+      chunk_limit_size 50m
+      total_limit_size #{50 * 1024 * 1024 * 128}
+      flush_at_shutdown yes
+    </buffer>
+  ]
+  JSON_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      @type json
+    </format>
+  ]
+  MSGPACK_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      @type msgpack
+    </format>
+  ]
+
+  CONFIG_COMPAT = %[
+    buffer_path #{TMP_DIR}/buffer
+    command cat >#{TMP_DIR}/out
+    localtime
+  ]
+  TSV_CONFIG_COMPAT = %[
+    keys "time,tag,k1"
+    tag_key "tag"
+    time_key "time"
+    time_format %Y-%m-%d %H:%M:%S
+  ]
+  BUFFER_CONFIG_COMPAT = %[
+    buffer_type memory
+    time_slice_format %Y%m%d%H
+    num_threads 5
+    buffer_chunk_limit 50m
+    buffer_queue_limit 128
+    flush_at_shutdown yes
+  ]
+  TSV_CONFIG_WITH_SUBSEC_COMPAT = %[
+    keys "time,tag,k1"
+    tag_key "tag"
+    time_key "time"
+    time_format %Y-%m-%d %H:%M:%S.%3N
+  ]
+
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'configure for tsv' do |conf|
+    d = create_driver(conf)
+
+    assert_equal ["time","tag","k1"], d.instance.formatter.keys
+    assert_equal "tag", d.instance.inject_config.tag_key
+    assert_equal "time", d.instance.inject_config.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
+    assert_equal true, d.instance.inject_config.localtime
+  end
+
+  data(
+    'with sections' => TSV_CONFIG_WITH_BUFFER,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT + BUFFER_CONFIG_COMPAT,
+  )
+  test 'configure_with_compat_buffer_parameters' do |conf|
+    d = create_driver(conf)
+    assert_equal 3600, d.instance.buffer_config.timekey
+    assert_equal 5, d.instance.buffer_config.flush_thread_count
+    assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
+    assert_equal 50*1024*1024*128, d.instance.buffer.total_limit_size
+    assert d.instance.buffer_config.flush_at_shutdown
+  end
+
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'format' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
+
+    d.run(default_tag: 'test') do
+      d.feed(time, records[0])
+      d.feed(time, records[1])
+    end
+
+    assert_equal %[2011-01-02 13:14:15\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15\ttest\tv2\n], d.formatted[1]
+  end
+
+  data(
+    'with sections' => JSON_CONFIG,
+    'traditional' => CONFIG_COMPAT + "format json",
+  )
+  test 'format_json' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
+
+    d.run(default_tag: 'test') do
+      d.feed(time, records[0])
+      d.feed(time, records[1])
+    end
+
+    assert_equal Yajl.dump(records[0]) + "\n", d.formatted[0]
+    assert_equal Yajl.dump(records[1]) + "\n", d.formatted[1]
+  end
+
+  data(
+    'with sections' => MSGPACK_CONFIG,
+    'traditional' => CONFIG_COMPAT + "format msgpack"
+  )
+  test 'format_msgpack' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
+
+    d.run(default_tag: 'test') do
+      d.feed(time, records[0])
+      d.feed(time, records[1])
+    end
+
+    assert_equal records[0].to_msgpack, d.formatted[0]
+    assert_equal records[1].to_msgpack, d.formatted[1]
+  end
+
+  data(
+    'with sections' => TSV_CONFIG_WITH_SUBSEC,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_WITH_SUBSEC_COMPAT,
+  )
+  test 'format subsecond time' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
+
+    d.run(default_tag: 'test') do
+      d.feed(time, records[0])
+      d.feed(time, records[1])
+    end
+
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv1\n], d.formatted[0]
+    assert_equal %[2011-01-02 13:14:15.123\ttest\tv2\n], d.formatted[1]
+  end
+
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'write' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
+
+    d.run(default_tag: 'test', flush: true) do
+      d.feed(time, records[0])
+      d.feed(time, records[1])
+    end
+
+    expect_path = "#{TMP_DIR}/out"
+
+    waiting(10, plugin: d.instance) do
+      sleep(0.1) until File.exist?(expect_path)
+    end
+
+    assert_equal true, File.exist?(expect_path)
+
+    data = File.read(expect_path)
+    expect_data =
+      %[2011-01-02 13:14:15\ttest\tv1\n] +
+      %[2011-01-02 13:14:15\ttest\tv2\n]
+    assert_equal expect_data, data
+  end
+
+  sub_test_case 'when executed process dies unexpectedly' do
+    setup do
+      @gen_config = ->(num){ <<EOC
+    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+EOC
+      }
+    end
+
+    test 'flushed chunk will be committed after child process successfully exits' do
+      d = create_driver(@gen_config.call(0))
+      time, records = create_test_data
+
+      expect_path = "#{TMP_DIR}/fail_out"
+
+      d.end_if{ File.exist?(expect_path) }
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+        d.feed(time, records[0])
+        d.feed(time, records[1])
+      end
+
+      assert{ File.exist?(expect_path) }
+
+      data = File.read(expect_path)
+      expect_data =
+        %[2011-01-02 13:14:15\ttest\tv1\n] +
+        %[2011-01-02 13:14:15\ttest\tv2\n]
+      assert_equal expect_data, data
+
+      assert{ d.instance.buffer.queue.empty? }
+      assert{ d.instance.dequeued_chunks.empty? }
+
+      d.instance_shutdown
+    end
+
+    test 'flushed chunk will be taken back after child process unexpectedly exits' do
+      d = create_driver(@gen_config.call(3))
+      time, records = create_test_data
+
+      expect_path = "#{TMP_DIR}/fail_out"
+
+      d.end_if{ d.instance.log.out.logs.any?{|line| line.include?("command exits with error code") } }
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+        d.feed(time, records[0])
+        d.feed(time, records[1])
+      end
+
+      assert{ d.instance.dequeued_chunks.empty? } # because it's already taken back
+      assert{ d.instance.buffer.queue.size == 1 }
+
+      logs = d.instance.log.out.logs
+      assert{ logs.any?{|line| line.include?("command exits with error code") && line.include?("status=3") } }
+
+      assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
+
+      d.instance_shutdown
+    end
+  end
+end