fluentd-hubspot 0.14.14.1

data/test/plugin/test_formatter_msgpack.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_msgpack'
+
+class MessagePackFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::MessagePackFormatter).configure(conf)
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal(record.to_msgpack, formatted)
+  end
+end

data/test/plugin/test_formatter_out_file.rb

@@ -0,0 +1,95 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_out_file'
+
+class OutFileFormatterTest < ::Test::Unit::TestCase
+  def setup
+    @time = event_time
+  end
+
+  def create_driver(conf = {})
+    d = Fluent::Test::Driver::Formatter.new(Fluent::Plugin::OutFileFormatter)
+    case conf
+    when Fluent::Config::Element
+      d.configure(conf)
+    when Hash
+      d.configure({'utc' => true}.merge(conf))
+    else
+      d.configure(conf)
+    end
+  end
+
+  def tag
+    "tag"
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  data('both true' => 'true', 'both false' => 'false')
+  def test_configured_with_both_of_utc_and_localtime(value)
+    assert_raise(Fluent::ConfigError.new("both of utc and localtime are specified, use only one of them")) do
+      create_driver({'utc' => value, 'localtime' => value})
+    end
+  end
+
+  time_i = Time.parse("2016-07-26 21:08:30 -0700").to_i
+  data(
+    'configured for localtime by localtime' => ['localtime', 'true',  time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for localtime by utc'       => ['utc',       'false', time_i, "2016-07-26T21:08:30-07:00"],
+    'configured for utc by localtime'       => ['localtime', 'false', time_i, "2016-07-27T04:08:30Z"],
+    'configured for utc by utc'             => ['utc',       'true',  time_i, "2016-07-27T04:08:30Z"],
+  )
+  def test_configured_with_utc_or_localtime(data)
+    key, value, time_i, expected = data
+    time = Time.at(time_i)
+    begin
+      oldtz, ENV['TZ'] = ENV['TZ'], "UTC+07"
+      d = create_driver(config_element('ROOT', '', {key => value}))
+      tag = 'test'
+      assert_equal "#{expected}\t#{tag}\t#{Yajl.dump(record)}\n", d.instance.format(tag, time, record)
+    ensure
+      ENV['TZ'] = oldtz
+    end
+  end
+
+  def test_format
+    d = create_driver({})
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time
+    d = create_driver('output_time' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_tag
+    d = create_driver('output_tag' => 'false')
+    formatted = d.instance.format(tag, @time, record)
+
+    assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag
+    d = create_driver('output_tag' => 'false', 'output_time' => 'false')
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+
+  def test_format_without_time_and_tag_against_string_literal_configure
+    d = create_driver(%[
+                        utc         true
+                        output_tag  false
+                        output_time false
+                      ])
+    formatted = d.instance.format('tag', @time, record)
+
+    assert_equal("#{Yajl.dump(record)}\n", formatted)
+  end
+end

data/test/plugin/test_formatter_single_value.rb

@@ -0,0 +1,38 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_single_value'
+
+class SingleValueFormatterTest < ::Test::Unit::TestCase
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::SingleValueFormatter).configure(conf)
+  end
+
+  def test_config_params
+    d = create_driver
+    assert_equal "message", d.instance.message_key
+  end
+
+  def test_config_params_message_key
+    d = create_driver('message_key' => 'foobar')
+    assert_equal "foobar", d.instance.message_key
+  end
+
+  def test_format
+    d = create_driver
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => 'false')
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome", formatted)
+  end
+
+  def test_format_with_message_key
+    d = create_driver('message_key' => 'foobar')
+    formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
+
+    assert_equal("foo\n", formatted)
+  end
+end

data/test/plugin/test_in_debug_agent.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_debug_agent'
+require 'fileutils'
+
+class DebugAgentInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/in_debug_agent")
+
+  def create_driver(conf = '')
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput).configure(conf)
+  end
+
+  def test_unix_path_writable
+    assert_nothing_raised do
+      create_driver %[unix_path #{TMP_DIR}/test_path]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[unix_path #{TMP_DIR}/does_not_exist/test_path]
+    end
+  end
+end

data/test/plugin/test_in_dummy.rb

@@ -0,0 +1,192 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_dummy'
+require 'fileutils'
+
+class DummyTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+  end
+
+  sub_test_case 'configure' do
+    test 'required parameters' do
+      assert_raise_message("'tag' parameter is required") do
+        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
+      end
+    end
+
+    test 'tag' do
+      d = create_driver(%[
+        tag dummy
+      ])
+      assert_equal "dummy", d.instance.tag
+    end
+
+    config = %[
+      tag dummy
+    ]
+
+    test 'auto_increment_key' do
+      d = create_driver(config + %[
+        auto_increment_key id
+      ])
+      assert_equal "id", d.instance.auto_increment_key
+    end
+
+    test 'rate' do
+      d = create_driver(config + %[
+        rate 10
+      ])
+      assert_equal 10, d.instance.rate
+    end
+
+    test 'dummy' do
+      # hash is okay
+      d = create_driver(config + %[dummy {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+
+      # array of hash is okay
+      d = create_driver(config + %[dummy [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+
+      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
+        create_driver(config + %[dummy "foo"])
+      end
+
+      assert_raise_message(/is not a hash/) do
+        create_driver(config + %[dummy ["foo"]])
+      end
+    end
+  end
+
+  sub_test_case "emit" do
+    config = %[
+      tag dummy
+      rate 10
+      dummy {"foo":"bar"}
+    ]
+
+    test 'simple' do
+      d = create_driver(config)
+      d.run(timeout: 0.5)
+
+      d.events.each do |tag, time, record|
+        assert_equal("dummy", tag)
+        assert_equal({"foo"=>"bar"}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'with auto_increment_key' do
+      d = create_driver(config + %[auto_increment_key id])
+      d.run(timeout: 0.5)
+
+      d.events.each_with_index do |(tag, _time, record), i|
+        assert_equal("dummy", tag)
+        assert_equal({"foo"=>"bar", "id"=>i}, record)
+      end
+    end
+  end
+
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
+
+  sub_test_case "doesn't suspend internal counters in default" do
+    config1 = {
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => false,
+    }
+    conf1 = config_element('ROOT', '', config1, [])
+    test "value of auto increment key is not suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d1 = create_driver(conf1)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      events = d1.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
+
+      first_id1 = events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d2 = create_driver(conf1)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+
+      events = d2.events.sort{|a,b| a[2]['id'] <=> b[2]['id'] }
+
+      first_id2 = events.first[2]['id']
+      assert_equal 0, first_id2
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+    end
+  end
+
+  sub_test_case "suspend internal counters if suspend is true" do
+    setup do
+      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
+      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+    end
+
+    config2 = {
+      '@id' => 'test-02',
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => true,
+    }
+    conf2 = config_element('ROOT', '', config2, [
+              config_element(
+                'storage', '',
+                {'@type' => 'local',
+                 '@id' => 'test-02',
+                 'path' => File.join(TEST_PLUGIN_STORAGE_PATH,
+                                     'json', 'test-02.json'),
+                 'persistent' => true,
+                })
+            ])
+    test "value of auto increment key is suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d1 = create_driver(conf2)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      first_id1 = d1.events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = d1.events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(conf2)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+      d2.events
+
+      first_id2 = d2.events.first[2]['id']
+      assert_equal last_id1 + 1, first_id2
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+    end
+  end
+end

data/test/plugin/test_in_exec.rb

@@ -0,0 +1,245 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_exec'
+require 'timecop'
+
+class ExecInputTest < Test::Unit::TestCase
+  SCRIPT_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
+  TEST_TIME = "2011-01-02 13:14:15"
+  TEST_UNIX_TIME = Time.parse(TEST_TIME)
+
+  def setup
+    Fluent::Test.setup
+    @test_time = event_time()
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)
+  end
+
+  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
+    run_interval 1s
+    tag "my.test.data"
+    <parse>
+      keys ["k1", "k2", "k3"]
+    </parse>
+  ]
+
+  TSV_CONFIG = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
+    run_interval 1s
+    <parse>
+      @type tsv
+      keys time, tag, k1
+    </parse>
+    <extract>
+      tag_key tag
+      time_key time
+      time_type string
+      time_format %Y-%m-%d %H:%M:%S
+    </extract>
+  ]
+
+  JSON_CONFIG = %[
+    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
+    run_interval 1s
+    <parse>
+      @type json
+    </parse>
+    <extract>
+      tag_key tag
+      time_key time
+      time_type unixtime
+    </extract>
+  ]
+
+  MSGPACK_CONFIG = %[
+    command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
+    run_interval 1s
+    <parse>
+      @type msgpack
+    </parse>
+    <extract>
+      tag_key tagger
+      time_key datetime
+      time_type unixtime
+    </extract>
+  ]
+
+  # here document for not de-quoting backslashes
+  REGEXP_CONFIG = %[
+    command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
+    run_interval 1s
+    tag regex_tag
+] + <<'EOC'
+    <parse>
+      @type regexp
+      expression "(?<time>[^\\]]*) (?<message>[^ ]*)"
+      time_key time
+      time_type string
+      time_format %Y-%m-%d %H:%M:%S
+    </parse>
+EOC
+
+  sub_test_case 'with configuration with sections' do
+    test 'configure with default tsv format without extract' do
+      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal "my.test.data", d.instance.tag
+      assert_equal ["k1", "k2", "k3"], d.instance.parser.keys
+    end
+
+    test 'configure raises error if both of tag and extract.tag_key are missing' do
+      assert_raise Fluent::ConfigError.new("'tag' or 'tag_key' option is required on exec input") do
+        create_driver %[
+          command ruby -e 'puts "yay"'
+          <parse>
+            keys y1
+          </parse>
+        ]
+      end
+    end
+
+    test 'configure for tsv' do
+      d = create_driver TSV_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal ["time", "tag", "k1"], d.instance.parser.keys
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal :string, d.instance.extract_config.time_type
+      assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
+    end
+
+    test 'configure for json' do
+      d = create_driver JSON_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal :unixtime, d.instance.extract_config.time_type
+    end
+
+    test 'configure for msgpack' do
+      d = create_driver MSGPACK_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
+      assert_equal "tagger", d.instance.extract_config.tag_key
+      assert_equal "datetime", d.instance.extract_config.time_key
+      assert_equal :unixtime, d.instance.extract_config.time_type
+    end
+
+    test 'configure for regexp' do
+      d = create_driver REGEXP_CONFIG
+      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
+      assert_equal "regex_tag", d.instance.tag
+      expression = <<'EXP'.chomp
+(?<time>[^\]]*) (?<message>[^ ]*)
+EXP
+      assert_equal expression, d.instance.parser.expression
+      assert_nil d.instance.extract_config
+    end
+  end
+
+  TSV_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
+      keys time,tag,k1
+      time_key time
+      tag_key tag
+      time_format %Y-%m-%d %H:%M:%S
+      run_interval 1s
+  ]
+
+  JSON_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
+      format json
+      tag_key tag
+      time_key time
+      run_interval 1s
+  ]
+
+  MSGPACK_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
+      format msgpack
+      tag_key tagger
+      time_key datetime
+      run_interval 1s
+  ]
+
+  REGEXP_CONFIG_COMPAT = %[
+      command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
+      format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
+      tag regex_tag
+      run_interval 1s
+  ]
+
+  sub_test_case 'with traditional configuration' do
+    test 'configure' do
+      d = create_driver TSV_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
+      assert_equal ["time","tag","k1"], d.instance.parser.keys
+      assert_equal "tag", d.instance.extract_config.tag_key
+      assert_equal "time", d.instance.extract_config.time_key
+      assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
+    end
+
+    test 'configure_with_json' do
+      d = create_driver JSON_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
+    end
+
+    test 'configure_with_msgpack' do
+      d = create_driver MSGPACK_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
+    end
+
+    test 'configure_with_regexp' do
+      d = create_driver REGEXP_CONFIG_COMPAT
+      assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
+      assert_equal '(?<time>[^\]]*) (?<message>[^ ]*)', d.instance.parser.expression
+      assert_equal 'regex_tag', d.instance.tag
+    end
+  end
+
+  sub_test_case 'with default configuration' do
+    setup do
+      @current_event_time = event_time('2016-10-31 20:01:30.123 -0700')
+      Timecop.freeze(Time.at(@current_event_time))
+    end
+
+    teardown do
+      Timecop.return
+    end
+
+    test 'emits events with current timestamp if time key is not specified' do
+      d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+      d.run(expect_records: 2, timeout: 10)
+
+      assert{ d.events.length > 0 }
+      d.events.each do |event|
+        assert_equal ["my.test.data", @current_event_time, {"k1"=>"2011-01-02 13:14:15", "k2"=>"tag1", "k3"=>"ok"}], event
+      end
+    end
+  end
+
+  data(
+    'default' => [TSV_CONFIG,     "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'json'    => [JSON_CONFIG,    "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'msgpack' => [MSGPACK_CONFIG, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'regexp'  => [REGEXP_CONFIG,  "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
+    'default_c' => [TSV_CONFIG_COMPAT,     "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'json_c'    => [JSON_CONFIG_COMPAT,    "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'msgpack_c' => [MSGPACK_CONFIG_COMPAT, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
+    'regexp_c'  => [REGEXP_CONFIG_COMPAT,  "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
+  )
+  test 'emit with formats' do |data|
+    config, tag, time, record = data
+    d = create_driver(config)
+
+    d.run(expect_emits: 2, timeout: 10)
+
+    assert{ d.events.length > 0 }
+    d.events.each_with_index {|event, i|
+      assert_equal_event_time(time, event[1])
+      assert_equal [tag, time, record], event
+    }
+  end
+end