dtomasgu-fluentd 1.14.7.pre.dev

data/test/plugin/test_out_http.rb

@@ -0,0 +1,428 @@
+require_relative "../helper"
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_http'
+
+require 'webrick'
+require 'webrick/https'
+require 'net/http'
+require 'uri'
+require 'json'
+
+# WEBrick's ProcHandler doesn't handle PUT by default
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_PUT do_GET
+  end
+end
+
+class HTTPOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  TMP_DIR = File.join(__dir__, "../tmp/out_http#{ENV['TEST_ENV_NUMBER']}")
+  DEFAULT_LOGGER = ::WEBrick::Log.new(::STDOUT, ::WEBrick::BasicLog::FATAL)
+
+  class << self
+    # Use class variable to reduce server start/shutdown time
+    def startup
+      @@result = nil
+      @@auth_handler = nil
+      @@http_server_thread = nil
+    end
+
+    def shutdown
+      @@http_server_thread.kill
+      @@http_server_thread.join
+    rescue
+    end
+  end
+
+  def server_port
+    19880
+  end
+
+  def base_endpoint
+    "http://127.0.0.1:#{server_port}"
+  end
+
+  def server_config
+    config = {BindAddress: '127.0.0.1', Port: server_port}
+    # Suppress webrick logs
+    config[:Logger] = DEFAULT_LOGGER
+    config[:AccessLog] = []
+    config
+  end
+
+  def http_client(**opts, &block)
+    opts = opts.merge(open_timeout: 1, read_timeout: 1)
+    if block_given?
+      Net::HTTP.start('127.0.0.1', server_port, **opts, &block)
+    else
+      Net::HTTP.start('127.0.0.1', server_port, **opts)
+    end
+  end
+
+  def run_http_server
+    server = ::WEBrick::HTTPServer.new(server_config)
+    server.mount_proc('/test') { |req, res|
+      if @@auth_handler
+        @@auth_handler.call(req, res)
+      end
+
+      @@result.method = req.request_method
+      @@result.content_type = req.content_type
+      req.each do |key, value|
+        @@result.headers[key] = value
+      end
+
+      data = []
+      case req.content_type
+      when 'application/x-ndjson'
+        req.body.each_line { |l|
+          data << JSON.parse(l)
+        }
+      when 'application/json'
+        data = JSON.parse(req.body)
+      when 'text/plain'
+        # Use single_value in this test
+        req.body.each_line { |line|
+          data << line.chomp
+        }
+      else
+        data << req.body
+      end
+      @@result.data = data
+
+      res.status = 200
+      res.body = "success"
+    }
+    server.mount_proc('/503') { |_, res|
+      res.status = 503
+      res.body = 'Service Unavailable'
+    }
+    server.mount_proc('/404') { |_, res|
+      res.status = 404
+      res.body = 'Not Found'
+    }
+    # For start check
+    server.mount_proc('/') { |_, res|
+      res.status = 200
+      res.body = 'Hello Fluentd!'
+    }
+    server.start
+  ensure
+    server.shutdown rescue nil
+  end
+
+  Result = Struct.new("Result", :method, :content_type, :headers, :data)
+
+  setup do
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+
+    @@result = Result.new(nil, nil, {}, nil)
+    @@http_server_thread ||= Thread.new do
+      run_http_server
+    end
+
+    now = Time.now
+    started = false
+    until started
+      raise "Server not started" if (Time.now - now > 10.0)
+      begin
+        http_client { |c| c.request_get('/') }
+        started = true
+      rescue
+        sleep 0.5
+      end
+    end
+  end
+
+  teardown do
+    @@result = nil
+    @@auth_handler = nil
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::HTTPOutput).configure(conf)
+  end
+
+  def config
+    %[
+      endpoint #{base_endpoint}/test
+    ]
+  end
+
+  def test_events
+    [
+      {"message" => "hello", "num" => 10, "bool" => true},
+      {"message" => "hello", "num" => 11, "bool" => false}
+    ]
+  end
+
+  def test_configure
+    d = create_driver(config)
+    assert_equal "http://127.0.0.1:#{server_port}/test", d.instance.endpoint
+    assert_equal :post, d.instance.http_method
+    assert_equal 'application/x-ndjson', d.instance.content_type
+    assert_equal [503], d.instance.retryable_response_codes
+    assert_true d.instance.error_response_as_unrecoverable
+    assert_nil d.instance.proxy
+    assert_nil d.instance.headers
+  end
+
+  def test_configure_with_warn
+    d = create_driver(config)
+    assert_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
+  end
+
+  def test_configure_without_warn
+    d = create_driver(<<~CONFIG)
+      endpoint #{base_endpoint}/test
+      retryable_response_codes [503]
+    CONFIG
+    assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
+  end
+
+  # Check if an exception is raised on not JSON format use
+  data('not_json' => 'msgpack')
+  def test_configure_with_json_array_err(format_type)
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        json_array true
+        <format>
+          @type #{format_type}
+        </format>
+      ])
+    end
+  end
+
+  data('json' => ['json', 'application/x-ndjson'],
+       'ltsv' => ['ltsv', 'text/tab-separated-values'],
+       'msgpack' => ['msgpack', 'application/x-msgpack'],
+       'single_value' => ['single_value', 'text/plain'])
+  def test_configure_content_type(types)
+    format_type, content_type = types
+    d = create_driver(config + %[
+      <format>
+        @type #{format_type}
+      </format>
+    ])
+    assert_equal content_type, d.instance.content_type
+  end
+
+  # Check that json_array setting sets content_type = application/json
+  data('json' => 'application/json')
+  def test_configure_content_type_json_array(content_type)
+    d = create_driver(config + "json_array true")
+
+    assert_equal content_type, d.instance.content_type
+  end
+
+  data('PUT' => 'put', 'POST' => 'post')
+  def test_write_with_method(method)
+    d = create_driver(config + "http_method #{method}")
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal method.upcase, result.method
+    assert_equal 'application/x-ndjson', result.content_type
+    assert_equal test_events, result.data
+    assert_not_empty result.headers
+  end
+
+  # Check that JSON at HTTP request body is valid
+  def test_write_with_json_array_setting
+    d = create_driver(config + "json_array true")
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal 'application/json', result.content_type
+    assert_equal test_events, result.data
+    assert_not_empty result.headers
+  end
+
+  def test_write_with_single_value_format
+    d = create_driver(config + %[
+      <format>
+        @type single_value
+      </format>
+    ])
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal 'text/plain', result.content_type
+    assert_equal (test_events.map { |e| e['message'] }), result.data
+    assert_not_empty result.headers
+  end
+
+  def test_write_with_headers
+    d = create_driver(config + 'headers {"test_header":"fluentd!"}')
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_true result.headers.has_key?('test_header')
+    assert_equal "fluentd!", result.headers['test_header']
+  end
+
+  def test_write_with_headers_from_placeholders
+    d = create_driver(config + %[
+      headers_from_placeholders {"x-test":"${$.foo.bar}-test","x-tag":"${tag}"}
+      <buffer tag,$.foo.bar>
+      </buffer>
+    ])
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        ev = event.dup
+        ev['foo'] = {'bar' => 'abcd'}
+        d.feed(ev)
+      }
+    end
+
+    result = @@result
+    assert_equal "abcd-test", result.headers['x-test']
+    assert_equal "test.http", result.headers['x-tag']
+  end
+
+  def test_write_with_retryable_response
+    old_report_on_exception = Thread.report_on_exception
+    Thread.report_on_exception = false # thread finished as invalid state since RetryableResponse raises.
+
+    d = create_driver("endpoint #{base_endpoint}/503")
+    assert_raise(Fluent::Plugin::HTTPOutput::RetryableResponse) do
+      d.run(default_tag: 'test.http', shutdown: false) do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+    end
+
+    d.instance_shutdown(log: $log)
+  ensure
+    Thread.report_on_exception = old_report_on_exception
+  end
+
+  def test_write_with_disabled_unrecoverable
+    d = create_driver(%[
+      endpoint #{base_endpoint}/404
+      error_response_as_unrecoverable false
+    ])
+    d.run(default_tag: 'test.http', shutdown: false) do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+    assert_match(/got error response from.*404 Not Found Not Found/, d.instance.log.out.logs.join)
+    d.instance_shutdown
+  end
+
+  sub_test_case 'basic auth' do
+    setup do
+      FileUtils.mkdir_p(TMP_DIR)
+      htpd = WEBrick::HTTPAuth::Htpasswd.new(File.join(TMP_DIR, 'dot.htpasswd'))
+      htpd.set_passwd(nil, 'test', 'hey')
+      authenticator = WEBrick::HTTPAuth::BasicAuth.new(:UserDB => htpd, :Realm => 'test', :Logger => DEFAULT_LOGGER)
+      @@auth_handler = Proc.new { |req, res| authenticator.authenticate(req, res) }
+    end
+
+    teardown do
+      FileUtils.rm_rf(TMP_DIR)
+    end
+
+    def server_port
+      19881
+    end
+
+    def test_basic_auth
+      d = create_driver(config + %[
+        <auth>
+          method basic
+          username test
+          password hey
+        </auth>
+      ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+    end
+
+    # This test includes `error_response_as_unrecoverable true` behaviour check
+    def test_basic_auth_with_invalid_auth
+      d = create_driver(config + %[
+        <auth>
+          method basic
+          username ayaya
+          password hello?
+        </auth>
+      ])
+      d.run(default_tag: 'test.http', shutdown: false) do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+      assert_match(/got unrecoverable error/, d.instance.log.out.logs.join)
+
+      d.instance_shutdown
+    end
+  end
+
+  sub_test_case 'HTTPS' do
+    def server_port
+      19882
+    end
+
+    def server_config
+      config = super
+      # WEBrick supports self-generated self-signed certificate
+      config[:SSLEnable] = true
+      config[:SSLCertName] = [["CN", WEBrick::Utils::getservername]]
+      config
+    end
+
+    def http_client(&block)
+      super(use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE, &block)
+    end
+
+    def test_write_with_https
+      d = create_driver(%[
+        endpoint https://127.0.0.1:#{server_port}/test
+        tls_verify_mode none
+        ssl_timeout 2s
+      ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+    end
+  end
+end

data/test/plugin/test_out_null.rb

@@ -0,0 +1,105 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_null'
+
+class NullOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
+  end
+
+  sub_test_case 'non-buffered' do
+    test 'configure' do
+      assert_nothing_raised do
+        create_driver
+      end
+    end
+
+    test 'process' do
+      d = create_driver
+      assert_nothing_raised do
+        d.run do
+          d.feed("test", Fluent::EventTime.now, {"test" => "null"})
+        end
+      end
+    assert_equal([], d.events(tag: "test"))
+    end
+  end
+
+  sub_test_case 'buffered' do
+    test 'default chunk limit size is 100' do
+      d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
+      assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
+      assert d.instance.buffer_config.flush_at_shutdown
+      assert_equal ['tag'], d.instance.buffer_config.chunk_keys
+      assert d.instance.chunk_key_tag
+      assert !d.instance.chunk_key_time
+      assert_equal [], d.instance.chunk_keys
+    end
+
+    test 'writes standard formatted chunks' do
+      d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 3, d.instance.emit_count
+      assert_equal 3, d.instance.emit_records
+    end
+
+    test 'check for chunk passed to #write' do
+      d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      _, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+    end
+
+    test 'check for chunk passed to #try_write' do
+      d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+      d.instance.delayed = true
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      chunk_id, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+
+      assert_equal [chunk_id], d.instance.buffer.dequeued.keys
+
+      d.instance.commit_write(chunk_id)
+
+      assert_equal [], d.instance.buffer.dequeued.keys
+
+      d.instance_shutdown
+    end
+  end
+end

data/test/plugin/test_out_relabel.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_relabel'
+
+class RelabelOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def default_config
+    config_element('ROOT', '', {"@type"=>"relabel", "@label"=>"@RELABELED"})
+  end
+
+  def create_driver(conf = default_config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RelabelOutput).configure(conf)
+  end
+
+  def test_process
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    assert_equal [["test", time, {"a"=>1}], ["test", time, {"a"=>2}]], d.events
+  end
+end

data/test/plugin/test_out_roundrobin.rb

@@ -0,0 +1,146 @@
+require_relative '../helper'
+require 'fluent/test/driver/multi_output'
+require 'fluent/plugin/out_roundrobin'
+
+class RoundRobinOutputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
+      require 'fluent/plugin/out_test'
+      require 'fluent/plugin/out_test2'
+    end
+
+    def shutdown
+      $LOAD_PATH.shift
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    <store>
+      @type test
+      name c0
+    </store>
+    <store>
+      @type test2
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+  CONFIG_WITH_WEIGHT = %[
+    <store>
+      @type test
+      name c0
+      weight 3
+    </store>
+    <store>
+      @type test2
+      name c1
+      weight 3
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::MultiOutput.new(Fluent::Plugin::RoundRobinOutput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    outputs = d.instance.outputs
+    assert_equal 3, outputs.size
+
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+
+    assert !outputs[0].has_router?
+    assert outputs[1].has_router?
+    assert outputs[1].router
+    assert !outputs[2].has_router?
+
+    assert_equal "c0", outputs[0].name
+    assert_equal "c1", outputs[1].name
+    assert_equal "c2", outputs[2].name
+
+    weights = d.instance.weights
+    assert_equal 3, weights.size
+    assert_equal 1, weights[0]
+    assert_equal 1, weights[1]
+    assert_equal 1, weights[2]
+
+    d = create_driver(CONFIG_WITH_WEIGHT)
+
+    outputs = d.instance.outputs
+    assert_equal 3, outputs.size
+
+    assert_equal Fluent::Plugin::TestOutput, outputs[0].class
+    assert_equal Fluent::Plugin::Test2Output, outputs[1].class
+    assert_equal Fluent::Plugin::TestOutput, outputs[2].class
+
+    assert_equal "c0", outputs[0].name
+    assert_equal "c1", outputs[1].name
+    assert_equal "c2", outputs[2].name
+
+    weights = d.instance.weights
+    assert_equal 3, weights.size
+    assert_equal 3, weights[0]
+    assert_equal 3, weights[1]
+    assert_equal 1, weights[2]
+  end
+
+  def test_events_feeded_to_plugins_by_roundrobin
+    d = create_driver
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      d.feed(time, {"a" => 1})
+      d.feed(time, {"a" => 2})
+      d.feed(time, {"a" => 3})
+      d.feed(time, {"a" => 4})
+    end
+
+    os = d.instance.outputs
+
+    assert_equal [
+        [time, {"a"=>1}],
+        [time, {"a"=>4}],
+      ], os[0].events
+
+    assert_equal [
+        [time, {"a"=>2}],
+      ], os[1].events
+
+    assert_equal [
+        [time, {"a"=>3}],
+      ], os[2].events
+  end
+
+  def test_events_feeded_with_specified_weights
+    d = create_driver(CONFIG_WITH_WEIGHT)
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: 'test') do
+      14.times do |i|
+        d.feed(time, {"a" => i})
+      end
+    end
+
+    os = d.instance.outputs
+
+    assert_equal 6, os[0].events.size  # weight=3
+    assert_equal 6, os[1].events.size  # weight=3
+    assert_equal 2, os[2].events.size  # weight=1
+  end
+end
+