fluentd222 1.16.2-x86_64-linux

data/test/plugin_helper/test_formatter.rb

@@ -0,0 +1,255 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/formatter'
+require 'fluent/plugin/base'
+
+class FormatterHelperTest < Test::Unit::TestCase
+  class ExampleFormatter < Fluent::Plugin::Formatter
+    Fluent::Plugin.register_formatter('example', self)
+    def format(tag, time, record)
+      "#{tag},#{time.to_i},#{record.keys.sort.join(',')}" # hey, you miss values! :P
+    end
+  end
+  class Example2Formatter < Fluent::Plugin::Formatter
+    Fluent::Plugin.register_formatter('example2', self)
+    def format(tag, time, record)
+      "#{tag},#{time.to_i},#{record.values.sort.join(',')}" # key...
+    end
+  end
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :formatter
+    config_section :format do
+      config_set_default :@type, 'example'
+    end
+  end
+
+  class Dummy2 < Fluent::Plugin::TestBase
+    helpers :formatter
+    config_section :format do
+      config_set_default :@type, 'example2'
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any formatters at first' do
+    d = Dummy.new
+    assert_equal 0, d._formatters.size
+  end
+
+  test 'can override default configuration parameters, but not overwrite whole definition' do
+    d = Dummy.new
+    assert_equal [], d.formatter_configs
+
+    d = Dummy2.new
+    d.configure(config_element('ROOT', '', {}, [config_element('format', '', {}, [])]))
+    assert_raise NoMethodError do
+      d.format
+    end
+    assert_equal 1, d.formatter_configs.size
+    assert_equal 'example2', d.formatter_configs.first[:@type]
+  end
+
+  test 'creates instance of type specified by conf, or default_type if @type is missing in conf' do
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: config_element('format', '', {'@type' => 'example'}), default_type: 'example2')
+    assert{ i.is_a?(ExampleFormatter) }
+
+    d = Dummy2.new
+    d.configure(config_element())
+    i = d.formatter_create(conf: nil, default_type: 'example2')
+    assert{ i.is_a?(Example2Formatter) }
+  end
+
+  test 'raises config error if config section is specified, but @type is not specified' do
+    d = Dummy2.new
+    d.configure(config_element())
+    assert_raise Fluent::ConfigError.new("@type is required in <format>") do
+      d.formatter_create(conf: config_element('format', '', {}), default_type: 'example2')
+    end
+  end
+
+  test 'can be configured with default type without format sections' do
+    d = Dummy.new
+    assert_nothing_raised do
+      d.configure(config_element())
+    end
+    assert_equal 1, d._formatters.size
+  end
+
+  test 'can be configured with a format section' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', '', {'@type' => 'example'})
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 1, d._formatters.size
+    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }
+  end
+
+  test 'can be configured with 2 or more format sections with different usages with each other' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'default', {'@type' => 'example'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+      ])
+    assert_nothing_raised do
+      d.configure(conf)
+    end
+    assert_equal 2, d._formatters.size
+    assert{ d._formatters.values.all?{ |formatter| !formatter.started? } }
+  end
+
+  test 'cannot be configured with 2 format sections with same usage' do
+    d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'default', {'@type' => 'example'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+        config_element('format', 'extra', {'@type' => 'example2'}),
+      ])
+    assert_raises Fluent::ConfigError do
+      d.configure(conf)
+    end
+  end
+
+  test 'creates a format plugin instance which is already configured without usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', '', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    formatter = d.formatter_create
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates a formatter plugin instance which is already configured with usage' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'mydata', {'@type' => 'example'})
+      ])
+    d.configure(conf)
+    d.start
+
+    formatter = d.formatter_create(usage: 'mydata')
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates a formatter plugin without configurations' do
+    @d = d = Dummy.new
+    d.configure(config_element())
+    d.start
+
+    formatter = d.formatter_create(usage: 'mydata', type: 'example', conf: config_element('format', 'mydata'))
+    assert{ formatter.is_a? ExampleFormatter }
+    assert formatter.started?
+  end
+
+  test 'creates 2 or more formatter plugin instances' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [
+        config_element('format', 'mydata', {'@type' => 'example'}),
+        config_element('format', 'secret', {'@type' => 'example2'})
+      ])
+    d.configure(conf)
+    d.start
+
+    p1 = d.formatter_create(usage: 'mydata')
+    p2 = d.formatter_create(usage: 'secret')
+    assert{ p1.is_a? ExampleFormatter }
+    assert p1.started?
+    assert{ p2.is_a? Example2Formatter }
+    assert p2.started?
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    conf = config_element('ROOT', '', {}, [ config_element('format', '', {'@type' => 'example'}), config_element('format', 'e2', {'@type' => 'example'}) ])
+    d.configure(conf)
+    d.start
+
+    i1 = d.formatter_create(usage: '')
+    i2 = d.formatter_create(usage: 'e2')
+    i3 = d.formatter_create(usage: 'e3', type: 'example2')
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end

data/test/plugin_helper/test_http_server_helper.rb

@@ -0,0 +1,372 @@
+require_relative '../helper'
+require 'flexmock/test_unit'
+require 'fluent/plugin_helper/http_server'
+require 'fluent/plugin/output'
+require 'fluent/event'
+require 'net/http'
+require 'uri'
+require 'openssl'
+require 'async'
+
+class HttpHelperTest < Test::Unit::TestCase
+  NULL_LOGGER = Logger.new(nil)
+  CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
+  CERT_CA_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
+
+  def setup
+    @port = unused_port
+  end
+
+  def teardown
+    @port = nil
+  end
+
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :http_server
+  end
+
+  def on_driver(config = nil)
+    config ||= Fluent::Config.parse(config || '', '(name)', '')
+    Fluent::Test.setup
+    driver = Dummy.new
+    driver.configure(config)
+    driver.start
+    driver.after_start
+
+    yield(driver)
+  ensure
+    unless driver.stopped?
+      driver.stop rescue nil
+    end
+
+    unless driver.before_shutdown?
+      driver.before_shutdown rescue nil
+    end
+
+    unless driver.shutdown?
+      driver.shutdown rescue nil
+    end
+
+    unless driver.after_shutdown?
+      driver.after_shutdown rescue nil
+    end
+
+    unless driver.closed?
+      driver.close rescue nil
+    end
+
+    unless driver.terminated?
+      driver.terminated rescue nil
+    end
+  end
+
+  def on_driver_transport(opts = {}, &block)
+    transport_conf = config_element('transport', 'tls', opts)
+    c = config_element('ROOT', '', {}, [transport_conf])
+    on_driver(c, &block)
+  end
+
+  %w[get head].each do |n|
+    define_method(n) do |uri, header = {}|
+      url = URI.parse(uri)
+      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)
+      req = Net::HTTP.const_get(n.capitalize).new(url, headers)
+      Net::HTTP.start(url.host, url.port) do |http|
+        http.request(req)
+      end
+    end
+
+    define_method("secure_#{n}") do |uri, header = {}, verify: true, cert_path: nil, selfsigned: true, hostname: false|
+      url = URI.parse(uri)
+      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)
+      start_https_request(url.host, url.port, verify: verify, cert_path: cert_path, selfsigned: selfsigned) do |https|
+        https.send(n, url.path, headers.to_a)
+      end
+    end
+  end
+
+  %w[post put patch delete options trace].each do |n|
+    define_method(n) do |uri, body = '', header = {}|
+      url = URI.parse(uri)
+      headers = { 'Content-Type' => 'application/x-www-form-urlencoded/' }.merge(header)
+      req = Net::HTTP.const_get(n.capitalize).new(url, headers)
+      req.body = body
+      Net::HTTP.start(url.host, url.port) do |http|
+        http.request(req)
+      end
+    end
+  end
+
+  # wrapper for net/http
+  Response = Struct.new(:code, :body, :headers)
+
+  # Use async-http as http client since net/http can't be set verify_hostname= now
+  # will be replaced when net/http supports verify_hostname=
+  def start_https_request(addr, port, verify: true, cert_path: nil, selfsigned: true, hostname: nil)
+    context = OpenSSL::SSL::SSLContext.new
+    context.set_params({})
+    if verify
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.set_default_paths
+      if selfsigned && OpenSSL::X509.const_defined?('V_FLAG_CHECK_SS_SIGNATURE')
+        cert_store.flags = OpenSSL::X509::V_FLAG_CHECK_SS_SIGNATURE
+      end
+
+      if cert_path
+        cert_store.add_file(cert_path)
+      end
+
+      context.cert_store = cert_store
+      if !hostname
+        context.verify_hostname = false # In test code, using hostname to be connected is very difficult
+      end
+
+      context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    else
+      context.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+
+    client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
+    reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
+
+    resp = nil
+    error = nil
+
+    reactor.run do
+      begin
+        response = yield(client)
+      rescue => e               # Async::Reactor rescue all error. handle it by myself
+        error = e
+      end
+
+      if response
+        resp = Response.new(response.status.to_s, response.body.read, response.headers)
+      end
+    end
+
+    if error
+      raise error
+    else
+      resp
+    end
+  end
+
+  sub_test_case 'Create a HTTP server' do
+    test 'monunt given path' do
+      on_driver do |driver|
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+          s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+          s.post('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello post'] }
+          s.head('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello head'] }
+          s.put('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello put'] }
+          s.patch('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello patch'] }
+          s.delete('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello delete'] }
+          s.trace('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello trace'] }
+          s.options('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello options'] }
+        end
+
+        resp = head("http://127.0.0.1:#{@port}/example/hello")
+        assert_equal('200', resp.code)
+        assert_equal(nil, resp.body)
+        assert_equal('text/plain', resp['Content-Type'])
+
+        %w[get put post put delete options trace].each do |n|
+          resp = send(n, "http://127.0.0.1:#{@port}/example/hello")
+          assert_equal('200', resp.code)
+          assert_equal("hello #{n}", resp.body)
+          assert_equal('text/plain', resp['Content-Type'])
+        end
+      end
+    end
+
+    test 'when path does not start with `/` or ends with `/`' do
+      on_driver do |driver|
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+          s.get('example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+          s.get('/example/hello2/') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+        end
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello")
+        assert_equal('404', resp.code)
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello2")
+        assert_equal('200', resp.code)
+      end
+    end
+
+    test 'when error raised' do
+      on_driver do |driver|
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+          s.get('/example/hello') { raise 'error!' }
+        end
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello")
+        assert_equal('500', resp.code)
+      end
+    end
+
+    test 'when path is not found' do
+      on_driver do |driver|
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+          s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+        end
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello/not_found")
+        assert_equal('404', resp.code)
+      end
+    end
+
+    test 'params and body' do
+      on_driver do |driver|
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+          s.get('/example/hello') do |req|
+            assert_equal(req.query_string, nil)
+            assert_equal(req.body, nil)
+            [200, { 'Content-Type' => 'text/plain' }, 'hello get']
+          end
+
+          s.post('/example/hello') do |req|
+            assert_equal(req.query_string, nil)
+            assert_equal(req.body, 'this is body')
+            [200, { 'Content-Type' => 'text/plain' }, 'hello post']
+          end
+
+          s.get('/example/hello/params') do |req|
+            assert_equal(req.query_string, 'test=true')
+            assert_equal(req.body, nil)
+            [200, { 'Content-Type' => 'text/plain' }, 'hello get']
+          end
+
+          s.post('/example/hello/params') do |req|
+            assert_equal(req.query_string, 'test=true')
+            assert_equal(req.body, 'this is body')
+            [200, { 'Content-Type' => 'text/plain' }, 'hello post']
+          end
+        end
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello")
+        assert_equal('200', resp.code)
+
+        resp = post("http://127.0.0.1:#{@port}/example/hello", 'this is body')
+        assert_equal('200', resp.code)
+
+        resp = get("http://127.0.0.1:#{@port}/example/hello/params?test=true")
+        assert_equal('200', resp.code)
+
+        resp = post("http://127.0.0.1:#{@port}/example/hello/params?test=true", 'this is body')
+        assert_equal('200', resp.code)
+      end
+    end
+
+    sub_test_case 'create a HTTPS server' do
+      test '#configure' do
+        driver = Dummy.new
+
+        transport_conf = config_element('transport', 'tls', { 'version' => 'TLSv1_1' })
+        driver.configure(config_element('ROOT', '', {}, [transport_conf]))
+        assert_equal :tls, driver.transport_config.protocol
+        assert_equal :TLSv1_1, driver.transport_config.version
+      end
+
+      sub_test_case '#http_server_create_https_server' do
+        test 'can overwrite settings by using tls_context' do
+          on_driver_transport({ 'insecure' => 'false' }) do |driver|
+            tls = { 'insecure' => 'true' } # overwrite
+            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER, tls_opts: tls) do |s|
+              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+            end
+
+            resp = secure_get("https://127.0.0.1:#{@port}/example/hello", verify: false)
+            assert_equal('200', resp.code)
+            assert_equal('hello get', resp.body)
+          end
+        end
+
+        test 'with insecure in transport section' do
+          on_driver_transport({ 'insecure' => 'true' }) do |driver|
+            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+            end
+            omit "TLS connection should be aborted due to `Errno::ECONNABORTED`. Need to debug." if Fluent.windows?
+
+            resp = secure_get("https://127.0.0.1:#{@port}/example/hello", verify: false)
+            assert_equal('200', resp.code)
+            assert_equal('hello get', resp.body)
+
+            assert_raise OpenSSL::SSL::SSLError do
+              secure_get("https://127.0.0.1:#{@port}/example/hello")
+            end
+          end
+        end
+
+        data(
+          'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem'],
+          'without passphrase' => [nil, 'cert.pem', 'cert-key.pem'])
+        test 'load self-signed cert/key pair, verified from clients using cert files' do |(passphrase, cert, private_key)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
+          cert_path = File.join(CERT_DIR, cert)
+          private_key_path = File.join(CERT_DIR, private_key)
+          opt = { 'insecure' => 'false', 'private_key_path' => private_key_path, 'cert_path' => cert_path }
+          if passphrase
+            opt['private_key_passphrase'] = passphrase
+          end
+
+          on_driver_transport(opt) do |driver|
+            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+            end
+
+            resp = secure_get("https://127.0.0.1:#{@port}/example/hello", cert_path: cert_path)
+            assert_equal('200', resp.code)
+            assert_equal('hello get', resp.body)
+          end
+        end
+
+        data(
+          'with passphrase' => ['apple', 'cert-pass.pem', 'cert-key-pass.pem', 'ca-cert-pass.pem'],
+          'without passphrase' => [nil, 'cert.pem', 'cert-key.pem', 'ca-cert.pem'])
+        test 'load cert by private CA cert file, verified from clients using CA cert file' do |(passphrase, cert, cert_key, ca_cert)|
+          omit "Self signed certificate blocks TLS connection. Need to debug." if Fluent.windows?
+
+          cert_path = File.join(CERT_CA_DIR, cert)
+          private_key_path = File.join(CERT_CA_DIR, cert_key)
+
+          ca_cert_path = File.join(CERT_CA_DIR, ca_cert)
+
+          opt = { 'insecure' => 'false', 'cert_path' => cert_path, 'private_key_path' => private_key_path }
+          if passphrase
+            opt['private_key_passphrase'] = passphrase
+          end
+
+          on_driver_transport(opt) do |driver|
+            driver.http_server_create_https_server(:http_server_helper_test_tls, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do |s|
+              s.get('/example/hello') { [200, { 'Content-Type' => 'text/plain' }, 'hello get'] }
+            end
+
+            resp = secure_get("https://127.0.0.1:#{@port}/example/hello", cert_path: ca_cert_path)
+            assert_equal('200', resp.code)
+            assert_equal('hello get', resp.body)
+          end
+        end
+      end
+    end
+
+    test 'must be called #start and #stop' do
+      on_driver do |driver|
+        server = flexmock('Server') do |watcher|
+          watcher.should_receive(:start).once.and_return do |que|
+            que.push(:start)
+          end
+          watcher.should_receive(:stop).once
+        end
+
+        stub(Fluent::PluginHelper::HttpServer::Server).new(addr: anything, port: anything, logger: anything, default_app: anything) { server }
+        driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: @port, logger: NULL_LOGGER) do
+          # nothing
+        end
+        driver.stop
+      end
+    end
+  end
+end