fluent-plugin-out-http-ext-ignore-ssl-errors 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 44998e4900ae688e02a1c43658ff89405b0c4c67
+  data.tar.gz: 9f27da2e6130e79d1323563fe5ac1ab3e23e8361
+SHA512:
+  metadata.gz: 6c4bd94668ce17d69136af53c59c2ec54283f3ceb44d849d1bc0093e72189352bcd531f0e77194b5aee34ad255e9c7cdfa16274735052d5afa1ff41f52ce8aff
+  data.tar.gz: fcc40c34c6115f4cbd9548c2cc1ab0235a09797ef097eedb999dd1a825df0fa1380866541df5686611d3fda1adcb78b9e9b001ee03d14266207aad493e58e6e5

data/.coveralls.yml

@@ -0,0 +1,2 @@
+service_name: travis-ci
+repo_token: 9tG18WMwC1xpPdDBEGznon01RCp9NusHL

data/.gitignore

@@ -0,0 +1,9 @@
+*~
+\#*
+.\#*
+*.gem
+.bundle
+.ruby-version
+Gemfile.lock
+vendor
+coverage/

data/.travis.yml

@@ -0,0 +1,18 @@
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+  - 2.2
+  - ruby-head
+
+os:
+  - linux
+
+gemfile:
+  - Gemfile
+
+script: bundle exec rake test
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/CHANGELOG.md

@@ -0,0 +1,24 @@
+# Changelog
+
+## 0.1.5
+* Add headers directive
+* Add use_ssl directive
+
+## 0.1.4
+* #11 Updated Fluentd dependency to:  [">= 0.10.0", "< 2"]
+* #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)
+
+## 0.1.3
+* Added a new configuration option: `raise_on_error` (default: true)
+  * In order to let the plugin raise exceptions like it did in 0.1.1: keep using your configuration as-is
+  * In order to suppress all exceptions: add `raise_on_error false` to your configuration
+
+## 0.1.2
+* #6 Catch all `StandardError`s during HTTP request to prevent td-agent from freezing
+
+## 0.1.1
+* #2 Use yajl instead of json as json serializer
+* #1 Fix a bug where a nil HTTP response caused the plugin to stop working
+
+## 0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-out-http.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# fluent-plugin-out-http-ext, a plugin for [Fluentd](http://fluentd.org)
+
+[![Build Status](https://travis-ci.org/kawasakitoshiya/fluent-plugin-out-http-ext.svg)](https://travis-ci.org/kawasakitoshiya/fluent-plugin-out-http-ext)
+[![Coverage Status](https://coveralls.io/repos/kawasakitoshiya/fluent-plugin-out-http-ext/badge.svg?branch=master&service=github)](https://coveralls.io/github/kawasakitoshiya/fluent-plugin-out-http-ext?branch=master)
+
+**This is a fork of [ento / fluent-plugin-out-http](https://github.com/ento/fluent-plugin-out-http)**
+
+A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint
+
+## Configuration options
+
+    <match *>
+      type http_ext
+      endpoint_url          http://localhost.local/api/<data.id> # <data.id> refres to data.id in the record like {"data"=> {"id"=> 1, "name"=> "foo"}}
+      http_method           put                                  # default: post
+      serializer            json                                 # default: form
+      rate_limit_msec       100                                  # default: 0 = no rate limiting
+      open_timeout          5                                    # default: nil = no timeout
+      read_timeout          10                                   # default: 60
+      raise_on_error        false                                # default: true
+      raise_on_http_failure true                                 # default: false
+      ignore_http_status_code 300,400..499                        # default: nil # do not raise on these http_hstatus codes
+      authentication        basic                                # default: none
+      username              alice                                # default: ''
+      password              bobpop                               # default: '', secret: true
+      use_ssl               true                                 # default: false
+      <headers>
+        HeaderExample1 header1
+        HeaderExample2 header2
+      </headers>
+    </match>
+
+## Usage notes
+
+If you'd like to retry failed requests, consider using [fluent-plugin-bufferize][3].
+
+----
+
+Heavily based on [fluent-plugin-growthforecast][2]
+
+  [1]: http://fluentd.org/
+  [2]: https://github.com/tagomoris/fluent-plugin-growthforecast
+  [3]: https://github.com/sabottenda/fluent-plugin-bufferize

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-out-http-ext.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-out-http-ext-ignore-ssl-errors"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Joyce Chan"]
+  gem.email         = ["none@none.com"]
+  gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL and Header option, extended from kawasakitoshiya@gmail.com's similarily named gem'}
+  gem.description   = gem.summary
+  gem.licenses      = ["Apache-2.0"]
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
+  gem.add_runtime_dependency "fluentd", [">= 0.10.0", "< 2"]
+  gem.add_development_dependency "bundler"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", ">= 3.1.0"
+  gem.add_development_dependency "coveralls"
+end

data/lib/fluent/plugin/out_http_ext.rb

@@ -0,0 +1,254 @@
+require 'set'
+
+class Array
+  def to_set
+    Set.new(self)
+  end
+end
+
+
+class Hash
+  """
+  each traverse in hash
+  """
+  def each_deep(&proc)
+    self.each_deep_detail([], &proc)
+  end
+
+  def each_deep_detail(directory, &proc)
+    self.each do |k, v|
+      current = directory + [k]
+      if v.kind_of?(Hash)
+        v.each_deep_detail(current, &proc)
+      else
+        yield(current, v)
+      end
+    end
+  end
+
+end
+
+class StatusCodeParser
+  """
+  parse status code string to array of codes
+  """
+  def self.range?(str)
+     # i.e. 200..399 => return true
+    return /^\d{3}..\d{3}$/ =~ str ? true : false
+  end
+
+  def self.number?(str)
+    return /^\d{3}$/ =~ str ? true : false
+  end
+
+  def self.get_array(str)
+    if self.range?(str)
+      ends = str.split('..').map{|d| Integer(d)}
+      return (ends[0]..ends[1]).to_a
+    elsif self.number?(str)
+      return [str.to_i]
+    else
+      raise "invalid status code range format"
+    end
+  end
+
+  def self.convert(range_str)
+    elems = range_str.split(',')
+    status_codes = elems.flat_map do |elem|
+      self.get_array(elem)
+    end
+    return status_codes.to_set
+  end
+end
+
+class Fluent::HTTPOutput < Fluent::Output
+  Fluent::Plugin.register_output('http_ext', self)
+
+  def initialize
+    super
+    require 'net/http'
+    require 'uri'
+    require 'yajl'
+    require 'set'
+  end
+
+  # Endpoint URL ex. localhost.local/api/
+  config_param :endpoint_url, :string
+
+  # HTTP method
+  config_param :http_method, :string, :default => :post
+
+  # form | json
+  config_param :serializer, :string, :default => :form
+
+  # true | false
+  config_param :use_ssl, :bool, :default => false
+
+  config_param :open_timeout, :integer, :default => nil
+  config_param :read_timeout, :integer, :default => 60
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # Raise errors that were rescued during HTTP requests?
+  config_param :raise_on_error, :bool, :default => true
+
+  # Raise errors when HTTP response code was not successful.
+  config_param :raise_on_http_failure, :bool, :default => false
+  config_param :ignore_http_status_code, :string, :default => nil
+  # nil | 'none' | 'basic'
+  config_param :authentication, :string, :default => nil
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => '', :secret => true
+
+  def configure(conf)
+    super
+
+    serializers = [:json, :form]
+    @serializer = if serializers.include? @serializer.intern
+                    @serializer.intern
+                  else
+                    :form
+                  end
+
+    http_methods = [:get, :put, :post, :delete]
+    @http_method = if http_methods.include? @http_method.intern
+                    @http_method.intern
+                  else
+                    :post
+                  end
+
+    @ignore_http_status_code = if @ignore_http_status_code.nil?
+                          [].to_set
+                        else
+                          StatusCodeParser.convert(@ignore_http_status_code)
+                        end
+
+    @auth = case @authentication
+            when 'basic' then :basic
+            else
+              :none
+            end
+    @headers = {}
+    conf.elements.each do |element|
+      if element.name == 'headers'
+        @headers = element.to_hash
+      end
+    end
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    '''
+    replace format string to value
+    example
+      /test/<data> =(use {data: 1})> /test/1
+      /test/<hash.data> =(use {hash:{data:2}})> /test/2
+    '''
+    result_url = @endpoint_url
+    record.each_deep do |key_dir, value|
+      result_url = result_url.gsub(/<#{key_dir.join(".")}>/, value.to_s)
+    end
+    return result_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    @headers.each do |key, value|
+      req[key] = value
+    end
+    req
+  end
+
+  def set_json_body(req, data)
+    req.body = Yajl.dump(data)
+    req['Content-Type'] = 'application/json'
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    return req, uri
+  end
+
+  def send_request(req, uri)
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      $log.info('Dropped request due to rate limiting')
+      return
+    end
+
+    res = nil
+
+    begin
+      if @auth and @auth == :basic
+        req.basic_auth(@username, @password)
+      end
+      @last_request_time = Time.now.to_f
+      client = Net::HTTP.new(uri.host, uri.port)
+      if @use_ssl
+        client.use_ssl = true
+        client.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE
+        client.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      res = client.start {|http|
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+        http.request(req)
+      }
+    rescue => e # rescue all StandardErrors
+      # server didn't respond
+      $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+      raise e if @raise_on_error
+    else
+       unless res and res.is_a?(Net::HTTPSuccess)
+          res_summary = if res
+                           "#{res.code} #{res.message} #{res.body}"
+                        else
+                           "res=nil"
+                        end
+          warning = "failed to #{req.method} #{uri} (#{res_summary})"
+          $log.warn warning
+          if @raise_on_http_failure
+            unless @ignore_http_status_code.include?(res.code.to_i)
+              raise warning
+            else
+              $log.debug "ignore http status code #{req.method}"
+            end
+          end
+
+       end #end unless
+    end # end begin
+  end # end send_request
+
+  def handle_record(tag, time, record)
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def emit(tag, es, chain)
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+    chain.next
+  end
+end

data/lib/fluent/test/http_output_test.rb

@@ -0,0 +1,54 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+class Test::Unit::TestCase
+end
+
+require 'webrick'
+
+# to handle POST/PUT/DELETE ...
+module WEBrick::HTTPServlet
+  class ProcHandler < AbstractServlet
+    alias do_POST   do_GET
+    alias do_PUT    do_GET
+    alias do_DELETE do_GET
+  end
+end
+
+def get_code(server, port, path, headers={})
+  require 'net/http' 
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).code
+  } 
+end 
+def get_content(server, port, path, headers={})
+  require 'net/http'
+  Net::HTTP.start(server, port){|http|
+    http.get(path, headers).body
+  } 
+end

data/test/plugin/test_helper.rb

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+require 'coveralls'
+Coveralls.wear!

data/test/plugin/test_out_http_ext.rb

@@ -0,0 +1,511 @@
+# -*- coding: utf-8 -*-
+require 'uri'
+require 'yajl'
+require 'fluent/test/http_output_test'
+require 'fluent/plugin/out_http_ext'
+
+
+TEST_LISTEN_PORT = 5126
+
+
+class HTTPOutputTestBase < Test::Unit::TestCase
+  # setup / teardown for servers
+  def setup
+    Fluent::Test.setup
+    @posts = []
+    @puts = []
+    @prohibited = 0
+    @requests = 0
+    @auth = false
+    @status = 200
+    @dummy_server_thread = Thread.new do
+      srv = if ENV['VERBOSE']
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
+            else
+              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
+              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
+            end
+      begin
+        allowed_methods = %w(POST PUT)
+        srv.mount_proc('/api/') { |req,res|
+          @requests += 1
+          unless allowed_methods.include? req.request_method
+            res.status = 405
+            res.body = 'request method mismatch'
+            next
+          end
+          if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!'
+            # ok, authorized
+          elsif @auth
+            res.status = 403
+            @prohibited += 1
+            next
+          else
+            # ok, authorization not required
+          end
+
+          record = {:auth => nil}
+          if req.content_type == 'application/json'
+            record[:json] = Yajl.load(req.body)
+          else
+            record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)]
+          end
+
+          instance_variable_get("@#{req.request_method.downcase}s").push(record)
+
+          res.status = @status
+        }
+        srv.mount_proc('/') { |req,res|
+          res.status = 200
+          res.body = 'running'
+        }
+        srv.mount_proc('/slow_5') { |req,res|
+          sleep 5
+          res.status = 200
+          res.body = 'slow_5'
+        }
+        srv.mount_proc('/slow_10') { |req,res|
+          sleep 10
+          res.status = 200
+          res.body = 'slow_10'
+        }
+        srv.mount_proc('/status_code') { |req,res|
+          r = Yajl.load(req.body)
+          code = r["code"]
+          res.status = code.to_s
+          res.body = ''
+        }
+
+        srv.start
+      ensure
+        srv.shutdown
+      end
+    end
+
+    # to wait completion of dummy server.start()
+    require 'thread'
+    cv = ConditionVariable.new
+    watcher = Thread.new {
+      connected = false
+      while not connected
+        begin
+          get_content('localhost', TEST_LISTEN_PORT, '/')
+          connected = true
+        rescue Errno::ECONNREFUSED
+          sleep 0.1
+        rescue StandardError => e
+          p e
+          sleep 0.1
+        end
+      end
+      cv.signal
+    }
+    mutex = Mutex.new
+    mutex.synchronize {
+      cv.wait(mutex)
+    }
+  end
+
+  def test_dummy_server
+    host = '127.0.0.1'
+    port = TEST_LISTEN_PORT
+    client = Net::HTTP.start(host, port)
+
+    assert_equal '200', client.request_get('/').code
+    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '1', @posts[0][:form]['number']
+    assert_equal 'gauge', @posts[0][:form]['mode']
+    assert_nil @posts[0][:auth]
+
+    @auth = true
+
+    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
+
+    req_with_auth = lambda do |number, mode, user, pass|
+      url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
+      req = Net::HTTP::Post.new(url.path)
+      req.basic_auth user, pass
+      req.set_form_data({'number'=>number, 'mode'=>mode})
+      req
+    end
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code
+
+    assert_equal 1, @posts.size
+
+    assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code
+
+    assert_equal 2, @posts.size
+
+  end
+
+  def teardown
+    @dummy_server_thread.kill
+    @dummy_server_thread.join
+  end
+end
+
+class HTTPOutputTest < HTTPOutputTestBase
+  CONFIG = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+  ]
+
+  CONFIG_JSON = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    serializer json
+  ]
+
+  CONFIG_PUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    http_method put
+  ]
+
+  CONFIG_HTTP_ERROR = %[
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+  ]
+
+  CONFIG_HTTP_ERROR_SUPPRESSED = %[
+    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+    raise_on_error false
+  ]
+
+  CONFIG_RAISE_ON_HTTP_FAILURE = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    raise_on_http_failure true
+  ]
+
+  RATE_LIMIT_MSEC = 1200
+
+  CONFIG_RATE_LIMIT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    rate_limit_msec #{RATE_LIMIT_MSEC}
+  ]
+
+  CONFIG_NOT_READ_TIMEOUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/slow_5/
+    read_timeout 7
+  ]
+  CONFIG_READ_TIMEOUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/slow_10/
+    read_timeout 7
+  ]
+  CONFIG_IGNORE_NONE = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+  ]
+  CONFIG_IGNORE_409 = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 409
+  ]
+  CONFIG_IGNORE_4XX = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 400..499
+  ]
+  CONFIG_IGNORE_4XX_5XX = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 400..599
+  ]
+
+  def create_driver(conf=CONFIG, tag='test.metrics')
+    Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :form, d.instance.serializer
+
+    d = create_driver CONFIG_JSON
+    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal :json, d.instance.serializer
+  end
+
+  def test_emit_form
+    d = create_driver
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_equal '20', record[:form]['field2']
+    assert_equal '10', record[:form]['field3']
+    assert_equal '1', record[:form]['otherfield']
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run
+
+    assert_equal 2, @posts.size
+  end
+
+  def test_emit_form_put
+    d = create_driver CONFIG_PUT
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @puts.size
+    record = @puts[0]
+
+    assert_equal '50', record[:form]['field1']
+    assert_nil record[:auth]
+
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @puts.size
+  end
+
+  def test_emit_json
+    binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
+    d = create_driver CONFIG_JSON
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal 50, record[:json]['field1']
+    assert_equal 20, record[:json]['field2']
+    assert_equal 10, record[:json]['field3']
+    assert_equal 1, record[:json]['otherfield']
+    assert_equal binary_string, record[:json]['binary']
+    assert_nil record[:auth]
+  end
+
+  def test_http_error_is_raised
+    d = create_driver CONFIG_HTTP_ERROR
+    assert_raise Errno::ECONNREFUSED do
+      d.emit({ 'field1' => 50 })
+    end
+  end
+
+  def test_http_error_is_suppressed_with_raise_on_error_false
+    d = create_driver CONFIG_HTTP_ERROR_SUPPRESSED
+    d.emit({ 'field1' => 50 })
+    d.run
+    # drive asserts the next output chain is called;
+    # so no exception means our plugin handled the error
+
+    assert_equal 0, @requests
+  end
+
+  def test_http_failure_is_not_raised_on_http_failure_true_and_status_201
+    @status = 201
+
+    d = create_driver CONFIG_RAISE_ON_HTTP_FAILURE
+    assert_nothing_raised do
+      d.emit({ 'field1' => 50 })
+    end
+
+    @status = 200
+  end
+
+  def test_http_failure_is_raised_on_http_failure_true
+    @status = 500
+
+    d = create_driver CONFIG_RAISE_ON_HTTP_FAILURE
+    assert_raise RuntimeError do
+      d.emit({ 'field1' => 50 })
+    end
+
+    @status = 200
+  end
+
+  def test_rate_limiting
+    d = create_driver CONFIG_RATE_LIMIT
+    record = { :k => 1 }
+
+    last_emit = _current_msec
+    d.emit(record)
+    d.run
+
+    assert_equal 1, @posts.size
+
+    d.emit({})
+    d.run
+    assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval"
+    assert_equal 1, @posts.size
+
+    wait_msec = 500
+    sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
+
+    assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
+    d.emit(record)
+    d.run
+    assert_equal 2, @posts.size
+  end
+
+  def test_read_timeout
+    d = create_driver CONFIG_READ_TIMEOUT
+    assert_equal 7, d.instance.read_timeout
+    err = Net.const_defined?(:ReadTimeout) ? Net::ReadTimeout : Timeout::Error
+    assert_raise err do
+      d.emit({})
+      d.run
+    end
+  end
+
+  def test_not_read_timeout
+    d = create_driver CONFIG_NOT_READ_TIMEOUT
+    assert_equal 7, d.instance.read_timeout
+    assert_nothing_raised do
+      d.emit({})
+      d.run
+    end
+  end
+
+  def test_ignore_none
+    d = create_driver CONFIG_IGNORE_NONE
+    assert_equal [].to_set, d.instance.ignore_http_status_code
+
+    assert_raise do
+      d.emit({:code=> 409})
+      d.run
+    end
+
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_409
+    d = create_driver CONFIG_IGNORE_409
+    assert_equal [409].to_set, d.instance.ignore_http_status_code
+
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_4XX
+    d = create_driver CONFIG_IGNORE_4XX
+    assert_equal (400..499).to_a.to_set, d.instance.ignore_http_status_code
+
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_4XX_5XX
+    d = create_driver CONFIG_IGNORE_4XX_5XX
+    assert_equal (400..599).to_a.to_set, d.instance.ignore_http_status_code
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def _current_msec
+    Time.now.to_f * 1000
+  end
+
+  def test_auth
+    @auth = true # enable authentication of dummy server
+
+    d = create_driver(CONFIG, 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 1, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password wrong_password
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 0, @posts.size
+    assert_equal 2, @prohibited
+
+    d = create_driver(CONFIG + %[
+      authentication basic
+      username alice
+      password secret!
+    ], 'test.metrics')
+    d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 })
+    d.run # failed in background, and output warn log
+
+    assert_equal 1, @posts.size
+    assert_equal 2, @prohibited
+  end
+
+  def test_status_code_parser()
+    assert_equal (400..409).to_a.to_set, StatusCodeParser.convert("400..409")
+    assert_equal ((400..409).to_a + [300]).to_set, StatusCodeParser.convert("400..409,300")
+    assert_equal ((400..409).to_a + [300]).to_set, StatusCodeParser.convert("300,400..409")
+    assert_equal [404, 409].to_set, StatusCodeParser.convert("404,409")
+    assert_equal [404, 409, 300, 301, 302, 303].to_set, StatusCodeParser.convert("404,409,300..303")
+    assert_equal [409].to_set, StatusCodeParser.convert("409")
+    assert_equal [].to_set, StatusCodeParser.convert("")
+    assert_raise do
+       StatusCodeParser.convert("400...499")
+    end
+    assert_raise do
+      StatusCodeParser.convert("10..20")
+    end
+    assert_raise do
+      StatusCodeParser.convert("4XX")
+    end
+    assert_raise do
+      StatusCodeParser.convert("4XX..5XX")
+    end
+    assert_raise do
+      StatusCodeParser.convert("200.0..400")
+    end
+    assert_raise do
+      StatusCodeParser.convert("-200..400")
+    end
+
+  end
+
+  def test_array_extend
+    assert_equal [].to_set, Set.new([])
+    assert_equal [1, 2].to_set, Set.new([1, 2])
+  end
+end

metadata

@@ -0,0 +1,152 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-out-http-ext-ignore-ssl-errors
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joyce Chan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-10-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A generic Fluentd output plugin to send logs to an HTTP endpoint with
+  SSL and Header option, extended from kawasakitoshiya@gmail.com's similarily named
+  gem'
+email:
+- none@none.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".coveralls.yml"
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-out-http-ext.gemspec
+- lib/fluent/plugin/out_http_ext.rb
+- lib/fluent/test/http_output_test.rb
+- test/plugin/test_helper.rb
+- test/plugin/test_out_http_ext.rb
+homepage: 
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL
+  and Header option, extended from kawasakitoshiya@gmail.com's similarily named gem'
+test_files:
+- test/plugin/test_helper.rb
+- test/plugin/test_out_http_ext.rb