fluent-plugin-out-http-ext 0.1.6 → 0.1.7

data/.coveralls.yml

@@ -1,2 +1,2 @@
-service_name: travis-pro
+service_name: travis-ci
 repo_token: 9tG18WMwC1xpPdDBEGznon01RCp9NusHL

data/README.md

@@ -15,8 +15,11 @@ A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint
       http_method           put                                  # default: post
       serializer            json                                 # default: form
       rate_limit_msec       100                                  # default: 0 = no rate limiting
+      open_timeout          5                                    # default: nil = no timeout
+      read_timeout          10                                   # default: 60
       raise_on_error        false                                # default: true
       raise_on_http_failure true                                 # default: false
+      ignore_http_status_code 300,400..499                        # default: nil # do not raise on these http_hstatus codes
       authentication        basic                                # default: none
       username              alice                                # default: ''
       password              bobpop                               # default: '', secret: true

data/fluent-plugin-out-http-ext.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-out-http-ext"
-  gem.version       = "0.1.6"
+  gem.version       = "0.1.7"
   gem.authors       = ["Toshiya Kawasaki"]
   gem.email         = ["kawasakitoshiya@gmail.com"]
   gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL and Header option}

data/lib/fluent/plugin/out_http_ext.rb

@@ -1,3 +1,12 @@
+require 'set'
+
+class Array
+  def to_set
+    Set.new(self)
+  end
+end
+
+
 class Hash
   """
   each traverse in hash
@@ -19,6 +28,39 @@ class Hash
 
 end
 
+class StatusCodeParser
+  """
+  parse status code string to array of codes
+  """
+  def self.range?(str)
+     # i.e. 200..399 => return true
+    return /^\d{3}..\d{3}$/ =~ str ? true : false
+  end
+
+  def self.number?(str)
+    return /^\d{3}$/ =~ str ? true : false
+  end
+
+  def self.get_array(str)
+    if self.range?(str)
+      ends = str.split('..').map{|d| Integer(d)}
+      return (ends[0]..ends[1]).to_a
+    elsif self.number?(str)
+      return [str.to_i]
+    else
+      raise "invalid status code range format"
+    end
+  end
+
+  def self.convert(range_str)
+    elems = range_str.split(',')
+    status_codes = elems.flat_map do |elem|
+      self.get_array(elem)
+    end
+    return status_codes.to_set
+  end
+end
+
 class Fluent::HTTPOutput < Fluent::Output
   Fluent::Plugin.register_output('http_ext', self)
 
@@ -27,6 +69,7 @@ class Fluent::HTTPOutput < Fluent::Output
     require 'net/http'
     require 'uri'
     require 'yajl'
+    require 'set'
   end
 
   # Endpoint URL ex. localhost.local/api/
@@ -41,6 +84,9 @@ class Fluent::HTTPOutput < Fluent::Output
   # true | false
   config_param :use_ssl, :bool, :default => false
 
+  config_param :open_timeout, :integer, :default => nil
+  config_param :read_timeout, :integer, :default => 60
+
   # Simple rate limiting: ignore any records within `rate_limit_msec`
   # since the last one.
   config_param :rate_limit_msec, :integer, :default => 0
@@ -50,8 +96,7 @@ class Fluent::HTTPOutput < Fluent::Output
 
   # Raise errors when HTTP response code was not successful.
   config_param :raise_on_http_failure, :bool, :default => false
-
-
+  config_param :ignore_http_status_code, :string, :default => nil
   # nil | 'none' | 'basic'
   config_param :authentication, :string, :default => nil
   config_param :username, :string, :default => ''
@@ -74,6 +119,12 @@ class Fluent::HTTPOutput < Fluent::Output
                     :post
                   end
 
+    @ignore_http_status_code = if @ignore_http_status_code.nil?
+                          [].to_set
+                        else
+                          StatusCodeParser.convert(@ignore_http_status_code)
+                        end
+
     @auth = case @authentication
             when 'basic' then :basic
             else
@@ -158,7 +209,11 @@ class Fluent::HTTPOutput < Fluent::Output
         client.use_ssl = true
         client.ca_file = OpenSSL::X509::DEFAULT_CERT_FILE
       end
-      res = client.start {|http| http.request(req) }
+      res = client.start {|http|
+        http.open_timeout = @open_timeout
+        http.read_timeout = @read_timeout
+        http.request(req)
+      }
     rescue => e # rescue all StandardErrors
       # server didn't respond
       $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
@@ -172,7 +227,14 @@ class Fluent::HTTPOutput < Fluent::Output
                         end
           warning = "failed to #{req.method} #{uri} (#{res_summary})"
           $log.warn warning
-          raise warning if @raise_on_http_failure
+          if @raise_on_http_failure
+            unless @ignore_http_status_code.include?(res.code.to_i)
+              raise warning
+            else
+              $log.debug "ignore http status code #{req.method}"
+            end
+          end
+
        end #end unless
     end # end begin
   end # end send_request

data/test/plugin/test_out_http_ext.rb

@@ -59,6 +59,23 @@ class HTTPOutputTestBase < Test::Unit::TestCase
           res.status = 200
           res.body = 'running'
         }
+        srv.mount_proc('/slow_5') { |req,res|
+          sleep 5
+          res.status = 200
+          res.body = 'slow_5'
+        }
+        srv.mount_proc('/slow_10') { |req,res|
+          sleep 10
+          res.status = 200
+          res.body = 'slow_10'
+        }
+        srv.mount_proc('/status_code') { |req,res|
+          r = Yajl.load(req.body)
+          code = r["code"]
+          res.status = code.to_s
+          res.body = ''
+        }
+
         srv.start
       ensure
         srv.shutdown
@@ -169,6 +186,38 @@ class HTTPOutputTest < HTTPOutputTestBase
     rate_limit_msec #{RATE_LIMIT_MSEC}
   ]
 
+  CONFIG_NOT_READ_TIMEOUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/slow_5/
+    read_timeout 7
+  ]
+  CONFIG_READ_TIMEOUT = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/slow_10/
+    read_timeout 7
+  ]
+  CONFIG_IGNORE_NONE = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+  ]
+  CONFIG_IGNORE_409 = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 409
+  ]
+  CONFIG_IGNORE_4XX = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 400..499
+  ]
+  CONFIG_IGNORE_4XX_5XX = %[
+    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/status_code/
+    serializer json
+    raise_on_http_failure true
+    ignore_http_status_code 400..599
+  ]
+
   def create_driver(conf=CONFIG, tag='test.metrics')
     Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
   end
@@ -302,6 +351,93 @@ class HTTPOutputTest < HTTPOutputTestBase
     assert_equal 2, @posts.size
   end
 
+  def test_read_timeout
+    d = create_driver CONFIG_READ_TIMEOUT
+    assert_equal 7, d.instance.read_timeout
+    err = Net.const_defined?(:ReadTimeout) ? Net::ReadTimeout : Timeout::Error
+    assert_raise err do
+      d.emit({})
+      d.run
+    end
+  end
+
+  def test_not_read_timeout
+    d = create_driver CONFIG_NOT_READ_TIMEOUT
+    assert_equal 7, d.instance.read_timeout
+    assert_nothing_raised do
+      d.emit({})
+      d.run
+    end
+  end
+
+  def test_ignore_none
+    d = create_driver CONFIG_IGNORE_NONE
+    assert_equal [].to_set, d.instance.ignore_http_status_code
+
+    assert_raise do
+      d.emit({:code=> 409})
+      d.run
+    end
+
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_409
+    d = create_driver CONFIG_IGNORE_409
+    assert_equal [409].to_set, d.instance.ignore_http_status_code
+
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_4XX
+    d = create_driver CONFIG_IGNORE_4XX
+    assert_equal (400..499).to_a.to_set, d.instance.ignore_http_status_code
+
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_raise do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
+  def test_ignore_4XX_5XX
+    d = create_driver CONFIG_IGNORE_4XX_5XX
+    assert_equal (400..599).to_a.to_set, d.instance.ignore_http_status_code
+    assert_nothing_raised do
+      d.emit({:code => 409})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 404})
+      d.run
+    end
+    assert_nothing_raised do
+      d.emit({:code => 500})
+      d.run
+    end
+  end
+
   def _current_msec
     Time.now.to_f * 1000
   end
@@ -339,4 +475,37 @@ class HTTPOutputTest < HTTPOutputTestBase
     assert_equal 2, @prohibited
   end
 
+  def test_status_code_parser()
+    assert_equal (400..409).to_a.to_set, StatusCodeParser.convert("400..409")
+    assert_equal ((400..409).to_a + [300]).to_set, StatusCodeParser.convert("400..409,300")
+    assert_equal ((400..409).to_a + [300]).to_set, StatusCodeParser.convert("300,400..409")
+    assert_equal [404, 409].to_set, StatusCodeParser.convert("404,409")
+    assert_equal [404, 409, 300, 301, 302, 303].to_set, StatusCodeParser.convert("404,409,300..303")
+    assert_equal [409].to_set, StatusCodeParser.convert("409")
+    assert_equal [].to_set, StatusCodeParser.convert("")
+    assert_raise do
+       StatusCodeParser.convert("400...499")
+    end
+    assert_raise do
+      StatusCodeParser.convert("10..20")
+    end
+    assert_raise do
+      StatusCodeParser.convert("4XX")
+    end
+    assert_raise do
+      StatusCodeParser.convert("4XX..5XX")
+    end
+    assert_raise do
+      StatusCodeParser.convert("200.0..400")
+    end
+    assert_raise do
+      StatusCodeParser.convert("-200..400")
+    end
+
+  end
+
+  def test_array_extend
+    assert_equal [].to_set, Set.new([])
+    assert_equal [1, 2].to_set, Set.new([1, 2])
+  end
 end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-http-ext
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
+  prerelease: 
 platform: ruby
 authors:
 - Toshiya Kawasaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-16 00:00:00.000000000 Z
+date: 2016-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,8 +30,9 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.0
     - - <
@@ -37,8 +41,9 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.10.0
     - - <
@@ -47,57 +52,65 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: A generic Fluentd output plugin to send logs to an HTTP endpoint with
@@ -124,26 +137,27 @@ files:
 homepage: 
 licenses:
 - Apache-2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: A generic Fluentd output plugin to send logs to an HTTP endpoint with SSL
   and Header option
 test_files:

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 3e9c08d1a809ddd3df73b5ab13fe55ac7ed79305
-  data.tar.gz: 2af17adfcbb3375ab1079dfcc56d51d2a64d0b06
-SHA512:
-  metadata.gz: 4c445f8eb6d35bf52705eeb683ff48f15d96b5eadc8625736ce98cd2b52d9f207a4e08ba6abd2ac8306b025d0626eae55e3204ab3806638fdd4468c19b7699d4
-  data.tar.gz: 66977734544084539b681b6690e82fef9cb595badfb57cd509f97f1cdf196afd3e314e0697bda3a25175f119bdb78b278237ee8ee1c7d2e3e515bb50d99f7bb6