fluent-plugin-out-http 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0fceb91f15222821701aa3d08f07da9de768fb31
-  data.tar.gz: 03d04b3948f1e113af96f57049c50c3546e1e1dc
+  metadata.gz: 12f3735f1d540b81b847e6fadf5bdca62cf963bf
+  data.tar.gz: 75df66f76592807c49682def91dd3a88374e13f7
 SHA512:
-  metadata.gz: aa98fe26f647b56c21d5fc9981db3f0c82e79643759f36dead4e561ecd280b92644e642c241dac2f214065720191b76ee9240d3f596330669adc13e1bcef4d12
-  data.tar.gz: bfe8b534adcad17bdab5502bc1e9963c4dc5f1307dfd48d39e4cf1bb313bdb779cc50ad100263f5ff5c3ce04026aedda93557a8ddf51d844a4586fe567d67e24
+  metadata.gz: 9bee4d8fac451d23d10f84fc2e1458b848d44fb70a8a3521b5a8b23f18c8101b480316c1bafcf5783091fc5e659ed915b74cd0da6cdc30e693bfdfae72d7f3dc
+  data.tar.gz: eb73a64c59eb8c690bc9d5750c2f22e3e67356a26ebf30bdf67593d4df99f2fda2f4dc355b67624a5c1f0fbd737bcfa6ede122962da00277c4baaf90990b003e

data/.travis.yml

@@ -1,15 +1,16 @@
 rvm:
-  - 1.9.3
-  - 2.0.0
   - 2.1
   - 2.2
+  - 2.3
+  - 2.4.0
   - ruby-head
-  - rbx-2
 
 os:
   - linux
   - osx
 
+dist: trusty
+
 gemfile:
   - Gemfile
 
@@ -18,4 +19,3 @@ script: bundle exec rake test
 matrix:
   allow_failures:
     - rvm: ruby-head
-    - rvm: rbx-2

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 0.2.0
+### Added
+* SSL is now supported if `endpoint_url` uses the `https` scheme (uses ruby-2.1 syntax internally)
+* New config: set `ssl_no_verify` to `true` to bypass SSL certificate verification.
+  Use at your own risk.
+### Changed
+* Fixed tests:
+  * Removed some warnings
+  * Fixed failing binary test to use UTF-8
+### Removed
+* Dropped support of Ruby 1.9-2.0
+
 ## 0.1.4
 * #11 Updated Fluentd dependency to:  [">= 0.10.0", "< 2"]
 * #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604)

data/fluent-plugin-out-http.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-out-http"
-  gem.version       = "0.1.4"
+  gem.version       = "0.2.0"
   gem.authors       = ["Marica Odagaki"]
   gem.email         = ["ento.entotto@gmail.com"]
   gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint}
@@ -15,6 +15,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.required_ruby_version  = '>= 2.1.0'
+
   gem.add_runtime_dependency "yajl-ruby", "~> 1.0"
   gem.add_runtime_dependency "fluentd", [">= 0.10.0", "< 2"]
   gem.add_development_dependency "bundler"

data/lib/fluent/plugin/out_http.rb

@@ -8,12 +8,15 @@ class Fluent::HTTPOutput < Fluent::Output
     require 'yajl'
   end
 
-  # Endpoint URL ex. localhost.local/api/
+  # Endpoint URL ex. http://localhost.local/api/
   config_param :endpoint_url, :string
 
+  # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
+  config_param :ssl_no_verify, :bool, :default => false
+
   # HTTP method
   config_param :http_method, :string, :default => :post
-  
+
   # form | json
   config_param :serializer, :string, :default => :form
 
@@ -25,13 +28,19 @@ class Fluent::HTTPOutput < Fluent::Output
   config_param :raise_on_error, :bool, :default => true
 
   # nil | 'none' | 'basic'
-  config_param :authentication, :string, :default => nil 
+  config_param :authentication, :string, :default => nil
   config_param :username, :string, :default => ''
   config_param :password, :string, :default => '', :secret => true
 
   def configure(conf)
     super
 
+    @ssl_verify_mode = if @ssl_no_verify
+                         OpenSSL::SSL::VERIFY_NONE
+                       else
+                         OpenSSL::SSL::VERIFY_PEER
+                       end
+
     serializers = [:json, :form]
     @serializer = if serializers.include? @serializer.intern
                     @serializer.intern
@@ -51,6 +60,8 @@ class Fluent::HTTPOutput < Fluent::Output
             else
               :none
             end
+
+    @last_request_time = nil
   end
 
   def start
@@ -92,13 +103,21 @@ class Fluent::HTTPOutput < Fluent::Output
     return req, uri
   end
 
-  def send_request(req, uri)    
+  def http_opts(uri)
+      opts = {
+        :use_ssl => uri.scheme == 'https'
+      }
+      opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
+      opts
+  end
+
+  def send_request(req, uri)
     is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
     if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
       $log.info('Dropped request due to rate limiting')
       return
     end
-    
+
     res = nil
 
     begin
@@ -106,7 +125,7 @@ class Fluent::HTTPOutput < Fluent::Output
         req.basic_auth(@username, @password)
       end
       @last_request_time = Time.now.to_f
-      res = Net::HTTP.new(uri.host, uri.port).start {|http| http.request(req) }
+      res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
     rescue => e # rescue all StandardErrors
       # server didn't respond
       $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"

data/lib/fluent/test/http_output_test.rb

@@ -30,6 +30,7 @@ class Test::Unit::TestCase
 end
 
 require 'webrick'
+require 'webrick/https'
 
 # to handle POST/PUT/DELETE ...
 module WEBrick::HTTPServlet
@@ -39,16 +40,3 @@ module WEBrick::HTTPServlet
     alias do_DELETE do_GET
   end
 end
-
-def get_code(server, port, path, headers={})
-  require 'net/http' 
-  Net::HTTP.start(server, port){|http|
-    http.get(path, headers).code
-  } 
-end 
-def get_content(server, port, path, headers={})
-  require 'net/http'
-  Net::HTTP.start(server, port){|http|
-    http.get(path, headers).body
-  } 
-end

data/test/plugin/test_out_http.rb

@@ -1,14 +1,31 @@
 # -*- coding: utf-8 -*-
+require 'net/http'
 require 'uri'
 require 'yajl'
 require 'fluent/test/http_output_test'
 require 'fluent/plugin/out_http'
 
 
-TEST_LISTEN_PORT = 5126
+class HTTPOutputTestBase < Test::Unit::TestCase
+  def self.port
+    5126
+  end
 
+  def self.server_config
+    config = {BindAddress: '127.0.0.1', Port: port}
+    if ENV['VERBOSE']
+      logger = WEBrick::Log.new(STDOUT, WEBrick::BasicLog::DEBUG)
+      config[:Logger] = logger
+      config[:AccessLog] = []
+    end
+    config
+  end
+
+  def self.test_http_client(**opts)
+    opts = opts.merge(open_timeout: 1, read_timeout: 1)
+    Net::HTTP.start('127.0.0.1', port, **opts)
+  end
 
-class HTTPOutputTestBase < Test::Unit::TestCase
   # setup / teardown for servers
   def setup
     Fluent::Test.setup
@@ -18,12 +35,7 @@ class HTTPOutputTestBase < Test::Unit::TestCase
     @requests = 0
     @auth = false
     @dummy_server_thread = Thread.new do
-      srv = if ENV['VERBOSE']
-              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT})
-            else
-              logger = WEBrick::Log.new('/dev/null', WEBrick::BasicLog::DEBUG)
-              WEBrick::HTTPServer.new({:BindAddress => '127.0.0.1', :Port => TEST_LISTEN_PORT, :Logger => logger, :AccessLog => []})
-            end
+      srv = WEBrick::HTTPServer.new(self.class.server_config)
       begin
         allowed_methods = %w(POST PUT)
         srv.mount_proc('/api/') { |req,res|
@@ -71,7 +83,8 @@ class HTTPOutputTestBase < Test::Unit::TestCase
       connected = false
       while not connected
         begin
-          get_content('localhost', TEST_LISTEN_PORT, '/')
+          client = self.class.test_http_client
+          client.request_get('/')
           connected = true
         rescue Errno::ECONNREFUSED
           sleep 0.1
@@ -81,20 +94,19 @@ class HTTPOutputTestBase < Test::Unit::TestCase
         end
       end
       cv.signal
-    } 
+    }
     mutex = Mutex.new
     mutex.synchronize {
       cv.wait(mutex)
-    } 
+    }
   end
 
   def test_dummy_server
-    host = '127.0.0.1'
-    port = TEST_LISTEN_PORT
-    client = Net::HTTP.start(host, port)
+    client = self.class.test_http_client
+    post_header = { 'Content-Type' => 'application/x-www-form-urlencoded' }
 
     assert_equal '200', client.request_get('/').code
-    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge').code
+    assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge', post_header).code
 
     assert_equal 1, @posts.size
 
@@ -104,11 +116,11 @@ class HTTPOutputTestBase < Test::Unit::TestCase
 
     @auth = true
 
-    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge').code
+    assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge', post_header).code
 
     req_with_auth = lambda do |number, mode, user, pass|
-      url = URI.parse("http://#{host}:#{port}/api/service/metrics/pos")
-      req = Net::HTTP::Post.new(url.path)
+      req = Net::HTTP::Post.new("/api/service/metrics/pos")
+      req.content_type = 'application/x-www-form-urlencoded'
       req.basic_auth user, pass
       req.set_form_data({'number'=>number, 'mode'=>mode})
       req
@@ -130,55 +142,55 @@ class HTTPOutputTestBase < Test::Unit::TestCase
     @dummy_server_thread.kill
     @dummy_server_thread.join
   end
+
+  def create_driver(conf, tag='test.metrics')
+    Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
+  end
 end
 
 class HTTPOutputTest < HTTPOutputTestBase
   CONFIG = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    endpoint_url http://127.0.0.1:#{port}/api/
   ]
 
   CONFIG_JSON = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    endpoint_url http://127.0.0.1:#{port}/api/
     serializer json
   ]
 
   CONFIG_PUT = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    endpoint_url http://127.0.0.1:#{port}/api/
     http_method put
   ]
 
   CONFIG_HTTP_ERROR = %[
-    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+    endpoint_url https://127.0.0.1:#{port - 1}/api/
   ]
 
   CONFIG_HTTP_ERROR_SUPPRESSED = %[
-    endpoint_url https://127.0.0.1:#{TEST_LISTEN_PORT + 1}/api/
+    endpoint_url https://127.0.0.1:#{port - 1}/api/
     raise_on_error false
   ]
 
   RATE_LIMIT_MSEC = 1200
 
   CONFIG_RATE_LIMIT = %[
-    endpoint_url http://127.0.0.1:#{TEST_LISTEN_PORT}/api/
+    endpoint_url http://127.0.0.1:#{port}/api/
     rate_limit_msec #{RATE_LIMIT_MSEC}
   ]
 
-  def create_driver(conf=CONFIG, tag='test.metrics')
-    Fluent::Test::OutputTestDriver.new(Fluent::HTTPOutput, tag).configure(conf)
-  end
-
   def test_configure
-    d = create_driver
-    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    d = create_driver CONFIG
+    assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url
     assert_equal :form, d.instance.serializer
 
     d = create_driver CONFIG_JSON
-    assert_equal "http://127.0.0.1:#{TEST_LISTEN_PORT}/api/", d.instance.endpoint_url
+    assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url
     assert_equal :json, d.instance.serializer
   end
 
   def test_emit_form
-    d = create_driver
+    d = create_driver CONFIG
     d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") })
     d.run
 
@@ -218,7 +230,7 @@ class HTTPOutputTest < HTTPOutputTestBase
   end
 
   def test_emit_json
-    binary_string = "\xe3\x81\x82".force_encoding("ascii-8bit")
+    binary_string = "\xe3\x81\x82"
     d = create_driver CONFIG_JSON
     d.emit({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
     d.run
@@ -268,7 +280,7 @@ class HTTPOutputTest < HTTPOutputTestBase
 
     wait_msec = 500
     sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001
-    
+
     assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval"
     d.emit(record)
     d.run
@@ -278,7 +290,7 @@ class HTTPOutputTest < HTTPOutputTestBase
   def _current_msec
     Time.now.to_f * 1000
   end
-  
+
   def test_auth
     @auth = true # enable authentication of dummy server
 
@@ -313,3 +325,61 @@ class HTTPOutputTest < HTTPOutputTestBase
   end
 
 end
+
+class HTTPSOutputTest < HTTPOutputTestBase
+  def self.port
+    5127
+  end
+
+  def self.server_config
+    config = super
+    config[:SSLEnable] = true
+    config[:SSLCertName] = [["CN", WEBrick::Utils::getservername]]
+    config
+  end
+
+  def self.test_http_client
+    super(
+      use_ssl: true,
+      verify_mode: OpenSSL::SSL::VERIFY_NONE,
+    )
+  end
+
+  def test_configure
+    test_uri = URI.parse("https://127.0.0.1/")
+
+    ssl_config = %[
+    endpoint_url https://127.0.0.1:#{self.class.port}/api/
+    ]
+    d = create_driver ssl_config
+    expected_endpoint_url = "https://127.0.0.1:#{self.class.port}/api/"
+    assert_equal expected_endpoint_url, d.instance.endpoint_url
+    http_opts = d.instance.http_opts(test_uri)
+    assert_equal true, http_opts[:use_ssl]
+    assert_equal OpenSSL::SSL::VERIFY_PEER, http_opts[:verify_mode]
+
+    no_verify_config = %[
+    endpoint_url https://127.0.0.1:#{self.class.port}/api/
+    ssl_no_verify true
+    ]
+    d = create_driver no_verify_config
+    http_opts = d.instance.http_opts(test_uri)
+    assert_equal true, http_opts[:use_ssl]
+    assert_equal OpenSSL::SSL::VERIFY_NONE, http_opts[:verify_mode]
+  end
+
+  def test_emit_form_ssl
+    config = %[
+    endpoint_url https://127.0.0.1:#{self.class.port}/api/
+    ssl_no_verify true
+    ]
+    d = create_driver config
+    d.emit({ 'field1' => 50 })
+    d.run
+
+    assert_equal 1, @posts.size
+    record = @posts[0]
+
+    assert_equal '50', record[:form]['field1']
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-http
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Marica Odagaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-16 00:00:00.000000000 Z
+date: 2017-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
@@ -116,7 +116,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: A generic Fluentd output plugin to send logs to an HTTP endpoint