fluent-plugin-http-pull 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3de0f90b59370bd0d780479ad4326cb3f1cb7a4d
-  data.tar.gz: 5bea27b0f9b1c3fb5e3a593b40c9366f4a651e2a
+  metadata.gz: 533830e127ee4cbc297f6dcfd1c4a3a08882ca9d
+  data.tar.gz: a689af041c24cc88e64fd61b14c54b42dbfe6450
 SHA512:
-  metadata.gz: f832e52834daa32d1be42df2be0cd836526dadf149a5c9973cc44eb3225b16217ca1926037a2553fdd024450e0eed748c9c02fecb94e1e5f901cc7ab42c031fa
-  data.tar.gz: dca6fe53b789b3b6ed8eb277af015461b0216d2804bf9720244dccfbd8cb817386e46ee520f40c63af7f8dfe192743bbd744f623167e7591000c5a5e2d804f55
+  metadata.gz: a813fd4dbee8607a9188676e0fa01f477aee8747064f848767a06949825973a09c1f233d767dd410911d6471c0d73714d586cf6cc2d0d518a9e63adb6342de5d
+  data.tar.gz: 616f089b1f0ebacc78fe871c21f4ee9131c6746e1bc88a905dee489f835f8cb5ffac409863b0f930b0235099926a9b115b8def19deb8c8a78585b3d6cf44f002

data/.gitignore

@@ -4,3 +4,4 @@ TODO
 Gemfile.lock
 coverage/**/*
 *.gem
+*.log

data/README.md

@@ -139,6 +139,18 @@ If status_only is true, body is not parsed.
 
 The timeout of each request.
 
+### proxy (string) (optional, default: nil)
+
+The HTTP proxy URL to use for each requests
+
+### user (string) (optional, default: nil)
+
+The user for basic auth
+
+### password (string) (optional, default: nil)
+
+The password for basic auth
+
 ## In case of remote error
 
 ### Can receive response from remote

data/Rakefile

@@ -1,13 +1,24 @@
-require "bundler"
-Bundler::GemHelper.install_tasks
-
-require "rake/testtask"
-
-Rake::TestTask.new(:test) do |t|
-  t.libs.push("lib", "test")
-  t.test_files = FileList["test/**/test_*.rb"]
-  t.verbose = true
-  t.warning = true
-end
-
-task default: [:test]
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+require "fileutils"
+
+Rake::TestTask.new(:test) do |t|
+  if File.exists? "stub_server.log"
+    puts "clear stub_server.log"
+    FileUtils.rm "stub_server.log"
+  end
+
+  if File.exists? "stub_proxy.log"
+    puts "clear stub_proxy.log"
+    FileUtils.rm "stub_proxy.log"
+  end
+
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-http-pull.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-http-pull"
-  spec.version = "0.3.0"
+  spec.version = "0.4.0"
   spec.authors = ["filepang"]
   spec.email   = ["filepang@gmail.com"]
 
@@ -20,6 +20,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 2.1'
+
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "simplecov", "~> 0.7"
   spec.add_development_dependency "test-unit", "~> 3.0"

data/lib/fluent/plugin/in_http_pull.rb

@@ -35,8 +35,15 @@ module Fluent
       config_param :interval, :time
       desc 'status_only'
       config_param :status_only, :bool, default: false
-      desc 'timeout second of each request'
+      desc 'The timeout stime of each request'
       config_param :timeout, :time, default: 10
+      desc 'The HTTP proxy URL to use for each requests'
+      config_param :proxy, :string, default: nil
+
+      desc 'user of basic auth'
+      config_param :user, :string, default: nil
+      desc 'password of basic auth'
+      config_param :password, :string, default: nil
 
       def configure(conf)
         compat_parameters_convert(conf, :parser)
@@ -55,9 +62,13 @@ module Fluent
         record = { "url" => @url }
 
         begin
-          res = RestClient::Request.execute(method: :get,
-                                            url: @url,
-                                            timeout: @timeout)
+          request_options = { method: :get, url: @url, timeout: @timeout }
+
+          request_options[:proxy] = @proxy if @proxy
+          request_options[:user] = @user if @user
+          request_options[:password] = @password if @password
+
+          res = RestClient::Request.execute request_options
           record["status"] = res.code
           record["body"] = res.body
         rescue StandardError => err

data/test/helper.rb

@@ -21,5 +21,9 @@ require "fluent/test/helpers"
 
 require "test/unit/rr"
 
+# require stub_server
+require "test/helper/stub_server"
+require "test/helper/stub_proxy"
+
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/helper/stub_proxy.rb

@@ -0,0 +1,39 @@
+require 'webrick'
+require 'webrick/httpproxy'
+
+class StubProxy
+  def initialize
+    create_proxy
+  end
+
+  def start
+    @thread = Thread.new { @proxy.start }
+  end
+
+  def shutdown
+    @proxy.shutdown
+
+    # wait until webrick was shutting down
+    while true
+      break if @thread.status == false
+
+      # issue webrick shutdown once more
+      @proxy.shutdown
+      sleep 1
+    end
+
+    # then exit thread
+    @thread.exit
+  end
+
+  private
+  def create_proxy
+    @log_file = File.open("stub_proxy.log", "a+")
+    @log = WEBrick::Log.new @log_file
+    @access_log = [
+      [@log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT],
+    ]
+
+    @proxy = WEBrick::HTTPProxyServer.new :Port => 4040, :Logger => @log, :AccessLog => @access_log
+  end
+end

data/test/helper/stub_server.rb

@@ -0,0 +1,84 @@
+require 'webrick'
+
+class StubServer
+  def initialize
+    create_server
+
+    # mount handler
+    @server.mount_proc '/', &method(:ok)
+    @server.mount_proc '/not_exist', &method(:not_exist)
+    @server.mount_proc '/timeout', &method(:timeout)
+    @server.mount_proc '/internal_error', &method(:internal_error)
+    @server.mount_proc '/redirect', &method(:redirect)
+    @server.mount_proc '/protected', &method(:protected)
+  end
+
+  def start
+    @thread = Thread.new { @server.start }
+  end
+
+  def shutdown
+    @server.shutdown
+
+    # wait until webrick was shutting down
+    while true
+      break if @thread.status == false
+
+      # issue webrick shutdown once more
+      @server.shutdown
+      sleep 1
+    end
+
+    # then exit thread
+    @thread.exit
+  end
+
+  private
+  def create_server
+    @log_file = File.open("stub_server.log", "a+")
+    @log = WEBrick::Log.new @log_file
+    @access_log = [
+      [@log_file, WEBrick::AccessLog::COMBINED_LOG_FORMAT],
+    ]
+
+    @server = WEBrick::HTTPServer.new :Port => 3939, :Logger => @log, :AccessLog => @access_log
+  end
+
+  def ok(req, res)
+    res.status = 200
+    res['Content-Type'] = 'application/json'
+    res.body = '{ "status": "OK" }'
+  end
+
+  def not_exist(req, res)
+    res.status = 404
+    res.body = ''
+  end
+
+  def timeout(req, res)
+    sleep 3
+
+    res.status = 200
+    res['Content-Type'] = 'application/json'
+    res.body = '{ "status": "OK" }'
+  end
+
+  def internal_error(req, res)
+    res.status = 500
+    res.body = ''
+  end
+
+  def redirect(req, res)
+    res.set_redirect WEBrick::HTTPStatus::TemporaryRedirect, "http://127.0.0.1:3939/"
+  end
+
+  def protected(req, res)
+    WEBrick::HTTPAuth.basic_auth(req, res, 'protected') do |user, password|
+      user == 'HatsuneMiku' && password == '3939'
+    end
+
+    res.status = 200
+    res['Content-Type'] = 'application/json'
+    res.body = '{ "status": "OK" }'
+  end
+end

data/test/plugin/test_in_http_pull.rb

@@ -4,14 +4,21 @@ require "fluent/plugin/in_http_pull.rb"
 require 'ostruct'
 
 class HttpPullInputTest < Test::Unit::TestCase
+  @stub_server = nil
+
   setup do
-    Fluent::Test.setup
+    @stub_server = StubServer.new
+    @stub_server.start
+  end
+
+  teardown do
+    @stub_server.shutdown
   end
 
   sub_test_case "default value of each options" do
     TEST_DEFAULT_VALUE_CONFIG = %[
       tag test
-      url http://127.0.0.1
+      url http://127.0.0.1:3939
 
       interval 3s
       format json
@@ -30,13 +37,33 @@ class HttpPullInputTest < Test::Unit::TestCase
 
       assert_equal(10, d.instance.timeout)
     end
+
+    test 'user' do
+      d = create_driver TEST_DEFAULT_VALUE_CONFIG
+      assert_equal("test", d.instance.tag)
+
+      assert_equal(nil, d.instance.user)
+    end
+
+    test 'password' do
+      d = create_driver TEST_DEFAULT_VALUE_CONFIG
+      assert_equal("test", d.instance.tag)
+
+      assert_equal(nil, d.instance.password)
+    end
+
+    test 'proxy' do
+      d = create_driver TEST_DEFAULT_VALUE_CONFIG
+      assert_equal("test", d.instance.tag)
+
+      assert_equal(nil, d.instance.proxy)
+    end
   end
 
-  sub_test_case "success case with status only" do
+  sub_test_case "success case" do
     TEST_INTERVAL_3_CONFIG = %[
       tag test
-      url http://127.0.0.1
-      timeout 10
+      url http://127.0.0.1:3939
 
       interval 3s
       format none
@@ -45,29 +72,26 @@ class HttpPullInputTest < Test::Unit::TestCase
 
     TEST_INTERVAL_5_CONFIG = %[
       tag test
-      url http://127.0.0.1
-      timeout 10
+      url http://127.0.0.1:3939
 
       interval 5s
       format json
     ]
 
-    setup do
-      mock(RestClient::Request).
-        execute(method: :get,
-                url: "http://127.0.0.1",
-                timeout: 10).
-        times(2) do
-          OpenStruct.new({code: 200, body: '{"status": "OK"}'})
-        end
-    end
+    TEST_INTERVAL_3_REDIRECT_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/redirect
+
+      interval 3s
+      format json
+    ]
 
     test 'interval 3 with status_only' do
       d = create_driver TEST_INTERVAL_3_CONFIG
       assert_equal("test", d.instance.tag)
       assert_equal(3, d.instance.interval)
 
-      d.run(timeout: 5) do
+      d.run(timeout: 8) do
         sleep 7
       end
       assert_equal(2, d.events.size)
@@ -75,7 +99,7 @@ class HttpPullInputTest < Test::Unit::TestCase
       d.events.each do |tag, time, record|
         assert_equal("test", tag)
 
-        assert_equal({"url"=>"http://127.0.0.1","status"=>200}, record)
+        assert_equal({"url"=>"http://127.0.0.1:3939","status"=>200}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
     end
@@ -85,7 +109,7 @@ class HttpPullInputTest < Test::Unit::TestCase
       assert_equal("test", d.instance.tag)
       assert_equal(5, d.instance.interval)
 
-      d.run(timeout: 7) do
+      d.run(timeout: 12) do
         sleep 11
       end
       assert_equal(2, d.events.size)
@@ -93,12 +117,92 @@ class HttpPullInputTest < Test::Unit::TestCase
       d.events.each do |tag, time, record|
         assert_equal("test", tag)
 
-        assert_equal({"url"=>"http://127.0.0.1","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert_equal({"url"=>"http://127.0.0.1:3939","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'interval 3 with redirect' do
+      d = create_driver TEST_INTERVAL_3_REDIRECT_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal({"url"=>"http://127.0.0.1:3939/redirect","status"=>200, "message"=>{"status"=>"OK"}}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
     end
   end
 
+  sub_test_case "fail when not 200 OK" do
+    TEST_404_INTERVAL_3_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/not_exist
+
+      interval 3s
+      format none
+      status_only true
+    ]
+
+    TEST_500_INTERVAL_3_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/internal_error
+
+      interval 3s
+      format none
+      status_only true
+    ]
+
+    test '404' do
+      d = create_driver TEST_404_INTERVAL_3_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal("http://127.0.0.1:3939/not_exist", record["url"])
+        assert(time.is_a?(Fluent::EventTime))
+
+        assert_equal(404, record["status"])
+        assert_not_nil(record["error"])
+      end
+    end
+
+    test '500' do
+      d = create_driver TEST_500_INTERVAL_3_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal("http://127.0.0.1:3939/internal_error", record["url"])
+        assert(time.is_a?(Fluent::EventTime))
+
+        assert_equal(500, record["status"])
+        assert_not_nil(record["error"])
+      end
+    end
+  end
+
   sub_test_case "fail when remote down" do
     TEST_REFUSED_CONFIG = %[
       tag test
@@ -111,7 +215,7 @@ class HttpPullInputTest < Test::Unit::TestCase
       d = create_driver TEST_REFUSED_CONFIG
       assert_equal("test", d.instance.tag)
 
-      d.run(timeout: 2) do
+      d.run(timeout: 4) do
         sleep 3
       end
 
@@ -131,30 +235,19 @@ class HttpPullInputTest < Test::Unit::TestCase
   sub_test_case "fail when remote timeout" do
     TEST_TIMEOUT_FAIL_CONFIG = %[
       tag test
-      url http://127.0.0.1
+      url http://127.0.0.1:3939/timeout
       timeout 2s
 
       interval 3s
       format json
     ]
 
-    setup do
-      mock(RestClient::Request).
-        execute(method: :get,
-                url: "http://127.0.0.1",
-                timeout: 2).
-        times(2) do
-          sleep 2
-          raise RestClient::Exceptions::Timeout.new
-        end
-    end
-
     test "timeout" do
       d = create_driver TEST_TIMEOUT_FAIL_CONFIG
       assert_equal("test", d.instance.tag)
       assert_equal(2, d.instance.timeout)
 
-      d.run(timeout: 5) do
+      d.run(timeout: 8) do
         sleep 7
       end
       assert_equal(2, d.events.size)
@@ -162,7 +255,7 @@ class HttpPullInputTest < Test::Unit::TestCase
       d.events.each do |tag, time, record|
         assert_equal("test", tag)
 
-        assert_equal("http://127.0.0.1", record["url"])
+        assert_equal("http://127.0.0.1:3939/timeout", record["url"])
         assert(time.is_a?(Fluent::EventTime))
 
         assert_equal(0, record["status"])
@@ -171,6 +264,195 @@ class HttpPullInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "remote is prtected by basic auth" do
+    TEST_AUTH_SUCCESS_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/protected
+      timeout 2s
+      user HatsuneMiku
+      password 3939
+
+      interval 3s
+      format json
+    ]
+
+    TEST_AUTH_FAIL_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/protected
+      timeout 2s
+      user HatsuneMiku
+      password wrong_password
+
+      interval 3s
+      format json
+    ]
+
+    TEST_AUTH_FAIL_NOT_GIVEN_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/protected
+      timeout 2s
+
+      interval 3s
+      format json
+    ]
+
+    test 'interval 3 with corrent password' do
+      d = create_driver TEST_AUTH_SUCCESS_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal({"url"=>"http://127.0.0.1:3939/protected","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'interval 3 with wrong password' do
+      d = create_driver TEST_AUTH_FAIL_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal("http://127.0.0.1:3939/protected", record["url"])
+        assert(time.is_a?(Fluent::EventTime))
+
+        assert_equal(401, record["status"])
+        assert_not_nil(record["error"])
+      end
+    end
+
+    test 'interval 3 without auth info' do
+      d = create_driver TEST_AUTH_FAIL_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal("http://127.0.0.1:3939/protected", record["url"])
+        assert(time.is_a?(Fluent::EventTime))
+
+        assert_equal(401, record["status"])
+        assert_not_nil(record["error"])
+      end
+    end
+  end
+
+  sub_test_case "success case behind proxy" do
+    TEST_INTERVAL_3_PROXY_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939
+      proxy http://127.0.0.1:4040
+
+      interval 3s
+      format none
+      status_only true
+    ]
+
+    TEST_INTERVAL_3_REDIRECT_PROXY_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/redirect
+      proxy http://127.0.0.1:4040
+
+      interval 3s
+      format json
+    ]
+
+    TEST_AUTH_SUCCESS_PROXY_CONFIG = %[
+      tag test
+      url http://127.0.0.1:3939/protected
+      proxy http://127.0.0.1:4040
+      timeout 2s
+      user HatsuneMiku
+      password 3939
+
+      interval 3s
+      format json
+    ]
+
+    setup do
+      @proxy_server = StubProxy.new
+      @proxy_server.start
+    end
+
+    teardown do
+      @proxy_server.shutdown
+    end
+
+    test 'interval 3 with status_only' do
+      d = create_driver TEST_INTERVAL_3_PROXY_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal({"url"=>"http://127.0.0.1:3939","status"=>200}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'interval 3 with redirect' do
+      d = create_driver TEST_INTERVAL_3_REDIRECT_PROXY_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal({"url"=>"http://127.0.0.1:3939/redirect","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'interval 3 with corrent password' do
+      d = create_driver TEST_AUTH_SUCCESS_PROXY_CONFIG
+      assert_equal("test", d.instance.tag)
+      assert_equal(3, d.instance.interval)
+
+      d.run(timeout: 8) do
+        sleep 7
+      end
+      assert_equal(2, d.events.size)
+
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+
+        assert_equal({"url"=>"http://127.0.0.1:3939/protected","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+  end
+
   private
 
   def create_driver(conf)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http-pull
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - filepang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-30 00:00:00.000000000 Z
+date: 2017-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -144,6 +144,8 @@ files:
 - fluent-plugin-http-pull.gemspec
 - lib/fluent/plugin/in_http_pull.rb
 - test/helper.rb
+- test/helper/stub_proxy.rb
+- test/helper/stub_server.rb
 - test/plugin/test_in_http_pull.rb
 homepage: https://github.com/HatsuneMiku3939/fluent-plugin-http-pull
 licenses:
@@ -157,7 +159,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -171,4 +173,6 @@ specification_version: 4
 summary: fluent-plugin-http-pull
 test_files:
 - test/helper.rb
+- test/helper/stub_proxy.rb
+- test/helper/stub_server.rb
 - test/plugin/test_in_http_pull.rb