fluent-plugin-http_shadow 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b47da229788f20b9d79742ffde107634b120adcd
-  data.tar.gz: 85c3d6ea8c4a5520d2dca87b0b781bbb0cc9052a
+  metadata.gz: 85934dddb2f9ec1e02ec98997f0fb71dff86153e
+  data.tar.gz: 41cc427fce05f732a149e5196f012507fea3d5b3
 SHA512:
-  metadata.gz: 4f623027f369842bb639e94a16eaa4ed8e6fd0eb83db983f30f16cf91c10818505196093daf7dfa483edde3f28f2fdb0c6a27540d8430e9940c162ae7fcd6d21
-  data.tar.gz: 7e88e174de9677945838a62956ce945114de8643c1a2fea3255e097b824f7b986a21782869ac2e4b4b972dae919737ea01d272fa09e9e50a8ef971e1f404ed86
+  metadata.gz: 4fe72dfeaa974d5bcfe7ae08bbad021fa149b24c62440d954a854b6993679aae74fd983ddeca2da1500265107b40c2468ae926b87f8977cfd905d2d285a9b300
+  data.tar.gz: 84ff822909dc2b4cf154a75e8d7b62c3d4a9a97949122cbaecc254b70cd9189503512d1844ba24755071aa1c475080ee54a65e6e87111fe5dfebccbcc8eed6e4

data/fluent-plugin-http_shadow.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-http_shadow"
-  gem.version       = "0.0.1"
+  gem.version       = "0.0.2"
   gem.summary       = %q{copy http request. use shadow proxy server.}
   gem.description   = %q{copy http request. use shadow proxy server.}
   gem.license       = "MIT"
@@ -15,6 +15,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
+  gem.add_runtime_dependency "typhoeus"
+  gem.add_runtime_dependency "addressable"
   gem.add_development_dependency 'bundler', '~> 1.7.2'
   gem.add_development_dependency 'fluentd', '~> 0.10.58'
   gem.add_development_dependency 'pry', '~> 0.10.1'

data/lib/fluent/plugin/out_http_shadow.rb

@@ -1,34 +1,32 @@
-require 'json'
 module Fluent
-  class HttpShadowOutput < Fluent::Output
+  class HttpShadowOutput < Fluent::BufferedOutput
     Fluent::Plugin.register_output('http_shadow', self)
 
     def initialize
       super
       require 'uri'
-      require 'net/http'
       require 'erb'
+      require 'typhoeus'
+      require "addressable/uri"
     end
 
     config_param :host, :string, :default => nil
     config_param :host_key, :string, :default => nil
     config_param :host_hash, :hash, :default => nil
     config_param :path_format, :string
-    config_param :method_key, :string
+    config_param :method_key, :string, :default => nil
     config_param :header_hash, :hash, :default => nil
     config_param :cookie_hash, :hash, :default => nil
     config_param :params_key, :string, :default => nil
+    config_param :max_concurrency, :integer, :default => 10
+    config_param :timeout, :integer, :default => 5
+    config_param :username, :string, :default => nil
+    config_param :password, :string, :default => nil
 
     def configure(conf)
       super
-      if host
-        @http = Net::HTTP.new(host)
-      else
-        @host_hash = Hash[@host_hash.map { |k,v| [k, Net::HTTP.new(v)] }]
-      end
       @regexp = /\$\{([^}]+)\}/
-      @path_format = @path_format.gsub(@regexp, "<%=record['" + '\1' + "'] %>")
-      @path_format = ERB.new(@path_format)
+      @path_format = ERB.new(@path_format.gsub(@regexp, "<%=record['" + '\1' + "'] %>"))
 
       @headers = get_formatter(@header_hash)
       @cookies = get_formatter(@cookie_hash)
@@ -42,38 +40,49 @@ module Fluent
       super
     end
 
-    def emit(tag, es, chain)
-      chain.next
-      es.each {|time,record|
-        http = @http || @host_hash[record[@host_key]]
-        next if http.nil?
-        send_request(http, record)
-      }
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
     end
 
-    def send_request(http, record)
-      method = record[@method_key] || 'GET'
-
-      path = @path_format.result(binding)
-      params = record[@params_key]
-      unless params.nil?
-        path = add_query_string(path, record, params) if method !~ /POST/i
+    def write(chunk)
+      records = []
+      chunk.msgpack_each do |tag, time, record|
+        records << record
       end
-      req = Net::HTTP.const_get(method.capitalize).new(path)
-      unless params.nil?
-        req.set_form_data(params, "&") if method =~ /POST/i
+      send_request_parallel(records)
+    end
+
+    private
+
+    def send_request_parallel(records)
+      hydra = Typhoeus::Hydra.new(max_concurrency: @max_concurrency)
+      records.each do |record|
+        host = @host || @host_hash[record[@host_key]]
+        next if host.nil?
+        hydra.queue(get_request(host, record))
       end
-      req = set_header(req, record)
-      req['Cookie'] = get_cookie_string(record) if @cookie_hash
-      response = http.request(req)
+      hydra.run
     end
 
-    def add_query_string(path, record, params)
-      uri = URI.parse(path)
-      params_string = params.collect { |k,v| "#{k}=#{CGI::escape(v.to_s)}" }.join('&')
-      join_string = uri.query.nil? ? '?' : '&'
-      path = path + join_string + params_string
-      path
+    def get_request(host, record)
+      method = (record[@method_key] || 'get').downcase.to_sym
+      path = @path_format.result(binding)
+
+      url = "http://" + host + path
+      uri = Addressable::URI.parse(url)
+      params = uri.query_values
+      params.merge(record[@params_key]) unless record[@params_key].nil?
+
+      option = {
+        timeout: @timeout * 1000,
+        followlocation: true,
+        method: method,
+        params: params,
+        headers: get_header(record)
+      }
+      option[:userpwd] = "#{@username}:#{@password}" if @username
+
+      Typhoeus::Request.new("http://" + host + uri.path, option)
     end
 
     def get_formatter(hash)
@@ -86,17 +95,27 @@ module Fluent
       formatter
     end
 
-    def set_header(req, record)
+    def get_params(query, record_params)
+      params = query.nil? ? {} : Hash[URI::decode_www_form(query)]
+      if record_params
+        params = params.merge(record_params)
+      end
+      params
+    end
+
+    def get_header(record)
+      header = {}
       @headers.each do |k, v|
-        req[k] = v.result(binding)
+        header[k] = v.result(binding)
       end
-      req
+      header['Cookie'] = get_cookie_string(record) if @cookie_hash
+      header
     end
 
     def get_cookie_string(record)
       @cookies.map{|k, v|
         "#{k}=#{v.result(binding)}"
-      }.join(';')
+      }.join('; ')
     end
   end
 end

data/test/plugin/test_out_http_shadow.rb

@@ -4,29 +4,12 @@ class HttpShadowOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  CONFIG = %[
-    host staging.exsample.com
-    path_format ${path}
-    method_key method
-  ]
-
-  def create_driver(conf = CONFIG, tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::HttpShadowOutput, tag).configure(conf)
+  def create_driver(conf)
   end
 
   def test_configure
-    d = create_driver
-    assert_equal 'method', d.instance.method_key
-    assert_equal nil, d.instance.header_hash
-  end
-
-  def test_configure_error
-    config = %[
-      method_key method
-    ]
-    assert_raise(Fluent::ConfigError) do
-      d = create_driver(config)
-    end
   end
 
 end
+
+

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http_shadow
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Hiroshi Toyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-26 00:00:00.000000000 Z
+date: 2015-02-10 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement