fluent-plugin-http_shadow 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3a9d29147a3f99994547fb1378583e77634ae8d3
-  data.tar.gz: bb7dc39257f466df902b250231b8e3a16ea9b50d
+  metadata.gz: cd40eb5720f4b03216dc6640d8e2980c7cbf3700
+  data.tar.gz: 4e61d4e93b20fa2431e0217f070ed63cda1cfa8f
 SHA512:
-  metadata.gz: 579f5881ef955bc48fb5f700c2c129b9250fdd2481e360afd57e7b32996d34fc2756d88116facc22bf95ced80ba4260559e137fe33a48a8677f92681f00b19d1
-  data.tar.gz: 2cecd61dde598ab66c0a12d4da24c88674f581aed98ab86a009dd289f879ee9a738a8b0287f038ba6daa2eefd77bab2d6393d189a567bbbc2e1a94a3d08764f8
+  metadata.gz: 1396a7f83d5362b9fa4dacc94a9c4bdaa9006526de8359b00a4a9692ad68475cca9a5bb12d0e748dc3e5496362c65fcad5722925d982b08afad1313691676561
+  data.tar.gz: 932b4de2cecf599822aa2630425a5bc1776a533852049747139e774269b6da6b5bca7142d6d672d8e4f9dd5998cfc1369b202d6eafba29333a02d7413a6f67a4

data/.gitignore

@@ -7,3 +7,4 @@ pkg/
 vendor/bundle/
 .tags
 tmp/
+.idea

data/.travis.yml

@@ -1,15 +1,20 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1
-
-os:
-  - linux
-  - osx
+  - 2.0.*
+  - 2.1.*
+  - 2.2.*
+  - 2.3.0
 
 gemfile:
   - Gemfile
+  - Gemfile.fluentd.0.12
+
+matrix:
+  exclude:
+    - rvm: 2.0.*
+      gemfile: Gemfile
+
+before_install: gem update bundler
 
 script: 'bundle exec rake test'

data/Gemfile.fluentd.0.12

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.12.0'
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/README.md

@@ -88,6 +88,43 @@ GET http://staging.exsample.com/hoge/?id=1
 </match>
 ```
 
+## Examples(use rate_per_method_hash)
+```
+<match http_shadow.exsample>
+  type http_shadow
+  host_hash {
+    "www.example.com": "staging.example.com",
+    "api.example.com": "api-staging.example.com",
+    "blog.ipros.jp": "blog-staging.ipros.jp"
+  }
+  host_key host
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
+  rate_per_method_hash {
+    "get": 30, # This means 30% requests of GET will be sent. Default(when not defined) value is 100.
+    "post": 90
+  }
+</match>
+```
+
+## Examples(use support_methods)
+```
+<match http_shadow.exsample>
+  type http_shadow
+  host_hash {
+    "www.example.com": "staging.example.com",
+    "api.example.com": "api-staging.example.com",
+    "blog.ipros.jp": "blog-staging.ipros.jp"
+  }
+  host_key host
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
+  support_methods [ "get", "post" ] # It means that only GET and POST are sent. By default all methods are sent.
+</match>
+```
+
 ## note
 
 default GET Request.

data/fluent-plugin-http_shadow.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-http_shadow"
-  gem.version       = "0.1.0"
+  gem.version       = "0.1.1"
   gem.summary       = %q{copy http request. use shadow proxy server.}
   gem.description   = %q{copy http request. use shadow proxy server.}
   gem.license       = "MIT"
@@ -25,4 +25,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rubocop', '~> 0.24.1'
   gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
   gem.add_development_dependency 'yard', '~> 0.8'
+  gem.add_development_dependency 'test-unit'
 end

data/lib/fluent/plugin/out_http_shadow.rb

@@ -26,9 +26,11 @@ module Fluent
     config_param :username, :string, :default => nil
     config_param :password, :string, :default => nil
     config_param :rate, :integer, :default => 100
+    config_param :rate_per_method_hash, :hash, :default => nil
     config_param :replace_hash, :hash, :default => nil
     config_param :protocol_format, :string, :default => 'http'
     config_param :no_send_header_pattern, :string, :default => nil
+    config_param :support_methods, :array, :default => nil, :value_type => :string
 
     def configure(conf)
       super
@@ -81,7 +83,10 @@ module Fluent
       records.each do |record|
         host = @host || @host_hash[record[@host_key]]
         next if host.nil?
-        hydra.queue(get_request(host, record))
+        request = get_request(host, record)
+        method = request.options[:method]
+        next unless supported?(method)
+        hydra.queue(request) if rate_per_method(method)
       end
       hydra.run
     end
@@ -161,5 +166,17 @@ module Fluent
       end
       cookie.join('; ')
     end
+
+    def supported?(method)
+      return true unless @support_methods
+      return @support_methods.include?(method.to_s)
+    end
+
+    def rate_per_method(method)
+      return true unless @rate_per_method_hash
+
+      rate_per_method = @rate_per_method_hash[method.to_s] || 100
+      return Random.rand(100) < rate_per_method
+    end
   end
 end

data/test/plugin/test_out_http_shadow.rb

@@ -37,6 +37,137 @@ class HttpShadowOutputTest < Test::Unit::TestCase
       ]
     end
   end
+
+  def test_get_cookie_string
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
+      cookie_hash { "cookie1": "${cookie1}", "cookie2": "${cookie2}"}
+      flush_interval 10
+    ]
+    d.instance.start
+    cookie_string = d.instance.send(:get_cookie_string, { 'cookie1' => 'value1'})
+    assert_equal cookie_string, "cookie1=value1; cookie2="
+
+    cookie_string = d.instance.send(:get_cookie_string, { 'cookie1' => 'value1', 'cookie2' => 'value2' })
+    assert_equal cookie_string, "cookie1=value1; cookie2=value2"
+  end
+
+  def test_get_cookie_string_no_send_header_pattern
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
+      cookie_hash { "cookie1": "${cookie1}", "cookie2": "${cookie2}", "cookie3": "${cookie3}" }
+      no_send_header_pattern ^(-|)$
+      flush_interval 10
+    ]
+    d.instance.start
+
+    cookie_string = d.instance.send(:get_cookie_string, { 'cookie1' => '-', 'cookie2' => '', 'cookie3' => 'value3' })
+    assert_equal cookie_string, "cookie3=value3"
+
+    cookie_string = d.instance.send(:get_cookie_string, { 'cookie1' => '-', 'cookie2' => '', 'cookie3' => '' })
+    assert_equal cookie_string, ""
+  end
+
+  def test_get_header
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}?hub=param", "X-Forwarded-For": "${ip_address}", "User-Agent": "${user_agent}" }
+      cookie_hash { "cookie1": "${cookie1}"}
+      flush_interval 10
+    ]
+    d.instance.start
+    header = d.instance.send(
+      :get_header,
+      {
+        'referer' => 'http://eetimes.jp/ee/articles/1407/30/news071.html',
+        'ip_address' => '10.10.10.10',
+        'user_agent' => 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36',
+        'cookie1' => 'value1'
+      }
+    )
+    assert_equal header['Referer'], "http://eetimes.jp/ee/articles/1407/30/news071.html?hub=param"
+    assert_equal header['X-Forwarded-For'], "10.10.10.10"
+    assert_equal header['User-Agent'], "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.124 Safari/537.36"
+    assert_equal header['Cookie'], "cookie1=value1"
+  end
+
+  def test_get_header_no_send_header_pattern
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}", "User-Agent": "${user_agent}" }
+      cookie_hash { "cookie1": "${cookie1}"}
+      no_send_header_pattern ^(-|)$
+      flush_interval 10
+    ]
+    d.instance.start
+    header = d.instance.send(
+      :get_header,
+      {
+        'referer' => '-',
+        'ip_address' => '10.10.10.10',
+        'user_agent' => '',
+        'cookie1' => 'value1'
+      }
+    )
+    assert_equal header['Referer'], nil
+    assert_equal header['X-Forwarded-For'], "10.10.10.10"
+    assert_equal header['User-Agent'], nil
+    assert_equal header['Cookie'], "cookie1=value1"
+  end
+
+  def test_supported?
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
+      cookie_hash { "iij-stg1_session": "${session_id}", "___IPROS_UUID_": "${uuid}"}
+      flush_interval 10
+      support_methods [ "get", "post" ]
+    ]
+    d.instance.start
+    assert_true d.instance.send(:supported?, :get)
+    assert_true d.instance.send(:supported?, :post)
+    assert_false d.instance.send(:supported?, :put)
+  end
+
+  def test_rate_per_method
+    d = create_driver %[
+      host google.com
+      path_format ${path}
+      method_key method
+      path_format ${url}
+      method_key method
+      header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
+      cookie_hash { "iij-stg1_session": "${session_id}", "___IPROS_UUID_": "${uuid}"}
+      flush_interval 10
+      rate_per_method_hash { "get": 100, "post": 0 }
+    ]
+    d.instance.start
+    assert_true d.instance.send(:rate_per_method, :get)
+    assert_false d.instance.send(:rate_per_method, :post)
+    assert_true d.instance.send(:rate_per_method, :put)
+  end
 end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http_shadow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Hiroshi Toyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-07 00:00:00.000000000 Z
+date: 2017-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -136,6 +136,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: copy http request. use shadow proxy server.
 email: toyama0919@gmail.com
 executables: []
@@ -147,6 +161,7 @@ files:
 - ".travis.yml"
 - ".yardopts"
 - Gemfile
+- Gemfile.fluentd.0.12
 - README.md
 - Rakefile
 - exsample/exsample.conf
@@ -174,11 +189,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: copy http request. use shadow proxy server.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_http_shadow.rb
-has_rdoc: