fluent-plugin-http_shadow 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b47da229788f20b9d79742ffde107634b120adcd
+  data.tar.gz: 85c3d6ea8c4a5520d2dca87b0b781bbb0cc9052a
+SHA512:
+  metadata.gz: 4f623027f369842bb639e94a16eaa4ed8e6fd0eb83db983f30f16cf91c10818505196093daf7dfa483edde3f28f2fdb0c6a27540d8430e9940c162ae7fcd6d21
+  data.tar.gz: 7e88e174de9677945838a62956ce945114de8643c1a2fea3255e097b824f7b986a21782869ac2e4b4b972dae919737ea01d272fa09e9e50a8ef971e1f404ed86

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,9 @@
+Gemfile.lock
+doc/
+pkg/
+**/*.gem
+.yardoc/
+.bundle/
+vendor/bundle/
+.tags
+tmp/

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+
+os:
+  - linux
+  - osx
+
+gemfile:
+  - Gemfile
+
+script: 'bundle exec rake test'

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "fluent-plugin-http_shadow Documentation" --protected

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/README.md

@@ -0,0 +1,120 @@
+# fluent-plugin-http_shadow [![Build Status](https://secure.travis-ci.org/toyama0919/fluent-plugin-http_shadow.png?branch=master)](http://travis-ci.org/toyama0919/fluent-plugin-http_shadow)
+
+copy http request. use shadow proxy server.
+
+## Examples
+```
+<source>
+  type tail
+  format apache
+  path /var/log/httpd/access_log
+  pos_file /var/log/td-agent/access.pos
+  tag apache.access
+</source>
+
+<match apache.access>
+  type http_shadow
+  host staging.exsample.com
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "User-Agent": "${agent}" }
+</match>
+```
+
+Assume following input is coming:
+
+```
+  {
+    "host": "exsample.com",
+    "ip_address": "127.0.0.1",
+    "server": "10.0.0.11",
+    "remote": "-",
+    "time": "22/Dec/2014:03:20:26 +0900",
+    "method": "GET",
+    "path": "/hoge/?id=1",
+    "code": "200",
+    "size": "1578",
+    "referer": "http://exsample.com/other/",
+    "agent": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko"
+  }
+```
+
+then result becomes as below (indented):
+
+```
+GET http://staging.exsample.com/hoge/?id=1
+#=>  HTTP HEADER
+#=>  "referer": "http://exsample.com/other/"
+#=>  "agent": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko"
+```
+
+## Examples(Virtual Host)
+```
+<match http_shadow.exsample>
+  type http_shadow
+  host_hash { 
+    "www.example.com": "staging.example.com", 
+    "api.example.com": "api-staging.example.com", 
+    "blog.ipros.jp": "blog-staging.ipros.jp"
+  }
+  host_key host
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
+</match>
+```
+
+## Examples(use cookie)
+```
+<match http_shadow.exsample>
+  type http_shadow
+  host_hash { 
+    "www.example.com": "staging.example.com", 
+    "api.example.com": "api-staging.example.com", 
+    "blog.ipros.jp": "blog-staging.ipros.jp"
+  }
+  host_key host
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
+  cookie_hash {"rails-app_session": "${session_id}"}
+</match>
+```
+
+## note
+
+default GET Request.
+
+## parameter
+
+TODO
+
+## todo
+
+more test
+
+
+## Installation
+```
+fluent-gem install fluent-plugin-http_shadow
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Information
+
+* [Homepage](https://github.com/toyama0919/fluent-plugin-http_shadow)
+* [Issues](https://github.com/toyama0919/fluent-plugin-http_shadow/issues)
+* [Documentation](http://rubydoc.info/gems/fluent-plugin-http_shadow/frames)
+* [Email](mailto:toyama0919@gmail.com)
+
+## Copyright
+
+Copyright (c) 2015 Hiroshi Toyama
+

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/exsample/exsample.conf

@@ -0,0 +1,20 @@
+<source>
+  type forward
+  port 24224
+</source>
+
+<source>
+  type tail
+  format apache
+  path /var/log/httpd/access_log
+  pos_file /var/log/td-agent/access.pos
+  tag apache.access
+</source>
+
+<match apache.access>
+  type http_shadow
+  host staging.exsample.com
+  path_format ${path}
+  method_key method
+  header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
+</match>

data/fluent-plugin-http_shadow.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-http_shadow"
+  gem.version       = "0.0.1"
+  gem.summary       = %q{copy http request. use shadow proxy server.}
+  gem.description   = %q{copy http request. use shadow proxy server.}
+  gem.license       = "MIT"
+  gem.authors       = ["Hiroshi Toyama"]
+  gem.email         = "toyama0919@gmail.com"
+  gem.homepage      = "https://github.com/toyama0919/fluent-plugin-http_shadow"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'bundler', '~> 1.7.2'
+  gem.add_development_dependency 'fluentd', '~> 0.10.58'
+  gem.add_development_dependency 'pry', '~> 0.10.1'
+  gem.add_development_dependency 'rake', '~> 10.3.2'
+  gem.add_development_dependency 'rubocop', '~> 0.24.1'
+  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  gem.add_development_dependency 'yard', '~> 0.8'
+end

data/lib/fluent/plugin/out_http_shadow.rb

@@ -0,0 +1,102 @@
+require 'json'
+module Fluent
+  class HttpShadowOutput < Fluent::Output
+    Fluent::Plugin.register_output('http_shadow', self)
+
+    def initialize
+      super
+      require 'uri'
+      require 'net/http'
+      require 'erb'
+    end
+
+    config_param :host, :string, :default => nil
+    config_param :host_key, :string, :default => nil
+    config_param :host_hash, :hash, :default => nil
+    config_param :path_format, :string
+    config_param :method_key, :string
+    config_param :header_hash, :hash, :default => nil
+    config_param :cookie_hash, :hash, :default => nil
+    config_param :params_key, :string, :default => nil
+
+    def configure(conf)
+      super
+      if host
+        @http = Net::HTTP.new(host)
+      else
+        @host_hash = Hash[@host_hash.map { |k,v| [k, Net::HTTP.new(v)] }]
+      end
+      @regexp = /\$\{([^}]+)\}/
+      @path_format = @path_format.gsub(@regexp, "<%=record['" + '\1' + "'] %>")
+      @path_format = ERB.new(@path_format)
+
+      @headers = get_formatter(@header_hash)
+      @cookies = get_formatter(@cookie_hash)
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      chain.next
+      es.each {|time,record|
+        http = @http || @host_hash[record[@host_key]]
+        next if http.nil?
+        send_request(http, record)
+      }
+    end
+
+    def send_request(http, record)
+      method = record[@method_key] || 'GET'
+
+      path = @path_format.result(binding)
+      params = record[@params_key]
+      unless params.nil?
+        path = add_query_string(path, record, params) if method !~ /POST/i
+      end
+      req = Net::HTTP.const_get(method.capitalize).new(path)
+      unless params.nil?
+        req.set_form_data(params, "&") if method =~ /POST/i
+      end
+      req = set_header(req, record)
+      req['Cookie'] = get_cookie_string(record) if @cookie_hash
+      response = http.request(req)
+    end
+
+    def add_query_string(path, record, params)
+      uri = URI.parse(path)
+      params_string = params.collect { |k,v| "#{k}=#{CGI::escape(v.to_s)}" }.join('&')
+      join_string = uri.query.nil? ? '?' : '&'
+      path = path + join_string + params_string
+      path
+    end
+
+    def get_formatter(hash)
+      formatter = {}
+      return formatter unless hash
+      hash.each do |k, v|
+        format = v.gsub(@regexp,  "<%=record['" + '\1' + "'] %>")
+        formatter[k] = ERB.new(format)
+      end
+      formatter
+    end
+
+    def set_header(req, record)
+      @headers.each do |k, v|
+        req[k] = v.result(binding)
+      end
+      req
+    end
+
+    def get_cookie_string(record)
+      @cookies.map{|k, v|
+        "#{k}=#{v.result(binding)}"
+      }.join(';')
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/out_http_shadow'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_http_shadow.rb

@@ -0,0 +1,32 @@
+require 'helper'
+class HttpShadowOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    host staging.exsample.com
+    path_format ${path}
+    method_key method
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::OutputTestDriver.new(Fluent::HttpShadowOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'method', d.instance.method_key
+    assert_equal nil, d.instance.header_hash
+  end
+
+  def test_configure_error
+    config = %[
+      method_key method
+    ]
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(config)
+    end
+  end
+
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http_shadow
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: copy http request. use shadow proxy server.
+email: toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- README.md
+- Rakefile
+- exsample/exsample.conf
+- fluent-plugin-http_shadow.gemspec
+- lib/fluent/plugin/out_http_shadow.rb
+- test/helper.rb
+- test/plugin/test_out_http_shadow.rb
+homepage: https://github.com/toyama0919/fluent-plugin-http_shadow
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: copy http request. use shadow proxy server.
+test_files:
+- test/helper.rb
+- test/plugin/test_out_http_shadow.rb
+has_rdoc: