fluent-plugin-http_shadow 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.document +3 -0
- data/.gitignore +9 -0
- data/.travis.yml +15 -0
- data/.yardopts +1 -0
- data/Gemfile +7 -0
- data/README.md +120 -0
- data/Rakefile +11 -0
- data/exsample/exsample.conf +20 -0
- data/fluent-plugin-http_shadow.gemspec +25 -0
- data/lib/fluent/plugin/out_http_shadow.rb +102 -0
- data/test/helper.rb +18 -0
- data/test/plugin/test_out_http_shadow.rb +32 -0
- metadata +156 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b47da229788f20b9d79742ffde107634b120adcd
|
4
|
+
data.tar.gz: 85c3d6ea8c4a5520d2dca87b0b781bbb0cc9052a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4f623027f369842bb639e94a16eaa4ed8e6fd0eb83db983f30f16cf91c10818505196093daf7dfa483edde3f28f2fdb0c6a27540d8430e9940c162ae7fcd6d21
|
7
|
+
data.tar.gz: 7e88e174de9677945838a62956ce945114de8643c1a2fea3255e097b824f7b986a21782869ac2e4b4b972dae919737ea01d272fa09e9e50a8ef971e1f404ed86
|
data/.document
ADDED
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/.yardopts
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--markup markdown --title "fluent-plugin-http_shadow Documentation" --protected
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
# fluent-plugin-http_shadow [](http://travis-ci.org/toyama0919/fluent-plugin-http_shadow)
|
2
|
+
|
3
|
+
copy http request. use shadow proxy server.
|
4
|
+
|
5
|
+
## Examples
|
6
|
+
```
|
7
|
+
<source>
|
8
|
+
type tail
|
9
|
+
format apache
|
10
|
+
path /var/log/httpd/access_log
|
11
|
+
pos_file /var/log/td-agent/access.pos
|
12
|
+
tag apache.access
|
13
|
+
</source>
|
14
|
+
|
15
|
+
<match apache.access>
|
16
|
+
type http_shadow
|
17
|
+
host staging.exsample.com
|
18
|
+
path_format ${path}
|
19
|
+
method_key method
|
20
|
+
header_hash { "Referer": "${referer}", "User-Agent": "${agent}" }
|
21
|
+
</match>
|
22
|
+
```
|
23
|
+
|
24
|
+
Assume following input is coming:
|
25
|
+
|
26
|
+
```
|
27
|
+
{
|
28
|
+
"host": "exsample.com",
|
29
|
+
"ip_address": "127.0.0.1",
|
30
|
+
"server": "10.0.0.11",
|
31
|
+
"remote": "-",
|
32
|
+
"time": "22/Dec/2014:03:20:26 +0900",
|
33
|
+
"method": "GET",
|
34
|
+
"path": "/hoge/?id=1",
|
35
|
+
"code": "200",
|
36
|
+
"size": "1578",
|
37
|
+
"referer": "http://exsample.com/other/",
|
38
|
+
"agent": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko"
|
39
|
+
}
|
40
|
+
```
|
41
|
+
|
42
|
+
then result becomes as below (indented):
|
43
|
+
|
44
|
+
```
|
45
|
+
GET http://staging.exsample.com/hoge/?id=1
|
46
|
+
#=> HTTP HEADER
|
47
|
+
#=> "referer": "http://exsample.com/other/"
|
48
|
+
#=> "agent": "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko"
|
49
|
+
```
|
50
|
+
|
51
|
+
## Examples(Virtual Host)
|
52
|
+
```
|
53
|
+
<match http_shadow.exsample>
|
54
|
+
type http_shadow
|
55
|
+
host_hash {
|
56
|
+
"www.example.com": "staging.example.com",
|
57
|
+
"api.example.com": "api-staging.example.com",
|
58
|
+
"blog.ipros.jp": "blog-staging.ipros.jp"
|
59
|
+
}
|
60
|
+
host_key host
|
61
|
+
path_format ${path}
|
62
|
+
method_key method
|
63
|
+
header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
|
64
|
+
</match>
|
65
|
+
```
|
66
|
+
|
67
|
+
## Examples(use cookie)
|
68
|
+
```
|
69
|
+
<match http_shadow.exsample>
|
70
|
+
type http_shadow
|
71
|
+
host_hash {
|
72
|
+
"www.example.com": "staging.example.com",
|
73
|
+
"api.example.com": "api-staging.example.com",
|
74
|
+
"blog.ipros.jp": "blog-staging.ipros.jp"
|
75
|
+
}
|
76
|
+
host_key host
|
77
|
+
path_format ${path}
|
78
|
+
method_key method
|
79
|
+
header_hash { "Referer": "${referer}", "User-Agent": "${user_agent}" }
|
80
|
+
cookie_hash {"rails-app_session": "${session_id}"}
|
81
|
+
</match>
|
82
|
+
```
|
83
|
+
|
84
|
+
## note
|
85
|
+
|
86
|
+
default GET Request.
|
87
|
+
|
88
|
+
## parameter
|
89
|
+
|
90
|
+
TODO
|
91
|
+
|
92
|
+
## todo
|
93
|
+
|
94
|
+
more test
|
95
|
+
|
96
|
+
|
97
|
+
## Installation
|
98
|
+
```
|
99
|
+
fluent-gem install fluent-plugin-http_shadow
|
100
|
+
```
|
101
|
+
|
102
|
+
## Contributing
|
103
|
+
|
104
|
+
1. Fork it
|
105
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
106
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
107
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
108
|
+
5. Create new [Pull Request](../../pull/new/master)
|
109
|
+
|
110
|
+
## Information
|
111
|
+
|
112
|
+
* [Homepage](https://github.com/toyama0919/fluent-plugin-http_shadow)
|
113
|
+
* [Issues](https://github.com/toyama0919/fluent-plugin-http_shadow/issues)
|
114
|
+
* [Documentation](http://rubydoc.info/gems/fluent-plugin-http_shadow/frames)
|
115
|
+
* [Email](mailto:toyama0919@gmail.com)
|
116
|
+
|
117
|
+
## Copyright
|
118
|
+
|
119
|
+
Copyright (c) 2015 Hiroshi Toyama
|
120
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
<source>
|
2
|
+
type forward
|
3
|
+
port 24224
|
4
|
+
</source>
|
5
|
+
|
6
|
+
<source>
|
7
|
+
type tail
|
8
|
+
format apache
|
9
|
+
path /var/log/httpd/access_log
|
10
|
+
pos_file /var/log/td-agent/access.pos
|
11
|
+
tag apache.access
|
12
|
+
</source>
|
13
|
+
|
14
|
+
<match apache.access>
|
15
|
+
type http_shadow
|
16
|
+
host staging.exsample.com
|
17
|
+
path_format ${path}
|
18
|
+
method_key method
|
19
|
+
header_hash { "Referer": "${referer}", "X-Forwarded-For": "${ip_address}" }
|
20
|
+
</match>
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |gem|
|
4
|
+
gem.name = "fluent-plugin-http_shadow"
|
5
|
+
gem.version = "0.0.1"
|
6
|
+
gem.summary = %q{copy http request. use shadow proxy server.}
|
7
|
+
gem.description = %q{copy http request. use shadow proxy server.}
|
8
|
+
gem.license = "MIT"
|
9
|
+
gem.authors = ["Hiroshi Toyama"]
|
10
|
+
gem.email = "toyama0919@gmail.com"
|
11
|
+
gem.homepage = "https://github.com/toyama0919/fluent-plugin-http_shadow"
|
12
|
+
|
13
|
+
gem.files = `git ls-files`.split($/)
|
14
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
15
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
16
|
+
gem.require_paths = ['lib']
|
17
|
+
|
18
|
+
gem.add_development_dependency 'bundler', '~> 1.7.2'
|
19
|
+
gem.add_development_dependency 'fluentd', '~> 0.10.58'
|
20
|
+
gem.add_development_dependency 'pry', '~> 0.10.1'
|
21
|
+
gem.add_development_dependency 'rake', '~> 10.3.2'
|
22
|
+
gem.add_development_dependency 'rubocop', '~> 0.24.1'
|
23
|
+
gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
|
24
|
+
gem.add_development_dependency 'yard', '~> 0.8'
|
25
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
require 'json'
|
2
|
+
module Fluent
|
3
|
+
class HttpShadowOutput < Fluent::Output
|
4
|
+
Fluent::Plugin.register_output('http_shadow', self)
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
super
|
8
|
+
require 'uri'
|
9
|
+
require 'net/http'
|
10
|
+
require 'erb'
|
11
|
+
end
|
12
|
+
|
13
|
+
config_param :host, :string, :default => nil
|
14
|
+
config_param :host_key, :string, :default => nil
|
15
|
+
config_param :host_hash, :hash, :default => nil
|
16
|
+
config_param :path_format, :string
|
17
|
+
config_param :method_key, :string
|
18
|
+
config_param :header_hash, :hash, :default => nil
|
19
|
+
config_param :cookie_hash, :hash, :default => nil
|
20
|
+
config_param :params_key, :string, :default => nil
|
21
|
+
|
22
|
+
def configure(conf)
|
23
|
+
super
|
24
|
+
if host
|
25
|
+
@http = Net::HTTP.new(host)
|
26
|
+
else
|
27
|
+
@host_hash = Hash[@host_hash.map { |k,v| [k, Net::HTTP.new(v)] }]
|
28
|
+
end
|
29
|
+
@regexp = /\$\{([^}]+)\}/
|
30
|
+
@path_format = @path_format.gsub(@regexp, "<%=record['" + '\1' + "'] %>")
|
31
|
+
@path_format = ERB.new(@path_format)
|
32
|
+
|
33
|
+
@headers = get_formatter(@header_hash)
|
34
|
+
@cookies = get_formatter(@cookie_hash)
|
35
|
+
end
|
36
|
+
|
37
|
+
def start
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
def shutdown
|
42
|
+
super
|
43
|
+
end
|
44
|
+
|
45
|
+
def emit(tag, es, chain)
|
46
|
+
chain.next
|
47
|
+
es.each {|time,record|
|
48
|
+
http = @http || @host_hash[record[@host_key]]
|
49
|
+
next if http.nil?
|
50
|
+
send_request(http, record)
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
54
|
+
def send_request(http, record)
|
55
|
+
method = record[@method_key] || 'GET'
|
56
|
+
|
57
|
+
path = @path_format.result(binding)
|
58
|
+
params = record[@params_key]
|
59
|
+
unless params.nil?
|
60
|
+
path = add_query_string(path, record, params) if method !~ /POST/i
|
61
|
+
end
|
62
|
+
req = Net::HTTP.const_get(method.capitalize).new(path)
|
63
|
+
unless params.nil?
|
64
|
+
req.set_form_data(params, "&") if method =~ /POST/i
|
65
|
+
end
|
66
|
+
req = set_header(req, record)
|
67
|
+
req['Cookie'] = get_cookie_string(record) if @cookie_hash
|
68
|
+
response = http.request(req)
|
69
|
+
end
|
70
|
+
|
71
|
+
def add_query_string(path, record, params)
|
72
|
+
uri = URI.parse(path)
|
73
|
+
params_string = params.collect { |k,v| "#{k}=#{CGI::escape(v.to_s)}" }.join('&')
|
74
|
+
join_string = uri.query.nil? ? '?' : '&'
|
75
|
+
path = path + join_string + params_string
|
76
|
+
path
|
77
|
+
end
|
78
|
+
|
79
|
+
def get_formatter(hash)
|
80
|
+
formatter = {}
|
81
|
+
return formatter unless hash
|
82
|
+
hash.each do |k, v|
|
83
|
+
format = v.gsub(@regexp, "<%=record['" + '\1' + "'] %>")
|
84
|
+
formatter[k] = ERB.new(format)
|
85
|
+
end
|
86
|
+
formatter
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_header(req, record)
|
90
|
+
@headers.each do |k, v|
|
91
|
+
req[k] = v.result(binding)
|
92
|
+
end
|
93
|
+
req
|
94
|
+
end
|
95
|
+
|
96
|
+
def get_cookie_string(record)
|
97
|
+
@cookies.map{|k, v|
|
98
|
+
"#{k}=#{v.result(binding)}"
|
99
|
+
}.join(';')
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
require 'fluent/test'
|
15
|
+
require 'fluent/plugin/out_http_shadow'
|
16
|
+
|
17
|
+
class Test::Unit::TestCase
|
18
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'helper'
|
2
|
+
class HttpShadowOutputTest < Test::Unit::TestCase
|
3
|
+
def setup
|
4
|
+
Fluent::Test.setup
|
5
|
+
end
|
6
|
+
|
7
|
+
CONFIG = %[
|
8
|
+
host staging.exsample.com
|
9
|
+
path_format ${path}
|
10
|
+
method_key method
|
11
|
+
]
|
12
|
+
|
13
|
+
def create_driver(conf = CONFIG, tag='test')
|
14
|
+
Fluent::Test::OutputTestDriver.new(Fluent::HttpShadowOutput, tag).configure(conf)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_configure
|
18
|
+
d = create_driver
|
19
|
+
assert_equal 'method', d.instance.method_key
|
20
|
+
assert_equal nil, d.instance.header_hash
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_configure_error
|
24
|
+
config = %[
|
25
|
+
method_key method
|
26
|
+
]
|
27
|
+
assert_raise(Fluent::ConfigError) do
|
28
|
+
d = create_driver(config)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
metadata
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-http_shadow
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Hiroshi Toyama
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-01-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.7.2
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.7.2
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.10.58
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.10.58
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: pry
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.10.1
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.10.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 10.3.2
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 10.3.2
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rubocop
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.24.1
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.24.1
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rubygems-tasks
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.2'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.2'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: yard
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.8'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.8'
|
111
|
+
description: copy http request. use shadow proxy server.
|
112
|
+
email: toyama0919@gmail.com
|
113
|
+
executables: []
|
114
|
+
extensions: []
|
115
|
+
extra_rdoc_files: []
|
116
|
+
files:
|
117
|
+
- ".document"
|
118
|
+
- ".gitignore"
|
119
|
+
- ".travis.yml"
|
120
|
+
- ".yardopts"
|
121
|
+
- Gemfile
|
122
|
+
- README.md
|
123
|
+
- Rakefile
|
124
|
+
- exsample/exsample.conf
|
125
|
+
- fluent-plugin-http_shadow.gemspec
|
126
|
+
- lib/fluent/plugin/out_http_shadow.rb
|
127
|
+
- test/helper.rb
|
128
|
+
- test/plugin/test_out_http_shadow.rb
|
129
|
+
homepage: https://github.com/toyama0919/fluent-plugin-http_shadow
|
130
|
+
licenses:
|
131
|
+
- MIT
|
132
|
+
metadata: {}
|
133
|
+
post_install_message:
|
134
|
+
rdoc_options: []
|
135
|
+
require_paths:
|
136
|
+
- lib
|
137
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
138
|
+
requirements:
|
139
|
+
- - ">="
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
143
|
+
requirements:
|
144
|
+
- - ">="
|
145
|
+
- !ruby/object:Gem::Version
|
146
|
+
version: '0'
|
147
|
+
requirements: []
|
148
|
+
rubyforge_project:
|
149
|
+
rubygems_version: 2.4.2
|
150
|
+
signing_key:
|
151
|
+
specification_version: 4
|
152
|
+
summary: copy http request. use shadow proxy server.
|
153
|
+
test_files:
|
154
|
+
- test/helper.rb
|
155
|
+
- test/plugin/test_out_http_shadow.rb
|
156
|
+
has_rdoc:
|