abhay-typhoeus 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/README.textile +212 -0
- data/Rakefile +45 -0
- data/VERSION +1 -0
- data/benchmarks/profile.rb +25 -0
- data/benchmarks/vs_nethttp.rb +35 -0
- data/examples/twitter.rb +21 -0
- data/ext/typhoeus/.gitignore +6 -0
- data/ext/typhoeus/extconf.rb +23 -0
- data/ext/typhoeus/native.c +11 -0
- data/ext/typhoeus/native.h +20 -0
- data/ext/typhoeus/typhoeus_easy.c +206 -0
- data/ext/typhoeus/typhoeus_easy.h +19 -0
- data/ext/typhoeus/typhoeus_multi.c +213 -0
- data/ext/typhoeus/typhoeus_multi.h +16 -0
- data/lib/typhoeus.rb +51 -0
- data/lib/typhoeus/.gitignore +1 -0
- data/lib/typhoeus/easy.rb +210 -0
- data/lib/typhoeus/filter.rb +28 -0
- data/lib/typhoeus/multi.rb +34 -0
- data/lib/typhoeus/remote.rb +306 -0
- data/lib/typhoeus/remote_method.rb +108 -0
- data/lib/typhoeus/remote_proxy_object.rb +48 -0
- data/lib/typhoeus/response.rb +17 -0
- data/lib/typhoeus/service.rb +20 -0
- data/profilers/valgrind.rb +24 -0
- data/spec/fixtures/result_set.xml +60 -0
- data/spec/servers/app.rb +24 -0
- data/spec/servers/delay_fixture_server.rb +66 -0
- data/spec/servers/method_server.rb +51 -0
- data/spec/spec.opts +2 -0
- data/spec/spec_helper.rb +29 -0
- data/spec/typhoeus/easy_spec.rb +148 -0
- data/spec/typhoeus/filter_spec.rb +35 -0
- data/spec/typhoeus/multi_spec.rb +82 -0
- data/spec/typhoeus/remote_method_spec.rb +141 -0
- data/spec/typhoeus/remote_proxy_object_spec.rb +73 -0
- data/spec/typhoeus/remote_spec.rb +699 -0
- data/spec/typhoeus/response_spec.rb +31 -0
- data/typhoeus.gemspec +88 -0
- metadata +104 -0
@@ -0,0 +1,48 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
class RemoteProxyObject
|
3
|
+
instance_methods.each { |m| undef_method m unless m =~ /^__/ }
|
4
|
+
|
5
|
+
def initialize(clear_memoized_store_proc, easy, options = {})
|
6
|
+
@clear_memoized_store_proc = clear_memoized_store_proc
|
7
|
+
@easy = easy
|
8
|
+
@success = options[:on_success]
|
9
|
+
@failure = options[:on_failure]
|
10
|
+
@cache = options.delete(:cache)
|
11
|
+
@cache_key = options.delete(:cache_key)
|
12
|
+
@timeout = options.delete(:cache_timeout)
|
13
|
+
Typhoeus.add_easy_request(@easy)
|
14
|
+
end
|
15
|
+
|
16
|
+
def method_missing(sym, *args, &block)
|
17
|
+
unless @proxied_object
|
18
|
+
if @cache && @cache_key
|
19
|
+
@proxied_object = @cache.get(@cache_key) rescue nil
|
20
|
+
end
|
21
|
+
|
22
|
+
unless @proxied_object
|
23
|
+
Typhoeus.perform_easy_requests
|
24
|
+
response = Response.new(:code => @easy.response_code,
|
25
|
+
:headers => @easy.response_header,
|
26
|
+
:body => @easy.response_body,
|
27
|
+
:time => @easy.total_time_taken,
|
28
|
+
:requested_url => @easy.url,
|
29
|
+
:requested_http_method => @easy.method,
|
30
|
+
:start_time => @easy.start_time)
|
31
|
+
if @easy.response_code >= 200 && @easy.response_code < 300
|
32
|
+
Typhoeus.release_easy_object(@easy)
|
33
|
+
@proxied_object = @success.nil? ? response : @success.call(response)
|
34
|
+
|
35
|
+
if @cache && @cache_key
|
36
|
+
@cache.set(@cache_key, @proxied_object, @timeout)
|
37
|
+
end
|
38
|
+
else
|
39
|
+
@proxied_object = @failure.nil? ? response : @failure.call(response)
|
40
|
+
end
|
41
|
+
@clear_memoized_store_proc.call
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
@proxied_object.__send__(sym, *args, &block)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
class Response
|
3
|
+
attr_reader :code, :headers, :body, :time,
|
4
|
+
:requested_url, :requested_remote_method,
|
5
|
+
:requested_http_method, :start_time
|
6
|
+
|
7
|
+
def initialize(params = {})
|
8
|
+
@code = params[:code]
|
9
|
+
@headers = params[:headers]
|
10
|
+
@body = params[:body]
|
11
|
+
@time = params[:time]
|
12
|
+
@requested_url = params[:requested_url]
|
13
|
+
@requested_http_method = params[:requested_http_method]
|
14
|
+
@start_time = params[:start_time]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
class Service
|
3
|
+
def initialize(host, port)
|
4
|
+
@host = host
|
5
|
+
@port = port
|
6
|
+
end
|
7
|
+
|
8
|
+
def get(resource, params)
|
9
|
+
end
|
10
|
+
|
11
|
+
def put(resource, params)
|
12
|
+
end
|
13
|
+
|
14
|
+
def post(resource, params)
|
15
|
+
end
|
16
|
+
|
17
|
+
def delete(resource, params)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# go to ext/typhoeus and run ruby extconf.rb && make before running
|
3
|
+
# this.
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__) + "/../ext")
|
6
|
+
require File.dirname(__FILE__) + "/../lib/typhoeus"
|
7
|
+
|
8
|
+
klass = Class.new { include Typhoeus }
|
9
|
+
|
10
|
+
loops = ENV['LOOPS'].to_i
|
11
|
+
url = ARGV.first || (raise "requires URL!")
|
12
|
+
|
13
|
+
loops.times do |i|
|
14
|
+
puts "On loop #{i}" if i % 10 == 0
|
15
|
+
results = []
|
16
|
+
5.times do
|
17
|
+
results << klass.get(url)
|
18
|
+
end
|
19
|
+
|
20
|
+
# fire requests
|
21
|
+
results[0].code
|
22
|
+
end
|
23
|
+
|
24
|
+
puts "Ran #{loops} loops on #{url}!"
|
@@ -0,0 +1,60 @@
|
|
1
|
+
<result_set>
|
2
|
+
<ttl>20</ttl>
|
3
|
+
<result>
|
4
|
+
<id>1</id>
|
5
|
+
<name>hello</name>
|
6
|
+
<description>
|
7
|
+
this is a long description for a text field of some kind.
|
8
|
+
this is a long description for a text field of some kind.
|
9
|
+
this is a long description for a text field of some kind.
|
10
|
+
this is a long description for a text field of some kind.
|
11
|
+
this is a long description for a text field of some kind.
|
12
|
+
this is a long description for a text field of some kind.
|
13
|
+
this is a long description for a text field of some kind.
|
14
|
+
this is a long description for a text field of some kind.
|
15
|
+
this is a long description for a text field of some kind.
|
16
|
+
this is a long description for a text field of some kind.
|
17
|
+
this is a long description for a text field of some kind.
|
18
|
+
this is a long description for a text field of some kind.
|
19
|
+
this is a long description for a text field of some kind.
|
20
|
+
</description>
|
21
|
+
</result>
|
22
|
+
<result>
|
23
|
+
<id>2</id>
|
24
|
+
<name>hello</name>
|
25
|
+
<description>
|
26
|
+
this is a long description for a text field of some kind.
|
27
|
+
this is a long description for a text field of some kind.
|
28
|
+
this is a long description for a text field of some kind.
|
29
|
+
this is a long description for a text field of some kind.
|
30
|
+
this is a long description for a text field of some kind.
|
31
|
+
this is a long description for a text field of some kind.
|
32
|
+
this is a long description for a text field of some kind.
|
33
|
+
this is a long description for a text field of some kind.
|
34
|
+
this is a long description for a text field of some kind.
|
35
|
+
this is a long description for a text field of some kind.
|
36
|
+
this is a long description for a text field of some kind.
|
37
|
+
this is a long description for a text field of some kind.
|
38
|
+
this is a long description for a text field of some kind.
|
39
|
+
</description>
|
40
|
+
</result>
|
41
|
+
<result>
|
42
|
+
<id>3</id>
|
43
|
+
<name>hello</name>
|
44
|
+
<description>
|
45
|
+
this is a long description for a text field of some kind.
|
46
|
+
this is a long description for a text field of some kind.
|
47
|
+
this is a long description for a text field of some kind.
|
48
|
+
this is a long description for a text field of some kind.
|
49
|
+
this is a long description for a text field of some kind.
|
50
|
+
this is a long description for a text field of some kind.
|
51
|
+
this is a long description for a text field of some kind.
|
52
|
+
this is a long description for a text field of some kind.
|
53
|
+
this is a long description for a text field of some kind.
|
54
|
+
this is a long description for a text field of some kind.
|
55
|
+
this is a long description for a text field of some kind.
|
56
|
+
this is a long description for a text field of some kind.
|
57
|
+
this is a long description for a text field of some kind.
|
58
|
+
</description>
|
59
|
+
</result>
|
60
|
+
</result_set>
|
data/spec/servers/app.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'sinatra'
|
3
|
+
|
4
|
+
get '/**' do
|
5
|
+
puts request.inspect
|
6
|
+
puts "**#{request.body.read}**"
|
7
|
+
sleep 3
|
8
|
+
"hello world"
|
9
|
+
end
|
10
|
+
|
11
|
+
put '/**' do
|
12
|
+
puts request.inspect
|
13
|
+
puts "**#{request.body.read}**"
|
14
|
+
end
|
15
|
+
|
16
|
+
post '/**' do
|
17
|
+
puts request.inspect
|
18
|
+
puts "**#{request.body.read}**"
|
19
|
+
end
|
20
|
+
|
21
|
+
delete '/**' do
|
22
|
+
puts request.inspect
|
23
|
+
puts "**#{request.body.read}**"
|
24
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# this server simply accepts requests and blocks for a passed in interval before returning a passed in reqeust value to
|
2
|
+
# the client
|
3
|
+
require 'rubygems'
|
4
|
+
require 'eventmachine'
|
5
|
+
require 'evma_httpserver'
|
6
|
+
|
7
|
+
class DelayFixtureServer < EventMachine::Connection
|
8
|
+
include EventMachine::HttpServer
|
9
|
+
|
10
|
+
def process_http_request
|
11
|
+
EventMachine.stop if ENV["PATH_INFO"] == "/die"
|
12
|
+
puts "got a request #{ENV['PATH_INFO']}"
|
13
|
+
resp = EventMachine::DelegatedHttpResponse.new( self )
|
14
|
+
|
15
|
+
# Block which fulfills the request
|
16
|
+
operation = proc do
|
17
|
+
sleep DelayFixtureServer.response_delay
|
18
|
+
|
19
|
+
resp.status = 200
|
20
|
+
resp.content = "whatever"
|
21
|
+
end
|
22
|
+
|
23
|
+
# Callback block to execute once the request is fulfilled
|
24
|
+
callback = proc do |res|
|
25
|
+
resp.send_response
|
26
|
+
end
|
27
|
+
|
28
|
+
# Let the thread pool (20 Ruby threads) handle request
|
29
|
+
EM.defer(operation, callback)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.response_fixture
|
33
|
+
@response_fixture ||= ""
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.response_fixture=(val)
|
37
|
+
@response_fixture = val
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.response_delay
|
41
|
+
@response_delay ||= 0
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.response_delay=(val)
|
45
|
+
@response_delay = val
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.reponse_number
|
49
|
+
@response_number
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.response_number=(val)
|
53
|
+
@response_number = val
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
port = (ARGV[0] || 3000).to_i
|
58
|
+
|
59
|
+
DelayFixtureServer.response_delay = 0.5
|
60
|
+
DelayFixtureServer.response_number = 0
|
61
|
+
#DelayFixtureServer.response_fixture = File.read(File.dirname(__FILE__) + "/../fixtures/result_set.xml")
|
62
|
+
|
63
|
+
EventMachine::run {
|
64
|
+
EventMachine.epoll
|
65
|
+
EventMachine::start_server("0.0.0.0", port, DelayFixtureServer)
|
66
|
+
}
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# this server simply is for testing out the different http methods. it echoes back the passed in info
|
2
|
+
require 'rubygems'
|
3
|
+
require 'eventmachine'
|
4
|
+
require 'evma_httpserver'
|
5
|
+
|
6
|
+
class MethodServer < EventMachine::Connection
|
7
|
+
include EventMachine::HttpServer
|
8
|
+
|
9
|
+
def process_http_request
|
10
|
+
EventMachine.stop if ENV["PATH_INFO"] == "/die"
|
11
|
+
|
12
|
+
resp = EventMachine::DelegatedHttpResponse.new( self )
|
13
|
+
|
14
|
+
# Block which fulfills the request
|
15
|
+
operation = proc do
|
16
|
+
sleep MethodServer.sleep_time
|
17
|
+
resp.status = 200
|
18
|
+
resp.content = request_params + "\n#{@http_post_content}"
|
19
|
+
end
|
20
|
+
|
21
|
+
# Callback block to execute once the request is fulfilled
|
22
|
+
callback = proc do |res|
|
23
|
+
resp.send_response
|
24
|
+
end
|
25
|
+
|
26
|
+
# Let the thread pool (20 Ruby threads) handle request
|
27
|
+
EM.defer(operation, callback)
|
28
|
+
end
|
29
|
+
|
30
|
+
def request_params
|
31
|
+
%w( PATH_INFO QUERY_STRING HTTP_COOKIE IF_NONE_MATCH CONTENT_TYPE REQUEST_METHOD REQUEST_URI ).collect do |param|
|
32
|
+
"#{param}=#{ENV[param]}"
|
33
|
+
end.join("\n")
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.sleep_time=(val)
|
37
|
+
@sleep_time = val
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.sleep_time
|
41
|
+
@sleep_time || 0
|
42
|
+
end
|
43
|
+
end
|
44
|
+
#
|
45
|
+
# port = (ARGV[0] || 3000).to_i
|
46
|
+
# #Process.fork do
|
47
|
+
# EventMachine::run {
|
48
|
+
# EventMachine.epoll
|
49
|
+
# EventMachine::start_server("0.0.0.0", port, MethodServer)
|
50
|
+
# }
|
51
|
+
# #end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require "rubygems"
|
2
|
+
require "spec"
|
3
|
+
|
4
|
+
# gem install redgreen for colored test output
|
5
|
+
begin require "redgreen" unless ENV['TM_CURRENT_LINE']; rescue LoadError; end
|
6
|
+
|
7
|
+
path = File.expand_path(File.dirname(__FILE__) + "/../lib/")
|
8
|
+
$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
|
9
|
+
|
10
|
+
require "lib/typhoeus"
|
11
|
+
|
12
|
+
# local servers for running tests
|
13
|
+
require File.dirname(__FILE__) + "/servers/method_server.rb"
|
14
|
+
|
15
|
+
def start_method_server(port, sleep = 0)
|
16
|
+
MethodServer.sleep_time = sleep
|
17
|
+
pid = Process.fork do
|
18
|
+
EventMachine::run {
|
19
|
+
EventMachine.epoll
|
20
|
+
EventMachine::start_server("0.0.0.0", port, MethodServer)
|
21
|
+
}
|
22
|
+
end
|
23
|
+
sleep 0.2
|
24
|
+
pid
|
25
|
+
end
|
26
|
+
|
27
|
+
def stop_method_server(pid)
|
28
|
+
Process.kill("HUP", pid)
|
29
|
+
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/../spec_helper'
|
2
|
+
|
3
|
+
describe Typhoeus::Easy do
|
4
|
+
before(:all) do
|
5
|
+
@pid = start_method_server(3002)
|
6
|
+
end
|
7
|
+
|
8
|
+
after(:all) do
|
9
|
+
stop_method_server(@pid)
|
10
|
+
end
|
11
|
+
|
12
|
+
describe "options" do
|
13
|
+
it "should allow for following redirects"
|
14
|
+
it "should allow you to set the user agent"
|
15
|
+
it "should provide a timeout in milliseconds" do
|
16
|
+
pid = start_method_server(3001, 5)
|
17
|
+
e = Typhoeus::Easy.new
|
18
|
+
e.url = "http://localhost:3001"
|
19
|
+
e.method = :get
|
20
|
+
e.timeout = 50
|
21
|
+
e.perform
|
22
|
+
puts e.response_code
|
23
|
+
puts e.total_time_taken
|
24
|
+
# e.timed_out?.should == true
|
25
|
+
stop_method_server(pid)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe "get" do
|
30
|
+
it "should perform a get" do
|
31
|
+
easy = Typhoeus::Easy.new
|
32
|
+
easy.url = "http://localhost:3002"
|
33
|
+
easy.method = :get
|
34
|
+
easy.perform
|
35
|
+
easy.response_code.should == 200
|
36
|
+
easy.response_body.should include("REQUEST_METHOD=GET")
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should send a request body" do
|
40
|
+
easy = Typhoeus::Easy.new
|
41
|
+
easy.url = "http://localhost:3002"
|
42
|
+
easy.method = :get
|
43
|
+
easy.request_body = "this is a body!"
|
44
|
+
easy.perform
|
45
|
+
easy.response_code.should == 200
|
46
|
+
easy.response_body.should include("this is a body!")
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe "start_time" do
|
51
|
+
it "should be get/settable" do
|
52
|
+
time = Time.now
|
53
|
+
easy = Typhoeus::Easy.new
|
54
|
+
easy.start_time.should be_nil
|
55
|
+
easy.start_time = time
|
56
|
+
easy.start_time.should == time
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
describe "params=" do
|
61
|
+
it "should handle arrays of params" do
|
62
|
+
easy = Typhoeus::Easy.new
|
63
|
+
easy.url = "http://localhost:3002/index.html"
|
64
|
+
easy.method = :get
|
65
|
+
easy.request_body = "this is a body!"
|
66
|
+
easy.params = {
|
67
|
+
:foo => 'bar',
|
68
|
+
:username => ['dbalatero', 'dbalatero2']
|
69
|
+
}
|
70
|
+
|
71
|
+
easy.url.should =~ /\?.*username=dbalatero&username=dbalatero2/
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
describe "put" do
|
77
|
+
it "should perform a put" do
|
78
|
+
easy = Typhoeus::Easy.new
|
79
|
+
easy.url = "http://localhost:3002"
|
80
|
+
easy.method = :put
|
81
|
+
easy.perform
|
82
|
+
easy.response_code.should == 200
|
83
|
+
easy.response_body.should include("REQUEST_METHOD=PUT")
|
84
|
+
end
|
85
|
+
|
86
|
+
it "should send a request body" do
|
87
|
+
easy = Typhoeus::Easy.new
|
88
|
+
easy.url = "http://localhost:3002"
|
89
|
+
easy.method = :put
|
90
|
+
easy.request_body = "this is a body!"
|
91
|
+
easy.perform
|
92
|
+
easy.response_code.should == 200
|
93
|
+
easy.response_body.should include("this is a body!")
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
describe "post" do
|
98
|
+
it "should perform a post" do
|
99
|
+
easy = Typhoeus::Easy.new
|
100
|
+
easy.url = "http://localhost:3002"
|
101
|
+
easy.method = :post
|
102
|
+
easy.perform
|
103
|
+
easy.response_code.should == 200
|
104
|
+
easy.response_body.should include("REQUEST_METHOD=POST")
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should send a request body" do
|
108
|
+
easy = Typhoeus::Easy.new
|
109
|
+
easy.url = "http://localhost:3002"
|
110
|
+
easy.method = :post
|
111
|
+
easy.request_body = "this is a body!"
|
112
|
+
easy.perform
|
113
|
+
easy.response_code.should == 200
|
114
|
+
easy.response_body.should include("this is a body!")
|
115
|
+
end
|
116
|
+
|
117
|
+
it "should handle params" do
|
118
|
+
easy = Typhoeus::Easy.new
|
119
|
+
easy.url = "http://localhost:3002"
|
120
|
+
easy.method = :post
|
121
|
+
easy.params = {:foo => "bar"}
|
122
|
+
easy.perform
|
123
|
+
easy.response_code.should == 200
|
124
|
+
easy.response_body.should include("foo=bar")
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
describe "delete" do
|
129
|
+
it "should perform a delete" do
|
130
|
+
easy = Typhoeus::Easy.new
|
131
|
+
easy.url = "http://localhost:3002"
|
132
|
+
easy.method = :delete
|
133
|
+
easy.perform
|
134
|
+
easy.response_code.should == 200
|
135
|
+
easy.response_body.should include("REQUEST_METHOD=DELETE")
|
136
|
+
end
|
137
|
+
|
138
|
+
it "should send a request body" do
|
139
|
+
easy = Typhoeus::Easy.new
|
140
|
+
easy.url = "http://localhost:3002"
|
141
|
+
easy.method = :delete
|
142
|
+
easy.request_body = "this is a body!"
|
143
|
+
easy.perform
|
144
|
+
easy.response_code.should == 200
|
145
|
+
easy.response_body.should include("this is a body!")
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|