typhoeus 0.4.2 → 0.5.0.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +86 -28
- data/Gemfile +17 -1
- data/README.md +20 -422
- data/Rakefile +21 -12
- data/lib/typhoeus.rb +58 -41
- data/lib/typhoeus/config.rb +14 -0
- data/lib/typhoeus/errors.rb +9 -0
- data/lib/typhoeus/errors/no_stub.rb +12 -0
- data/lib/typhoeus/errors/typhoeus_error.rb +8 -0
- data/lib/typhoeus/expectation.rb +126 -0
- data/lib/typhoeus/hydra.rb +31 -236
- data/lib/typhoeus/hydras/block_connection.rb +33 -0
- data/lib/typhoeus/hydras/easy_factory.rb +67 -0
- data/lib/typhoeus/hydras/easy_pool.rb +40 -0
- data/lib/typhoeus/hydras/memoizable.rb +53 -0
- data/lib/typhoeus/hydras/queueable.rb +46 -0
- data/lib/typhoeus/hydras/runnable.rb +18 -0
- data/lib/typhoeus/hydras/stubbable.rb +27 -0
- data/lib/typhoeus/request.rb +68 -243
- data/lib/typhoeus/requests/actions.rb +101 -0
- data/lib/typhoeus/requests/block_connection.rb +31 -0
- data/lib/typhoeus/requests/callbacks.rb +82 -0
- data/lib/typhoeus/requests/marshal.rb +21 -0
- data/lib/typhoeus/requests/memoizable.rb +36 -0
- data/lib/typhoeus/requests/operations.rb +52 -0
- data/lib/typhoeus/requests/responseable.rb +29 -0
- data/lib/typhoeus/requests/stubbable.rb +29 -0
- data/lib/typhoeus/response.rb +24 -118
- data/lib/typhoeus/responses/header.rb +50 -0
- data/lib/typhoeus/responses/informations.rb +43 -0
- data/lib/typhoeus/responses/legacy.rb +27 -0
- data/lib/typhoeus/responses/status.rb +78 -0
- data/lib/typhoeus/version.rb +3 -1
- metadata +34 -141
- data/lib/typhoeus/curl.rb +0 -453
- data/lib/typhoeus/easy.rb +0 -115
- data/lib/typhoeus/easy/auth.rb +0 -14
- data/lib/typhoeus/easy/callbacks.rb +0 -33
- data/lib/typhoeus/easy/ffi_helper.rb +0 -61
- data/lib/typhoeus/easy/infos.rb +0 -90
- data/lib/typhoeus/easy/options.rb +0 -115
- data/lib/typhoeus/easy/proxy.rb +0 -20
- data/lib/typhoeus/easy/ssl.rb +0 -82
- data/lib/typhoeus/filter.rb +0 -28
- data/lib/typhoeus/form.rb +0 -61
- data/lib/typhoeus/header.rb +0 -54
- data/lib/typhoeus/hydra/callbacks.rb +0 -24
- data/lib/typhoeus/hydra/connect_options.rb +0 -61
- data/lib/typhoeus/hydra/stubbing.rb +0 -68
- data/lib/typhoeus/hydra_mock.rb +0 -131
- data/lib/typhoeus/multi.rb +0 -146
- data/lib/typhoeus/param_processor.rb +0 -43
- data/lib/typhoeus/remote.rb +0 -306
- data/lib/typhoeus/remote_method.rb +0 -108
- data/lib/typhoeus/remote_proxy_object.rb +0 -50
- data/lib/typhoeus/utils.rb +0 -50
data/Rakefile
CHANGED
@@ -1,13 +1,18 @@
|
|
1
|
-
|
1
|
+
require "bundler"
|
2
|
+
Bundler.setup
|
2
3
|
|
3
|
-
require
|
4
|
-
|
4
|
+
require "rake"
|
5
|
+
require "rspec/core/rake_task"
|
6
|
+
$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
|
7
|
+
require "typhoeus/version"
|
8
|
+
|
9
|
+
task :gem => :build
|
10
|
+
task :build do
|
11
|
+
system "gem build typhoeus.gemspec"
|
5
12
|
end
|
6
13
|
|
7
|
-
task :install do
|
8
|
-
|
9
|
-
puts `gem build typhoeus.gemspec`
|
10
|
-
puts `gem install typhoeus-*.gem`
|
14
|
+
task :install => :build do
|
15
|
+
system "gem install typhoeus-#{Typhoeus::VERSION}.gem"
|
11
16
|
end
|
12
17
|
|
13
18
|
task :release => :build do
|
@@ -16,14 +21,18 @@ task :release => :build do
|
|
16
21
|
system "gem push typhoeus-#{Typhoeus::VERSION}.gem"
|
17
22
|
end
|
18
23
|
|
24
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
25
|
+
t.verbose = false
|
26
|
+
t.ruby_opts = "-W -I./spec -rspec_helper"
|
27
|
+
end
|
28
|
+
|
19
29
|
desc "Start up the test servers"
|
20
|
-
task :
|
21
|
-
|
30
|
+
task :start do
|
31
|
+
require_relative 'spec/support/boot'
|
22
32
|
begin
|
23
|
-
|
33
|
+
Boot.start_servers(:rake)
|
24
34
|
rescue Exception
|
25
35
|
end
|
26
36
|
end
|
27
37
|
|
28
|
-
|
29
|
-
task :default => [:spec]
|
38
|
+
task :default => :spec
|
data/lib/typhoeus.rb
CHANGED
@@ -1,56 +1,73 @@
|
|
1
1
|
require 'digest/sha2'
|
2
|
+
require 'ethon'
|
2
3
|
|
3
|
-
require 'typhoeus/
|
4
|
-
require 'typhoeus/
|
5
|
-
require 'typhoeus/
|
6
|
-
require 'typhoeus/easy'
|
7
|
-
require 'typhoeus/form'
|
8
|
-
require 'typhoeus/multi'
|
9
|
-
require 'typhoeus/filter'
|
10
|
-
require 'typhoeus/param_processor'
|
11
|
-
require 'typhoeus/remote'
|
12
|
-
require 'typhoeus/remote_proxy_object'
|
13
|
-
require 'typhoeus/response'
|
14
|
-
require 'typhoeus/request'
|
4
|
+
require 'typhoeus/config'
|
5
|
+
require 'typhoeus/errors'
|
6
|
+
require 'typhoeus/expectation'
|
15
7
|
require 'typhoeus/hydra'
|
16
|
-
require 'typhoeus/
|
8
|
+
require 'typhoeus/request'
|
9
|
+
require 'typhoeus/response'
|
17
10
|
require 'typhoeus/version'
|
18
11
|
|
12
|
+
# Typhoeus is a http client library based on Ethon which
|
13
|
+
# wraps libcurl.
|
14
|
+
#
|
15
|
+
# If you want to make a single request, go with:
|
16
|
+
# Typhoeus.get("www.example.com")
|
17
|
+
#
|
18
|
+
# When you looking for firing a bunch of requests automatically
|
19
|
+
# choose the hydra:
|
20
|
+
#
|
21
|
+
# hydra = Typhoeus::Hydra.new
|
22
|
+
# requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
|
23
|
+
# requests.each{ |request| hydra.queue(request) }
|
24
|
+
# hydra.run
|
19
25
|
module Typhoeus
|
20
|
-
|
21
|
-
|
22
|
-
|
26
|
+
extend self
|
27
|
+
extend Hydras::EasyPool
|
28
|
+
extend Requests::Actions
|
29
|
+
extend Requests::Callbacks::Types
|
23
30
|
|
24
|
-
|
25
|
-
|
26
|
-
easy_object_pool << Typhoeus::Easy.new
|
27
|
-
end
|
28
|
-
end
|
31
|
+
# The default typhoeus user agent.
|
32
|
+
USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
|
29
33
|
|
30
|
-
|
31
|
-
|
32
|
-
|
34
|
+
# Set the Typhoeus configuration options by passing a block.
|
35
|
+
#
|
36
|
+
# @example Set the configuration options.
|
37
|
+
# Typhoeus.configure do |config|
|
38
|
+
# config.verbose = true
|
39
|
+
# end
|
40
|
+
#
|
41
|
+
# @return [ Config ] The configuration object.
|
42
|
+
def configure
|
43
|
+
yield Config
|
33
44
|
end
|
34
45
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
46
|
+
# Stub out specific request.
|
47
|
+
#
|
48
|
+
# @example Stub.
|
49
|
+
# Typhoeus.stub("www.example.com").and_return(Typhoeus::Response.new)
|
50
|
+
#
|
51
|
+
# @param [ String ] url The url to stub out.
|
52
|
+
# @param [ Hash ] options The options to stub out.
|
53
|
+
#
|
54
|
+
# @return [ Expection ] The expection.
|
55
|
+
def stub(url, options = {})
|
56
|
+
expectation = Expectation.all.find{ |e| e.url == url && e.options == options }
|
57
|
+
return expectation if expectation
|
42
58
|
|
43
|
-
|
44
|
-
|
45
|
-
|
59
|
+
Expectation.new(url, options).tap do |new_expectation|
|
60
|
+
Expectation.all << new_expectation
|
61
|
+
end
|
46
62
|
end
|
47
63
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
64
|
+
# Execute given block as if block connection is turned off.
|
65
|
+
#
|
66
|
+
# @param [ Block ] block The block to execute.
|
67
|
+
def with_connection
|
68
|
+
old = Config.block_connection
|
69
|
+
Config.block_connection = false
|
70
|
+
yield if block_given?
|
71
|
+
Config.block_connection = old
|
55
72
|
end
|
56
73
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
|
3
|
+
# The Typhoeus configuration used to set global
|
4
|
+
# options. Available options:
|
5
|
+
# * block_connection: only stubbed requests are
|
6
|
+
# allowed, raises NoStub error when trying to
|
7
|
+
# do a real request.
|
8
|
+
# * verbose: show curls debug out
|
9
|
+
# * memoize: memoize GET requests.
|
10
|
+
module Config
|
11
|
+
extend self
|
12
|
+
attr_accessor :block_connection, :memoize, :verbose
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
|
3
|
+
# This class represents an expectation. It is part
|
4
|
+
# of the stubbing mechanism. An expectation contains
|
5
|
+
# an url and options, like a request. They were compared
|
6
|
+
# to the request url and options in order to evaluate
|
7
|
+
# wether they match. If thats the case, the attached
|
8
|
+
# responses were returned one by one.
|
9
|
+
class Expectation
|
10
|
+
attr_reader :url, :options
|
11
|
+
|
12
|
+
class << self
|
13
|
+
|
14
|
+
# Returns all expectations.
|
15
|
+
#
|
16
|
+
# @example Return expectations.
|
17
|
+
# Typhoeus::Expectation.all
|
18
|
+
#
|
19
|
+
# @return [ Array ] The expectations.
|
20
|
+
def all
|
21
|
+
@expectations ||= []
|
22
|
+
end
|
23
|
+
|
24
|
+
# Clears expectations.
|
25
|
+
#
|
26
|
+
# @example Clear expectations.
|
27
|
+
# Typhoeus:;Expectation.clear
|
28
|
+
def clear
|
29
|
+
all.clear
|
30
|
+
end
|
31
|
+
|
32
|
+
# Returns expecation matching the provided
|
33
|
+
# request.
|
34
|
+
#
|
35
|
+
# @example Find expectation.
|
36
|
+
# Typhoeus::Expectation.find_by(request)
|
37
|
+
#
|
38
|
+
# @return [ Expectation ] The matching expectation.
|
39
|
+
def find_by(request)
|
40
|
+
all.find do |expectation|
|
41
|
+
expectation.matches?(request)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Creates an expactation.
|
47
|
+
#
|
48
|
+
# @example Create expactation.
|
49
|
+
# Typhoeus::Expectation.new(url)
|
50
|
+
#
|
51
|
+
# @return [ Expectation ] The created expactation.
|
52
|
+
def initialize(url, options = {})
|
53
|
+
@url = url
|
54
|
+
@options = options
|
55
|
+
@response_counter = 0
|
56
|
+
end
|
57
|
+
|
58
|
+
# Specify what should be returned,
|
59
|
+
# when this expactation is hit.
|
60
|
+
#
|
61
|
+
# @example Add response.
|
62
|
+
# expectation.and_return(response)
|
63
|
+
def and_return(response)
|
64
|
+
responses << response
|
65
|
+
end
|
66
|
+
|
67
|
+
# Checks wether this expectation matches
|
68
|
+
# the provided request.
|
69
|
+
#
|
70
|
+
# @example Check if request matches.
|
71
|
+
# expectation.matches? request
|
72
|
+
#
|
73
|
+
# @param [ Request ] The request to check.
|
74
|
+
#
|
75
|
+
# @return [ Boolean ] True when matches, else false.
|
76
|
+
def matches?(request)
|
77
|
+
url_match?(request.url) &&
|
78
|
+
(options ? options.all?{ |k,v| request.original_options[k] == v } : true)
|
79
|
+
end
|
80
|
+
|
81
|
+
# Return canned responses.
|
82
|
+
#
|
83
|
+
# @example Return responses.
|
84
|
+
# expectation.responses
|
85
|
+
#
|
86
|
+
# @return [ Array ] The responses.
|
87
|
+
def responses
|
88
|
+
@responses ||= []
|
89
|
+
end
|
90
|
+
|
91
|
+
# Return the response. When there are
|
92
|
+
# multiple responses, they were returned one
|
93
|
+
# by one.
|
94
|
+
#
|
95
|
+
# @example Return response.
|
96
|
+
# expectation.response
|
97
|
+
#
|
98
|
+
# @return [ Response ] The response.
|
99
|
+
def response
|
100
|
+
response = responses.fetch(@response_counter, responses.last)
|
101
|
+
@response_counter += 1
|
102
|
+
response
|
103
|
+
end
|
104
|
+
|
105
|
+
private
|
106
|
+
|
107
|
+
# Check wether the url matches the request url.
|
108
|
+
# The url can be a string, regex or nil. String and
|
109
|
+
# regexp were checked, nil is always true. Else false.
|
110
|
+
#
|
111
|
+
# Nil serves as a placeholder in case you want to match
|
112
|
+
# all urls.
|
113
|
+
def url_match?(request_url)
|
114
|
+
case url
|
115
|
+
when String
|
116
|
+
url == request_url
|
117
|
+
when Regexp
|
118
|
+
!!request_url.match(url)
|
119
|
+
when nil
|
120
|
+
true
|
121
|
+
else
|
122
|
+
false
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
data/lib/typhoeus/hydra.rb
CHANGED
@@ -1,247 +1,42 @@
|
|
1
|
-
require 'typhoeus/
|
2
|
-
require 'typhoeus/
|
3
|
-
require 'typhoeus/
|
1
|
+
require 'typhoeus/hydras/block_connection'
|
2
|
+
require 'typhoeus/hydras/easy_factory'
|
3
|
+
require 'typhoeus/hydras/easy_pool'
|
4
|
+
require 'typhoeus/hydras/memoizable'
|
5
|
+
require 'typhoeus/hydras/queueable'
|
6
|
+
require 'typhoeus/hydras/runnable'
|
7
|
+
require 'typhoeus/hydras/stubbable'
|
4
8
|
|
5
9
|
module Typhoeus
|
6
|
-
class Hydra
|
7
|
-
include ConnectOptions
|
8
|
-
include Stubbing
|
9
|
-
extend Callbacks
|
10
|
-
|
11
|
-
def initialize(options = {})
|
12
|
-
@memoize_requests = true
|
13
|
-
@multi = Multi.new
|
14
|
-
@easy_pool = []
|
15
|
-
initial_pool_size = options[:initial_pool_size] || 10
|
16
|
-
@max_concurrency = options[:max_concurrency] || 200
|
17
|
-
initial_pool_size.times { @easy_pool << Easy.new }
|
18
|
-
@memoized_requests = {}
|
19
|
-
@retrieved_from_cache = {}
|
20
|
-
@queued_requests = []
|
21
|
-
@running_requests = 0
|
22
10
|
|
23
|
-
|
24
|
-
|
25
|
-
|
11
|
+
# Hydra manages making parallel HTTP requests. This
|
12
|
+
# is achieved by using libcurls multi interface:
|
13
|
+
# http://curl.haxx.se/libcurl/c/libcurl-multi.html
|
14
|
+
# The benefits are that you don't have to worry running
|
15
|
+
# the requests by yourself.
|
16
|
+
class Hydra
|
17
|
+
include Hydras::EasyPool
|
18
|
+
include Hydras::Queueable
|
19
|
+
include Hydras::Runnable
|
20
|
+
include Hydras::Memoizable
|
21
|
+
include Hydras::BlockConnection
|
22
|
+
include Hydras::Stubbable
|
26
23
|
|
27
|
-
|
28
|
-
@hydra ||= new
|
29
|
-
end
|
30
|
-
|
31
|
-
def self.hydra=(val)
|
32
|
-
@hydra = val
|
33
|
-
end
|
24
|
+
attr_reader :max_concurrency, :multi
|
34
25
|
|
26
|
+
# Create a new hydra.
|
35
27
|
#
|
36
|
-
#
|
28
|
+
# @example Create a hydra.
|
29
|
+
# Typhoeus::Hydra.new
|
37
30
|
#
|
38
|
-
#
|
39
|
-
# however it won't fire the rest of the queued requests so the run
|
40
|
-
# will be aborted as soon as possible...
|
31
|
+
# @param [ Hash ] options The options hash.
|
41
32
|
#
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
@
|
48
|
-
@
|
49
|
-
end
|
50
|
-
|
51
|
-
def fire_and_forget
|
52
|
-
@queued_requests.each {|r| queue(r, false)}
|
53
|
-
@multi.fire_and_forget
|
54
|
-
end
|
55
|
-
|
56
|
-
def queue(request, obey_concurrency_limit = true)
|
57
|
-
return if assign_to_stub(request)
|
58
|
-
|
59
|
-
# At this point, we are running over live HTTP. Make sure we haven't
|
60
|
-
# disabled live requests.
|
61
|
-
check_allow_net_connect!(request)
|
62
|
-
|
63
|
-
if @running_requests >= @max_concurrency && obey_concurrency_limit
|
64
|
-
@queued_requests << request
|
65
|
-
else
|
66
|
-
if request.method == :get
|
67
|
-
if @memoize_requests && @memoized_requests.key?(request.url)
|
68
|
-
if response = @retrieved_from_cache[request.url]
|
69
|
-
request.response = response
|
70
|
-
request.call_handlers
|
71
|
-
else
|
72
|
-
@memoized_requests[request.url] << request
|
73
|
-
end
|
74
|
-
else
|
75
|
-
@memoized_requests[request.url] = [] if @memoize_requests
|
76
|
-
get_from_cache_or_queue(request)
|
77
|
-
end
|
78
|
-
else
|
79
|
-
get_from_cache_or_queue(request)
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
83
|
-
|
84
|
-
def run
|
85
|
-
while !@active_stubs.empty?
|
86
|
-
m = @active_stubs.first
|
87
|
-
while request = m.requests.shift
|
88
|
-
response = m.response
|
89
|
-
response.request = request
|
90
|
-
handle_request(request, response)
|
91
|
-
end
|
92
|
-
@active_stubs.delete(m)
|
93
|
-
end
|
94
|
-
|
95
|
-
@multi.perform
|
96
|
-
ensure
|
97
|
-
@multi.reset_easy_handles{|easy| release_easy_object(easy)}
|
98
|
-
@memoized_requests = {}
|
99
|
-
@retrieved_from_cache = {}
|
100
|
-
@running_requests = 0
|
101
|
-
end
|
102
|
-
|
103
|
-
def disable_memoization
|
104
|
-
@memoize_requests = false
|
105
|
-
end
|
106
|
-
|
107
|
-
def cache_getter(&block)
|
108
|
-
@cache_getter = block
|
109
|
-
end
|
110
|
-
|
111
|
-
def cache_setter(&block)
|
112
|
-
@cache_setter = block
|
113
|
-
end
|
114
|
-
|
115
|
-
def on_complete(&block)
|
116
|
-
@on_complete = block
|
117
|
-
end
|
118
|
-
|
119
|
-
def on_complete=(proc)
|
120
|
-
@on_complete = proc
|
121
|
-
end
|
122
|
-
|
123
|
-
private
|
124
|
-
|
125
|
-
def get_from_cache_or_queue(request)
|
126
|
-
if @cache_getter
|
127
|
-
val = @cache_getter.call(request)
|
128
|
-
if val
|
129
|
-
@retrieved_from_cache[request.url] = val
|
130
|
-
queue_next
|
131
|
-
handle_request(request, val, false)
|
132
|
-
else
|
133
|
-
@multi.add(get_easy_object(request))
|
134
|
-
end
|
135
|
-
else
|
136
|
-
@multi.add(get_easy_object(request))
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
def get_easy_object(request)
|
141
|
-
@running_requests += 1
|
142
|
-
|
143
|
-
easy = @easy_pool.pop || Easy.new
|
144
|
-
easy.verbose = request.verbose
|
145
|
-
if request.username || request.password
|
146
|
-
auth = { :username => request.username, :password => request.password }
|
147
|
-
auth[:method] = request.auth_method if request.auth_method
|
148
|
-
easy.auth = auth
|
149
|
-
end
|
150
|
-
|
151
|
-
if request.proxy
|
152
|
-
proxy = { :server => request.proxy }
|
153
|
-
proxy[:type] = request.proxy_type if request.proxy_type
|
154
|
-
easy.proxy = proxy if request.proxy
|
155
|
-
end
|
156
|
-
|
157
|
-
if request.proxy_username || request.proxy_password
|
158
|
-
auth = { :username => request.proxy_username, :password => request.proxy_password }
|
159
|
-
auth[:method] = request.proxy_auth_method if request.proxy_auth_method
|
160
|
-
easy.proxy_auth = auth
|
161
|
-
end
|
162
|
-
|
163
|
-
easy.url = request.url
|
164
|
-
easy.method = request.method
|
165
|
-
easy.params = request.params if [:post, :put].include?(request.method) && !request.params.nil?
|
166
|
-
easy.headers = request.headers if request.headers
|
167
|
-
easy.request_body = request.body if [:post, :put].include?(request.method) && !request.body.nil?
|
168
|
-
easy.timeout = request.timeout if request.timeout
|
169
|
-
easy.connect_timeout = request.connect_timeout if request.connect_timeout
|
170
|
-
easy.interface = request.interface if request.interface
|
171
|
-
easy.follow_location = request.follow_location if request.follow_location
|
172
|
-
easy.max_redirects = request.max_redirects if request.max_redirects
|
173
|
-
easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
|
174
|
-
easy.disable_ssl_host_verification if request.disable_ssl_host_verification
|
175
|
-
easy.ssl_cert = request.ssl_cert
|
176
|
-
easy.ssl_cert_type = request.ssl_cert_type
|
177
|
-
easy.ssl_key = request.ssl_key
|
178
|
-
easy.ssl_key_type = request.ssl_key_type
|
179
|
-
easy.ssl_key_password = request.ssl_key_password
|
180
|
-
easy.ssl_cacert = request.ssl_cacert
|
181
|
-
easy.ssl_capath = request.ssl_capath
|
182
|
-
easy.ssl_version = request.ssl_version || :default
|
183
|
-
easy.verbose = request.verbose
|
184
|
-
|
185
|
-
easy.on_success do |easy|
|
186
|
-
queue_next
|
187
|
-
handle_request(request, response_from_easy(easy, request))
|
188
|
-
release_easy_object(easy)
|
189
|
-
end
|
190
|
-
easy.on_failure do |easy|
|
191
|
-
queue_next
|
192
|
-
handle_request(request, response_from_easy(easy, request))
|
193
|
-
release_easy_object(easy)
|
194
|
-
end
|
195
|
-
easy.set_headers
|
196
|
-
easy
|
197
|
-
end
|
198
|
-
|
199
|
-
def queue_next
|
200
|
-
@running_requests -= 1
|
201
|
-
queue(@queued_requests.shift) unless @queued_requests.empty?
|
202
|
-
end
|
203
|
-
|
204
|
-
def release_easy_object(easy)
|
205
|
-
easy.reset
|
206
|
-
@easy_pool.push easy
|
207
|
-
end
|
208
|
-
|
209
|
-
def handle_request(request, response, live_request = true)
|
210
|
-
request.response = response
|
211
|
-
|
212
|
-
self.class.run_global_hooks_for(:after_request_before_on_complete,
|
213
|
-
request)
|
214
|
-
|
215
|
-
if live_request && request.cache_timeout && @cache_setter
|
216
|
-
@cache_setter.call(request)
|
217
|
-
end
|
218
|
-
@on_complete.call(response) if @on_complete
|
219
|
-
|
220
|
-
request.call_handlers
|
221
|
-
if requests = @memoized_requests[request.url]
|
222
|
-
requests.each do |r|
|
223
|
-
r.response = response
|
224
|
-
r.call_handlers
|
225
|
-
end
|
226
|
-
end
|
227
|
-
end
|
228
|
-
|
229
|
-
def response_from_easy(easy, request)
|
230
|
-
Response.new(:code => easy.response_code,
|
231
|
-
:headers => easy.response_header,
|
232
|
-
:body => easy.response_body,
|
233
|
-
:time => easy.total_time_taken,
|
234
|
-
:start_transfer_time => easy.start_transfer_time,
|
235
|
-
:app_connect_time => easy.app_connect_time,
|
236
|
-
:pretransfer_time => easy.pretransfer_time,
|
237
|
-
:connect_time => easy.connect_time,
|
238
|
-
:name_lookup_time => easy.name_lookup_time,
|
239
|
-
:effective_url => easy.effective_url,
|
240
|
-
:primary_ip => easy.primary_ip,
|
241
|
-
:curl_return_code => easy.curl_return_code,
|
242
|
-
:curl_error_message => easy.curl_error_message,
|
243
|
-
:redirect_count => easy.redirect_count,
|
244
|
-
:request => request)
|
33
|
+
# @option options :max_concurrency [ Integer ] Number
|
34
|
+
# of max concurrent connections to create. Default is
|
35
|
+
# 200.
|
36
|
+
def initialize(options = {})
|
37
|
+
@options = options
|
38
|
+
@max_concurrency = @options.fetch(:max_concurrency, 200)
|
39
|
+
@multi = Ethon::Multi.new
|
245
40
|
end
|
246
41
|
end
|
247
42
|
end
|