hhry-typhoeus 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +93 -0
- data/Gemfile +3 -0
- data/LICENSE +22 -0
- data/README.md +455 -0
- data/Rakefile +23 -0
- data/lib/typhoeus.rb +57 -0
- data/lib/typhoeus/curl.rb +453 -0
- data/lib/typhoeus/easy.rb +115 -0
- data/lib/typhoeus/easy/auth.rb +14 -0
- data/lib/typhoeus/easy/callbacks.rb +33 -0
- data/lib/typhoeus/easy/ffi_helper.rb +61 -0
- data/lib/typhoeus/easy/infos.rb +86 -0
- data/lib/typhoeus/easy/options.rb +115 -0
- data/lib/typhoeus/easy/proxy.rb +20 -0
- data/lib/typhoeus/easy/ssl.rb +82 -0
- data/lib/typhoeus/filter.rb +28 -0
- data/lib/typhoeus/form.rb +61 -0
- data/lib/typhoeus/header.rb +54 -0
- data/lib/typhoeus/hydra.rb +246 -0
- data/lib/typhoeus/hydra/callbacks.rb +24 -0
- data/lib/typhoeus/hydra/connect_options.rb +61 -0
- data/lib/typhoeus/hydra/stubbing.rb +68 -0
- data/lib/typhoeus/hydra_mock.rb +131 -0
- data/lib/typhoeus/multi.rb +146 -0
- data/lib/typhoeus/param_processor.rb +43 -0
- data/lib/typhoeus/remote.rb +310 -0
- data/lib/typhoeus/remote_method.rb +108 -0
- data/lib/typhoeus/remote_proxy_object.rb +50 -0
- data/lib/typhoeus/request.rb +278 -0
- data/lib/typhoeus/response.rb +122 -0
- data/lib/typhoeus/utils.rb +58 -0
- data/lib/typhoeus/version.rb +3 -0
- metadata +178 -0
@@ -0,0 +1,82 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
module EasyFu
|
3
|
+
module SSL
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(ClassMethods)
|
6
|
+
end
|
7
|
+
|
8
|
+
module ClassMethods
|
9
|
+
def valid_ssl_version(version)
|
10
|
+
Typhoeus::Easy::SSL_VERSIONS.key?(version.to_sym)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def ssl_version
|
15
|
+
@ssl_version
|
16
|
+
end
|
17
|
+
|
18
|
+
def ssl_version=(version)
|
19
|
+
raise "Invalid SSL version: '#{version}' supplied! Please supply one as listed in Typhoeus::Easy::SSL_VERSIONS" unless self.class.valid_ssl_version(version)
|
20
|
+
@ssl_version = version
|
21
|
+
|
22
|
+
set_option(:sslversion, Typhoeus::Easy::SSL_VERSIONS[version])
|
23
|
+
end
|
24
|
+
|
25
|
+
def disable_ssl_peer_verification
|
26
|
+
set_option(:verifypeer, 0)
|
27
|
+
end
|
28
|
+
|
29
|
+
def disable_ssl_host_verification
|
30
|
+
set_option(:verifyhost, 0)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Set SSL certificate
|
34
|
+
# " The string should be the file name of your certificate. "
|
35
|
+
# The default format is "PEM" and can be changed with ssl_cert_type=
|
36
|
+
def ssl_cert=(cert)
|
37
|
+
set_option(:sslcert, cert)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Set SSL certificate type
|
41
|
+
# " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
|
42
|
+
def ssl_cert_type=(cert_type)
|
43
|
+
raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER p12).include?(cert_type)
|
44
|
+
set_option(:sslcerttype, cert_type)
|
45
|
+
end
|
46
|
+
|
47
|
+
# Set SSL Key file
|
48
|
+
# " The string should be the file name of your private key. "
|
49
|
+
# The default format is "PEM" and can be changed with ssl_key_type=
|
50
|
+
#
|
51
|
+
def ssl_key=(key)
|
52
|
+
set_option(:sslkey, key)
|
53
|
+
end
|
54
|
+
|
55
|
+
# Set SSL Key type
|
56
|
+
# " The string should be the format of your private key. Supported formats are "PEM", "DER" and "ENG". "
|
57
|
+
#
|
58
|
+
def ssl_key_type=(key_type)
|
59
|
+
raise "Invalid ssl key type : '#{key_type}'..." if key_type and !%w(PEM DER ENG).include?(key_type)
|
60
|
+
set_option(:sslkeytype, key_type)
|
61
|
+
end
|
62
|
+
|
63
|
+
def ssl_key_password=(key_password)
|
64
|
+
set_option(:keypasswd, key_password)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Set SSL CACERT
|
68
|
+
# " File holding one or more certificates to verify the peer with. "
|
69
|
+
#
|
70
|
+
def ssl_cacert=(cacert)
|
71
|
+
set_option(:cainfo, cacert)
|
72
|
+
end
|
73
|
+
|
74
|
+
# Set CAPATH
|
75
|
+
# " directory holding multiple CA certificates to verify the peer with. The certificate directory must be prepared using the openssl c_rehash utility. "
|
76
|
+
#
|
77
|
+
def ssl_capath=(capath)
|
78
|
+
set_option(:capath, capath)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
class Filter
|
3
|
+
attr_reader :method_name
|
4
|
+
|
5
|
+
def initialize(method_name, options = {})
|
6
|
+
@method_name = method_name
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def apply_filter?(method_name)
|
11
|
+
if @options[:only]
|
12
|
+
if @options[:only].instance_of? Symbol
|
13
|
+
@options[:only] == method_name
|
14
|
+
else
|
15
|
+
@options[:only].include?(method_name)
|
16
|
+
end
|
17
|
+
elsif @options[:except]
|
18
|
+
if @options[:except].instance_of? Symbol
|
19
|
+
@options[:except] != method_name
|
20
|
+
else
|
21
|
+
!@options[:except].include?(method_name)
|
22
|
+
end
|
23
|
+
else
|
24
|
+
true
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'mime/types'
|
2
|
+
|
3
|
+
module Typhoeus
|
4
|
+
class Form
|
5
|
+
attr_accessor :params
|
6
|
+
attr_reader :first, :traversal
|
7
|
+
|
8
|
+
def initialize(params = {})
|
9
|
+
@params = params
|
10
|
+
@first = ::FFI::MemoryPointer.new(:pointer)
|
11
|
+
@last = ::FFI::MemoryPointer.new(:pointer)
|
12
|
+
|
13
|
+
ObjectSpace.define_finalizer(self, self.class.finalizer(self))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.finalizer(form)
|
17
|
+
proc { Curl.formfree(form.first) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def traversal
|
21
|
+
@traversal ||= Typhoeus::Utils.traverse_params_hash(params)
|
22
|
+
end
|
23
|
+
|
24
|
+
def formadd_param(name, contents)
|
25
|
+
Curl.formadd(@first, @last,
|
26
|
+
:form_option, :copyname, :pointer, name,
|
27
|
+
:form_option, :namelength, :long, Utils.bytesize(name),
|
28
|
+
:form_option, :copycontents, :pointer, contents,
|
29
|
+
:form_option, :contentslength, :long, Utils.bytesize(contents),
|
30
|
+
:form_option, :end)
|
31
|
+
end
|
32
|
+
private :formadd_param
|
33
|
+
|
34
|
+
def formadd_file(name, filename, contenttype, file)
|
35
|
+
Curl.formadd(@first, @last,
|
36
|
+
:form_option, :copyname, :pointer, name,
|
37
|
+
:form_option, :namelength, :long, Utils.bytesize(name),
|
38
|
+
:form_option, :file, :string, file,
|
39
|
+
:form_option, :filename, :string, filename,
|
40
|
+
:form_option, :contenttype, :string, contenttype,
|
41
|
+
:form_option, :end)
|
42
|
+
end
|
43
|
+
private :formadd_file
|
44
|
+
|
45
|
+
def process!
|
46
|
+
# add params
|
47
|
+
traversal[:params].each { |p| formadd_param(p[0], p[1]) }
|
48
|
+
|
49
|
+
# add files
|
50
|
+
traversal[:files].each { |file_args| formadd_file(*file_args) }
|
51
|
+
end
|
52
|
+
|
53
|
+
def multipart?
|
54
|
+
!traversal[:files].empty?
|
55
|
+
end
|
56
|
+
|
57
|
+
def to_s
|
58
|
+
Typhoeus::Utils.traversal_to_param_string(traversal, false)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Typhoeus
|
2
|
+
class Header < ::Hash
|
3
|
+
def initialize(constructor = {})
|
4
|
+
if constructor.is_a?(Hash)
|
5
|
+
super
|
6
|
+
update(constructor)
|
7
|
+
else
|
8
|
+
super(constructor)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def fetch(key, *extras)
|
13
|
+
super(convert_key(key), *extras)
|
14
|
+
end
|
15
|
+
|
16
|
+
def key?(key)
|
17
|
+
super(convert_key(key))
|
18
|
+
end
|
19
|
+
|
20
|
+
def [](key)
|
21
|
+
super(convert_key(key))
|
22
|
+
end
|
23
|
+
|
24
|
+
def []=(key, value)
|
25
|
+
super(convert_key(key), value)
|
26
|
+
end
|
27
|
+
|
28
|
+
def update(other_hash)
|
29
|
+
other_hash.each_pair do |key, value|
|
30
|
+
self[convert_key(key)] = value
|
31
|
+
end
|
32
|
+
self
|
33
|
+
end
|
34
|
+
|
35
|
+
alias_method :merge!, :update
|
36
|
+
|
37
|
+
def dup
|
38
|
+
self.class.new(Marshal.load(Marshal.dump(self)))
|
39
|
+
end
|
40
|
+
|
41
|
+
def merge(hash)
|
42
|
+
self.dup.update(hash)
|
43
|
+
end
|
44
|
+
|
45
|
+
def delete(key)
|
46
|
+
super(convert_key(key))
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
def convert_key(key)
|
51
|
+
key.to_s.split(/_|-/).map { |segment| segment.capitalize }.join("-")
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,246 @@
|
|
1
|
+
require 'typhoeus/hydra/callbacks'
|
2
|
+
require 'typhoeus/hydra/connect_options'
|
3
|
+
require 'typhoeus/hydra/stubbing'
|
4
|
+
|
5
|
+
module Typhoeus
|
6
|
+
class Hydra
|
7
|
+
include ConnectOptions
|
8
|
+
include Stubbing
|
9
|
+
extend Callbacks
|
10
|
+
|
11
|
+
def initialize(options = {})
|
12
|
+
@memoize_requests = true
|
13
|
+
@multi = Multi.new
|
14
|
+
@easy_pool = []
|
15
|
+
initial_pool_size = options[:initial_pool_size] || 10
|
16
|
+
@max_concurrency = options[:max_concurrency] || 200
|
17
|
+
initial_pool_size.times { @easy_pool << Easy.new }
|
18
|
+
@memoized_requests = {}
|
19
|
+
@retrieved_from_cache = {}
|
20
|
+
@queued_requests = []
|
21
|
+
@running_requests = 0
|
22
|
+
|
23
|
+
self.stubs = []
|
24
|
+
@active_stubs = []
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.hydra
|
28
|
+
@hydra ||= new
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.hydra=(val)
|
32
|
+
@hydra = val
|
33
|
+
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Abort the run on a best-effort basis.
|
37
|
+
#
|
38
|
+
# It won't abort the current burst of @max_concurrency requests,
|
39
|
+
# however it won't fire the rest of the queued requests so the run
|
40
|
+
# will be aborted as soon as possible...
|
41
|
+
#
|
42
|
+
def abort
|
43
|
+
@queued_requests.clear
|
44
|
+
end
|
45
|
+
|
46
|
+
def clear_cache_callbacks
|
47
|
+
@cache_setter = nil
|
48
|
+
@cache_getter = nil
|
49
|
+
end
|
50
|
+
|
51
|
+
def fire_and_forget
|
52
|
+
@queued_requests.each {|r| queue(r, false)}
|
53
|
+
@multi.fire_and_forget
|
54
|
+
end
|
55
|
+
|
56
|
+
def queue(request, obey_concurrency_limit = true)
|
57
|
+
return if assign_to_stub(request)
|
58
|
+
|
59
|
+
# At this point, we are running over live HTTP. Make sure we haven't
|
60
|
+
# disabled live requests.
|
61
|
+
check_allow_net_connect!(request)
|
62
|
+
|
63
|
+
if @running_requests >= @max_concurrency && obey_concurrency_limit
|
64
|
+
@queued_requests << request
|
65
|
+
else
|
66
|
+
if request.method == :get
|
67
|
+
if @memoize_requests && @memoized_requests.key?(request.url)
|
68
|
+
if response = @retrieved_from_cache[request.url]
|
69
|
+
request.response = response
|
70
|
+
request.call_handlers
|
71
|
+
else
|
72
|
+
@memoized_requests[request.url] << request
|
73
|
+
end
|
74
|
+
else
|
75
|
+
@memoized_requests[request.url] = [] if @memoize_requests
|
76
|
+
get_from_cache_or_queue(request)
|
77
|
+
end
|
78
|
+
else
|
79
|
+
get_from_cache_or_queue(request)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def run
|
85
|
+
while !@active_stubs.empty?
|
86
|
+
m = @active_stubs.first
|
87
|
+
while request = m.requests.shift
|
88
|
+
response = m.response
|
89
|
+
response.request = request
|
90
|
+
handle_request(request, response)
|
91
|
+
end
|
92
|
+
@active_stubs.delete(m)
|
93
|
+
end
|
94
|
+
|
95
|
+
@multi.perform
|
96
|
+
ensure
|
97
|
+
@multi.reset_easy_handles{|easy| release_easy_object(easy)}
|
98
|
+
@memoized_requests = {}
|
99
|
+
@retrieved_from_cache = {}
|
100
|
+
@running_requests = 0
|
101
|
+
end
|
102
|
+
|
103
|
+
def disable_memoization
|
104
|
+
@memoize_requests = false
|
105
|
+
end
|
106
|
+
|
107
|
+
def cache_getter(&block)
|
108
|
+
@cache_getter = block
|
109
|
+
end
|
110
|
+
|
111
|
+
def cache_setter(&block)
|
112
|
+
@cache_setter = block
|
113
|
+
end
|
114
|
+
|
115
|
+
def on_complete(&block)
|
116
|
+
@on_complete = block
|
117
|
+
end
|
118
|
+
|
119
|
+
def on_complete=(proc)
|
120
|
+
@on_complete = proc
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
def get_from_cache_or_queue(request)
|
126
|
+
if @cache_getter
|
127
|
+
val = @cache_getter.call(request)
|
128
|
+
if val
|
129
|
+
@retrieved_from_cache[request.url] = val
|
130
|
+
queue_next
|
131
|
+
handle_request(request, val, false)
|
132
|
+
else
|
133
|
+
@multi.add(get_easy_object(request))
|
134
|
+
end
|
135
|
+
else
|
136
|
+
@multi.add(get_easy_object(request))
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def get_easy_object(request)
|
141
|
+
@running_requests += 1
|
142
|
+
|
143
|
+
easy = @easy_pool.pop || Easy.new
|
144
|
+
easy.verbose = request.verbose
|
145
|
+
if request.username || request.password
|
146
|
+
auth = { :username => request.username, :password => request.password }
|
147
|
+
auth[:method] = request.auth_method if request.auth_method
|
148
|
+
easy.auth = auth
|
149
|
+
end
|
150
|
+
|
151
|
+
if request.proxy
|
152
|
+
proxy = { :server => request.proxy }
|
153
|
+
proxy[:type] = request.proxy_type if request.proxy_type
|
154
|
+
easy.proxy = proxy if request.proxy
|
155
|
+
end
|
156
|
+
|
157
|
+
if request.proxy_username || request.proxy_password
|
158
|
+
auth = { :username => request.proxy_username, :password => request.proxy_password }
|
159
|
+
auth[:method] = request.proxy_auth_method if request.proxy_auth_method
|
160
|
+
easy.proxy_auth = auth
|
161
|
+
end
|
162
|
+
|
163
|
+
easy.url = request.url
|
164
|
+
easy.method = request.method
|
165
|
+
easy.params = request.params if request.method == :post && request.params.present?
|
166
|
+
easy.headers = request.headers if request.headers
|
167
|
+
easy.request_body = request.body if request.method == :post && request.body.present?
|
168
|
+
easy.timeout = request.timeout if request.timeout
|
169
|
+
easy.connect_timeout = request.connect_timeout if request.connect_timeout
|
170
|
+
easy.interface = request.interface if request.interface
|
171
|
+
easy.follow_location = request.follow_location if request.follow_location
|
172
|
+
easy.max_redirects = request.max_redirects if request.max_redirects
|
173
|
+
easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
|
174
|
+
easy.disable_ssl_host_verification if request.disable_ssl_host_verification
|
175
|
+
easy.ssl_cert = request.ssl_cert
|
176
|
+
easy.ssl_cert_type = request.ssl_cert_type
|
177
|
+
easy.ssl_key = request.ssl_key
|
178
|
+
easy.ssl_key_type = request.ssl_key_type
|
179
|
+
easy.ssl_key_password = request.ssl_key_password
|
180
|
+
easy.ssl_cacert = request.ssl_cacert
|
181
|
+
easy.ssl_capath = request.ssl_capath
|
182
|
+
easy.ssl_version = request.ssl_version || :default
|
183
|
+
easy.verbose = request.verbose
|
184
|
+
|
185
|
+
easy.on_success do |easy|
|
186
|
+
queue_next
|
187
|
+
handle_request(request, response_from_easy(easy, request))
|
188
|
+
release_easy_object(easy)
|
189
|
+
end
|
190
|
+
easy.on_failure do |easy|
|
191
|
+
queue_next
|
192
|
+
handle_request(request, response_from_easy(easy, request))
|
193
|
+
release_easy_object(easy)
|
194
|
+
end
|
195
|
+
easy.set_headers
|
196
|
+
easy
|
197
|
+
end
|
198
|
+
|
199
|
+
def queue_next
|
200
|
+
@running_requests -= 1
|
201
|
+
queue(@queued_requests.shift) unless @queued_requests.empty?
|
202
|
+
end
|
203
|
+
|
204
|
+
def release_easy_object(easy)
|
205
|
+
easy.reset
|
206
|
+
@easy_pool.push easy
|
207
|
+
end
|
208
|
+
|
209
|
+
def handle_request(request, response, live_request = true)
|
210
|
+
request.response = response
|
211
|
+
|
212
|
+
self.class.run_global_hooks_for(:after_request_before_on_complete,
|
213
|
+
request)
|
214
|
+
|
215
|
+
if live_request && request.cache_timeout && @cache_setter
|
216
|
+
@cache_setter.call(request)
|
217
|
+
end
|
218
|
+
@on_complete.call(response) if @on_complete
|
219
|
+
|
220
|
+
request.call_handlers
|
221
|
+
if requests = @memoized_requests[request.url]
|
222
|
+
requests.each do |r|
|
223
|
+
r.response = response
|
224
|
+
r.call_handlers
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def response_from_easy(easy, request)
|
230
|
+
Response.new(:code => easy.response_code,
|
231
|
+
:headers => easy.response_header,
|
232
|
+
:body => easy.response_body,
|
233
|
+
:time => easy.total_time_taken,
|
234
|
+
:start_transfer_time => easy.start_transfer_time,
|
235
|
+
:app_connect_time => easy.app_connect_time,
|
236
|
+
:pretransfer_time => easy.pretransfer_time,
|
237
|
+
:connect_time => easy.connect_time,
|
238
|
+
:name_lookup_time => easy.name_lookup_time,
|
239
|
+
:effective_url => easy.effective_url,
|
240
|
+
:primary_ip => easy.primary_ip,
|
241
|
+
:curl_return_code => easy.curl_return_code,
|
242
|
+
:curl_error_message => easy.curl_error_message,
|
243
|
+
:request => request)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|