typhoeus 0.4.2 → 0.5.0.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/CHANGELOG.md +86 -28
  2. data/Gemfile +17 -1
  3. data/README.md +20 -422
  4. data/Rakefile +21 -12
  5. data/lib/typhoeus.rb +58 -41
  6. data/lib/typhoeus/config.rb +14 -0
  7. data/lib/typhoeus/errors.rb +9 -0
  8. data/lib/typhoeus/errors/no_stub.rb +12 -0
  9. data/lib/typhoeus/errors/typhoeus_error.rb +8 -0
  10. data/lib/typhoeus/expectation.rb +126 -0
  11. data/lib/typhoeus/hydra.rb +31 -236
  12. data/lib/typhoeus/hydras/block_connection.rb +33 -0
  13. data/lib/typhoeus/hydras/easy_factory.rb +67 -0
  14. data/lib/typhoeus/hydras/easy_pool.rb +40 -0
  15. data/lib/typhoeus/hydras/memoizable.rb +53 -0
  16. data/lib/typhoeus/hydras/queueable.rb +46 -0
  17. data/lib/typhoeus/hydras/runnable.rb +18 -0
  18. data/lib/typhoeus/hydras/stubbable.rb +27 -0
  19. data/lib/typhoeus/request.rb +68 -243
  20. data/lib/typhoeus/requests/actions.rb +101 -0
  21. data/lib/typhoeus/requests/block_connection.rb +31 -0
  22. data/lib/typhoeus/requests/callbacks.rb +82 -0
  23. data/lib/typhoeus/requests/marshal.rb +21 -0
  24. data/lib/typhoeus/requests/memoizable.rb +36 -0
  25. data/lib/typhoeus/requests/operations.rb +52 -0
  26. data/lib/typhoeus/requests/responseable.rb +29 -0
  27. data/lib/typhoeus/requests/stubbable.rb +29 -0
  28. data/lib/typhoeus/response.rb +24 -118
  29. data/lib/typhoeus/responses/header.rb +50 -0
  30. data/lib/typhoeus/responses/informations.rb +43 -0
  31. data/lib/typhoeus/responses/legacy.rb +27 -0
  32. data/lib/typhoeus/responses/status.rb +78 -0
  33. data/lib/typhoeus/version.rb +3 -1
  34. metadata +34 -141
  35. data/lib/typhoeus/curl.rb +0 -453
  36. data/lib/typhoeus/easy.rb +0 -115
  37. data/lib/typhoeus/easy/auth.rb +0 -14
  38. data/lib/typhoeus/easy/callbacks.rb +0 -33
  39. data/lib/typhoeus/easy/ffi_helper.rb +0 -61
  40. data/lib/typhoeus/easy/infos.rb +0 -90
  41. data/lib/typhoeus/easy/options.rb +0 -115
  42. data/lib/typhoeus/easy/proxy.rb +0 -20
  43. data/lib/typhoeus/easy/ssl.rb +0 -82
  44. data/lib/typhoeus/filter.rb +0 -28
  45. data/lib/typhoeus/form.rb +0 -61
  46. data/lib/typhoeus/header.rb +0 -54
  47. data/lib/typhoeus/hydra/callbacks.rb +0 -24
  48. data/lib/typhoeus/hydra/connect_options.rb +0 -61
  49. data/lib/typhoeus/hydra/stubbing.rb +0 -68
  50. data/lib/typhoeus/hydra_mock.rb +0 -131
  51. data/lib/typhoeus/multi.rb +0 -146
  52. data/lib/typhoeus/param_processor.rb +0 -43
  53. data/lib/typhoeus/remote.rb +0 -306
  54. data/lib/typhoeus/remote_method.rb +0 -108
  55. data/lib/typhoeus/remote_proxy_object.rb +0 -50
  56. data/lib/typhoeus/utils.rb +0 -50
@@ -0,0 +1,33 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # This module handles the blocked connection request mode on
5
+ # the hydra side, where only stubbed requests
6
+ # are allowed.
7
+ # Connection blocking needs to be turned on:
8
+ # Typhoeus.configure do |config|
9
+ # config.block_connection = true
10
+ # end
11
+ #
12
+ # When trying to do real requests a NoStub error
13
+ # is raised.
14
+ module BlockConnection
15
+
16
+ # Overrides queue in order to check before if block connection
17
+ # is turned on. If thats the case a NoStub error is
18
+ # raised.
19
+ #
20
+ # @example Queue the request.
21
+ # hydra.queue(request)
22
+ #
23
+ # @param [ Request ] request The request to enqueue.
24
+ def queue(request)
25
+ if Typhoeus::Config.block_connection
26
+ raise Typhoeus::Errors::NoStub.new(request)
27
+ else
28
+ super
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,67 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # This is a Factory for easies to be used in the hydra.
5
+ # Before an easy is ready to be added to a multi, it needs
6
+ # to be prepared and the on_complete callback to be set.
7
+ # This is done by this class.
8
+ class EasyFactory
9
+ attr_reader :request, :hydra
10
+
11
+ # Create an easy factory.
12
+ #
13
+ # @example Create easy factory.
14
+ # Typhoeus::Hydras::EasyFactory.new(request, hydra)
15
+ #
16
+ # @param [ Request ] request The request to build an easy for.
17
+ # @param [ Hydra ] hydra The hydra to build an easy for.
18
+ def initialize(request, hydra)
19
+ @request = request
20
+ @hydra = hydra
21
+ end
22
+
23
+ # Return the easy in question.
24
+ #
25
+ # @example Return easy.
26
+ # easy_factory.easy
27
+ #
28
+ # @return [ Ethon::Easy ] The easy.
29
+ def easy
30
+ @easy ||= hydra.get_easy
31
+ end
32
+
33
+ # Fabricated and prepared easy.
34
+ #
35
+ # @example Prepared easy.
36
+ # easy_factory.get
37
+ #
38
+ # @return [ Ethon::Easy ] The prepared easy.
39
+ def get
40
+ easy.http_request(
41
+ request.url,
42
+ request.options.fetch(:method, :get),
43
+ request.options.reject{|k,_| k==:method}
44
+ )
45
+ easy.prepare
46
+ set_callback
47
+ easy
48
+ end
49
+
50
+ # Sets on_complete callback on easy in order to be able to
51
+ # track progress.
52
+ #
53
+ # @example Set callback.
54
+ # easy_factory.set_callback
55
+ #
56
+ # @return [ Ethon::Easy ] The easy.
57
+ def set_callback
58
+ easy.on_complete do |easy|
59
+ request.response = Response.new(easy.to_hash)
60
+ hydra.release_easy(easy)
61
+ hydra.queue(hydra.queued_requests.shift) unless hydra.queued_requests.empty?
62
+ request.execute_callbacks
63
+ end
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,40 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # The easy pool stores already initialized
5
+ # easy handles for future use. This is useful
6
+ # because creating them is quite expensive.
7
+ module EasyPool
8
+
9
+ # Return the easy pool.
10
+ #
11
+ # @example Return easy pool.
12
+ # hydra.easy_pool
13
+ #
14
+ # @return [ Array ] The easy pool.
15
+ def easy_pool
16
+ @easy_pool ||= []
17
+ end
18
+
19
+ # Releases easy into pool. The easy handle is
20
+ # resetted before it gets back in.
21
+ #
22
+ # @example Release easy.
23
+ # hydra.release_easy(easy)
24
+ def release_easy(easy)
25
+ easy.reset
26
+ easy_pool << easy
27
+ end
28
+
29
+ # Return an easy from pool.
30
+ #
31
+ # @example Return easy.
32
+ # hydra.get_easy
33
+ #
34
+ # @return [ Ethon::Easy ] The easy.
35
+ def get_easy
36
+ easy_pool.pop || Ethon::Easy.new
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,53 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # This module handles the GET request memoization
5
+ # on the hydra side. Memoization needs to be turned
6
+ # on:
7
+ # Typhoeus.configure do |config|
8
+ # config.memoize = true
9
+ # end
10
+ module Memoizable
11
+
12
+ # Return the memory.
13
+ #
14
+ # @example Return the memory.
15
+ # hydra.memory
16
+ #
17
+ # @return [ Hash ] The memory.
18
+ def memory
19
+ @memory ||= {}
20
+ end
21
+
22
+ # Overrides queue in order to check before if request
23
+ # is memoizable and already in memory. If thats the case,
24
+ # super is not called, instead the response is set and
25
+ # the on_complete callback called.
26
+ #
27
+ # @example Queue the request.
28
+ # hydra.queue(request)
29
+ #
30
+ # @param [ Request ] request The request to enqueue.
31
+ #
32
+ # @return [ Request ] The queued request.
33
+ def queue(request)
34
+ if request.memoizable? && memory.has_key?(request)
35
+ request.instance_variable_set(:@response, memory[request])
36
+ request.execute_callbacks
37
+ else
38
+ super
39
+ end
40
+ end
41
+
42
+ # Overrides run to make sure the memory is cleared after
43
+ # each run.
44
+ #
45
+ # @example Run hydra.
46
+ # hydra.run
47
+ def run
48
+ super
49
+ memory.clear
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,46 @@
1
+ module Typhoeus
2
+ module Hydras # :nodoc:
3
+
4
+ # This module handles the request queueing on
5
+ # hydra.
6
+ module Queueable
7
+
8
+ # Return the queued requests.
9
+ #
10
+ # @example Return queued requests.
11
+ # hydra.queued_requests
12
+ #
13
+ # @return [ Array ] The queued requests.
14
+ def queued_requests
15
+ @queued_requests ||= []
16
+ end
17
+
18
+ # Abort the current hydra run as good as
19
+ # possible. This means that it only
20
+ # clears the queued requests and can't do
21
+ # anything about already running requests.
22
+ #
23
+ # @example Abort hydra.
24
+ # hydra.abort
25
+ def abort
26
+ queued_requests.clear
27
+ end
28
+
29
+ # Enqueues a request in order to be performed
30
+ # by the hydra. This can even be done while
31
+ # the hydra is running. Also sets hydra on
32
+ # request.
33
+ #
34
+ # @example Queue request.
35
+ # hydra.queue(request)
36
+ def queue(request)
37
+ request.hydra = self
38
+ if multi.easy_handles.size < max_concurrency
39
+ multi.add(Hydras::EasyFactory.new(request, self).get)
40
+ else
41
+ queued_requests << request
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,18 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # This module contains logic to run a hydra.
5
+ module Runnable
6
+
7
+ # Start the hydra run.
8
+ #
9
+ # @example Start hydra run.
10
+ # hydra.run
11
+ #
12
+ # @return [ Symbol ] Return value from multi.perform.
13
+ def run
14
+ multi.perform
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,27 @@
1
+ module Typhoeus
2
+ module Hydras
3
+
4
+ # This module handles stubbing on the hydra side.
5
+ # It plays well with the block_connection configuration,
6
+ # which raises when you make a request which is not stubbed.
7
+ module Stubbable
8
+
9
+ # Override queue in order to check for matching expecations.
10
+ # When an expecation is found, super is not called. Instead a
11
+ # canned response is assigned to the request.
12
+ #
13
+ # @example Queue the request.
14
+ # hydra.queue(request)
15
+ def queue(request)
16
+ if expectation = Expectation.find_by(request)
17
+ request.response = expectation.response
18
+ request.response.mock = true
19
+ request.execute_callbacks
20
+ request.response
21
+ else
22
+ super
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -1,266 +1,91 @@
1
- require 'uri'
1
+ require 'typhoeus/requests/actions'
2
+ require 'typhoeus/requests/block_connection'
3
+ require 'typhoeus/requests/callbacks'
4
+ require 'typhoeus/requests/marshal'
5
+ require 'typhoeus/requests/memoizable'
6
+ require 'typhoeus/requests/operations'
7
+ require 'typhoeus/requests/responseable'
8
+ require 'typhoeus/requests/stubbable'
2
9
 
3
10
  module Typhoeus
4
- class Request
5
- ACCESSOR_OPTIONS = [
6
- :method,
7
- :params,
8
- :body,
9
- :headers,
10
- :cache_key_basis,
11
- :connect_timeout,
12
- :timeout,
13
- :user_agent,
14
- :response,
15
- :cache_timeout,
16
- :follow_location,
17
- :max_redirects,
18
- :proxy,
19
- :proxy_username,
20
- :proxy_password,
21
- :disable_ssl_peer_verification,
22
- :disable_ssl_host_verification,
23
- :interface,
24
- :ssl_cert,
25
- :ssl_cert_type,
26
- :ssl_key,
27
- :ssl_key_type,
28
- :ssl_key_password,
29
- :ssl_cacert,
30
- :ssl_capath,
31
- :ssl_version,
32
- :verbose,
33
- :username,
34
- :password,
35
- :auth_method,
36
- :proxy_auth_method,
37
- :proxy_type
38
- ]
39
-
40
- attr_accessor *ACCESSOR_OPTIONS
41
11
 
42
- # Initialize a new Request
12
+ # This class represents a request.
13
+ class Request
14
+ extend Requests::Actions
15
+ include Requests::Callbacks::Types
16
+ include Requests::Callbacks
17
+ include Requests::Marshal
18
+ include Requests::Operations
19
+ include Requests::Responseable
20
+ include Requests::Memoizable
21
+ include Requests::BlockConnection
22
+ include Requests::Stubbable
23
+
24
+ attr_accessor :options, :url, :hydra, :original_options
25
+
26
+ # Create a new request.
43
27
  #
44
- # Options:
45
- # * +url+ : Endpoint (URL) of the request
46
- # * +options+ : A hash containing options among :
47
- # ** +:method+ : :get (default) / :post / :put
48
- # ** +:params+ : params as a Hash
49
- # ** +:body+
50
- # ** +:timeout+ : timeout (ms)
51
- # ** +:interface+ : interface or ip address (string)
52
- # ** +:connect_timeout+ : connect timeout (ms)
53
- # ** +:headers+ : headers as Hash
54
- # ** +:cache_timeout+ : cache timeout (ms)
55
- # ** +:follow_location
56
- # ** +:max_redirects
57
- # ** +:proxy
58
- # ** +:disable_ssl_peer_verification
59
- # ** +:disable_ssl_host_verification
60
- # ** +:ssl_cert
61
- # ** +:ssl_cert_type
62
- # ** +:ssl_key
63
- # ** +:ssl_key_type
64
- # ** +:ssl_key_password
65
- # ** +:ssl_cacert
66
- # ** +:ssl_capath
67
- # ** +:verbose
68
- # ** +:username
69
- # ** +:password
70
- # ** +:auth_method
28
+ # @example Create a request.
29
+ # Request.new("www.example.com")
71
30
  #
31
+ # @param [ String ] url The url to request.
32
+ # @param [ Hash ] options The options.
33
+ #
34
+ # #return [ Request ] The new request.
72
35
  def initialize(url, options = {})
73
36
  @url = url
74
- @method = options[:method] || :get
75
- @params = options[:params]
76
- @body = options[:body]
77
- @timeout = safe_to_i(options[:timeout])
78
- @connect_timeout = safe_to_i(options[:connect_timeout])
79
- @interface = options[:interface]
80
- @headers = options[:headers] || {}
81
-
82
- @cache_timeout = safe_to_i(options[:cache_timeout])
83
- @follow_location = options[:follow_location]
84
- @max_redirects = options[:max_redirects]
85
- @proxy = options[:proxy]
86
- @proxy_type = options[:proxy_type]
87
- @proxy_username = options[:proxy_username]
88
- @proxy_password = options[:proxy_password]
89
- @proxy_auth_method = options[:proxy_auth_method]
90
- @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
91
- @disable_ssl_host_verification = options[:disable_ssl_host_verification]
92
- @ssl_cert = options[:ssl_cert]
93
- @ssl_cert_type = options[:ssl_cert_type]
94
- @ssl_key = options[:ssl_key]
95
- @ssl_key_type = options[:ssl_key_type]
96
- @ssl_key_password = options[:ssl_key_password]
97
- @ssl_cacert = options[:ssl_cacert]
98
- @ssl_capath = options[:ssl_capath]
99
- @ssl_version = options[:ssl_version]
100
- @verbose = options[:verbose]
101
- @username = options[:username]
102
- @password = options[:password]
103
- @auth_method = options[:auth_method]
104
-
105
- @on_complete = nil
106
- @after_complete = nil
107
- @handled_response = nil
108
- end
109
-
110
- LOCALHOST_ALIASES = %w[ localhost 127.0.0.1 0.0.0.0 ]
111
-
112
- def localhost?
113
- LOCALHOST_ALIASES.include?(parsed_uri.host)
114
- end
115
-
116
- def user_agent
117
- headers['User-Agent']
118
- end
119
-
120
- def url
121
- if [:post, :put].include?(@method)
122
- @url
123
- else
124
- url = "#{@url}?#{params_string}"
125
- url += "&#{URI.escape(@body)}" if @body
126
- url.gsub("?&", "?").gsub(/\?$/, '')
127
- end
128
- end
129
-
130
- def parsed_uri
131
- @parsed_uri ||= URI.parse(@url)
132
- end
37
+ @original_options = options
38
+ @options = options.dup
133
39
 
134
- def host
135
- slash_location = @url.index('/', 8)
136
- if slash_location
137
- @url.slice(0, slash_location)
138
- else
139
- query_string_location = @url.index('?')
140
- return query_string_location ? @url.slice(0, query_string_location) : @url
141
- end
142
- end
143
-
144
- def host_domain
145
- parsed_uri.host
146
- end
147
-
148
- def params_string
149
- return nil unless params
150
- traversal = Typhoeus::Utils.traverse_params_hash(params)
151
- Typhoeus::Utils.traversal_to_param_string(traversal)
152
- end
153
-
154
- def on_complete(&block)
155
- @on_complete = block
156
- end
157
-
158
- def on_complete=(proc)
159
- @on_complete = proc
160
- end
161
-
162
- def after_complete(&block)
163
- @after_complete = block
164
- end
165
-
166
- def after_complete=(proc)
167
- @after_complete = proc
168
- end
169
-
170
- def call_handlers
171
- if @on_complete
172
- @handled_response = @on_complete.call(response)
173
- call_after_complete
174
- end
40
+ set_defaults
175
41
  end
176
42
 
177
- def call_after_complete
178
- @after_complete.call(@handled_response) if @after_complete
179
- end
180
-
181
- def handled_response=(val)
182
- @handled_response = val
43
+ # Returns wether other is equal to self.
44
+ #
45
+ # @example Are request equal?
46
+ # request.eql?(other_request)
47
+ #
48
+ # @param [ Object ] other The object to check.
49
+ #
50
+ # @return [ Boolean ] Returns true if equals, else false.
51
+ def eql?(other)
52
+ self.class == other.class &&
53
+ self.url == other.url &&
54
+ fuzzy_hash_eql?(self.options, other.options)
183
55
  end
184
56
 
185
- def handled_response
186
- @handled_response || response
57
+ # Overrides Object#hash.
58
+ #
59
+ # @return [ Integer ] The integer representing the request.
60
+ def hash
61
+ [ self.class, self.url, self.options ].hash
187
62
  end
188
63
 
189
- def inspect
190
- result = ":method => #{self.method.inspect},\n" <<
191
- "\t:url => #{URI.parse(self.url).to_s}"
192
- if self.body and !self.body.empty?
193
- result << ",\n\t:body => #{self.body.inspect}"
194
- end
64
+ protected
195
65
 
196
- if self.params and !self.params.empty?
197
- result << ",\n\t:params => #{self.params.inspect}"
198
- end
66
+ # Checks if two hashes are equal or not, discarding first-level hash order
67
+ #
68
+ # @param [ Hash ] left
69
+ # @param [ Hash ] right hash to check for equality
70
+ #
71
+ # @return [ Boolean ] Returns true if hashes have same values for same keys and same length,
72
+ # even if the keys are given in a different order.
73
+ def fuzzy_hash_eql?(left, right)
74
+ return true if (left == right)
199
75
 
200
- if self.headers and !self.headers.empty?
201
- result << ",\n\t:headers => #{self.headers.inspect}"
76
+ (left.count == right.count) && left.inject(true) do |res, kvp|
77
+ res && (kvp[1] == right[kvp[0]])
202
78
  end
203
-
204
- result
205
- end
206
-
207
- def cache_key
208
- Digest::SHA1.hexdigest(cache_key_basis || url)
209
- end
210
-
211
- def self.run(url, params)
212
- r = new(url, params)
213
- Typhoeus::Hydra.hydra.queue r
214
- Typhoeus::Hydra.hydra.run
215
- r.response
216
- end
217
-
218
- def self.get(url, params = {})
219
- run(url, params.merge(:method => :get))
220
79
  end
221
80
 
222
- def self.post(url, params = {})
223
- run(url, params.merge(:method => :post))
224
- end
225
-
226
- def self.put(url, params = {})
227
- run(url, params.merge(:method => :put))
228
- end
229
-
230
- def self.delete(url, params = {})
231
- run(url, params.merge(:method => :delete))
232
- end
233
-
234
- def self.head(url, params = {})
235
- run(url, params.merge(:method => :head))
236
- end
237
-
238
- protected
239
-
240
- # Return the important data needed to serialize this Request, except the
241
- # `on_complete` and `after_complete` handlers, since they cannot be
242
- # marshalled.
243
- def marshal_dump
244
- (instance_variables - ['@on_complete', '@after_complete', :@on_complete, :@after_complete]).map do |name|
245
- [name, instance_variable_get(name)]
81
+ # Sets default header and verbose when turned on.
82
+ def set_defaults
83
+ if @options[:headers]
84
+ @options[:headers] = {'User-Agent' => Typhoeus::USER_AGENT}.merge(options[:headers])
85
+ else
86
+ @options[:headers] = {'User-Agent' => Typhoeus::USER_AGENT}
246
87
  end
247
- end
248
-
249
- def marshal_load(attributes)
250
- attributes.each { |name, value| instance_variable_set(name, value) }
251
- end
252
-
253
- def self.options
254
- ACCESSOR_OPTIONS
255
- end
256
-
257
- private
258
-
259
- def safe_to_i(value)
260
- return value if value.is_a?(Fixnum)
261
- return nil if value.nil?
262
- return nil if value.respond_to?(:empty?) && value.empty?
263
- value.to_i
88
+ @options[:verbose] = Typhoeus::Config.verbose if @options[:verbose].nil? && !Typhoeus::Config.verbose.nil?
264
89
  end
265
90
  end
266
91
  end