typhoeus 0.4.2 → 0.5.0.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. data/CHANGELOG.md +86 -28
  2. data/Gemfile +17 -1
  3. data/README.md +20 -422
  4. data/Rakefile +21 -12
  5. data/lib/typhoeus.rb +58 -41
  6. data/lib/typhoeus/config.rb +14 -0
  7. data/lib/typhoeus/errors.rb +9 -0
  8. data/lib/typhoeus/errors/no_stub.rb +12 -0
  9. data/lib/typhoeus/errors/typhoeus_error.rb +8 -0
  10. data/lib/typhoeus/expectation.rb +126 -0
  11. data/lib/typhoeus/hydra.rb +31 -236
  12. data/lib/typhoeus/hydras/block_connection.rb +33 -0
  13. data/lib/typhoeus/hydras/easy_factory.rb +67 -0
  14. data/lib/typhoeus/hydras/easy_pool.rb +40 -0
  15. data/lib/typhoeus/hydras/memoizable.rb +53 -0
  16. data/lib/typhoeus/hydras/queueable.rb +46 -0
  17. data/lib/typhoeus/hydras/runnable.rb +18 -0
  18. data/lib/typhoeus/hydras/stubbable.rb +27 -0
  19. data/lib/typhoeus/request.rb +68 -243
  20. data/lib/typhoeus/requests/actions.rb +101 -0
  21. data/lib/typhoeus/requests/block_connection.rb +31 -0
  22. data/lib/typhoeus/requests/callbacks.rb +82 -0
  23. data/lib/typhoeus/requests/marshal.rb +21 -0
  24. data/lib/typhoeus/requests/memoizable.rb +36 -0
  25. data/lib/typhoeus/requests/operations.rb +52 -0
  26. data/lib/typhoeus/requests/responseable.rb +29 -0
  27. data/lib/typhoeus/requests/stubbable.rb +29 -0
  28. data/lib/typhoeus/response.rb +24 -118
  29. data/lib/typhoeus/responses/header.rb +50 -0
  30. data/lib/typhoeus/responses/informations.rb +43 -0
  31. data/lib/typhoeus/responses/legacy.rb +27 -0
  32. data/lib/typhoeus/responses/status.rb +78 -0
  33. data/lib/typhoeus/version.rb +3 -1
  34. metadata +34 -141
  35. data/lib/typhoeus/curl.rb +0 -453
  36. data/lib/typhoeus/easy.rb +0 -115
  37. data/lib/typhoeus/easy/auth.rb +0 -14
  38. data/lib/typhoeus/easy/callbacks.rb +0 -33
  39. data/lib/typhoeus/easy/ffi_helper.rb +0 -61
  40. data/lib/typhoeus/easy/infos.rb +0 -90
  41. data/lib/typhoeus/easy/options.rb +0 -115
  42. data/lib/typhoeus/easy/proxy.rb +0 -20
  43. data/lib/typhoeus/easy/ssl.rb +0 -82
  44. data/lib/typhoeus/filter.rb +0 -28
  45. data/lib/typhoeus/form.rb +0 -61
  46. data/lib/typhoeus/header.rb +0 -54
  47. data/lib/typhoeus/hydra/callbacks.rb +0 -24
  48. data/lib/typhoeus/hydra/connect_options.rb +0 -61
  49. data/lib/typhoeus/hydra/stubbing.rb +0 -68
  50. data/lib/typhoeus/hydra_mock.rb +0 -131
  51. data/lib/typhoeus/multi.rb +0 -146
  52. data/lib/typhoeus/param_processor.rb +0 -43
  53. data/lib/typhoeus/remote.rb +0 -306
  54. data/lib/typhoeus/remote_method.rb +0 -108
  55. data/lib/typhoeus/remote_proxy_object.rb +0 -50
  56. data/lib/typhoeus/utils.rb +0 -50
data/Rakefile CHANGED
@@ -1,13 +1,18 @@
1
- $LOAD_PATH.unshift(File.dirname(__FILE__))
1
+ require "bundler"
2
+ Bundler.setup
2
3
 
3
- require 'rspec/core/rake_task'
4
- RSpec::Core::RakeTask.new do |t|
4
+ require "rake"
5
+ require "rspec/core/rake_task"
6
+ $LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
7
+ require "typhoeus/version"
8
+
9
+ task :gem => :build
10
+ task :build do
11
+ system "gem build typhoeus.gemspec"
5
12
  end
6
13
 
7
- task :install do
8
- rm_rf "*.gem"
9
- puts `gem build typhoeus.gemspec`
10
- puts `gem install typhoeus-*.gem`
14
+ task :install => :build do
15
+ system "gem install typhoeus-#{Typhoeus::VERSION}.gem"
11
16
  end
12
17
 
13
18
  task :release => :build do
@@ -16,14 +21,18 @@ task :release => :build do
16
21
  system "gem push typhoeus-#{Typhoeus::VERSION}.gem"
17
22
  end
18
23
 
24
+ RSpec::Core::RakeTask.new(:spec) do |t|
25
+ t.verbose = false
26
+ t.ruby_opts = "-W -I./spec -rspec_helper"
27
+ end
28
+
19
29
  desc "Start up the test servers"
20
- task :start_test_servers do
21
- require 'spec/support/typhoeus_localhost_server'
30
+ task :start do
31
+ require_relative 'spec/support/boot'
22
32
  begin
23
- TyphoeusLocalhostServer.start_servers!(:rake)
33
+ Boot.start_servers(:rake)
24
34
  rescue Exception
25
35
  end
26
36
  end
27
37
 
28
- desc "Build Typhoeus and run all the tests."
29
- task :default => [:spec]
38
+ task :default => :spec
@@ -1,56 +1,73 @@
1
1
  require 'digest/sha2'
2
+ require 'ethon'
2
3
 
3
- require 'typhoeus/utils'
4
- require 'typhoeus/header'
5
- require 'typhoeus/curl'
6
- require 'typhoeus/easy'
7
- require 'typhoeus/form'
8
- require 'typhoeus/multi'
9
- require 'typhoeus/filter'
10
- require 'typhoeus/param_processor'
11
- require 'typhoeus/remote'
12
- require 'typhoeus/remote_proxy_object'
13
- require 'typhoeus/response'
14
- require 'typhoeus/request'
4
+ require 'typhoeus/config'
5
+ require 'typhoeus/errors'
6
+ require 'typhoeus/expectation'
15
7
  require 'typhoeus/hydra'
16
- require 'typhoeus/hydra_mock'
8
+ require 'typhoeus/request'
9
+ require 'typhoeus/response'
17
10
  require 'typhoeus/version'
18
11
 
12
+ # Typhoeus is a http client library based on Ethon which
13
+ # wraps libcurl.
14
+ #
15
+ # If you want to make a single request, go with:
16
+ # Typhoeus.get("www.example.com")
17
+ #
18
+ # When you looking for firing a bunch of requests automatically
19
+ # choose the hydra:
20
+ #
21
+ # hydra = Typhoeus::Hydra.new
22
+ # requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
23
+ # requests.each{ |request| hydra.queue(request) }
24
+ # hydra.run
19
25
  module Typhoeus
20
- def self.easy_object_pool
21
- @easy_objects ||= []
22
- end
26
+ extend self
27
+ extend Hydras::EasyPool
28
+ extend Requests::Actions
29
+ extend Requests::Callbacks::Types
23
30
 
24
- def self.init_easy_object_pool
25
- 20.times do
26
- easy_object_pool << Typhoeus::Easy.new
27
- end
28
- end
31
+ # The default typhoeus user agent.
32
+ USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
29
33
 
30
- def self.release_easy_object(easy)
31
- easy.reset
32
- easy_object_pool << easy
34
+ # Set the Typhoeus configuration options by passing a block.
35
+ #
36
+ # @example Set the configuration options.
37
+ # Typhoeus.configure do |config|
38
+ # config.verbose = true
39
+ # end
40
+ #
41
+ # @return [ Config ] The configuration object.
42
+ def configure
43
+ yield Config
33
44
  end
34
45
 
35
- def self.get_easy_object
36
- if easy_object_pool.empty?
37
- Typhoeus::Easy.new
38
- else
39
- easy_object_pool.pop
40
- end
41
- end
46
+ # Stub out specific request.
47
+ #
48
+ # @example Stub.
49
+ # Typhoeus.stub("www.example.com").and_return(Typhoeus::Response.new)
50
+ #
51
+ # @param [ String ] url The url to stub out.
52
+ # @param [ Hash ] options The options to stub out.
53
+ #
54
+ # @return [ Expection ] The expection.
55
+ def stub(url, options = {})
56
+ expectation = Expectation.all.find{ |e| e.url == url && e.options == options }
57
+ return expectation if expectation
42
58
 
43
- def self.add_easy_request(easy_object)
44
- Thread.current[:curl_multi] ||= Typhoeus::Multi.new
45
- Thread.current[:curl_multi].add(easy_object)
59
+ Expectation.new(url, options).tap do |new_expectation|
60
+ Expectation.all << new_expectation
61
+ end
46
62
  end
47
63
 
48
- def self.perform_easy_requests
49
- multi = Thread.current[:curl_multi]
50
- start_time = Time.now
51
- multi.easy_handles.each do |easy|
52
- easy.start_time = start_time
53
- end
54
- multi.perform
64
+ # Execute given block as if block connection is turned off.
65
+ #
66
+ # @param [ Block ] block The block to execute.
67
+ def with_connection
68
+ old = Config.block_connection
69
+ Config.block_connection = false
70
+ yield if block_given?
71
+ Config.block_connection = old
55
72
  end
56
73
  end
@@ -0,0 +1,14 @@
1
+ module Typhoeus
2
+
3
+ # The Typhoeus configuration used to set global
4
+ # options. Available options:
5
+ # * block_connection: only stubbed requests are
6
+ # allowed, raises NoStub error when trying to
7
+ # do a real request.
8
+ # * verbose: show curls debug out
9
+ # * memoize: memoize GET requests.
10
+ module Config
11
+ extend self
12
+ attr_accessor :block_connection, :memoize, :verbose
13
+ end
14
+ end
@@ -0,0 +1,9 @@
1
+ require 'typhoeus/errors/typhoeus_error'
2
+ require 'typhoeus/errors/no_stub'
3
+
4
+ module Typhoeus
5
+
6
+ # This namespace contains all errors raised by typhoeus.
7
+ module Errors
8
+ end
9
+ end
@@ -0,0 +1,12 @@
1
+ module Typhoeus
2
+ module Errors
3
+
4
+ # Raises when block connection is turned on
5
+ # and making a real request.
6
+ class NoStub < TyphoeusError
7
+ def initialize(request)
8
+ super("The connection is blocked and no stub defined.")
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,8 @@
1
+ module Typhoeus
2
+ module Errors
3
+
4
+ # Default typhoeus error class for all custom errors.
5
+ class TyphoeusError < StandardError
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,126 @@
1
+ module Typhoeus
2
+
3
+ # This class represents an expectation. It is part
4
+ # of the stubbing mechanism. An expectation contains
5
+ # an url and options, like a request. They were compared
6
+ # to the request url and options in order to evaluate
7
+ # wether they match. If thats the case, the attached
8
+ # responses were returned one by one.
9
+ class Expectation
10
+ attr_reader :url, :options
11
+
12
+ class << self
13
+
14
+ # Returns all expectations.
15
+ #
16
+ # @example Return expectations.
17
+ # Typhoeus::Expectation.all
18
+ #
19
+ # @return [ Array ] The expectations.
20
+ def all
21
+ @expectations ||= []
22
+ end
23
+
24
+ # Clears expectations.
25
+ #
26
+ # @example Clear expectations.
27
+ # Typhoeus:;Expectation.clear
28
+ def clear
29
+ all.clear
30
+ end
31
+
32
+ # Returns expecation matching the provided
33
+ # request.
34
+ #
35
+ # @example Find expectation.
36
+ # Typhoeus::Expectation.find_by(request)
37
+ #
38
+ # @return [ Expectation ] The matching expectation.
39
+ def find_by(request)
40
+ all.find do |expectation|
41
+ expectation.matches?(request)
42
+ end
43
+ end
44
+ end
45
+
46
+ # Creates an expactation.
47
+ #
48
+ # @example Create expactation.
49
+ # Typhoeus::Expectation.new(url)
50
+ #
51
+ # @return [ Expectation ] The created expactation.
52
+ def initialize(url, options = {})
53
+ @url = url
54
+ @options = options
55
+ @response_counter = 0
56
+ end
57
+
58
+ # Specify what should be returned,
59
+ # when this expactation is hit.
60
+ #
61
+ # @example Add response.
62
+ # expectation.and_return(response)
63
+ def and_return(response)
64
+ responses << response
65
+ end
66
+
67
+ # Checks wether this expectation matches
68
+ # the provided request.
69
+ #
70
+ # @example Check if request matches.
71
+ # expectation.matches? request
72
+ #
73
+ # @param [ Request ] The request to check.
74
+ #
75
+ # @return [ Boolean ] True when matches, else false.
76
+ def matches?(request)
77
+ url_match?(request.url) &&
78
+ (options ? options.all?{ |k,v| request.original_options[k] == v } : true)
79
+ end
80
+
81
+ # Return canned responses.
82
+ #
83
+ # @example Return responses.
84
+ # expectation.responses
85
+ #
86
+ # @return [ Array ] The responses.
87
+ def responses
88
+ @responses ||= []
89
+ end
90
+
91
+ # Return the response. When there are
92
+ # multiple responses, they were returned one
93
+ # by one.
94
+ #
95
+ # @example Return response.
96
+ # expectation.response
97
+ #
98
+ # @return [ Response ] The response.
99
+ def response
100
+ response = responses.fetch(@response_counter, responses.last)
101
+ @response_counter += 1
102
+ response
103
+ end
104
+
105
+ private
106
+
107
+ # Check wether the url matches the request url.
108
+ # The url can be a string, regex or nil. String and
109
+ # regexp were checked, nil is always true. Else false.
110
+ #
111
+ # Nil serves as a placeholder in case you want to match
112
+ # all urls.
113
+ def url_match?(request_url)
114
+ case url
115
+ when String
116
+ url == request_url
117
+ when Regexp
118
+ !!request_url.match(url)
119
+ when nil
120
+ true
121
+ else
122
+ false
123
+ end
124
+ end
125
+ end
126
+ end
@@ -1,247 +1,42 @@
1
- require 'typhoeus/hydra/callbacks'
2
- require 'typhoeus/hydra/connect_options'
3
- require 'typhoeus/hydra/stubbing'
1
+ require 'typhoeus/hydras/block_connection'
2
+ require 'typhoeus/hydras/easy_factory'
3
+ require 'typhoeus/hydras/easy_pool'
4
+ require 'typhoeus/hydras/memoizable'
5
+ require 'typhoeus/hydras/queueable'
6
+ require 'typhoeus/hydras/runnable'
7
+ require 'typhoeus/hydras/stubbable'
4
8
 
5
9
  module Typhoeus
6
- class Hydra
7
- include ConnectOptions
8
- include Stubbing
9
- extend Callbacks
10
-
11
- def initialize(options = {})
12
- @memoize_requests = true
13
- @multi = Multi.new
14
- @easy_pool = []
15
- initial_pool_size = options[:initial_pool_size] || 10
16
- @max_concurrency = options[:max_concurrency] || 200
17
- initial_pool_size.times { @easy_pool << Easy.new }
18
- @memoized_requests = {}
19
- @retrieved_from_cache = {}
20
- @queued_requests = []
21
- @running_requests = 0
22
10
 
23
- self.stubs = []
24
- @active_stubs = []
25
- end
11
+ # Hydra manages making parallel HTTP requests. This
12
+ # is achieved by using libcurls multi interface:
13
+ # http://curl.haxx.se/libcurl/c/libcurl-multi.html
14
+ # The benefits are that you don't have to worry running
15
+ # the requests by yourself.
16
+ class Hydra
17
+ include Hydras::EasyPool
18
+ include Hydras::Queueable
19
+ include Hydras::Runnable
20
+ include Hydras::Memoizable
21
+ include Hydras::BlockConnection
22
+ include Hydras::Stubbable
26
23
 
27
- def self.hydra
28
- @hydra ||= new
29
- end
30
-
31
- def self.hydra=(val)
32
- @hydra = val
33
- end
24
+ attr_reader :max_concurrency, :multi
34
25
 
26
+ # Create a new hydra.
35
27
  #
36
- # Abort the run on a best-effort basis.
28
+ # @example Create a hydra.
29
+ # Typhoeus::Hydra.new
37
30
  #
38
- # It won't abort the current burst of @max_concurrency requests,
39
- # however it won't fire the rest of the queued requests so the run
40
- # will be aborted as soon as possible...
31
+ # @param [ Hash ] options The options hash.
41
32
  #
42
- def abort
43
- @queued_requests.clear
44
- end
45
-
46
- def clear_cache_callbacks
47
- @cache_setter = nil
48
- @cache_getter = nil
49
- end
50
-
51
- def fire_and_forget
52
- @queued_requests.each {|r| queue(r, false)}
53
- @multi.fire_and_forget
54
- end
55
-
56
- def queue(request, obey_concurrency_limit = true)
57
- return if assign_to_stub(request)
58
-
59
- # At this point, we are running over live HTTP. Make sure we haven't
60
- # disabled live requests.
61
- check_allow_net_connect!(request)
62
-
63
- if @running_requests >= @max_concurrency && obey_concurrency_limit
64
- @queued_requests << request
65
- else
66
- if request.method == :get
67
- if @memoize_requests && @memoized_requests.key?(request.url)
68
- if response = @retrieved_from_cache[request.url]
69
- request.response = response
70
- request.call_handlers
71
- else
72
- @memoized_requests[request.url] << request
73
- end
74
- else
75
- @memoized_requests[request.url] = [] if @memoize_requests
76
- get_from_cache_or_queue(request)
77
- end
78
- else
79
- get_from_cache_or_queue(request)
80
- end
81
- end
82
- end
83
-
84
- def run
85
- while !@active_stubs.empty?
86
- m = @active_stubs.first
87
- while request = m.requests.shift
88
- response = m.response
89
- response.request = request
90
- handle_request(request, response)
91
- end
92
- @active_stubs.delete(m)
93
- end
94
-
95
- @multi.perform
96
- ensure
97
- @multi.reset_easy_handles{|easy| release_easy_object(easy)}
98
- @memoized_requests = {}
99
- @retrieved_from_cache = {}
100
- @running_requests = 0
101
- end
102
-
103
- def disable_memoization
104
- @memoize_requests = false
105
- end
106
-
107
- def cache_getter(&block)
108
- @cache_getter = block
109
- end
110
-
111
- def cache_setter(&block)
112
- @cache_setter = block
113
- end
114
-
115
- def on_complete(&block)
116
- @on_complete = block
117
- end
118
-
119
- def on_complete=(proc)
120
- @on_complete = proc
121
- end
122
-
123
- private
124
-
125
- def get_from_cache_or_queue(request)
126
- if @cache_getter
127
- val = @cache_getter.call(request)
128
- if val
129
- @retrieved_from_cache[request.url] = val
130
- queue_next
131
- handle_request(request, val, false)
132
- else
133
- @multi.add(get_easy_object(request))
134
- end
135
- else
136
- @multi.add(get_easy_object(request))
137
- end
138
- end
139
-
140
- def get_easy_object(request)
141
- @running_requests += 1
142
-
143
- easy = @easy_pool.pop || Easy.new
144
- easy.verbose = request.verbose
145
- if request.username || request.password
146
- auth = { :username => request.username, :password => request.password }
147
- auth[:method] = request.auth_method if request.auth_method
148
- easy.auth = auth
149
- end
150
-
151
- if request.proxy
152
- proxy = { :server => request.proxy }
153
- proxy[:type] = request.proxy_type if request.proxy_type
154
- easy.proxy = proxy if request.proxy
155
- end
156
-
157
- if request.proxy_username || request.proxy_password
158
- auth = { :username => request.proxy_username, :password => request.proxy_password }
159
- auth[:method] = request.proxy_auth_method if request.proxy_auth_method
160
- easy.proxy_auth = auth
161
- end
162
-
163
- easy.url = request.url
164
- easy.method = request.method
165
- easy.params = request.params if [:post, :put].include?(request.method) && !request.params.nil?
166
- easy.headers = request.headers if request.headers
167
- easy.request_body = request.body if [:post, :put].include?(request.method) && !request.body.nil?
168
- easy.timeout = request.timeout if request.timeout
169
- easy.connect_timeout = request.connect_timeout if request.connect_timeout
170
- easy.interface = request.interface if request.interface
171
- easy.follow_location = request.follow_location if request.follow_location
172
- easy.max_redirects = request.max_redirects if request.max_redirects
173
- easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
174
- easy.disable_ssl_host_verification if request.disable_ssl_host_verification
175
- easy.ssl_cert = request.ssl_cert
176
- easy.ssl_cert_type = request.ssl_cert_type
177
- easy.ssl_key = request.ssl_key
178
- easy.ssl_key_type = request.ssl_key_type
179
- easy.ssl_key_password = request.ssl_key_password
180
- easy.ssl_cacert = request.ssl_cacert
181
- easy.ssl_capath = request.ssl_capath
182
- easy.ssl_version = request.ssl_version || :default
183
- easy.verbose = request.verbose
184
-
185
- easy.on_success do |easy|
186
- queue_next
187
- handle_request(request, response_from_easy(easy, request))
188
- release_easy_object(easy)
189
- end
190
- easy.on_failure do |easy|
191
- queue_next
192
- handle_request(request, response_from_easy(easy, request))
193
- release_easy_object(easy)
194
- end
195
- easy.set_headers
196
- easy
197
- end
198
-
199
- def queue_next
200
- @running_requests -= 1
201
- queue(@queued_requests.shift) unless @queued_requests.empty?
202
- end
203
-
204
- def release_easy_object(easy)
205
- easy.reset
206
- @easy_pool.push easy
207
- end
208
-
209
- def handle_request(request, response, live_request = true)
210
- request.response = response
211
-
212
- self.class.run_global_hooks_for(:after_request_before_on_complete,
213
- request)
214
-
215
- if live_request && request.cache_timeout && @cache_setter
216
- @cache_setter.call(request)
217
- end
218
- @on_complete.call(response) if @on_complete
219
-
220
- request.call_handlers
221
- if requests = @memoized_requests[request.url]
222
- requests.each do |r|
223
- r.response = response
224
- r.call_handlers
225
- end
226
- end
227
- end
228
-
229
- def response_from_easy(easy, request)
230
- Response.new(:code => easy.response_code,
231
- :headers => easy.response_header,
232
- :body => easy.response_body,
233
- :time => easy.total_time_taken,
234
- :start_transfer_time => easy.start_transfer_time,
235
- :app_connect_time => easy.app_connect_time,
236
- :pretransfer_time => easy.pretransfer_time,
237
- :connect_time => easy.connect_time,
238
- :name_lookup_time => easy.name_lookup_time,
239
- :effective_url => easy.effective_url,
240
- :primary_ip => easy.primary_ip,
241
- :curl_return_code => easy.curl_return_code,
242
- :curl_error_message => easy.curl_error_message,
243
- :redirect_count => easy.redirect_count,
244
- :request => request)
33
+ # @option options :max_concurrency [ Integer ] Number
34
+ # of max concurrent connections to create. Default is
35
+ # 200.
36
+ def initialize(options = {})
37
+ @options = options
38
+ @max_concurrency = @options.fetch(:max_concurrency, 200)
39
+ @multi = Ethon::Multi.new
245
40
  end
246
41
  end
247
42
  end