typhoeus 0.2.4 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,3 +1,25 @@
1
+ 0.3.0
2
+ -----
3
+ * Fix array params to be consistent with HTTP spec [gridaphobe]
4
+ * traversal_to_params_hash should use the escape option [itsmeduncan]
5
+ * Fix > 1024 open file descriptors [mschulkind]
6
+ * Fixed a bug with internally queued requests being dropped [mschulkind]
7
+ * Use gemspec in bundler to avoid duplication [mschulkind]
8
+ * Run internally queued requests in FIFO order [mschulkind]
9
+ * Moved Typhoeus::VERSION to a separate file, to fix rake build_native [mschulkind]
10
+ * Fixed problems related to put requests with empty bodies [skaes, GH-84]
11
+ * Added CURLOPT_INTERFACE option via Request#interface=. [spiegela]
12
+ * Added Tempfile support to Form#process! [richievos]
13
+ * Hydra won't forget to accept gzip/deflate encoding [codesnik]
14
+ * Accept and convert strings to integers in Typhoeus::Request#initialize for timeout/cache_timeout/connect_timeout values when using ruby 1.9.x. [djnawara]
15
+ * Added interface for registering stub finders [myronmarston]
16
+ * Fixed header stubbing [myronmarston]
17
+ * Added PKCS12 support [jodell]
18
+ * Make a request with handlers marshallable [bernerdschaefer]
19
+ * Upgraded to RSpec 2 [bernerdschaefer]
20
+ * Fix HTTP status edge-case [balexis]
21
+ * Expose primary_ip to easy object [balexis]
22
+
1
23
  0.2.4
2
24
  -----
3
25
  * Fix form POSTs to only use multipart for file uploads, otherwise use application/x-www-form-urlencoded [dbalatero]
data/Gemfile CHANGED
@@ -1,11 +1,3 @@
1
1
  source :rubygems
2
2
 
3
- gem "mime-types"
4
-
5
- group :test do
6
- gem 'rspec', '1.3.1'
7
- gem 'jeweler'
8
- gem 'json'
9
- gem 'sinatra'
10
- gem 'diff-lcs'
11
- end
3
+ gemspec
@@ -1,32 +1,35 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ typhoeus (0.3.0)
5
+ mime-types
6
+
1
7
  GEM
2
8
  remote: http://rubygems.org/
3
9
  specs:
4
10
  diff-lcs (1.1.2)
5
- gemcutter (0.6.1)
6
- git (1.2.5)
7
- jeweler (1.4.0)
8
- gemcutter (>= 0.1.0)
9
- git (>= 1.2.5)
10
- rubyforge (>= 2.0.0)
11
- json (1.4.6)
12
- json_pure (1.4.6)
11
+ json (1.5.3)
13
12
  mime-types (1.16)
14
- rack (1.2.1)
15
- rspec (1.3.1)
16
- rubyforge (2.0.4)
17
- json_pure (>= 1.1.7)
18
- sinatra (1.1.0)
13
+ rack (1.3.0)
14
+ rspec (2.6.0)
15
+ rspec-core (~> 2.6.0)
16
+ rspec-expectations (~> 2.6.0)
17
+ rspec-mocks (~> 2.6.0)
18
+ rspec-core (2.6.4)
19
+ rspec-expectations (2.6.0)
20
+ diff-lcs (~> 1.1.2)
21
+ rspec-mocks (2.6.0)
22
+ sinatra (1.2.6)
19
23
  rack (~> 1.1)
20
- tilt (~> 1.1)
21
- tilt (1.1)
24
+ tilt (< 2.0, >= 1.2.2)
25
+ tilt (1.3.2)
22
26
 
23
27
  PLATFORMS
24
28
  ruby
25
29
 
26
30
  DEPENDENCIES
27
31
  diff-lcs
28
- jeweler
29
32
  json
30
- mime-types
31
- rspec (= 1.3.1)
33
+ rspec (~> 2.6)
32
34
  sinatra
35
+ typhoeus!
data/Rakefile CHANGED
@@ -1,8 +1,6 @@
1
1
  $LOAD_PATH.unshift(File.dirname(__FILE__))
2
2
 
3
- require "spec"
4
- require "spec/rake/spectask"
5
- require 'lib/typhoeus'
3
+ require "lib/typhoeus/version"
6
4
 
7
5
  begin
8
6
  require 'jeweler'
@@ -14,7 +12,7 @@ begin
14
12
  gemspec.homepage = "http://github.com/dbalatero/typhoeus"
15
13
  gemspec.authors = ["Paul Dix", "David Balatero"]
16
14
  gemspec.add_dependency "mime-types"
17
- gemspec.add_development_dependency "rspec"
15
+ gemspec.add_development_dependency "rspec", "~> 2.6"
18
16
  gemspec.add_development_dependency "jeweler"
19
17
  gemspec.add_development_dependency "diff-lcs"
20
18
  gemspec.add_development_dependency "sinatra"
@@ -26,9 +24,8 @@ rescue LoadError
26
24
  puts "Jeweler not available. Install it with: gem install jeweler"
27
25
  end
28
26
 
29
- Spec::Rake::SpecTask.new do |t|
30
- t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
31
- t.spec_files = FileList['spec/**/*_spec.rb']
27
+ require 'rspec/core/rake_task'
28
+ RSpec::Core::RakeTask.new do |t|
32
29
  end
33
30
 
34
31
  task :install do
@@ -37,5 +34,20 @@ task :install do
37
34
  puts `gem install typhoeus-#{Typhoeus::VERSION}.gem`
38
35
  end
39
36
 
37
+ desc "Builds the native code"
38
+ task :build_native do
39
+ system("cd ext/typhoeus && ruby extconf.rb && make")
40
+ end
41
+
42
+ desc "Start up the test servers"
43
+ task :start_test_servers do
44
+ puts "Starting 3 test servers"
45
+ (3000..3002).map do |port|
46
+ Thread.new do
47
+ system("ruby spec/servers/app.rb -p #{port}")
48
+ end
49
+ end.each(&:join)
50
+ end
51
+
40
52
  desc "Run all the tests"
41
53
  task :default => :spec
@@ -1,4 +1,10 @@
1
+ /* Make sure select() works with >1024 file handles open. */
2
+ #include <sys/types.h>
3
+ #undef __FD_SETSIZE
4
+ #define __FD_SETSIZE 524288
5
+
1
6
  #include <typhoeus_multi.h>
7
+ #include <errno.h>
2
8
 
3
9
  static void multi_read_info(VALUE self, CURLM *multi_handle);
4
10
 
@@ -158,7 +164,7 @@ static VALUE multi_perform(VALUE self) {
158
164
 
159
165
  rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
160
166
  if (rc < 0) {
161
- rb_raise(rb_eRuntimeError, "error on thread select");
167
+ rb_raise(rb_eRuntimeError, "error on thread select: %d", errno);
162
168
  }
163
169
  rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
164
170
 
@@ -16,10 +16,9 @@ require 'typhoeus/response'
16
16
  require 'typhoeus/request'
17
17
  require 'typhoeus/hydra'
18
18
  require 'typhoeus/hydra_mock'
19
+ require 'typhoeus/version'
19
20
 
20
21
  module Typhoeus
21
- VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").chomp
22
-
23
22
  def self.easy_object_pool
24
23
  @easy_objects ||= []
25
24
  end
@@ -20,6 +20,7 @@ module Typhoeus
20
20
  # [Only works on unix-style/SIGALRM operating systems. IOW, does
21
21
  # not work on Windows.
22
22
  :CURLOPT_CONNECTTIMEOUT_MS => 156,
23
+ :CURLOPT_INTERFACE => 10000 + 62,
23
24
  :CURLOPT_NOSIGNAL => 99,
24
25
  :CURLOPT_HTTPHEADER => 10023,
25
26
  :CURLOPT_FOLLOWLOCATION => 52,
@@ -32,6 +33,7 @@ module Typhoeus
32
33
  :CURLOPT_PROXYTYPE => 101,
33
34
  :CURLOPT_PROXYAUTH => 111,
34
35
  :CURLOPT_VERIFYPEER => 64,
36
+ :CURLOPT_VERIFYHOST => 81,
35
37
  :CURLOPT_NOBODY => 44,
36
38
  :CURLOPT_ENCODING => 10000 + 102,
37
39
  :CURLOPT_SSLCERT => 10025,
@@ -47,6 +49,7 @@ module Typhoeus
47
49
  :CURLINFO_TOTAL_TIME => 3145731,
48
50
  :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23,
49
51
  :CURLINFO_EFFECTIVE_URL => 0x100000 + 1,
52
+ :CURLINFO_PRIMARY_IP => 0x100000 + 32,
50
53
  :CURLINFO_NAMELOOKUP_TIME => 0x300000 + 4,
51
54
  :CURLINFO_CONNECT_TIME => 0x300000 + 5,
52
55
  :CURLINFO_PRETRANSFER_TIME => 0x300000 + 6,
@@ -75,6 +78,10 @@ module Typhoeus
75
78
  @method = :get
76
79
  @headers = {}
77
80
 
81
+ set_defaults
82
+ end
83
+
84
+ def set_defaults
78
85
  # Enable encoding/compression support
79
86
  set_option(OPTION_VALUES[:CURLOPT_ENCODING], '')
80
87
  end
@@ -82,6 +89,11 @@ module Typhoeus
82
89
  def headers=(hash)
83
90
  @headers = hash
84
91
  end
92
+
93
+ def interface=(interface)
94
+ @interface = interface
95
+ set_option(OPTION_VALUES[:CURLOPT_INTERFACE], interface)
96
+ end
85
97
 
86
98
  def proxy=(proxy)
87
99
  set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy[:server])
@@ -133,6 +145,10 @@ module Typhoeus
133
145
  def effective_url
134
146
  get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
135
147
  end
148
+
149
+ def primary_ip
150
+ get_info_string(INFO_VALUES[:CURLINFO_PRIMARY_IP])
151
+ end
136
152
 
137
153
  def response_code
138
154
  get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
@@ -173,9 +189,7 @@ module Typhoeus
173
189
  def request_body=(request_body)
174
190
  @request_body = request_body
175
191
  if @method == :put
176
- easy_set_request_body(@request_body)
177
- headers["Transfer-Encoding"] = ""
178
- headers["Expect"] = ""
192
+ easy_set_request_body(@request_body.to_s)
179
193
  else
180
194
  self.post_data = request_body
181
195
  end
@@ -194,6 +208,10 @@ module Typhoeus
194
208
  set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
195
209
  end
196
210
 
211
+ def disable_ssl_host_verification
212
+ set_option(OPTION_VALUES[:CURLOPT_VERIFYHOST], 0)
213
+ end
214
+
197
215
  def method=(method)
198
216
  @method = method
199
217
  if method == :get
@@ -203,7 +221,7 @@ module Typhoeus
203
221
  self.post_data = ""
204
222
  elsif method == :put
205
223
  set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
206
- self.request_body = "" unless @request_body
224
+ self.request_body = @request_body.to_s
207
225
  elsif method == :head
208
226
  set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
209
227
  else
@@ -246,7 +264,7 @@ module Typhoeus
246
264
  # Set SSL certificate type
247
265
  # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
248
266
  def ssl_cert_type=(cert_type)
249
- raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type)
267
+ raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER p12).include?(cert_type)
250
268
  set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
251
269
  end
252
270
 
@@ -344,7 +362,9 @@ module Typhoeus
344
362
  @response_code = 0
345
363
  @response_header = ""
346
364
  @response_body = ""
365
+ @request_body = ""
347
366
  easy_reset()
367
+ set_defaults
348
368
  end
349
369
 
350
370
  def get_info_string(option)
@@ -125,6 +125,7 @@ module Typhoeus
125
125
  val = @cache_getter.call(request)
126
126
  if val
127
127
  @retrieved_from_cache[request.url] = val
128
+ queue_next
128
129
  handle_request(request, val, false)
129
130
  else
130
131
  @multi.add(get_easy_object(request))
@@ -165,9 +166,11 @@ module Typhoeus
165
166
  easy.request_body = request.body if request.body
166
167
  easy.timeout = request.timeout if request.timeout
167
168
  easy.connect_timeout = request.connect_timeout if request.connect_timeout
169
+ easy.interface = request.interface if request.interface
168
170
  easy.follow_location = request.follow_location if request.follow_location
169
171
  easy.max_redirects = request.max_redirects if request.max_redirects
170
172
  easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
173
+ easy.disable_ssl_host_verification if request.disable_ssl_host_verification
171
174
  easy.ssl_cert = request.ssl_cert
172
175
  easy.ssl_cert_type = request.ssl_cert_type
173
176
  easy.ssl_key = request.ssl_key
@@ -194,7 +197,7 @@ module Typhoeus
194
197
 
195
198
  def queue_next
196
199
  @running_requests -= 1
197
- queue(@queued_requests.pop) unless @queued_requests.empty?
200
+ queue(@queued_requests.shift) unless @queued_requests.empty?
198
201
  end
199
202
  private :queue_next
200
203
 
@@ -236,6 +239,7 @@ module Typhoeus
236
239
  :connect_time => easy.connect_time,
237
240
  :name_lookup_time => easy.name_lookup_time,
238
241
  :effective_url => easy.effective_url,
242
+ :primary_ip => easy.primary_ip,
239
243
  :curl_return_code => easy.curl_return_code,
240
244
  :curl_error_message => easy.curl_error_message,
241
245
  :request => request)
@@ -11,10 +11,26 @@ module Typhoeus
11
11
  self.stubs = []
12
12
  end
13
13
 
14
+ def register_stub_finder(&block)
15
+ stub_finders << block
16
+ end
17
+
14
18
  def find_stub_from_request(request)
19
+ stub_finders.each do |finder|
20
+ if response = finder.call(request)
21
+ mock = HydraMock.new(/.*/, :any)
22
+ mock.and_return(response)
23
+ return mock
24
+ end
25
+ end
26
+
15
27
  stubs.detect { |stub| stub.matches?(request) }
16
28
  end
17
29
 
30
+ def stub_finders
31
+ @stub_finders ||= []
32
+ end
33
+
18
34
  def self.extended(base)
19
35
  class << base
20
36
  attr_accessor :stubs
@@ -7,7 +7,7 @@ module Typhoeus
7
7
  attr_accessor :method, :params, :body, :connect_timeout, :timeout,
8
8
  :user_agent, :response, :cache_timeout, :follow_location,
9
9
  :max_redirects, :proxy, :proxy_username,:proxy_password,
10
- :disable_ssl_peer_verification,
10
+ :disable_ssl_peer_verification, :disable_ssl_host_verification, :interface,
11
11
  :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_type,
12
12
  :ssl_key_password, :ssl_cacert, :ssl_capath, :verbose,
13
13
  :username, :password, :auth_method, :user_agent,
@@ -22,6 +22,7 @@ module Typhoeus
22
22
  # ** +:params+ : params as a Hash
23
23
  # ** +:body+
24
24
  # ** +:timeout+ : timeout (ms)
25
+ # ** +:interface+ : interface or ip address (string)
25
26
  # ** +:connect_timeout+ : connect timeout (ms)
26
27
  # ** +:headers+ : headers as Hash
27
28
  # ** +:user_agent+ : user agent (string)
@@ -30,6 +31,7 @@ module Typhoeus
30
31
  # ** +:max_redirects
31
32
  # ** +:proxy
32
33
  # ** +:disable_ssl_peer_verification
34
+ # ** +:disable_ssl_host_verification
33
35
  # ** +:ssl_cert
34
36
  # ** +:ssl_cert_type
35
37
  # ** +:ssl_key
@@ -46,11 +48,12 @@ module Typhoeus
46
48
  @method = options[:method] || :get
47
49
  @params = options[:params]
48
50
  @body = options[:body]
49
- @timeout = options[:timeout]
50
- @connect_timeout = options[:connect_timeout]
51
+ @timeout = safe_to_i(options[:timeout])
52
+ @connect_timeout = safe_to_i(options[:connect_timeout])
53
+ @interface = options[:interface]
51
54
  @headers = options[:headers] || {}
52
55
  @user_agent = options[:user_agent] || Typhoeus::USER_AGENT
53
- @cache_timeout = options[:cache_timeout]
56
+ @cache_timeout = safe_to_i(options[:cache_timeout])
54
57
  @follow_location = options[:follow_location]
55
58
  @max_redirects = options[:max_redirects]
56
59
  @proxy = options[:proxy]
@@ -59,6 +62,7 @@ module Typhoeus
59
62
  @proxy_password = options[:proxy_password]
60
63
  @proxy_auth_method = options[:proxy_auth_method]
61
64
  @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
65
+ @disable_ssl_host_verification = options[:disable_ssl_host_verification]
62
66
  @ssl_cert = options[:ssl_cert]
63
67
  @ssl_cert_type = options[:ssl_cert_type]
64
68
  @ssl_key = options[:ssl_key]
@@ -197,5 +201,29 @@ module Typhoeus
197
201
  def self.head(url, params = {})
198
202
  run(url, params.merge(:method => :head))
199
203
  end
204
+
205
+ protected
206
+
207
+ # Return the important data needed to serialize this Request, except the
208
+ # `on_complete` and `after_complete` handlers, since they cannot be
209
+ # marshalled.
210
+ def marshal_dump
211
+ (instance_variables - ['@on_complete', '@after_complete', :@on_complete, :@after_complete]).map do |name|
212
+ [name, instance_variable_get(name)]
213
+ end
214
+ end
215
+
216
+ def marshal_load(attributes)
217
+ attributes.each { |name, value| instance_variable_set(name, value) }
218
+ end
219
+
220
+ private
221
+
222
+ def safe_to_i(value)
223
+ return value if value.is_a?(Fixnum)
224
+ return nil if value.nil?
225
+ return nil if value.respond_to?(:empty?) && value.empty?
226
+ value.to_i
227
+ end
200
228
  end
201
229
  end
@@ -7,7 +7,8 @@ module Typhoeus
7
7
  :effective_url, :start_transfer_time,
8
8
  :app_connect_time, :pretransfer_time,
9
9
  :connect_time, :name_lookup_time,
10
- :curl_return_code, :curl_error_message
10
+ :curl_return_code, :curl_error_message,
11
+ :primary_ip
11
12
 
12
13
  attr_writer :headers_hash
13
14
 
@@ -17,7 +18,7 @@ module Typhoeus
17
18
  @curl_error_message = params[:curl_error_message]
18
19
  @status_message = params[:status_message]
19
20
  @http_version = params[:http_version]
20
- @headers = params[:headers] || ''
21
+ @headers = params[:headers]
21
22
  @body = params[:body]
22
23
  @time = params[:time]
23
24
  @requested_url = params[:requested_url]
@@ -30,6 +31,7 @@ module Typhoeus
30
31
  @name_lookup_time = params[:name_lookup_time]
31
32
  @request = params[:request]
32
33
  @effective_url = params[:effective_url]
34
+ @primary_ip = params[:primary_ip]
33
35
  @mock = params[:mock] || false # default
34
36
  @headers_hash = NormalizedHeaderHash.new(params[:headers_hash]) if params[:headers_hash]
35
37
  end
@@ -39,6 +41,10 @@ module Typhoeus
39
41
  @mock
40
42
  end
41
43
 
44
+ def headers
45
+ @headers ||= @headers_hash ? construct_header_string : ''
46
+ end
47
+
42
48
  def headers_hash
43
49
  @headers_hash ||= begin
44
50
  headers.split("\n").map {|o| o.strip}.inject(Typhoeus::NormalizedHeaderHash.new) do |hash, o|
@@ -62,8 +68,20 @@ module Typhoeus
62
68
  end
63
69
 
64
70
  def status_message
65
- # http://rubular.com/r/eAr1oVYsVa
66
- @status_message ||= first_header_line ? first_header_line[/\d{3} (.*)$/, 1].chomp : nil
71
+ return @status_message if @status_message != nil
72
+
73
+ # HTTP servers can choose not to include the explanation to HTTP codes. The RFC
74
+ # states this (http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4):
75
+ # Except when responding to a HEAD request, the server SHOULD include an entity containing
76
+ # an explanation of the error situation [...]
77
+ # This means 'HTTP/1.1 404' is as valid as 'HTTP/1.1 404 Not Found' and we have to handle it.
78
+
79
+ # Regexp doc: http://rubular.com/r/eAr1oVYsVa
80
+ if first_header_line != nil and first_header_line[/\d{3} (.*)$/, 1] != nil
81
+ @status_message = first_header_line[/\d{3} (.*)$/, 1].chomp
82
+ else
83
+ @status_message = nil
84
+ end
67
85
  end
68
86
 
69
87
  def http_version
@@ -85,7 +103,20 @@ module Typhoeus
85
103
  private
86
104
 
87
105
  def first_header_line
88
- @first_header_line ||= headers.split("\n").first
106
+ @first_header_line ||= @headers.to_s.split("\n").first
107
+ end
108
+
109
+ def construct_header_string
110
+ lines = ["HTTP/#{http_version} #{code} #{status_message}"]
111
+
112
+ @headers_hash.each do |key, values|
113
+ [values].flatten.each do |value|
114
+ lines << "#{key}: #{value}"
115
+ end
116
+ end
117
+
118
+ lines << '' << ''
119
+ lines.join("\r\n")
89
120
  end
90
121
  end
91
122
  end