excon 0.62.0 → 0.92.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (125) hide show
  1. checksums.yaml +5 -5
  2. data/CONTRIBUTING.md +0 -1
  3. data/LICENSE.md +1 -1
  4. data/README.md +7 -6
  5. data/data/cacert.pem +1220 -1828
  6. data/excon.gemspec +19 -3
  7. data/lib/excon/connection.rb +216 -139
  8. data/lib/excon/constants.rb +38 -13
  9. data/lib/excon/error.rb +15 -0
  10. data/lib/excon/headers.rb +4 -3
  11. data/lib/excon/instrumentors/logging_instrumentor.rb +5 -16
  12. data/lib/excon/instrumentors/standard_instrumentor.rb +2 -9
  13. data/lib/excon/middlewares/base.rb +6 -0
  14. data/lib/excon/middlewares/capture_cookies.rb +1 -1
  15. data/lib/excon/middlewares/decompress.rb +11 -4
  16. data/lib/excon/middlewares/expects.rb +7 -1
  17. data/lib/excon/middlewares/idempotent.rb +20 -3
  18. data/lib/excon/middlewares/instrumentor.rb +8 -0
  19. data/lib/excon/middlewares/mock.rb +12 -3
  20. data/lib/excon/middlewares/redirect_follower.rb +25 -3
  21. data/lib/excon/middlewares/response_parser.rb +3 -0
  22. data/lib/excon/pretty_printer.rb +1 -8
  23. data/lib/excon/response.rb +12 -9
  24. data/lib/excon/socket.rb +59 -42
  25. data/lib/excon/ssl_socket.rb +37 -15
  26. data/lib/excon/test/plugin/server/exec.rb +5 -2
  27. data/lib/excon/test/plugin/server/puma.rb +4 -1
  28. data/lib/excon/test/plugin/server/unicorn.rb +5 -0
  29. data/lib/excon/test/plugin/server/webrick.rb +4 -1
  30. data/lib/excon/test/server.rb +1 -1
  31. data/lib/excon/unix_socket.rb +1 -0
  32. data/lib/excon/utils.rb +59 -5
  33. data/lib/excon/version.rb +1 -1
  34. data/lib/excon.rb +25 -17
  35. metadata +27 -98
  36. data/.document +0 -5
  37. data/.gitignore +0 -13
  38. data/.rspec +0 -3
  39. data/.travis.yml +0 -29
  40. data/Gemfile +0 -19
  41. data/Rakefile +0 -41
  42. data/benchmarks/class_vs_lambda.rb +0 -50
  43. data/benchmarks/concat_vs_insert.rb +0 -21
  44. data/benchmarks/concat_vs_interpolate.rb +0 -22
  45. data/benchmarks/cr_lf.rb +0 -21
  46. data/benchmarks/downcase-eq-eq_vs_casecmp.rb +0 -169
  47. data/benchmarks/excon.rb +0 -69
  48. data/benchmarks/excon_vs.rb +0 -165
  49. data/benchmarks/for_vs_array_each.rb +0 -27
  50. data/benchmarks/for_vs_hash_each.rb +0 -27
  51. data/benchmarks/has_key-vs-lookup.rb +0 -177
  52. data/benchmarks/headers_case_sensitivity.rb +0 -83
  53. data/benchmarks/headers_split_vs_match.rb +0 -34
  54. data/benchmarks/implicit_block-vs-explicit_block.rb +0 -98
  55. data/benchmarks/merging.rb +0 -21
  56. data/benchmarks/single_vs_double_quotes.rb +0 -21
  57. data/benchmarks/string_ranged_index.rb +0 -87
  58. data/benchmarks/strip_newline.rb +0 -115
  59. data/benchmarks/vs_stdlib.rb +0 -82
  60. data/changelog.txt +0 -1083
  61. data/spec/excon/error_spec.rb +0 -139
  62. data/spec/excon/test/server_spec.rb +0 -28
  63. data/spec/excon_spec.rb +0 -7
  64. data/spec/helpers/file_path_helpers.rb +0 -22
  65. data/spec/requests/basic_spec.rb +0 -40
  66. data/spec/requests/eof_requests_spec.rb +0 -36
  67. data/spec/requests/unix_socket_spec.rb +0 -46
  68. data/spec/spec_helper.rb +0 -24
  69. data/spec/support/shared_contexts/test_server_context.rb +0 -83
  70. data/spec/support/shared_examples/shared_example_for_clients.rb +0 -218
  71. data/spec/support/shared_examples/shared_example_for_streaming_clients.rb +0 -20
  72. data/spec/support/shared_examples/shared_example_for_test_servers.rb +0 -16
  73. data/tests/authorization_header_tests.rb +0 -29
  74. data/tests/bad_tests.rb +0 -47
  75. data/tests/basic_tests.rb +0 -351
  76. data/tests/batch_requests.rb +0 -133
  77. data/tests/complete_responses.rb +0 -31
  78. data/tests/data/127.0.0.1.cert.crt +0 -20
  79. data/tests/data/127.0.0.1.cert.key +0 -27
  80. data/tests/data/excon.cert.crt +0 -20
  81. data/tests/data/excon.cert.key +0 -27
  82. data/tests/data/xs +0 -1
  83. data/tests/error_tests.rb +0 -145
  84. data/tests/header_tests.rb +0 -119
  85. data/tests/middlewares/canned_response_tests.rb +0 -34
  86. data/tests/middlewares/capture_cookies_tests.rb +0 -34
  87. data/tests/middlewares/decompress_tests.rb +0 -157
  88. data/tests/middlewares/escape_path_tests.rb +0 -36
  89. data/tests/middlewares/idempotent_tests.rb +0 -206
  90. data/tests/middlewares/instrumentation_tests.rb +0 -315
  91. data/tests/middlewares/mock_tests.rb +0 -304
  92. data/tests/middlewares/redirect_follower_tests.rb +0 -112
  93. data/tests/pipeline_tests.rb +0 -40
  94. data/tests/proxy_tests.rb +0 -306
  95. data/tests/query_string_tests.rb +0 -87
  96. data/tests/rackups/basic.rb +0 -41
  97. data/tests/rackups/basic.ru +0 -3
  98. data/tests/rackups/basic_auth.ru +0 -14
  99. data/tests/rackups/deflater.ru +0 -4
  100. data/tests/rackups/proxy.ru +0 -18
  101. data/tests/rackups/query_string.ru +0 -13
  102. data/tests/rackups/redirecting.ru +0 -23
  103. data/tests/rackups/redirecting_with_cookie.ru +0 -40
  104. data/tests/rackups/request_headers.ru +0 -15
  105. data/tests/rackups/request_methods.ru +0 -21
  106. data/tests/rackups/response_header.ru +0 -18
  107. data/tests/rackups/ssl.ru +0 -16
  108. data/tests/rackups/ssl_mismatched_cn.ru +0 -15
  109. data/tests/rackups/ssl_verify_peer.ru +0 -16
  110. data/tests/rackups/streaming.ru +0 -30
  111. data/tests/rackups/thread_safety.ru +0 -17
  112. data/tests/rackups/timeout.ru +0 -14
  113. data/tests/rackups/webrick_patch.rb +0 -34
  114. data/tests/request_headers_tests.rb +0 -21
  115. data/tests/request_method_tests.rb +0 -47
  116. data/tests/request_tests.rb +0 -59
  117. data/tests/response_tests.rb +0 -197
  118. data/tests/servers/bad.rb +0 -20
  119. data/tests/servers/eof.rb +0 -17
  120. data/tests/servers/error.rb +0 -20
  121. data/tests/servers/good.rb +0 -350
  122. data/tests/test_helper.rb +0 -306
  123. data/tests/thread_safety_tests.rb +0 -39
  124. data/tests/timeout_tests.rb +0 -12
  125. data/tests/utils_tests.rb +0 -81
data/excon.gemspec CHANGED
@@ -1,4 +1,5 @@
1
- require File.join(File.dirname(__FILE__), 'lib', 'excon', 'version')
1
+ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'lib')
2
+ require 'excon/version'
2
3
 
3
4
  Gem::Specification.new do |s|
4
5
  s.name = 'excon'
@@ -11,8 +12,13 @@ Gem::Specification.new do |s|
11
12
  s.license = 'MIT'
12
13
  s.rdoc_options = ["--charset=UTF-8"]
13
14
  s.extra_rdoc_files = %w[README.md CONTRIBUTORS.md CONTRIBUTING.md]
14
- s.files = `git ls-files -z`.split("\x0")
15
- s.test_files = s.files.select { |path| path =~ /^[spec|tests]\/.*_[spec|tests]\.rb/ }
15
+ s.files = `git ls-files -- data/* lib/*`.split("\n") + [
16
+ "CONTRIBUTING.md",
17
+ "CONTRIBUTORS.md",
18
+ "LICENSE.md",
19
+ "README.md",
20
+ "excon.gemspec"
21
+ ]
16
22
 
17
23
  s.add_development_dependency('rspec', '>= 3.5.0')
18
24
  s.add_development_dependency('activesupport')
@@ -26,4 +32,14 @@ Gem::Specification.new do |s|
26
32
  s.add_development_dependency('sinatra-contrib')
27
33
  s.add_development_dependency('json', '>= 1.8.5')
28
34
  s.add_development_dependency('puma')
35
+ s.add_development_dependency('webrick')
36
+
37
+ s.metadata = {
38
+ 'homepage_uri' => 'https://github.com/excon/excon',
39
+ 'bug_tracker_uri' => 'https://github.com/excon/excon/issues',
40
+ 'changelog_uri' => 'https://github.com/excon/excon/blob/master/changelog.txt',
41
+ 'documentation_uri' => 'https://github.com/excon/excon/blob/master/README.md',
42
+ 'source_code_uri' => 'https://github.com/excon/excon',
43
+ 'wiki_uri' => 'https://github.com/excon/excon/wiki'
44
+ }
29
45
  end
@@ -1,4 +1,6 @@
1
1
  # frozen_string_literal: true
2
+ require 'ipaddr'
3
+
2
4
  module Excon
3
5
  class Connection
4
6
  include Utils
@@ -38,27 +40,27 @@ module Excon
38
40
  end
39
41
  end
40
42
  def logger=(logger)
41
- Excon::LoggingInstrumentor.logger = logger
42
43
  @data[:instrumentor] = Excon::LoggingInstrumentor
44
+ @data[:logger] = logger
43
45
  end
44
46
 
45
47
  # Initializes a new Connection instance
46
- # @param [Hash<Symbol, >] params One or more optional params
47
- # @option params [String] :body Default text to be sent over a socket. Only used if :body absent in Connection#request params
48
- # @option params [Hash<Symbol, String>] :headers The default headers to supply in a request. Only used if params[:headers] is not supplied to Connection#request
49
- # @option params [String] :host The destination host's reachable DNS name or IP, in the form of a String. IPv6 addresses must be wrapped (e.g. [::1]). See URI#host.
50
- # @option params [String] :hostname Same as host, but usable for socket connections. IPv6 addresses must not be wrapped (e.g. ::1). See URI#hostname.
51
- # @option params [String] :path Default path; appears after 'scheme://host:port/'. Only used if params[:path] is not supplied to Connection#request
52
- # @option params [Fixnum] :port The port on which to connect, to the destination host
53
- # @option params [Hash] :query Default query; appended to the 'scheme://host:port/path/' in the form of '?key=value'. Will only be used if params[:query] is not supplied to Connection#request
54
- # @option params [String] :scheme The protocol; 'https' causes OpenSSL to be used
55
- # @option params [String] :socket The path to the unix socket (required for 'unix://' connections)
56
- # @option params [String] :ciphers Only use the specified SSL/TLS cipher suites; use OpenSSL cipher spec format e.g. 'HIGH:!aNULL:!3DES' or 'AES256-SHA:DES-CBC3-SHA'
57
- # @option params [String] :proxy Proxy server; e.g. 'http://myproxy.com:8888'
58
- # @option params [Fixnum] :retry_limit Set how many times we'll retry a failed request. (Default 4)
59
- # @option params [Fixnum] :retry_interval Set how long to wait between retries. (Default 0)
60
- # @option params [Class] :instrumentor Responds to #instrument as in ActiveSupport::Notifications
61
- # @option params [String] :instrumentor_name Name prefix for #instrument events. Defaults to 'excon'
48
+ # @param [Hash<Symbol, >] params One or more optional params
49
+ # @option params [String] :body Default text to be sent over a socket. Only used if :body absent in Connection#request params
50
+ # @option params [Hash<Symbol, String>] :headers The default headers to supply in a request. Only used if params[:headers] is not supplied to Connection#request
51
+ # @option params [String] :host The destination host's reachable DNS name or IP, in the form of a String. IPv6 addresses must be wrapped (e.g. [::1]). See URI#host.
52
+ # @option params [String] :hostname Same as host, but usable for socket connections. IPv6 addresses must not be wrapped (e.g. ::1). See URI#hostname.
53
+ # @option params [String] :path Default path; appears after 'scheme://host:port/'. Only used if params[:path] is not supplied to Connection#request
54
+ # @option params [Fixnum] :port The port on which to connect, to the destination host
55
+ # @option params [Hash] :query Default query; appended to the 'scheme://host:port/path/' in the form of '?key=value'. Will only be used if params[:query] is not supplied to Connection#request
56
+ # @option params [String] :scheme The protocol; 'https' causes OpenSSL to be used
57
+ # @option params [String] :socket The path to the unix socket (required for 'unix://' connections)
58
+ # @option params [String] :ciphers Only use the specified SSL/TLS cipher suites; use OpenSSL cipher spec format e.g. 'HIGH:!aNULL:!3DES' or 'AES256-SHA:DES-CBC3-SHA'
59
+ # @option params [String] :proxy Proxy server; e.g. 'http://myproxy.com:8888'
60
+ # @option params [Fixnum] :retry_limit Set how many times we'll retry a failed request. (Default 4)
61
+ # @option params [Fixnum] :retry_interval Set how long to wait between retries. (Default 0)
62
+ # @option params [Class] :instrumentor Responds to #instrument as in ActiveSupport::Notifications
63
+ # @option params [String] :instrumentor_name Name prefix for #instrument events. Defaults to 'excon'
62
64
  def initialize(params = {})
63
65
  @data = Excon.defaults.dup
64
66
  # merge does not deep-dup, so make sure headers is not the original
@@ -67,12 +69,17 @@ module Excon
67
69
  # the same goes for :middlewares
68
70
  @data[:middlewares] = @data[:middlewares].dup
69
71
 
70
- params = validate_params(:connection, params)
71
72
  @data.merge!(params)
73
+ validate_params(:connection, @data, @data[:middlewares])
74
+
75
+ if @data.key?(:host) && !@data.key?(:hostname)
76
+ Excon.display_warning('hostname is missing! For IPv6 support, provide both host and hostname: Excon::Connection#new(:host => uri.host, :hostname => uri.hostname, ...).')
77
+ @data[:hostname] = @data[:host]
78
+ end
72
79
 
73
80
  setup_proxy
74
81
 
75
- if ENV.has_key?('EXCON_STANDARD_INSTRUMENTOR')
82
+ if ENV.has_key?('EXCON_STANDARD_INSTRUMENTOR')
76
83
  @data[:instrumentor] = Excon::StandardInstrumentor
77
84
  end
78
85
 
@@ -108,7 +115,7 @@ module Excon
108
115
  # we already have data from a middleware, so bail
109
116
  return datum
110
117
  else
111
- socket.data = datum
118
+ socket(datum).data = datum
112
119
  # start with "METHOD /path"
113
120
  request = datum[:method].to_s.upcase + ' '
114
121
  if datum[:proxy] && datum[:scheme] != HTTPS
@@ -137,31 +144,25 @@ module Excon
137
144
  end
138
145
 
139
146
  # add headers to request
140
- datum[:headers].each do |key, values|
141
- [values].flatten.each do |value|
142
- request << key.to_s << ': ' << value.to_s << CR_NL
143
- end
144
- end
147
+ request << Utils.headers_hash_to_s(datum[:headers])
145
148
 
146
149
  # add additional "\r\n" to indicate end of headers
147
150
  request << CR_NL
148
151
 
149
152
  if datum.has_key?(:request_block)
150
- socket.write(request) # write out request + headers
153
+ socket(datum).write(request) # write out request + headers
151
154
  while true # write out body with chunked encoding
152
155
  chunk = datum[:request_block].call
153
- if FORCE_ENC
154
- chunk.force_encoding('BINARY')
155
- end
156
+ chunk = binary_encode(chunk)
156
157
  if chunk.length > 0
157
- socket.write(chunk.length.to_s(16) << CR_NL << chunk << CR_NL)
158
+ socket(datum).write(chunk.length.to_s(16) << CR_NL << chunk << CR_NL)
158
159
  else
159
- socket.write(String.new("0#{CR_NL}#{CR_NL}"))
160
+ socket(datum).write(String.new("0#{CR_NL}#{CR_NL}"))
160
161
  break
161
162
  end
162
163
  end
163
164
  elsif body.nil?
164
- socket.write(request) # write out request + headers
165
+ socket(datum).write(request) # write out request + headers
165
166
  else # write out body
166
167
  if body.respond_to?(:binmode) && !body.is_a?(StringIO)
167
168
  body.binmode
@@ -171,28 +172,29 @@ module Excon
171
172
  end
172
173
 
173
174
  # if request + headers is less than chunk size, fill with body
174
- if FORCE_ENC
175
- request.force_encoding('BINARY')
176
- end
175
+ request = binary_encode(request)
177
176
  chunk = body.read([datum[:chunk_size] - request.length, 0].max)
178
177
  if chunk
179
- if FORCE_ENC
180
- chunk.force_encoding('BINARY')
181
- end
182
- socket.write(request << chunk)
178
+ chunk = binary_encode(chunk)
179
+ socket(datum).write(request << chunk)
183
180
  else
184
- socket.write(request) # write out request + headers
181
+ socket(datum).write(request) # write out request + headers
185
182
  end
186
183
 
187
- while chunk = body.read(datum[:chunk_size])
188
- socket.write(chunk)
184
+ while (chunk = body.read(datum[:chunk_size]))
185
+ socket(datum).write(chunk)
189
186
  end
190
187
  end
191
188
  end
192
189
  rescue => error
193
190
  case error
194
- when Excon::Errors::StubNotFound, Excon::Errors::Timeout
191
+ when Excon::Errors::InvalidHeaderKey, Excon::Errors::InvalidHeaderValue, Excon::Errors::StubNotFound, Excon::Errors::Timeout
195
192
  raise(error)
193
+ when Errno::EPIPE
194
+ # Read whatever remains in the pipe to aid in debugging
195
+ response = socket.read
196
+ error = Excon::Error.new(response + error.message)
197
+ raise_socket_error(error)
196
198
  else
197
199
  raise_socket_error(error)
198
200
  end
@@ -216,18 +218,24 @@ module Excon
216
218
  end
217
219
 
218
220
  # Sends the supplied request to the destination host.
219
- # @yield [chunk] @see Response#self.parse
220
- # @param [Hash<Symbol, >] params One or more optional params, override defaults set in Connection.new
221
- # @option params [String] :body text to be sent over a socket
222
- # @option params [Hash<Symbol, String>] :headers The default headers to supply in a request
223
- # @option params [String] :path appears after 'scheme://host:port/'
224
- # @option params [Hash] :query appended to the 'scheme://host:port/path/' in the form of '?key=value'
221
+ # @yield [chunk] @see Response#self.parse
222
+ # @param [Hash<Symbol, >] params One or more optional params, override defaults set in Connection.new
223
+ # @option params [String] :body text to be sent over a socket
224
+ # @option params [Hash<Symbol, String>] :headers The default headers to supply in a request
225
+ # @option params [String] :path appears after 'scheme://host:port/'
226
+ # @option params [Hash] :query appended to the 'scheme://host:port/path/' in the form of '?key=value'
225
227
  def request(params={}, &block)
226
- params = validate_params(:request, params)
227
228
  # @data has defaults, merge in new params to override
228
229
  datum = @data.merge(params)
229
230
  datum[:headers] = @data[:headers].merge(datum[:headers] || {})
230
231
 
232
+ validate_params(:request, params, datum[:middlewares])
233
+ # If the user passed in new middleware, we want to validate that the original connection parameters
234
+ # are still valid with the provided middleware.
235
+ if params[:middlewares]
236
+ validate_params(:connection, @data, datum[:middlewares])
237
+ end
238
+
231
239
  if datum[:user] || datum[:password]
232
240
  user, pass = Utils.unescape_uri(datum[:user].to_s), Utils.unescape_uri(datum[:password].to_s)
233
241
  datum[:headers]['Authorization'] ||= 'Basic ' + ["#{user}:#{pass}"].pack('m').delete(Excon::CR_NL)
@@ -238,7 +246,17 @@ module Excon
238
246
  else
239
247
  datum[:headers]['Host'] ||= datum[:host] + port_string(datum)
240
248
  end
241
- datum[:retries_remaining] ||= datum[:retry_limit]
249
+
250
+ # RFC 7230, section 5.4, states that the Host header SHOULD be the first one # to be present.
251
+ # Some web servers will reject the request if it comes too late, so let's hoist it to the top.
252
+ if (host = datum[:headers].delete('Host'))
253
+ datum[:headers] = { 'Host' => host }.merge(datum[:headers])
254
+ end
255
+
256
+ # default to GET if no method specified
257
+ unless datum[:method]
258
+ datum[:method] = :get
259
+ end
242
260
 
243
261
  # if path is empty or doesn't start with '/', insert one
244
262
  unless datum[:path][0, 1] == '/'
@@ -247,11 +265,16 @@ module Excon
247
265
 
248
266
  if block_given?
249
267
  Excon.display_warning('Excon requests with a block are deprecated, pass :response_block instead.')
250
- datum[:response_block] = Proc.new
268
+ datum[:response_block] = block
251
269
  end
252
270
 
253
271
  datum[:connection] = self
254
272
 
273
+ # cleanup data left behind on persistent connection after interrupt
274
+ if datum[:persistent] && !@persistent_socket_reusable
275
+ reset
276
+ end
277
+
255
278
  datum[:stack] = datum[:middlewares].map do |middleware|
256
279
  lambda {|stack| middleware.new(stack)}
257
280
  end.reverse.inject(self) do |middlewares, middleware|
@@ -260,10 +283,12 @@ module Excon
260
283
  datum = datum[:stack].request_call(datum)
261
284
 
262
285
  unless datum[:pipeline]
286
+ @persistent_socket_reusable = false
263
287
  datum = response(datum)
288
+ @persistent_socket_reusable = true
264
289
 
265
290
  if datum[:persistent]
266
- if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Connection') == 0 }
291
+ if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Connection') == 0 })
267
292
  if datum[:response][:headers][key].casecmp('close') == 0
268
293
  reset
269
294
  end
@@ -278,6 +303,10 @@ module Excon
278
303
  end
279
304
  rescue => error
280
305
  reset
306
+
307
+ # If we didn't get far enough to initialize datum and the middleware stack, just raise
308
+ raise error if !datum
309
+
281
310
  datum[:error] = error
282
311
  if datum[:stack]
283
312
  datum[:stack].error_call(datum)
@@ -287,7 +316,7 @@ module Excon
287
316
  end
288
317
 
289
318
  # Sends the supplied requests to the destination host using pipelining.
290
- # @pipeline_params [Array<Hash>] pipeline_params An array of one or more optional params, override defaults set in Connection.new, see #request for details
319
+ # @param pipeline_params [Array<Hash>] An array of one or more optional params, override defaults set in Connection.new, see #request for details
291
320
  def requests(pipeline_params)
292
321
  pipeline_params.each {|params| params.merge!(:pipeline => true, :persistent => true) }
293
322
  pipeline_params.last.merge!(:persistent => @data[:persistent])
@@ -299,7 +328,7 @@ module Excon
299
328
  end
300
329
 
301
330
  if @data[:persistent]
302
- if key = responses.last[:headers].keys.detect {|k| k.casecmp('Connection') == 0 }
331
+ if (key = responses.last[:headers].keys.detect {|k| k.casecmp('Connection') == 0 })
303
332
  if responses.last[:headers][key].casecmp('close') == 0
304
333
  reset
305
334
  end
@@ -314,7 +343,7 @@ module Excon
314
343
  # Sends the supplied requests to the destination host using pipelining in
315
344
  # batches of @limit [Numeric] requests. This is your soft file descriptor
316
345
  # limit by default, typically 256.
317
- # @pipeline_params [Array<Hash>] pipeline_params An array of one or more optional params, override defaults set in Connection.new, see #request for details
346
+ # @param pipeline_params [Array<Hash>] An array of one or more optional params, override defaults set in Connection.new, see #request for details
318
347
  def batch_requests(pipeline_params, limit = nil)
319
348
  limit ||= Process.respond_to?(:getrlimit) ? Process.getrlimit(:NOFILE).first : 256
320
349
  responses = []
@@ -327,9 +356,10 @@ module Excon
327
356
  end
328
357
 
329
358
  def reset
330
- if old_socket = sockets.delete(@socket_key)
359
+ if (old_socket = sockets.delete(@socket_key))
331
360
  old_socket.close rescue nil
332
361
  end
362
+ @persistent_socket_reusable = true
333
363
  end
334
364
 
335
365
  # Generate HTTP request verb methods
@@ -355,15 +385,7 @@ module Excon
355
385
  vars = instance_variables.inject({}) do |accum, var|
356
386
  accum.merge!(var.to_sym => instance_variable_get(var))
357
387
  end
358
- if vars[:'@data'][:headers].has_key?('Authorization')
359
- vars[:'@data'] = vars[:'@data'].dup
360
- vars[:'@data'][:headers] = vars[:'@data'][:headers].dup
361
- vars[:'@data'][:headers]['Authorization'] = REDACTED
362
- end
363
- if vars[:'@data'][:password]
364
- vars[:'@data'] = vars[:'@data'].dup
365
- vars[:'@data'][:password] = REDACTED
366
- end
388
+ vars[:'@data'] = Utils.redact(vars[:'@data'])
367
389
  inspection = '#<Excon::Connection:'
368
390
  inspection += (object_id << 1).to_s(16)
369
391
  vars.each do |key, value|
@@ -373,6 +395,10 @@ module Excon
373
395
  inspection
374
396
  end
375
397
 
398
+ def valid_request_keys(middlewares)
399
+ valid_middleware_keys(middlewares) + Excon::VALID_REQUEST_KEYS
400
+ end
401
+
376
402
  private
377
403
 
378
404
  def detect_content_length(body)
@@ -387,60 +413,77 @@ module Excon
387
413
  end
388
414
  end
389
415
 
390
- def validate_params(validation, params)
416
+ def valid_middleware_keys(middlewares)
417
+ middlewares.flat_map do |middleware|
418
+ if middleware.respond_to?(:valid_parameter_keys)
419
+ middleware.valid_parameter_keys
420
+ else
421
+ Excon.display_warning(
422
+ "Excon middleware #{middleware} does not define #valid_parameter_keys"
423
+ )
424
+ []
425
+ end
426
+ end
427
+ end
428
+
429
+ def validate_params(validation, params, middlewares)
391
430
  valid_keys = case validation
392
431
  when :connection
393
- Excon::VALID_CONNECTION_KEYS
432
+ valid_middleware_keys(middlewares) + Excon::VALID_CONNECTION_KEYS
394
433
  when :request
395
- Excon::VALID_REQUEST_KEYS
434
+ valid_request_keys(middlewares)
435
+ else
436
+ raise ArgumentError.new("Invalid validation type '#{validation}'")
396
437
  end
438
+
397
439
  invalid_keys = params.keys - valid_keys
398
440
  unless invalid_keys.empty?
399
441
  Excon.display_warning("Invalid Excon #{validation} keys: #{invalid_keys.map(&:inspect).join(', ')}")
400
- # FIXME: for now, just warn, don't mutate, give things (ie fog) a chance to catch up
401
- #params = params.dup
402
- #invalid_keys.each {|key| params.delete(key) }
403
- end
404
442
 
405
- if validation == :connection && params.key?(:host) && !params.key?(:hostname)
406
- Excon.display_warning('hostname is missing! For IPv6 support, provide both host and hostname: Excon::Connection#new(:host => uri.host, :hostname => uri.hostname, ...).')
407
- params[:hostname] = params[:host]
443
+ if validation == :request
444
+ deprecated_keys = invalid_keys & Excon::DEPRECATED_VALID_REQUEST_KEYS.keys
445
+ mw_msg = deprecated_keys.map do |k|
446
+ "#{k}: #{Excon::DEPRECATED_VALID_REQUEST_KEYS[k]}"
447
+ end.join(', ')
448
+ Excon.display_warning(
449
+ "The following request keys are only valid with the associated middleware: #{mw_msg}"
450
+ )
451
+ end
408
452
  end
409
-
410
- params
411
453
  end
412
454
 
413
455
  def response(datum={})
414
456
  datum[:stack].response_call(datum)
415
457
  rescue => error
416
458
  case error
417
- when Excon::Errors::HTTPStatusError, Excon::Errors::Timeout
459
+ when Excon::Errors::HTTPStatusError, Excon::Errors::Timeout, Excon::Errors::TooManyRedirects
418
460
  raise(error)
419
461
  else
420
462
  raise_socket_error(error)
421
463
  end
422
464
  end
423
465
 
424
- def socket
425
- unix_proxy = @data[:proxy] ? @data[:proxy][:scheme] == UNIX : false
426
- sockets[@socket_key] ||= if @data[:scheme] == UNIX || unix_proxy
427
- Excon::UnixSocket.new(@data)
428
- elsif @data[:ssl_uri_schemes].include?(@data[:scheme])
429
- Excon::SSLSocket.new(@data)
466
+ def socket(datum = @data)
467
+ unix_proxy = datum[:proxy] ? datum[:proxy][:scheme] == UNIX : false
468
+ sockets[@socket_key] ||= if datum[:scheme] == UNIX || unix_proxy
469
+ Excon::UnixSocket.new(datum)
470
+ elsif datum[:ssl_uri_schemes].include?(datum[:scheme])
471
+ Excon::SSLSocket.new(datum)
430
472
  else
431
- Excon::Socket.new(@data)
473
+ Excon::Socket.new(datum)
432
474
  end
433
475
  end
434
476
 
435
477
  def sockets
436
478
  @_excon_sockets ||= {}
479
+ @_excon_sockets.compare_by_identity
437
480
 
438
481
  if @data[:thread_safe_sockets]
439
482
  # In a multi-threaded world, if the same connection is used by multiple
440
483
  # threads at the same time to connect to the same destination, they may
441
484
  # stomp on each other's sockets. This ensures every thread gets their
442
485
  # own socket cache, within the context of a single connection.
443
- @_excon_sockets[Thread.current.object_id] ||= {}
486
+ @_excon_sockets[Thread.current] ||= {}
444
487
  else
445
488
  @_excon_sockets
446
489
  end
@@ -454,6 +497,47 @@ module Excon
454
497
  end
455
498
  end
456
499
 
500
+ def proxy_match_host_port(host, port)
501
+ host_match = if host.is_a? IPAddr
502
+ begin
503
+ host.include? @data[:host]
504
+ rescue IPAddr::Error
505
+ false
506
+ end
507
+ else
508
+ /(^|\.)#{host}$/.match(@data[:host])
509
+ end
510
+ host_match && (port.nil? || port.to_i == @data[:port])
511
+ end
512
+
513
+ def proxy_from_env
514
+ if (no_proxy_env = ENV['no_proxy'] || ENV['NO_PROXY'])
515
+ no_proxy_list = no_proxy_env.scan(/\s*(?:\[([\dA-Fa-f:\/]+)\]|\*?\.?([^\s,:]+))(?::(\d+))?\s*/i).map { |e|
516
+ if e[0]
517
+ begin
518
+ [IPAddr.new(e[0]), e[2]]
519
+ rescue IPAddr::Error
520
+ nil
521
+ end
522
+ else
523
+ begin
524
+ [IPAddr.new(e[1]), e[2]]
525
+ rescue IPAddr::Error
526
+ [e[1], e[2]]
527
+ end
528
+ end
529
+ }.reject { |e| e.nil? || e[0].nil? }
530
+ end
531
+
532
+ unless no_proxy_env && no_proxy_list.index { |h| proxy_match_host_port(h[0], h[1]) }
533
+ if @data[:scheme] == HTTPS && (ENV.has_key?('https_proxy') || ENV.has_key?('HTTPS_PROXY'))
534
+ @data[:proxy] = ENV['https_proxy'] || ENV['HTTPS_PROXY']
535
+ elsif (ENV.has_key?('http_proxy') || ENV.has_key?('HTTP_PROXY'))
536
+ @data[:proxy] = ENV['http_proxy'] || ENV['HTTP_PROXY']
537
+ end
538
+ end
539
+ end
540
+
457
541
  def setup_proxy
458
542
  if @data[:disable_proxy]
459
543
  if @data[:proxy]
@@ -462,64 +546,57 @@ module Excon
462
546
  return
463
547
  end
464
548
 
465
- unless @data[:scheme] == UNIX
466
- if no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
467
- no_proxy_list = no_proxy_env.scan(/\*?\.?([^\s,:]+)(?::(\d+))?/i).map { |s| [s[0], s[1]] }
549
+ return if @data[:scheme] == UNIX
550
+
551
+ proxy_from_env
552
+
553
+ case @data[:proxy]
554
+ when nil
555
+ @data.delete(:proxy)
556
+ when ''
557
+ @data.delete(:proxy)
558
+ when Hash
559
+ # no processing needed
560
+ when String, URI
561
+ uri = @data[:proxy].is_a?(String) ? URI.parse(@data[:proxy]) : @data[:proxy]
562
+ @data[:proxy] = {
563
+ :host => uri.host,
564
+ :hostname => uri.hostname,
565
+ # path is only sensible for a Unix socket proxy
566
+ :path => uri.scheme == UNIX ? uri.path : nil,
567
+ :port => uri.port,
568
+ :scheme => uri.scheme,
569
+ }
570
+ if uri.password
571
+ @data[:proxy][:password] = uri.password
468
572
  end
469
-
470
- unless no_proxy_env && no_proxy_list.index { |h| /(^|\.)#{h[0]}$/.match(@data[:host]) && (h[1].nil? || h[1].to_i == @data[:port]) }
471
- if @data[:scheme] == HTTPS && (ENV.has_key?('https_proxy') || ENV.has_key?('HTTPS_PROXY'))
472
- @data[:proxy] = ENV['https_proxy'] || ENV['HTTPS_PROXY']
473
- elsif (ENV.has_key?('http_proxy') || ENV.has_key?('HTTP_PROXY'))
474
- @data[:proxy] = ENV['http_proxy'] || ENV['HTTP_PROXY']
475
- end
573
+ if uri.user
574
+ @data[:proxy][:user] = uri.user
476
575
  end
477
-
478
- case @data[:proxy]
479
- when nil
480
- @data.delete(:proxy)
481
- when ''
482
- @data.delete(:proxy)
483
- when Hash
484
- # no processing needed
485
- when String, URI
486
- uri = @data[:proxy].is_a?(String) ? URI.parse(@data[:proxy]) : @data[:proxy]
487
- @data[:proxy] = {
488
- :host => uri.host,
489
- :hostname => uri.hostname,
490
- # path is only sensible for a Unix socket proxy
491
- :path => uri.scheme == UNIX ? uri.path : nil,
492
- :port => uri.port,
493
- :scheme => uri.scheme,
494
- }
495
- if uri.password
496
- @data[:proxy][:password] = uri.password
497
- end
498
- if uri.user
499
- @data[:proxy][:user] = uri.user
500
- end
501
- if @data[:proxy][:scheme] == UNIX
502
- if @data[:proxy][:host]
503
- raise ArgumentError, "The `:host` parameter should not be set for `unix://` proxies.\n" +
504
- "When supplying a `unix://` URI, it should start with `unix:/` or `unix:///`."
505
- end
506
- else
507
- unless uri.host && uri.port && uri.scheme
508
- raise Excon::Errors::ProxyParse, "Proxy is invalid"
509
- end
576
+ if @data[:ssl_proxy_headers] && !@data[:ssl_uri_schemes].include?(@data[:scheme])
577
+ raise ArgumentError, "The `:ssl_proxy_headers` parameter should only be used with HTTPS requests."
578
+ end
579
+ if @data[:proxy][:scheme] == UNIX
580
+ if @data[:proxy][:host]
581
+ raise ArgumentError, "The `:host` parameter should not be set for `unix://` proxies.\n" +
582
+ "When supplying a `unix://` URI, it should start with `unix:/` or `unix:///`."
510
583
  end
511
584
  else
512
- raise Excon::Errors::ProxyParse, "Proxy is invalid"
585
+ unless uri.host && uri.port && uri.scheme
586
+ raise Excon::Errors::ProxyParse, "Proxy is invalid"
587
+ end
513
588
  end
589
+ else
590
+ raise Excon::Errors::ProxyParse, "Proxy is invalid"
591
+ end
514
592
 
515
- if @data.has_key?(:proxy) && @data[:scheme] == 'http'
516
- @data[:headers]['Proxy-Connection'] ||= 'Keep-Alive'
517
- # https credentials happen in handshake
518
- if @data[:proxy].has_key?(:user) || @data[:proxy].has_key?(:password)
519
- user, pass = Utils.unescape_form(@data[:proxy][:user].to_s), Utils.unescape_form(@data[:proxy][:password].to_s)
520
- auth = ["#{user}:#{pass}"].pack('m').delete(Excon::CR_NL)
521
- @data[:headers]['Proxy-Authorization'] = 'Basic ' + auth
522
- end
593
+ if @data.has_key?(:proxy) && @data[:scheme] == 'http'
594
+ @data[:headers]['Proxy-Connection'] ||= 'Keep-Alive'
595
+ # https credentials happen in handshake
596
+ if @data[:proxy].has_key?(:user) || @data[:proxy].has_key?(:password)
597
+ user, pass = Utils.unescape_form(@data[:proxy][:user].to_s), Utils.unescape_form(@data[:proxy][:password].to_s)
598
+ auth = ["#{user}:#{pass}"].pack('m').delete(Excon::CR_NL)
599
+ @data[:headers]['Proxy-Authorization'] = 'Basic ' + auth
523
600
  end
524
601
  end
525
602
  end