httpx 0.2.1 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4c5b48a5d65398810ef73e2361ca65c113091b91f1dcb88cf7602da6c680e1b1
4
- data.tar.gz: 9995ed5b22888e9e0a737be6c197d1d30f5b21d06a7b304242cd3c86efe2f386
3
+ metadata.gz: bed8cbd465eb9d1b565024395767f619aaaff1bf2ee35d33dc15e9fe4eeec13e
4
+ data.tar.gz: 65ac8fee9f8c773d6b9a7894e68641547f75ee69159f01e60c8cdc1f30b648f0
5
5
  SHA512:
6
- metadata.gz: bd3877426964516076b6516237169d06435ff9e39dc44ba977e993de3dfbdcbf90b5a7e486e5f76827f89032d0c9ac5e1d2bac144d025fbe888ff3e07384243d
7
- data.tar.gz: 91fcd80f50d0ea4ca564291f4568e78a991b36e52eeb10033dd4e054ff27e7895961b2b69d6d1058eae1be646ee037707e8e9e056994305920cab5a6f3ed61a3
6
+ metadata.gz: 1e891362a86b73f6d2670eb14a1fd0815a0fab0173f6adf58bdb8e8225d83012560492c5563291d10d03e05ab9e2d9c786ba0cc238e886832276639bab06a8aa
7
+ data.tar.gz: 539853ff0c6ca209f32d56b8f5eb6209ef60a4df02f0073a588b915a5a583753b7bebe66cd91b76e599a55bb214c6d56a9d1a4d47d6d7a4d5ea2e9b18254af3c
@@ -5,6 +5,7 @@ require "httpx/version"
5
5
  require "httpx/extensions"
6
6
 
7
7
  require "httpx/errors"
8
+ require "httpx/altsvc"
8
9
  require "httpx/callbacks"
9
10
  require "httpx/loggable"
10
11
  require "httpx/registry"
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module AltSvc
5
+ @altsvc_mutex = Mutex.new
6
+ @altsvcs = Hash.new { |h, k| h[k] = [] }
7
+
8
+ module_function
9
+
10
+ def cached_altsvc(origin)
11
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
12
+ @altsvc_mutex.synchronize do
13
+ lookup(origin, now)
14
+ end
15
+ end
16
+
17
+ def cached_altsvc_set(origin, entry)
18
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
19
+ @altsvc_mutex.synchronize do
20
+ return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
21
+ entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
22
+ @altsvcs[origin] << entry
23
+ entry
24
+ end
25
+ end
26
+
27
+ def lookup(origin, ttl)
28
+ return [] unless @altsvcs.key?(origin)
29
+ @altsvcs[origin] = @altsvcs[origin].select do |entry|
30
+ !entry.key?("TTL") || entry["TTL"] > ttl
31
+ end
32
+ @altsvcs[origin].reject { |entry| entry["noop"] }
33
+ end
34
+
35
+ def emit(request, response)
36
+ # Alt-Svc
37
+ return unless response.headers.key?("alt-svc")
38
+ origin = request.origin
39
+ host = request.uri.host
40
+ parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
41
+ alt_origin.host ||= host
42
+ yield(alt_origin, origin, alt_params)
43
+ end
44
+ end
45
+
46
+ def parse(altsvc)
47
+ return enum_for(__method__, altsvc) unless block_given?
48
+ alt_origins, *alt_params = altsvc.split(/ *; */)
49
+ alt_params = Hash[alt_params.map { |field| field.split("=") }]
50
+ alt_origins.split(/ *, */).each do |alt_origin|
51
+ yield(parse_altsvc_origin(alt_origin), alt_params)
52
+ end
53
+ end
54
+
55
+ if RUBY_VERSION < "2.2"
56
+ def parse_altsvc_origin(alt_origin)
57
+ alt_proto, alt_origin = alt_origin.split("=")
58
+ alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
59
+ if alt_origin.start_with?(":")
60
+ alt_origin = "dummy#{alt_origin}"
61
+ uri = URI.parse(alt_origin)
62
+ uri.host = nil
63
+ uri
64
+ else
65
+ URI.parse("#{alt_proto}://#{alt_origin}")
66
+ end
67
+ end
68
+ else
69
+ def parse_altsvc_origin(alt_origin)
70
+ alt_proto, alt_origin = alt_origin.split("=")
71
+ alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
72
+ URI.parse("#{alt_proto}://#{alt_origin}")
73
+ end
74
+ end
75
+ end
76
+ end
@@ -24,6 +24,10 @@ module HTTPX
24
24
  headers("accept" => String(type))
25
25
  end
26
26
 
27
+ def wrap(&blk)
28
+ branch(default_options).wrap(&blk)
29
+ end
30
+
27
31
  def plugin(*plugins)
28
32
  klass = is_a?(Client) ? self.class : Client
29
33
  klass = Class.new(klass)
@@ -35,6 +35,8 @@ module HTTPX
35
35
  include Loggable
36
36
  include Callbacks
37
37
 
38
+ using URIExtensions
39
+
38
40
  require "httpx/channel/http2"
39
41
  require "httpx/channel/http1"
40
42
 
@@ -44,10 +46,15 @@ module HTTPX
44
46
  def by(uri, options)
45
47
  type = options.transport || begin
46
48
  case uri.scheme
47
- when "http" then "tcp"
48
- when "https" then "ssl"
49
+ when "http"
50
+ "tcp"
51
+ when "https"
52
+ "ssl"
53
+ when "h2"
54
+ options = options.merge(ssl: { alpn_protocols: %(h2) })
55
+ "ssl"
49
56
  else
50
- raise Error, "#{uri}: #{uri.scheme}: unsupported URI scheme"
57
+ raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
51
58
  end
52
59
  end
53
60
  new(type, uri, options)
@@ -58,19 +65,19 @@ module HTTPX
58
65
 
59
66
  def_delegator :@write_buffer, :empty?
60
67
 
61
- attr_reader :uri, :state
68
+ attr_reader :uri, :state, :pending
62
69
 
63
70
  def initialize(type, uri, options)
64
71
  @type = type
65
72
  @uri = uri
66
- @hostnames = [@uri.host]
73
+ @origins = [@uri.origin]
67
74
  @options = Options.new(options)
68
75
  @window_size = @options.window_size
69
76
  @read_buffer = Buffer.new(BUFFER_SIZE)
70
77
  @write_buffer = Buffer.new(BUFFER_SIZE)
71
78
  @pending = []
72
- @state = :idle
73
79
  on(:error) { |ex| on_error(ex) }
80
+ transition(:idle)
74
81
  end
75
82
 
76
83
  def addresses=(addrs)
@@ -88,14 +95,12 @@ module HTTPX
88
95
  if @io.protocol == "h2" && @uri.scheme == "https"
89
96
  @io.verify_hostname(channel.uri.host)
90
97
  else
91
- @uri.host == channel.uri.host &&
92
- @uri.port == channel.uri.port &&
93
- @uri.scheme == channel.uri.scheme
98
+ @uri.origin == channel.uri.origin
94
99
  end
95
100
  end
96
101
 
97
102
  def merge(channel)
98
- @hostnames += channel.instance_variable_get(:@hostnames)
103
+ @origins += channel.instance_variable_get(:@origins)
99
104
  pending = channel.instance_variable_get(:@pending)
100
105
  pending.each do |req, args|
101
106
  send(req, args)
@@ -103,14 +108,20 @@ module HTTPX
103
108
  end
104
109
 
105
110
  def unmerge(channel)
106
- @hostnames -= channel.instance_variable_get(:@hostnames)
111
+ @origins -= channel.instance_variable_get(:@origins)
112
+ purge_pending do |request, args|
113
+ request.uri == channel.uri && begin
114
+ request.transition(:idle)
115
+ channel.send(request, *args)
116
+ true
117
+ end
118
+ end
119
+ end
120
+
121
+ def purge_pending
107
122
  [@parser.pending, @pending].each do |pending|
108
- pending.reject! do |request|
109
- request.uri == channel.uri && begin
110
- request.transition(:idle)
111
- channel.send(request)
112
- true
113
- end
123
+ pending.reject! do |request, *args|
124
+ yield(request, args)
114
125
  end
115
126
  end
116
127
  end
@@ -118,9 +129,20 @@ module HTTPX
118
129
  def match?(uri)
119
130
  return false if @state == :closing
120
131
 
121
- @hostnames.include?(uri.host) &&
122
- uri.port == @uri.port &&
123
- uri.scheme == @uri.scheme
132
+ @origins.include?(uri.origin) || match_altsvcs?(uri)
133
+ end
134
+
135
+ # checks if this is channel is an alternative service of
136
+ # +uri+
137
+ def match_altsvcs?(uri)
138
+ AltSvc.cached_altsvc(@uri.origin).any? do |altsvc|
139
+ origin = altsvc["origin"]
140
+ origin.altsvc_match?(uri.origin)
141
+ end
142
+ end
143
+
144
+ def connecting?
145
+ @state == :idle
124
146
  end
125
147
 
126
148
  def interests
@@ -157,6 +179,7 @@ module HTTPX
157
179
  if @error_response
158
180
  emit(:response, request, @error_response)
159
181
  elsif @parser && !@write_buffer.full?
182
+ request.headers["alt-used"] = @uri.authority if match_altsvcs?(request.uri)
160
183
  parser.send(request, **args)
161
184
  else
162
185
  @pending << [request, args]
@@ -164,6 +187,7 @@ module HTTPX
164
187
  end
165
188
 
166
189
  def call
190
+ @timeout = @timeout_threshold
167
191
  case @state
168
192
  when :closed
169
193
  return
@@ -182,6 +206,17 @@ module HTTPX
182
206
  @parser = build_parser(protocol)
183
207
  end
184
208
 
209
+ def handle_timeout_error(e)
210
+ return emit(:error, e) unless @timeout
211
+ @timeout -= e.timeout
212
+ return unless @timeout <= 0
213
+ if connecting?
214
+ emit(:error, e.to_connection_error)
215
+ else
216
+ emit(:error, e)
217
+ end
218
+ end
219
+
185
220
  private
186
221
 
187
222
  def consume
@@ -204,6 +239,7 @@ module HTTPX
204
239
  return if siz.zero?
205
240
  log { "READ: #{siz} bytes..." }
206
241
  parser << @read_buffer.to_s
242
+ return if @state == :closing || @state == :closed
207
243
  end
208
244
  end
209
245
 
@@ -219,6 +255,7 @@ module HTTPX
219
255
  end
220
256
  log { "WRITE: #{siz} bytes..." }
221
257
  return if siz.zero?
258
+ return if @state == :closing || @state == :closed
222
259
  end
223
260
  end
224
261
 
@@ -236,8 +273,15 @@ module HTTPX
236
273
  def build_parser(protocol = @io.protocol)
237
274
  parser = registry(protocol).new(@write_buffer, @options)
238
275
  parser.on(:response) do |*args|
276
+ AltSvc.emit(*args) do |alt_origin, origin, alt_params|
277
+ emit(:altsvc, alt_origin, origin, alt_params)
278
+ end
239
279
  emit(:response, *args)
240
280
  end
281
+ parser.on(:altsvc) do |alt_origin, origin, alt_params|
282
+ emit(:altsvc, alt_origin, origin, alt_params)
283
+ end
284
+
241
285
  parser.on(:promise) do |*args|
242
286
  emit(:promise, *args)
243
287
  end
@@ -248,10 +292,14 @@ module HTTPX
248
292
  transition(:closing)
249
293
  unless parser.empty?
250
294
  transition(:closed)
295
+ emit(:reset)
251
296
  transition(:idle)
252
297
  transition(:open)
253
298
  end
254
299
  end
300
+ parser.on(:timeout) do |timeout|
301
+ @timeout = timeout
302
+ end
255
303
  parser.on(:error) do |request, ex|
256
304
  case ex
257
305
  when MisdirectedRequestError
@@ -269,11 +317,15 @@ module HTTPX
269
317
  # when :idle
270
318
  when :idle
271
319
  @error_response = nil
320
+ @timeout_threshold = @options.timeout.connect_timeout
321
+ @timeout = @timeout_threshold
272
322
  when :open
273
323
  return if @state == :closed
274
324
  @io.connect
275
325
  return unless @io.connected?
276
326
  send_pending
327
+ @timeout_threshold = @options.timeout.operation_timeout
328
+ @timeout = @timeout_threshold
277
329
  emit(:open)
278
330
  when :closing
279
331
  return unless @state == :open
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "http_parser"
3
+ require "httpx/parser/http1"
4
4
 
5
5
  module HTTPX
6
6
  class Channel::HTTP1
@@ -12,18 +12,16 @@ module HTTPX
12
12
  def initialize(buffer, options)
13
13
  @options = Options.new(options)
14
14
  @max_concurrent_requests = @options.max_concurrent_requests
15
- @parser = HTTP::Parser.new(self)
16
- @parser.header_value_type = :arrays
15
+ @max_requests = Float::INFINITY
16
+ @parser = Parser::HTTP1.new(self)
17
17
  @buffer = buffer
18
18
  @version = [1, 1]
19
19
  @pending = []
20
20
  @requests = []
21
- @has_response = false
22
21
  end
23
22
 
24
23
  def reset
25
24
  @parser.reset!
26
- @has_response = false
27
25
  end
28
26
 
29
27
  def close
@@ -39,15 +37,18 @@ module HTTPX
39
37
 
40
38
  def <<(data)
41
39
  @parser << data
42
- dispatch if @has_response
43
40
  end
44
41
 
45
42
  def send(request, **)
46
- if @requests.size >= @max_concurrent_requests
43
+ if @max_requests.positive? &&
44
+ @requests.size >= @max_concurrent_requests
47
45
  @pending << request
48
46
  return
49
47
  end
50
- @requests << request unless @requests.include?(request)
48
+ unless @requests.include?(request)
49
+ @requests << request
50
+ @pipelining = true if @requests.size > 1
51
+ end
51
52
  handle(request)
52
53
  end
53
54
 
@@ -69,69 +70,59 @@ module HTTPX
69
70
  #
70
71
  # must be public methods, or else they won't be reachable
71
72
 
72
- def on_message_begin
73
+ def on_start
73
74
  log(level: 2) { "parsing begins" }
74
75
  end
75
76
 
76
- def on_headers_complete(h)
77
- return on_trailer_headers_complete(h) if @parser_trailers
78
- # Wait for fix: https://github.com/tmm1/http_parser.rb/issues/52
79
- # callback is called 2 times when chunked
80
- request = @requests.first
81
- return if request.response
77
+ def on_headers(h)
78
+ @request = @requests.first
79
+ return if @request.response
82
80
 
83
81
  log(level: 2) { "headers received" }
84
82
  headers = @options.headers_class.new(h)
85
- response = @options.response_class.new(@requests.last,
83
+ response = @options.response_class.new(@request,
86
84
  @parser.status_code,
87
85
  @parser.http_version.join("."),
88
86
  headers, @options)
89
87
  log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
90
88
  log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
91
89
 
92
- request.response = response
90
+ @request.response = response
91
+ on_complete if response.complete?
92
+ end
93
+
94
+ def on_trailers(h)
95
+ return unless @request
96
+ response = @request.response
97
+ log(level: 2) { "trailer headers received" }
93
98
 
94
- @has_response = true if response.complete?
99
+ log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
100
+ response.merge_headers(h)
95
101
  end
96
102
 
97
- def on_body(chunk)
103
+ def on_data(chunk)
104
+ return unless @request
98
105
  log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
99
106
  log(level: 2, color: :green) { "-> #{chunk.inspect}" }
100
- response = @requests.first.response
107
+ response = @request.response
101
108
 
102
109
  response << chunk
103
-
104
- @has_response = response.complete?
105
110
  end
106
111
 
107
- def on_message_complete
112
+ def on_complete
113
+ return unless @request
108
114
  log(level: 2) { "parsing complete" }
109
- request = @requests.first
110
- response = request.response
111
-
112
- if !@parser_trailers && response.headers.key?("trailer")
113
- @parser_trailers = true
114
- # this is needed, because the parser can't accept further headers.
115
- # we need to reset it and artificially move it to receive headers state,
116
- # hence the bogus headline
117
- #
118
- @parser.reset!
119
- @parser << "#{request.verb.to_s.upcase} #{request.path} HTTP/#{response.version}#{CRLF}"
120
- else
121
- @has_response = true
122
- end
123
- end
124
-
125
- def on_trailer_headers_complete(h)
126
- response = @requests.first.response
127
-
128
- response.merge_headers(h)
115
+ dispatch
129
116
  end
130
117
 
131
118
  def dispatch
132
- request = @requests.first
133
- return handle(request) if request.expects?
119
+ if @request.expects?
120
+ reset
121
+ return handle(@request)
122
+ end
134
123
 
124
+ request = @request
125
+ @request = nil
135
126
  @requests.shift
136
127
  response = request.response
137
128
  emit(:response, request, response)
@@ -140,28 +131,57 @@ module HTTPX
140
131
  response << @parser.upgrade_data
141
132
  throw(:called)
142
133
  end
134
+
143
135
  reset
136
+ @max_requests -= 1
144
137
  send(@pending.shift) unless @pending.empty?
145
- return unless response.headers["connection"] == "close"
146
- disable_concurrency
147
- emit(:reset)
138
+ manage_connection(response)
148
139
  end
149
140
 
150
141
  def handle_error(ex)
151
- @requests.each do |request|
152
- emit(:error, request, ex)
142
+ if @pipelining
143
+ disable_pipelining
144
+ emit(:reset)
145
+ throw(:called)
146
+ else
147
+ @requests.each do |request|
148
+ emit(:error, request, ex)
149
+ end
153
150
  end
154
151
  end
155
152
 
156
153
  private
157
154
 
158
- def disable_concurrency
155
+ def manage_connection(response)
156
+ connection = response.headers["connection"]
157
+ case connection
158
+ when /keep\-alive/i
159
+ keep_alive = response.headers["keep-alive"]
160
+ return unless keep_alive
161
+ parameters = Hash[keep_alive.split(/ *, */).map do |pair|
162
+ pair.split(/ *= */)
163
+ end]
164
+ @max_requests = parameters["max"].to_i if parameters.key?("max")
165
+ if parameters.key?("timeout")
166
+ keep_alive_timeout = parameters["timeout"].to_i
167
+ emit(:timeout, keep_alive_timeout)
168
+ end
169
+ # TODO: on keep alive timeout, reset
170
+ when /close/i, nil
171
+ disable_pipelining
172
+ @max_requests = Float::INFINITY
173
+ emit(:reset)
174
+ end
175
+ end
176
+
177
+ def disable_pipelining
159
178
  return if @requests.empty?
160
179
  @requests.each { |r| r.transition(:idle) }
161
180
  # server doesn't handle pipelining, and probably
162
181
  # doesn't support keep-alive. Fallback to send only
163
182
  # 1 keep alive request.
164
183
  @max_concurrent_requests = 1
184
+ @pipelining = false
165
185
  end
166
186
 
167
187
  def set_request_headers(request)
@@ -178,7 +198,6 @@ module HTTPX
178
198
  end
179
199
 
180
200
  def handle(request)
181
- @has_response = false
182
201
  set_request_headers(request)
183
202
  catch(:buffer_full) do
184
203
  request.transition(:headers)