httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -1,45 +1,47 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- class Client
4
+ class Session
5
5
  include Loggable
6
6
  include Chainable
7
7
 
8
8
  def initialize(options = {}, &blk)
9
9
  @options = self.class.default_options.merge(options)
10
- @connection = Connection.new(@options)
11
10
  @responses = {}
12
- @keep_open = false
11
+ @persistent = @options.persistent
13
12
  wrap(&blk) if block_given?
14
13
  end
15
14
 
16
15
  def wrap
17
16
  return unless block_given?
17
+
18
18
  begin
19
- prev_keep_open = @keep_open
20
- @keep_open = true
19
+ prev_persistent = @persistent
20
+ @persistent = true
21
21
  yield self
22
22
  ensure
23
- @keep_open = prev_keep_open
24
- close
23
+ @persistent = prev_persistent
25
24
  end
26
25
  end
27
26
 
28
- def close
29
- @connection.close
27
+ def close(*args)
28
+ pool.close(*args)
30
29
  end
31
30
 
32
- def request(*args, keep_open: @keep_open, **options)
33
- requests = __build_reqs(*args, **options)
34
- responses = __send_reqs(*requests, **options)
31
+ def request(*args, **options)
32
+ requests = build_requests(*args, options)
33
+ responses = send_requests(*requests, options)
35
34
  return responses.first if responses.size == 1
35
+
36
36
  responses
37
- ensure
38
- close unless keep_open
39
37
  end
40
38
 
41
39
  private
42
40
 
41
+ def pool
42
+ Thread.current[:httpx_connection_pool] ||= Pool.new
43
+ end
44
+
43
45
  def on_response(request, response)
44
46
  @responses[request] = response
45
47
  end
@@ -49,55 +51,54 @@ module HTTPX
49
51
  stream.refuse
50
52
  end
51
53
 
52
- def fetch_response(request)
54
+ def fetch_response(request, _, _)
53
55
  @responses.delete(request)
54
56
  end
55
57
 
56
- def find_channel(request, **options)
58
+ def find_connection(request, connections, options)
57
59
  uri = URI(request.uri)
58
- @connection.find_channel(uri) || build_channel(uri, options)
60
+ connection = pool.find_connection(uri, options) || build_connection(uri, options)
61
+ unless connections.nil? || connections.include?(connection)
62
+ connections << connection
63
+ set_connection_callbacks(connection, options)
64
+ end
65
+ connection
59
66
  end
60
67
 
61
- def set_channel_callbacks(channel, options)
62
- channel.on(:response, &method(:on_response))
63
- channel.on(:promise, &method(:on_promise))
64
- channel.on(:uncoalesce) do |uncoalesced_uri|
65
- other_channel = build_channel(uncoalesced_uri, options)
66
- channel.unmerge(other_channel)
68
+ def set_connection_callbacks(connection, options)
69
+ connection.on(:uncoalesce) do |uncoalesced_uri|
70
+ other_connection = build_connection(uncoalesced_uri, options)
71
+ connection.unmerge(other_connection)
67
72
  end
68
- channel.on(:altsvc) do |alt_origin, origin, alt_params|
69
- build_altsvc_channel(channel, alt_origin, origin, alt_params, options)
73
+ connection.on(:altsvc) do |alt_origin, origin, alt_params|
74
+ build_altsvc_connection(connection, alt_origin, origin, alt_params, options)
70
75
  end
71
76
  end
72
77
 
73
- def build_channel(uri, options)
74
- channel = @connection.build_channel(uri, **options)
75
- set_channel_callbacks(channel, options)
76
- channel
77
- end
78
-
79
- def build_altsvc_channel(existing_channel, alt_origin, origin, alt_params, options)
78
+ def build_altsvc_connection(existing_connection, alt_origin, origin, alt_params, options)
80
79
  altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
81
80
 
82
81
  # altsvc already exists, somehow it wasn't advertised, probably noop
83
82
  return unless altsvc
84
83
 
85
- channel = @connection.find_channel(alt_origin) || build_channel(alt_origin, options)
84
+ connection = pool.find_connection(alt_origin, options) || build_connection(alt_origin, options)
86
85
  # advertised altsvc is the same origin being used, ignore
87
- return if channel == existing_channel
86
+ return if connection == existing_connection
87
+
88
+ set_connection_callbacks(connection, options)
88
89
 
89
90
  log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
90
91
 
91
92
  # get uninitialized requests
92
93
  # incidentally, all requests will be re-routed to the first
93
94
  # advertised alt-svc, which incidentally follows the spec.
94
- existing_channel.purge_pending do |request, args|
95
+ existing_connection.purge_pending do |request|
95
96
  is_idle = request.origin == origin &&
96
97
  request.state == :idle &&
97
98
  !request.headers.key?("alt-used")
98
99
  if is_idle
99
100
  log(level: 1) { "#{origin} alt-svc: sending #{request.uri} to #{alt_origin}" }
100
- channel.send(request, args)
101
+ connection.send(request)
101
102
  end
102
103
  is_idle
103
104
  end
@@ -105,57 +106,92 @@ module HTTPX
105
106
  altsvc["noop"] = true
106
107
  end
107
108
 
108
- def __build_reqs(*args, **options)
109
+ def build_requests(*args, options)
110
+ request_options = @options.merge(options)
111
+
109
112
  requests = case args.size
110
113
  when 1
111
114
  reqs = args.first
112
115
  reqs.map do |verb, uri|
113
- __build_req(verb, uri, options)
116
+ build_request(verb, uri, request_options)
114
117
  end
115
118
  when 2, 3
116
119
  verb, uris = args
117
120
  if uris.respond_to?(:each)
118
121
  uris.map do |uri, **opts|
119
- __build_req(verb, uri, options.merge(opts))
122
+ build_request(verb, uri, request_options.merge(opts))
120
123
  end
121
124
  else
122
- [__build_req(verb, uris, options)]
125
+ [build_request(verb, uris, request_options)]
123
126
  end
124
127
  else
125
128
  raise ArgumentError, "unsupported number of arguments"
126
129
  end
127
130
  raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
131
+
128
132
  requests
129
133
  end
130
134
 
131
- def __send_reqs(*requests, **options)
135
+ def build_connection(uri, options)
136
+ type = options.transport || begin
137
+ case uri.scheme
138
+ when "http"
139
+ "tcp"
140
+ when "https"
141
+ "ssl"
142
+ when "h2"
143
+ options = options.merge(ssl: { alpn_protocols: %w[h2] })
144
+ "ssl"
145
+ else
146
+ raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
147
+ end
148
+ end
149
+ connection = options.connection_class.new(type, uri, options)
150
+ pool.init_connection(connection, options)
151
+ connection
152
+ end
153
+
154
+ def send_requests(*requests, options)
155
+ connections = []
156
+ request_options = @options.merge(options)
157
+ timeout = request_options.timeout
158
+
132
159
  requests.each do |request|
133
- channel = find_channel(request, **options)
134
- channel.send(request)
160
+ connection = find_connection(request, connections, request_options)
161
+ connection.send(request)
135
162
  end
163
+
136
164
  responses = []
137
165
 
138
- # guarantee ordered responses
139
- loop do
140
- begin
141
- request = requests.first
142
- @connection.next_tick until (response = fetch_response(request))
166
+ begin
167
+ # guarantee ordered responses
168
+ loop do
169
+ begin
170
+ request = requests.first
171
+ pool.next_tick(timeout) until (response = fetch_response(request, connections, request_options))
143
172
 
144
- responses << response
145
- requests.shift
173
+ responses << response
174
+ requests.shift
146
175
 
147
- break if requests.empty? || !@connection.running?
176
+ break if requests.empty? || pool.empty?
177
+ end
148
178
  end
179
+ responses
180
+ ensure
181
+ close(connections) unless @persistent
149
182
  end
150
- responses
151
183
  end
152
184
 
153
- def __build_req(verb, uri, options = {})
185
+ def build_request(verb, uri, options)
154
186
  rklass = @options.request_class
155
- rklass.new(verb, uri, @options.merge(options))
187
+ request = rklass.new(verb, uri, @options.merge(options))
188
+ request.on(:response, &method(:on_response).curry[request])
189
+ request.on(:promise, &method(:on_promise))
190
+ request
156
191
  end
157
192
 
158
193
  @default_options = Options.new
194
+ @default_options.freeze
159
195
  @plugins = []
160
196
 
161
197
  class << self
@@ -163,7 +199,7 @@ module HTTPX
163
199
 
164
200
  def inherited(klass)
165
201
  super
166
- klass.instance_variable_set(:@default_options, @default_options.dup)
202
+ klass.instance_variable_set(:@default_options, @default_options)
167
203
  klass.instance_variable_set(:@plugins, @plugins.dup)
168
204
  end
169
205
 
@@ -173,15 +209,13 @@ module HTTPX
173
209
  unless @plugins.include?(pl)
174
210
  @plugins << pl
175
211
  pl.load_dependencies(self, *args, &block) if pl.respond_to?(:load_dependencies)
212
+ @default_options = @default_options.dup
213
+ @default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
214
+
176
215
  include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
177
216
  extend(pl::ClassMethods) if defined?(pl::ClassMethods)
178
- if defined?(pl::OptionsMethods) || defined?(pl::OptionsClassMethods)
179
- options_klass = Class.new(@default_options.class)
180
- options_klass.extend(pl::OptionsClassMethods) if defined?(pl::OptionsClassMethods)
181
- options_klass.__send__(:include, pl::OptionsMethods) if defined?(pl::OptionsMethods)
182
- @default_options = options_klass.new
183
- end
184
- opts = default_options
217
+
218
+ opts = @default_options
185
219
  opts.request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
186
220
  opts.request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
187
221
  opts.response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
@@ -192,7 +226,10 @@ module HTTPX
192
226
  opts.request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
193
227
  opts.response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
194
228
  opts.response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
229
+ opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
195
230
  pl.configure(self, *args, &block) if pl.respond_to?(:configure)
231
+
232
+ @default_options.freeze
196
233
  end
197
234
  self
198
235
  end
@@ -9,6 +9,7 @@ module HTTPX
9
9
 
10
10
  def self.new(opts = {})
11
11
  return opts if opts.is_a?(Timeout)
12
+
12
13
  super
13
14
  end
14
15
 
@@ -26,12 +27,10 @@ module HTTPX
26
27
  @operation_timeout = loop_timeout
27
28
  end
28
29
  reset_counter
29
- @state = :idle # this is here not to trigger the log
30
- transition(:idle)
31
30
  end
32
31
 
33
- def timeout
34
- @timeout || @total_timeout
32
+ def total_timeout
33
+ @total_timeout
35
34
  ensure
36
35
  log_time
37
36
  end
@@ -63,16 +62,8 @@ module HTTPX
63
62
  end
64
63
  end
65
64
 
66
- def transition(nextstate)
67
- return if @state == nextstate
68
- case nextstate
69
- # when :idle
70
- when :idle
71
- @timeout = @connect_timeout
72
- when :open
73
- @timeout = @operation_timeout
74
- end
75
- @state = nextstate
65
+ def no_time_left?
66
+ @time_left <= 0
76
67
  end
77
68
 
78
69
  private
@@ -88,8 +79,9 @@ module HTTPX
88
79
  def log_time
89
80
  return unless @time_left
90
81
  return reset_timer unless @started
82
+
91
83
  @time_left -= (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started)
92
- raise TimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds") if @time_left <= 0
84
+ raise TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds") if no_time_left?
93
85
 
94
86
  reset_timer
95
87
  end
@@ -4,6 +4,8 @@ require "forwardable"
4
4
 
5
5
  module HTTPX::Transcoder
6
6
  module Body
7
+ Error = Class.new(HTTPX::Error)
8
+
7
9
  module_function
8
10
 
9
11
  class Encoder
@@ -22,6 +24,8 @@ module HTTPX::Transcoder
22
24
  @raw.map(&:bytesize).reduce(0, :+)
23
25
  elsif @raw.respond_to?(:size)
24
26
  @raw.size || Float::INFINITY
27
+ elsif @raw.respond_to?(:length)
28
+ @raw.length || Float::INFINITY
25
29
  elsif @raw.respond_to?(:each)
26
30
  Float::INFINITY
27
31
  else
@@ -18,6 +18,7 @@ module HTTPX::Transcoder
18
18
 
19
19
  def each
20
20
  return enum_for(__method__) unless block_given?
21
+
21
22
  @raw.each do |chunk|
22
23
  yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
23
24
  end
@@ -57,6 +58,7 @@ module HTTPX::Transcoder
57
58
  when :length
58
59
  index = @buffer.index(CRLF)
59
60
  return unless index && index.positive?
61
+
60
62
  # Read hex-length
61
63
  hexlen = @buffer.slice!(0, index)
62
64
  hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
@@ -69,11 +71,13 @@ module HTTPX::Transcoder
69
71
  # consume CRLF
70
72
  return if @buffer.bytesize < crlf_size
71
73
  raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
74
+
72
75
  @buffer.slice!(0, crlf_size)
73
76
  if @chunk_length.nil?
74
77
  nextstate(:length)
75
78
  else
76
79
  return if @finished
80
+
77
81
  nextstate(:data)
78
82
  end
79
83
  when :data
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- VERSION = "0.3.1"
4
+ VERSION = "0.4.0"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tiago Cardoso
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-12-27 00:00:00.000000000 Z
11
+ date: 2019-05-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: http-2
@@ -68,15 +68,14 @@ files:
68
68
  - LICENSE.txt
69
69
  - README.md
70
70
  - lib/httpx.rb
71
+ - lib/httpx/adapters/faraday.rb
71
72
  - lib/httpx/altsvc.rb
72
73
  - lib/httpx/buffer.rb
73
74
  - lib/httpx/callbacks.rb
74
75
  - lib/httpx/chainable.rb
75
- - lib/httpx/channel.rb
76
- - lib/httpx/channel/http1.rb
77
- - lib/httpx/channel/http2.rb
78
- - lib/httpx/client.rb
79
76
  - lib/httpx/connection.rb
77
+ - lib/httpx/connection/http1.rb
78
+ - lib/httpx/connection/http2.rb
80
79
  - lib/httpx/errors.rb
81
80
  - lib/httpx/extensions.rb
82
81
  - lib/httpx/headers.rb
@@ -99,13 +98,16 @@ files:
99
98
  - lib/httpx/plugins/follow_redirects.rb
100
99
  - lib/httpx/plugins/h2c.rb
101
100
  - lib/httpx/plugins/multipart.rb
101
+ - lib/httpx/plugins/persistent.rb
102
102
  - lib/httpx/plugins/proxy.rb
103
103
  - lib/httpx/plugins/proxy/http.rb
104
104
  - lib/httpx/plugins/proxy/socks4.rb
105
105
  - lib/httpx/plugins/proxy/socks5.rb
106
+ - lib/httpx/plugins/proxy/ssh.rb
106
107
  - lib/httpx/plugins/push_promise.rb
107
108
  - lib/httpx/plugins/retries.rb
108
109
  - lib/httpx/plugins/stream.rb
110
+ - lib/httpx/pool.rb
109
111
  - lib/httpx/registry.rb
110
112
  - lib/httpx/request.rb
111
113
  - lib/httpx/resolver.rb
@@ -116,6 +118,7 @@ files:
116
118
  - lib/httpx/resolver/system.rb
117
119
  - lib/httpx/response.rb
118
120
  - lib/httpx/selector.rb
121
+ - lib/httpx/session.rb
119
122
  - lib/httpx/timeout.rb
120
123
  - lib/httpx/transcoder.rb
121
124
  - lib/httpx/transcoder/body.rb
@@ -142,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
142
145
  - !ruby/object:Gem::Version
143
146
  version: '0'
144
147
  requirements: []
145
- rubyforge_project:
146
- rubygems_version: 2.7.8
148
+ rubygems_version: 3.0.1
147
149
  signing_key:
148
150
  specification_version: 4
149
151
  summary: HTTPX, to the future, and beyond