httpx 0.20.3 → 0.20.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 543163347ad58a18829298d36113f88da710c5b139c14968962cdd88d9d36c86
4
- data.tar.gz: 1a460e4e2f079f6b08ede7ad7076039ead2b6fc31cffb590e4c245d38e838dc5
3
+ metadata.gz: 3ae7fc4057762abf47253c487544c4e3cd07e31326adec158235f1b53130a8db
4
+ data.tar.gz: 0a09ef99c59c1b3fd1e516066b656b78d888ffd4eed1160c6c9d96f001669410
5
5
  SHA512:
6
- metadata.gz: 7bf4ccdbefc71b868b51ff130cb0892791935dade7829ffe09b3401dbba112fd354a8781da8d58015afb254206ac8a5187477ac65bdaf09b996909ef2e42f93c
7
- data.tar.gz: a06649e2a51aebe113e3604e739916f164a30aa643eb7e2a5f82871929d2ce1c9bb8b9f234c2ac2ce07f3bd995058e8ba7f1b42e5eaf5cb63fe205b753184b0b
6
+ metadata.gz: 61c786041e8e27c55d45c30e5f8558b1423470254c5be819c2a9daaa61b8bb049e7e892e0a2673799d26339435fd237fbe772bcb02a0373969e7f382eaa76186
7
+ data.tar.gz: f24c113213513d847aa693242902d334e1052deaacdfd944def81d91ea9545113ad8a6a973c5b97ec141fc6474f5513b2a3228c9ba644f8560c29806c10c2886
@@ -0,0 +1,17 @@
1
+ # 0.20.4
2
+
3
+ ## Improvements
4
+
5
+ The `:response_cache` plugin is now more compliant with how the RFC 2616 defines which behaviour caches shall have:
6
+
7
+ * it caches only responses with one of the following status codes: 200, 203, 300, 301, 410.
8
+ * it discards cached responses which become stale.
9
+ * it supports "cache-control" header directives to decided when to cache, to store, what the response "age" is.
10
+ * it can cache more than one response for the same request, provided that the request presents different header values for the headers declared in the "vary" response header (previously, it was only caching the first response, and discarding the remainder).
11
+
12
+
13
+
14
+ ## Bugfixes
15
+
16
+ * fixed DNS resolution bug which caused a loop when a failed connection attempt would cause a new DNS request to be triggered for the same domain, filling up and giving preference to the very IP which failed the attempt.
17
+ * response_cache: request verb is now taken into account, not causing HEAD/GET confusion for the same URL.
@@ -83,22 +83,41 @@ module HTTPX
83
83
  end
84
84
 
85
85
  module ArrayExtensions
86
- refine Array do
86
+ module FilterMap
87
+ refine Array do
87
88
 
88
- def filter_map
89
- return to_enum(:filter_map) unless block_given?
89
+ def filter_map
90
+ return to_enum(:filter_map) unless block_given?
90
91
 
91
- each_with_object([]) do |item, res|
92
- processed = yield(item)
93
- res << processed if processed
92
+ each_with_object([]) do |item, res|
93
+ processed = yield(item)
94
+ res << processed if processed
95
+ end
94
96
  end
95
97
  end unless Array.method_defined?(:filter_map)
98
+ end
96
99
 
97
- def sum(accumulator = 0, &block)
98
- values = block_given? ? map(&block) : self
99
- values.inject(accumulator, :+)
100
+ module Sum
101
+ refine Array do
102
+ def sum(accumulator = 0, &block)
103
+ values = block_given? ? map(&block) : self
104
+ values.inject(accumulator, :+)
105
+ end
100
106
  end unless Array.method_defined?(:sum)
101
107
  end
108
+
109
+ module Intersect
110
+ refine Array do
111
+ def intersect?(arr)
112
+ if size < arr.size
113
+ smaller = self
114
+ else
115
+ smaller, arr = arr, self
116
+ end
117
+ (array & smaller).size > 0
118
+ end
119
+ end unless Array.method_defined?(:intersect?)
120
+ end
102
121
  end
103
122
 
104
123
  module IOExtensions
@@ -72,7 +72,7 @@ module HTTPX
72
72
  end
73
73
 
74
74
  module ResponseBodyMethods
75
- using ArrayExtensions
75
+ using ArrayExtensions::FilterMap
76
76
 
77
77
  attr_reader :encodings
78
78
 
@@ -13,35 +13,38 @@ module HTTPX::Plugins
13
13
  @store = {}
14
14
  end
15
15
 
16
- def lookup(uri)
17
- @store[uri]
16
+ def lookup(request)
17
+ responses = @store[request.response_cache_key]
18
+
19
+ return unless responses
20
+
21
+ response = responses.find(&method(:match_by_vary?).curry(2)[request])
22
+
23
+ return unless response && response.fresh?
24
+
25
+ response
18
26
  end
19
27
 
20
- def cached?(uri)
21
- @store.key?(uri)
28
+ def cached?(request)
29
+ lookup(request)
22
30
  end
23
31
 
24
- def cache(uri, response)
25
- @store[uri] = response
32
+ def cache(request, response)
33
+ return unless ResponseCache.cacheable_request?(request) && ResponseCache.cacheable_response?(response)
34
+
35
+ responses = (@store[request.response_cache_key] ||= [])
36
+
37
+ responses.reject!(&method(:match_by_vary?).curry(2)[request])
38
+
39
+ responses << response
26
40
  end
27
41
 
28
42
  def prepare(request)
29
- cached_response = @store[request.uri]
43
+ cached_response = lookup(request)
30
44
 
31
45
  return unless cached_response
32
46
 
33
- original_request = cached_response.instance_variable_get(:@request)
34
-
35
- if (vary = cached_response.headers["vary"])
36
- if vary == "*"
37
- return unless request.headers.same_headers?(original_request.headers)
38
- else
39
- return unless vary.split(/ *, */).all? do |cache_field|
40
- cache_field.downcase!
41
- !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
42
- end
43
- end
44
- end
47
+ return unless match_by_vary?(request, cached_response)
45
48
 
46
49
  if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
47
50
  request.headers.add("if-modified-since", last_modified)
@@ -51,6 +54,23 @@ module HTTPX::Plugins
51
54
  request.headers.add("if-none-match", etag)
52
55
  end
53
56
  end
57
+
58
+ private
59
+
60
+ def match_by_vary?(request, response)
61
+ vary = response.vary
62
+
63
+ return true unless vary
64
+
65
+ original_request = response.instance_variable_get(:@request)
66
+
67
+ return request.headers.same_headers?(original_request.headers) if vary == %w[*]
68
+
69
+ vary.all? do |cache_field|
70
+ cache_field.downcase!
71
+ !original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
72
+ end
73
+ end
54
74
  end
55
75
  end
56
76
  end
@@ -9,7 +9,9 @@ module HTTPX
9
9
  #
10
10
  module ResponseCache
11
11
  CACHEABLE_VERBS = %i[get head].freeze
12
+ CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
12
13
  private_constant :CACHEABLE_VERBS
14
+ private_constant :CACHEABLE_STATUS_CODES
13
15
 
14
16
  class << self
15
17
  def load_dependencies(*)
@@ -17,14 +19,28 @@ module HTTPX
17
19
  end
18
20
 
19
21
  def cacheable_request?(request)
20
- CACHEABLE_VERBS.include?(request.verb)
22
+ CACHEABLE_VERBS.include?(request.verb) &&
23
+ (
24
+ !request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
25
+ )
21
26
  end
22
27
 
23
28
  def cacheable_response?(response)
24
29
  response.is_a?(Response) &&
25
- # partial responses shall not be cached, only full ones.
30
+ (
31
+ response.cache_control.nil? ||
32
+ # TODO: !response.cache_control.include?("private") && is shared cache
33
+ !response.cache_control.include?("no-store")
34
+ ) &&
35
+ CACHEABLE_STATUS_CODES.include?(response.status) &&
36
+ # RFC 2616 13.4 - A response received with a status code of 200, 203, 206, 300, 301 or
37
+ # 410 MAY be stored by a cache and used in reply to a subsequent
38
+ # request, subject to the expiration mechanism, unless a cache-control
39
+ # directive prohibits caching. However, a cache that does not support
40
+ # the Range and Content-Range headers MUST NOT cache 206 (Partial
41
+ # Content) responses.
26
42
  response.status != 206 && (
27
- response.headers.key?("etag") || response.headers.key?("last-modified-at")
43
+ response.headers.key?("etag") || response.headers.key?("last-modified-at") || response.fresh?
28
44
  )
29
45
  end
30
46
 
@@ -52,7 +68,7 @@ module HTTPX
52
68
 
53
69
  def build_request(*)
54
70
  request = super
55
- return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request.uri)
71
+ return request unless ResponseCache.cacheable_request?(request) && @options.response_cache_store.cached?(request)
56
72
 
57
73
  @options.response_cache_store.prepare(request)
58
74
 
@@ -62,25 +78,99 @@ module HTTPX
62
78
  def fetch_response(request, *)
63
79
  response = super
64
80
 
65
- if response && ResponseCache.cached_response?(response)
81
+ return unless response
82
+
83
+ if ResponseCache.cached_response?(response)
66
84
  log { "returning cached response for #{request.uri}" }
67
- cached_response = @options.response_cache_store.lookup(request.uri)
85
+ cached_response = @options.response_cache_store.lookup(request)
68
86
 
69
87
  response.copy_from_cached(cached_response)
70
- end
71
88
 
72
- @options.response_cache_store.cache(request.uri, response) if response && ResponseCache.cacheable_response?(response)
89
+ else
90
+ @options.response_cache_store.cache(request, response)
91
+ end
73
92
 
74
93
  response
75
94
  end
76
95
  end
77
96
 
97
+ module RequestMethods
98
+ def response_cache_key
99
+ @response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}#{@uri}")
100
+ end
101
+ end
102
+
78
103
  module ResponseMethods
79
104
  def copy_from_cached(other)
80
105
  @body = other.body
81
106
 
82
107
  @body.__send__(:rewind)
83
108
  end
109
+
110
+ # A response is fresh if its age has not yet exceeded its freshness lifetime.
111
+ def fresh?
112
+ if cache_control
113
+ return false if cache_control.include?("no-cache")
114
+
115
+ # check age: max-age
116
+ max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
117
+
118
+ max_age ||= cache_control.find { |directive| directive.start_with?("max-age") }
119
+
120
+ max_age = max_age[/age=(\d+)/, 1] if max_age
121
+
122
+ max_age = max_age.to_i if max_age
123
+
124
+ return max_age > age if max_age
125
+ end
126
+
127
+ # check age: expires
128
+ if @headers.key?("expires")
129
+ begin
130
+ expires = Time.httpdate(@headers["expires"])
131
+ rescue ArgumentError
132
+ return true
133
+ end
134
+
135
+ return (expires - Time.now).to_i.positive?
136
+ end
137
+
138
+ true
139
+ end
140
+
141
+ def cache_control
142
+ return @cache_control if defined?(@cache_control)
143
+
144
+ @cache_control = begin
145
+ return unless @headers.key?("cache-control")
146
+
147
+ @headers["cache-control"].split(/ *, */)
148
+ end
149
+ end
150
+
151
+ def vary
152
+ return @vary if defined?(@vary)
153
+
154
+ @vary = begin
155
+ return unless @headers.key?("vary")
156
+
157
+ @headers["vary"].split(/ *, */)
158
+ end
159
+ end
160
+
161
+ private
162
+
163
+ def age
164
+ return @headers["age"].to_i if @headers.key?("age")
165
+
166
+ (Time.now - date).to_i
167
+ end
168
+
169
+ def date
170
+ @date ||= Time.httpdate(@headers["date"])
171
+ rescue NoMethodError, ArgumentError
172
+ Time.now.httpdate
173
+ end
84
174
  end
85
175
  end
86
176
  register_plugin :response_cache, ResponseCache
data/lib/httpx/pool.rb CHANGED
@@ -7,7 +7,7 @@ require "httpx/resolver"
7
7
 
8
8
  module HTTPX
9
9
  class Pool
10
- using ArrayExtensions
10
+ using ArrayExtensions::FilterMap
11
11
  extend Forwardable
12
12
 
13
13
  def_delegator :@timers, :after
@@ -6,7 +6,7 @@ require "resolv"
6
6
  module HTTPX
7
7
  class Resolver::Multi
8
8
  include Callbacks
9
- using ArrayExtensions
9
+ using ArrayExtensions::FilterMap
10
10
 
11
11
  attr_reader :resolvers
12
12
 
@@ -8,6 +8,8 @@ module HTTPX
8
8
  include Callbacks
9
9
  include Loggable
10
10
 
11
+ using ArrayExtensions::Intersect
12
+
11
13
  RECORD_TYPES = {
12
14
  Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
13
15
  Socket::AF_INET => Resolv::DNS::Resource::IN::A,
@@ -48,6 +50,10 @@ module HTTPX
48
50
  addresses.map! do |address|
49
51
  address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
50
52
  end
53
+
54
+ # double emission check
55
+ return if connection.addresses && !addresses.intersect?(connection.addresses)
56
+
51
57
  log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
52
58
  if @pool && # if triggered by early resolve, pool may not be here yet
53
59
  !connection.io &&
@@ -56,8 +62,12 @@ module HTTPX
56
62
  addresses.first.to_s != connection.origin.host.to_s
57
63
  log { "resolver: A response, applying resolution delay..." }
58
64
  @pool.after(0.05) do
59
- connection.addresses = addresses
60
- emit(:resolve, connection)
65
+ # double emission check
66
+ unless connection.addresses && addresses.intersect?(connection.addresses)
67
+
68
+ connection.addresses = addresses
69
+ emit(:resolve, connection)
70
+ end
61
71
  end
62
72
  else
63
73
  connection.addresses = addresses
@@ -310,9 +310,12 @@ module HTTPX
310
310
 
311
311
  class ErrorResponse
312
312
  include Loggable
313
+ extend Forwardable
313
314
 
314
315
  attr_reader :request, :error
315
316
 
317
+ def_delegator :@request, :uri
318
+
316
319
  def initialize(request, error, options)
317
320
  @request = request
318
321
  @error = error
@@ -9,7 +9,7 @@ module HTTPX::Transcoder
9
9
  module_function
10
10
 
11
11
  class Encoder
12
- using HTTPX::ArrayExtensions
12
+ using HTTPX::ArrayExtensions::Sum
13
13
  extend Forwardable
14
14
 
15
15
  def_delegator :@raw, :to_s
data/lib/httpx/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- VERSION = "0.20.3"
4
+ VERSION = "0.20.4"
5
5
  end
@@ -8,15 +8,19 @@ module HTTPX
8
8
  def self?.cached_response?: (response response) -> bool
9
9
 
10
10
  class Store
11
- @store: Hash[URI::Generic, Response]
11
+ @store: Hash[String, Array[Response]]
12
12
 
13
- def lookup: (URI::Generic uri) -> Response?
13
+ def lookup: (Request request) -> Response?
14
14
 
15
- def cached?: (URI::Generic uri) -> bool
15
+ def cached?: (Request request) -> boolish
16
16
 
17
- def cache: (URI::Generic uri, Response response) -> void
17
+ def cache: (Request request, Response response) -> void
18
18
 
19
19
  def prepare: (Request request) -> void
20
+
21
+ private
22
+
23
+ def match_by_vary?: (Request request, Response response) -> bool
20
24
  end
21
25
 
22
26
  module InstanceMethods
@@ -25,8 +29,24 @@ module HTTPX
25
29
  def clear_response_cache: () -> void
26
30
  end
27
31
 
32
+ module RequestMethods
33
+ def response_cache_key: () -> String
34
+ end
35
+
28
36
  module ResponseMethods
29
37
  def copy_from_cached: (Response other) -> void
38
+
39
+ def fresh?: () -> bool
40
+
41
+ def cache_control: () -> Array[String]?
42
+
43
+ def vary: () -> Array[String]?
44
+
45
+ private
46
+
47
+ def age: () -> Integer
48
+
49
+ def date: () -> Time
30
50
  end
31
51
  end
32
52
 
data/sig/response.rbs CHANGED
@@ -96,6 +96,7 @@ module HTTPX
96
96
  class ErrorResponse
97
97
  include _Response
98
98
  include Loggable
99
+ extend Forwardable
99
100
 
100
101
  @options: Options
101
102
  @error: Exception
@@ -104,6 +105,8 @@ module HTTPX
104
105
 
105
106
  def status: () -> (Integer | _ToS)
106
107
 
108
+ def uri: () -> URI::Generic
109
+
107
110
  private
108
111
 
109
112
  def initialize: (Request, Exception, options) -> untyped
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.20.3
4
+ version: 0.20.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tiago Cardoso
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-07-03 00:00:00.000000000 Z
11
+ date: 2022-08-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: http-2-next
@@ -84,6 +84,7 @@ extra_rdoc_files:
84
84
  - doc/release_notes/0_20_1.md
85
85
  - doc/release_notes/0_20_2.md
86
86
  - doc/release_notes/0_20_3.md
87
+ - doc/release_notes/0_24_4.md
87
88
  - doc/release_notes/0_2_0.md
88
89
  - doc/release_notes/0_2_1.md
89
90
  - doc/release_notes/0_3_0.md
@@ -160,6 +161,7 @@ files:
160
161
  - doc/release_notes/0_20_1.md
161
162
  - doc/release_notes/0_20_2.md
162
163
  - doc/release_notes/0_20_3.md
164
+ - doc/release_notes/0_24_4.md
163
165
  - doc/release_notes/0_2_0.md
164
166
  - doc/release_notes/0_2_1.md
165
167
  - doc/release_notes/0_3_0.md
@@ -364,7 +366,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
364
366
  - !ruby/object:Gem::Version
365
367
  version: '0'
366
368
  requirements: []
367
- rubygems_version: 3.2.32
369
+ rubygems_version: 3.3.7
368
370
  signing_key:
369
371
  specification_version: 4
370
372
  summary: HTTPX, to the future, and beyond