micky 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +20 -0
- data/README.md +40 -1
- data/Rakefile +8 -0
- data/lib/micky/errors.rb +4 -0
- data/lib/micky/request.rb +76 -11
- data/lib/micky/response.rb +7 -1
- data/lib/micky/uri.rb +5 -2
- data/lib/micky/version.rb +1 -1
- data/lib/micky.rb +3 -0
- data/test/max_response_size_test.rb +89 -0
- data/test/redirect_test.rb +22 -0
- data/test/test_helper.rb +115 -0
- data/test/total_timeout_test.rb +44 -0
- data/test/uri_test.rb +26 -0
- metadata +71 -9
- data/.gitignore +0 -17
- data/Gemfile +0 -2
- data/micky.gemspec +0 -26
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ecea094f30e0551b84949260cd3c78b34673cb69bcefdcd746b506938adffeab
|
|
4
|
+
data.tar.gz: 98874aec481b78fdb48fe0ae50248ec51683e299b3a6c6770706a706b6266894
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6a0f768ceb90965a4203f30c1d68de2a82f328f3fa5cbf18a68ebe94f54c0018cd2df1c2ef5399af64bae8221868368407bd98d60cbdf78beb59c31ee80a7ae7
|
|
7
|
+
data.tar.gz: a277768598966377bd039e5a4df2340ac679d10728d9b913b45f3882159b1812d8eccf768424a43b6f4b4244a9b2e57c400a09b4dc685d2ab1eda56040a66f0f
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
strategy:
|
|
11
|
+
fail-fast: false
|
|
12
|
+
matrix:
|
|
13
|
+
ruby: ['3.3', '3.4', '4.0']
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: ruby/setup-ruby@v1
|
|
17
|
+
with:
|
|
18
|
+
ruby-version: ${{ matrix.ruby }}
|
|
19
|
+
bundler-cache: true
|
|
20
|
+
- run: bundle exec rake test
|
data/README.md
CHANGED
|
@@ -76,6 +76,46 @@ Micky.get('http://drpm.me/unwz.jpg', headers: { 'Accept' => 'text/html' })
|
|
|
76
76
|
Micky.get('http://urls.api.twitter.com/1/urls/count.json', query: { url: 'dropmeme.com' })
|
|
77
77
|
```
|
|
78
78
|
|
|
79
|
+
### Limiting what an untrusted server can cost you
|
|
80
|
+
|
|
81
|
+
By default Micky reads a response to its end, and `:timeout` bounds each socket
|
|
82
|
+
operation rather than the call as a whole. Against a server you don’t control,
|
|
83
|
+
neither is a limit: a body can be as large as the server cares to send, and a
|
|
84
|
+
server that answers slowly but never stalls is never timed out at all.
|
|
85
|
+
|
|
86
|
+
`:max_response_size` is the number of body bytes Micky will read. Past that it
|
|
87
|
+
stops reading and drops the connection, so an oversized response costs you the
|
|
88
|
+
limit and not the body. The call returns `nil`, or raises
|
|
89
|
+
`Micky::TooLargeResponse` when `:raise_errors` is set:
|
|
90
|
+
|
|
91
|
+
```ruby
|
|
92
|
+
Micky.get(url, max_response_size: 512 * 1024) # nil if the body is larger
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Pass `:truncate` to get the first `:max_response_size` bytes instead. A
|
|
96
|
+
truncated response is useful for some formats and useless for others, so this
|
|
97
|
+
is opt-in: fine for HTML, not for an image.
|
|
98
|
+
|
|
99
|
+
```ruby
|
|
100
|
+
response = Micky.get(url, max_response_size: 512 * 1024, truncate: true)
|
|
101
|
+
response.body.bytesize # at most 524288
|
|
102
|
+
response.truncated? # true if the server had more to send
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
`:total_timeout` is a wall clock for the whole call, redirects included. It is
|
|
106
|
+
the only option that bounds a server which drips one byte at a time, or a long
|
|
107
|
+
chain of individually-reasonable redirects:
|
|
108
|
+
|
|
109
|
+
```ruby
|
|
110
|
+
Micky.get(url, timeout: 5, total_timeout: 10)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
On expiry Micky returns `nil`, or raises `Micky::TotalTimeout` when
|
|
114
|
+
`:raise_errors` is set. It defaults to 20 seconds; pass `nil` for no limit.
|
|
115
|
+
|
|
116
|
+
`:max_response_size` has no default, since downloading a large file can be the
|
|
117
|
+
point. For most callers it is the only option worth setting explicitly.
|
|
118
|
+
|
|
79
119
|
### OAuth `Authorization` header
|
|
80
120
|
|
|
81
121
|
Micky supports creating a OAuth `Authorization` header with the help of the
|
|
@@ -162,7 +202,6 @@ end
|
|
|
162
202
|
## TODO
|
|
163
203
|
|
|
164
204
|
- Support :basic_auth and :digest_auth through [HTTPauth](https://github.com/Manfred/HTTPauth)
|
|
165
|
-
- Add tests
|
|
166
205
|
- Better document configuration options in README
|
|
167
206
|
|
|
168
207
|
## Contributing
|
data/Rakefile
CHANGED
data/lib/micky/errors.rb
CHANGED
data/lib/micky/request.rb
CHANGED
|
@@ -9,20 +9,40 @@ module Micky
|
|
|
9
9
|
value = opts.has_key?(name) ? opts[name] : Micky.public_send(name)
|
|
10
10
|
instance_variable_set "@#{name}", value
|
|
11
11
|
end
|
|
12
|
+
|
|
13
|
+
@truncated = false
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
def get(uri)
|
|
15
17
|
@request_class_name = 'Get'
|
|
16
|
-
request_with_redirect_handling(uri)
|
|
18
|
+
with_total_timeout { request_with_redirect_handling(uri) }
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
def head(uri)
|
|
20
22
|
@request_class_name = 'Head'
|
|
21
|
-
request_with_redirect_handling(uri)
|
|
23
|
+
with_total_timeout { request_with_redirect_handling(uri) }
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
private
|
|
25
27
|
|
|
28
|
+
# `timeout` bounds each socket operation, not the whole call. A slow drip or a
|
|
29
|
+
# long redirect chain never trips it. `total_timeout` is a wall clock for the
|
|
30
|
+
# whole call, redirects included.
|
|
31
|
+
# We pass our own exception class: Timeout only converts the instance it raised
|
|
32
|
+
# itself, so an enclosing block’s expiry passes through untouched and ours is
|
|
33
|
+
# never caught by a `rescue Timeout::Error` in between.
|
|
34
|
+
def with_total_timeout
|
|
35
|
+
return yield unless @total_timeout
|
|
36
|
+
|
|
37
|
+
begin
|
|
38
|
+
Timeout.timeout(@total_timeout, Micky::TotalTimeout) { yield }
|
|
39
|
+
rescue Micky::TotalTimeout => e
|
|
40
|
+
raise e if @raise_errors
|
|
41
|
+
log "Total timeout reached (#{@total_timeout}s)"
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
26
46
|
def request_with_redirect_handling(uri, redirect_count = 0)
|
|
27
47
|
if redirect_count >= @max_redirects
|
|
28
48
|
raise Micky::TooManyRedirects, "Max redirects reached (#{@max_redirects})" if @raise_errors
|
|
@@ -32,8 +52,15 @@ module Micky
|
|
|
32
52
|
|
|
33
53
|
case response = request(uri)
|
|
34
54
|
when Net::HTTPSuccess
|
|
55
|
+
if @truncated && !@truncate
|
|
56
|
+
raise Micky::TooLargeResponse, "Response larger than #{@max_response_size} bytes" if @raise_errors
|
|
57
|
+
log "Response larger than #{@max_response_size} bytes"
|
|
58
|
+
return nil
|
|
59
|
+
end
|
|
60
|
+
|
|
35
61
|
debug "#{response.code} success"
|
|
36
|
-
Response
|
|
62
|
+
log "Response truncated at #{@max_response_size} bytes" if @truncated
|
|
63
|
+
Response.new(response, @uri, truncated: @truncated)
|
|
37
64
|
when Net::HTTPRedirection
|
|
38
65
|
previous_uri = uri
|
|
39
66
|
uri = response['Location']
|
|
@@ -48,15 +75,19 @@ module Micky
|
|
|
48
75
|
if uri.start_with? '//'
|
|
49
76
|
# Protocol-relative
|
|
50
77
|
uri = Micky::URI(uri).to_s
|
|
51
|
-
elsif uri.start_with? '/'
|
|
52
|
-
# Host-relative
|
|
53
|
-
previous_uri = Micky::URI(previous_uri)
|
|
54
|
-
uri = File.join("#{previous_uri.scheme}://#{previous_uri.host}", uri)
|
|
55
78
|
else
|
|
56
|
-
# Path-relative
|
|
57
79
|
previous_uri = Micky::URI(previous_uri)
|
|
58
|
-
|
|
59
|
-
|
|
80
|
+
origin = "#{previous_uri.scheme}://#{previous_uri.host}"
|
|
81
|
+
origin += ":#{previous_uri.port}" unless previous_uri.port == previous_uri.default_port
|
|
82
|
+
|
|
83
|
+
if uri.start_with? '/'
|
|
84
|
+
# Host-relative
|
|
85
|
+
uri = File.join(origin, uri)
|
|
86
|
+
else
|
|
87
|
+
# Path-relative
|
|
88
|
+
previous_directory = previous_uri.path.sub(/[^\/]+\z/, '')
|
|
89
|
+
uri = File.join("#{origin}#{previous_directory}", uri)
|
|
90
|
+
end
|
|
60
91
|
end
|
|
61
92
|
end
|
|
62
93
|
|
|
@@ -156,7 +187,11 @@ module Micky
|
|
|
156
187
|
@headers.each { |k,v| request[k] = v }
|
|
157
188
|
|
|
158
189
|
begin
|
|
159
|
-
|
|
190
|
+
if @max_response_size
|
|
191
|
+
http.request(request) { |response| read_capped_body(response) }
|
|
192
|
+
else
|
|
193
|
+
http.request(request)
|
|
194
|
+
end
|
|
160
195
|
rescue Zlib::Error
|
|
161
196
|
request['Accept-Encoding'] = 'identity'
|
|
162
197
|
retry
|
|
@@ -171,6 +206,36 @@ module Micky
|
|
|
171
206
|
end
|
|
172
207
|
end
|
|
173
208
|
|
|
209
|
+
# Without a block, Net::HTTP reads the whole body into memory before we see any
|
|
210
|
+
# of it. Reading it ourselves lets us stop at `max_response_size` and drop the
|
|
211
|
+
# connection. Runs on redirect hops too, since a redirect can carry a body.
|
|
212
|
+
def read_capped_body(response)
|
|
213
|
+
# Net::HTTP yields nothing for a body-less response (204, HEAD) and for an
|
|
214
|
+
# empty one alike, so decide up front whether the body is nil or empty
|
|
215
|
+
body = +''.b if response.class.body_permitted? && @request_class_name == 'Get'
|
|
216
|
+
@truncated = false
|
|
217
|
+
|
|
218
|
+
response.read_body do |chunk|
|
|
219
|
+
room = @max_response_size - body.bytesize
|
|
220
|
+
|
|
221
|
+
if chunk.bytesize > room
|
|
222
|
+
body << chunk.byteslice(0, room)
|
|
223
|
+
@truncated = true
|
|
224
|
+
break
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
body << chunk
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# `read_body` only sets `@read` at the end of the body, and `break` skips that.
|
|
231
|
+
# Without it, `response.body` would resume reading where we stopped.
|
|
232
|
+
response.instance_variable_set(:@read, true)
|
|
233
|
+
response.body = body
|
|
234
|
+
|
|
235
|
+
# The server’s Content-Length no longer matches the body
|
|
236
|
+
response['content-length'] = body.bytesize.to_s if @truncated
|
|
237
|
+
end
|
|
238
|
+
|
|
174
239
|
def log(message, uri = @uri, severity = Logger::WARN)
|
|
175
240
|
message = "#{message.class}: #{message.message}" if message.is_a? Exception
|
|
176
241
|
@logger.add(severity, "[Micky] #{@request_class_name.upcase} #{uri} - #{message}")
|
data/lib/micky/response.rb
CHANGED
|
@@ -5,9 +5,15 @@ module Micky
|
|
|
5
5
|
class Response < SimpleDelegator
|
|
6
6
|
attr_reader :uri
|
|
7
7
|
|
|
8
|
-
def initialize(response, uri)
|
|
8
|
+
def initialize(response, uri, truncated: false)
|
|
9
9
|
super(response)
|
|
10
10
|
@uri = uri
|
|
11
|
+
@truncated = truncated
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# True when the body was cut at `max_response_size`
|
|
15
|
+
def truncated?
|
|
16
|
+
@truncated
|
|
11
17
|
end
|
|
12
18
|
|
|
13
19
|
def data
|
data/lib/micky/uri.rb
CHANGED
|
@@ -16,7 +16,9 @@ module Micky
|
|
|
16
16
|
::URI.parse(uri)
|
|
17
17
|
rescue ::URI::InvalidURIError
|
|
18
18
|
begin
|
|
19
|
-
::URI
|
|
19
|
+
# ::URI::Parser is the RFC3986 parser, whose #escape is a deprecated shim
|
|
20
|
+
# that delegates to the RFC2396 one and warns under $VERBOSE
|
|
21
|
+
::URI.parse(::URI::RFC2396_PARSER.escape(uri))
|
|
20
22
|
rescue ::URI::InvalidURIError
|
|
21
23
|
end
|
|
22
24
|
end
|
|
@@ -25,7 +27,8 @@ module Micky
|
|
|
25
27
|
|
|
26
28
|
module URI
|
|
27
29
|
def self.extract(text)
|
|
28
|
-
::URI.extract
|
|
30
|
+
# Same as above: ::URI.extract is a deprecated shim over this
|
|
31
|
+
::URI::RFC2396_PARSER.extract(text).select { |uri|
|
|
29
32
|
begin
|
|
30
33
|
::URI.parse(uri).is_a? ::URI::HTTP
|
|
31
34
|
rescue ::URI::InvalidURIError
|
data/lib/micky/version.rb
CHANGED
data/lib/micky.rb
CHANGED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
|
|
3
|
+
describe 'Micky max_response_size' do
|
|
4
|
+
it 'reads the whole body when the option is not set' do
|
|
5
|
+
response = Micky.get(server.url('/ok'))
|
|
6
|
+
assert_equal 1000, response.body.bytesize
|
|
7
|
+
refute response.truncated?
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
it 'leaves a body under the limit alone' do
|
|
11
|
+
response = Micky.get(server.url('/ok'), max_response_size: 100_000)
|
|
12
|
+
assert_equal 1000, response.body.bytesize
|
|
13
|
+
refute response.truncated?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
it 'accepts a body exactly at the limit' do
|
|
17
|
+
response = Micky.get(server.url('/ok'), max_response_size: 1000)
|
|
18
|
+
assert_equal 1000, response.body.bytesize
|
|
19
|
+
refute response.truncated?
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
describe 'when the server sends more than the limit' do
|
|
23
|
+
let(:limit) { 50_000 }
|
|
24
|
+
|
|
25
|
+
it 'returns nil' do
|
|
26
|
+
assert_nil Micky.get(server.url('/flood'), max_response_size: limit)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
it 'raises Micky::TooLargeResponse when asked to raise' do
|
|
30
|
+
error = assert_raises(Micky::TooLargeResponse) do
|
|
31
|
+
Micky.get(server.url('/flood'), max_response_size: limit, raise_errors: true)
|
|
32
|
+
end
|
|
33
|
+
assert_equal 'Response larger than 50000 bytes', error.message
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
it 'gives up as soon as it has read the limit, rather than draining the server' do
|
|
37
|
+
# Unbounded, this reads a gigabyte
|
|
38
|
+
assert_operator elapsed { Micky.get(server.url('/flood'), max_response_size: limit) }, :<, 1.0
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
describe 'with truncate: true' do
|
|
42
|
+
let(:response) { Micky.get(server.url('/flood'), max_response_size: limit, truncate: true) }
|
|
43
|
+
|
|
44
|
+
it 'returns the first bytes up to the limit' do
|
|
45
|
+
assert_equal limit, response.body.bytesize
|
|
46
|
+
assert response.truncated?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it 'does not resume reading when the body is asked for again' do
|
|
50
|
+
response.body
|
|
51
|
+
assert_equal limit, response.body.bytesize
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
it 'reports a content length that matches what it holds' do
|
|
55
|
+
assert_equal limit, response.content_length
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'returns as soon as it has enough, rather than draining the server' do
|
|
59
|
+
assert_operator elapsed { response }, :<, 1.0
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it 'counts what it keeps, not what came off the wire, when Net::HTTP decompresses' do
|
|
63
|
+
response = Micky.get(server.url('/gzipped-flood'), max_response_size: limit, truncate: true)
|
|
64
|
+
assert_equal limit, response.body.bytesize
|
|
65
|
+
assert response.body.start_with?('xxx')
|
|
66
|
+
assert response.truncated?
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
it 'caps a redirect body without treating it as too large' do
|
|
72
|
+
# The 302 itself carries a gigabyte; only the hop after it is a real response
|
|
73
|
+
time = elapsed {
|
|
74
|
+
response = Micky.get(server.url('/fat-redirect'), max_response_size: 50_000)
|
|
75
|
+
assert_equal 1000, response.body.bytesize
|
|
76
|
+
refute response.truncated?
|
|
77
|
+
}
|
|
78
|
+
assert_operator time, :<, 2.0
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
it 'keeps an empty body empty rather than nil' do
|
|
82
|
+
assert_equal '', Micky.get(server.url('/chunked-empty')).body
|
|
83
|
+
assert_equal '', Micky.get(server.url('/chunked-empty'), max_response_size: 100).body
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
it 'keeps a HEAD body nil' do
|
|
87
|
+
assert_nil Micky.head(server.url('/ok'), max_response_size: 100).body
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
|
|
3
|
+
describe 'Micky redirects' do
|
|
4
|
+
# The test server listens on a random port, so a redirect that loses the port
|
|
5
|
+
# goes to 127.0.0.1:80 and fails
|
|
6
|
+
it 'keeps the port when the Location is host-relative' do
|
|
7
|
+
response = Micky.get(server.url('/host-relative-redirect'))
|
|
8
|
+
assert_equal 1000, response.body.bytesize
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
it 'keeps the port when the Location is path-relative' do
|
|
12
|
+
response = Micky.get(server.url('/dir/path-relative-redirect'))
|
|
13
|
+
assert_equal 1000, response.body.bytesize
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
it 'redirects to the resolved URL' do
|
|
17
|
+
log = StringIO.new
|
|
18
|
+
Micky.logger = Logger.new(log, level: Logger::DEBUG)
|
|
19
|
+
Micky.get(server.url('/host-relative-redirect'))
|
|
20
|
+
assert_includes log.string, "302 redirect to #{server.url('/ok')}"
|
|
21
|
+
end
|
|
22
|
+
end
|
data/test/test_helper.rb
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
require 'minitest/autorun'
|
|
2
|
+
require 'micky'
|
|
3
|
+
require 'socket'
|
|
4
|
+
require 'logger'
|
|
5
|
+
|
|
6
|
+
# WebMock and friends intercept above Net::HTTP’s body reading, so they can’t
|
|
7
|
+
# test how many bytes come off the socket or how long we wait. This serves real
|
|
8
|
+
# HTTP over a real socket instead.
|
|
9
|
+
class TestServer
|
|
10
|
+
attr_reader :port
|
|
11
|
+
|
|
12
|
+
RESPONSES = {
|
|
13
|
+
# A small, ordinary response
|
|
14
|
+
'/ok' => ->(socket) {
|
|
15
|
+
body = 'hello' * 200
|
|
16
|
+
socket.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: #{body.bytesize}\r\n\r\n"
|
|
17
|
+
socket.write body
|
|
18
|
+
},
|
|
19
|
+
'/dir/ok' => ->(socket) { RESPONSES['/ok'].call(socket) },
|
|
20
|
+
# A 200 whose chunked body is empty; Net::HTTP yields no chunk for it
|
|
21
|
+
'/chunked-empty' => ->(socket) {
|
|
22
|
+
socket.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n"
|
|
23
|
+
},
|
|
24
|
+
# Redirects with a host-relative Location
|
|
25
|
+
'/host-relative-redirect' => ->(socket) {
|
|
26
|
+
socket.write "HTTP/1.1 302 Found\r\nLocation: /ok\r\nContent-Length: 0\r\n\r\n"
|
|
27
|
+
},
|
|
28
|
+
# Redirects with a path-relative Location, to /dir/ok
|
|
29
|
+
'/dir/path-relative-redirect' => ->(socket) {
|
|
30
|
+
socket.write "HTTP/1.1 302 Found\r\nLocation: ok\r\nContent-Length: 0\r\n\r\n"
|
|
31
|
+
},
|
|
32
|
+
# Streams for as long as anyone reads
|
|
33
|
+
'/flood' => ->(socket) {
|
|
34
|
+
socket.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 1000000000\r\n\r\n"
|
|
35
|
+
loop { socket.write('x' * 8192) }
|
|
36
|
+
},
|
|
37
|
+
# One byte at a time, never idle long enough to trip a per-read timeout
|
|
38
|
+
'/drip' => ->(socket) {
|
|
39
|
+
socket.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 100000\r\n\r\n"
|
|
40
|
+
loop { socket.write('x'); socket.flush; sleep 0.2 }
|
|
41
|
+
},
|
|
42
|
+
# A redirect that itself carries a huge body
|
|
43
|
+
'/fat-redirect' => ->(socket) {
|
|
44
|
+
socket.write "HTTP/1.1 302 Found\r\nLocation: /ok\r\nContent-Length: 1000000000\r\n\r\n"
|
|
45
|
+
loop { socket.write('x' * 8192) }
|
|
46
|
+
},
|
|
47
|
+
# gzip, so Net::HTTP’s transparent decoding is in the path
|
|
48
|
+
'/gzipped-flood' => ->(socket) {
|
|
49
|
+
require 'zlib'
|
|
50
|
+
socket.write "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\n\r\n"
|
|
51
|
+
writer = Zlib::GzipWriter.new(socket)
|
|
52
|
+
loop { writer.write('x' * 8192) }
|
|
53
|
+
},
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
def initialize
|
|
57
|
+
@server = TCPServer.new('127.0.0.1', 0)
|
|
58
|
+
@port = @server.addr[1]
|
|
59
|
+
@thread = Thread.new { accept_loop }
|
|
60
|
+
@thread.abort_on_exception = false
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def url(path) = "http://127.0.0.1:#{@port}#{path}"
|
|
64
|
+
|
|
65
|
+
def stop
|
|
66
|
+
@thread.kill
|
|
67
|
+
@server.close
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
def accept_loop
|
|
73
|
+
loop do
|
|
74
|
+
socket = @server.accept
|
|
75
|
+
Thread.start(socket) do |s|
|
|
76
|
+
begin
|
|
77
|
+
path = s.gets.to_s.split(' ')[1]
|
|
78
|
+
while (line = s.gets) && line != "\r\n"; end
|
|
79
|
+
RESPONSES.fetch(path, ->(so) { so.write("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") }).call(s)
|
|
80
|
+
rescue StandardError, Timeout::Error
|
|
81
|
+
# The client hung up, as most of these specs expect
|
|
82
|
+
ensure
|
|
83
|
+
s.close rescue nil
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
class MickyTest < Minitest::Spec
|
|
91
|
+
def self.server
|
|
92
|
+
@server ||= begin
|
|
93
|
+
server = TestServer.new
|
|
94
|
+
Minitest.after_run { server.stop }
|
|
95
|
+
server
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def server = self.class.server
|
|
100
|
+
|
|
101
|
+
def setup
|
|
102
|
+
Micky::DEFAULTS.each { |key, value| Micky.public_send("#{key}=", value) }
|
|
103
|
+
Micky.logger = Logger.new(nil)
|
|
104
|
+
Micky.skip_resolve = true
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def elapsed
|
|
108
|
+
t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
109
|
+
yield
|
|
110
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Every `describe` block inherits the server and the defaults reset above
|
|
115
|
+
Minitest::Spec.register_spec_type(//, MickyTest)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
|
|
3
|
+
describe 'Micky total_timeout' do
|
|
4
|
+
it 'defaults to 20 seconds' do
|
|
5
|
+
assert_equal 20, Micky.total_timeout
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
it 'leaves a fast response alone' do
|
|
9
|
+
assert_equal 1000, Micky.get(server.url('/ok'), total_timeout: 5).body.bytesize
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
describe 'against a server that drips bytes slower than it is read' do
|
|
13
|
+
# Every read succeeds, so `timeout` never fires however low it is set
|
|
14
|
+
let(:opts) { { timeout: 5, total_timeout: 1 } }
|
|
15
|
+
|
|
16
|
+
it 'gives up at the deadline' do
|
|
17
|
+
assert_in_delta 1, elapsed { Micky.get(server.url('/drip'), **opts) }, 0.5
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it 'returns nil by default' do
|
|
21
|
+
assert_nil Micky.get(server.url('/drip'), **opts)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it 'raises Micky::TotalTimeout when asked to raise' do
|
|
25
|
+
assert_raises(Micky::TotalTimeout) do
|
|
26
|
+
Micky.get(server.url('/drip'), raise_errors: true, **opts)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
it 'never returns without the deadline, which is what makes it necessary' do
|
|
31
|
+
assert_raises(Timeout::Error) do
|
|
32
|
+
Timeout.timeout(2) { Micky.get(server.url('/drip'), timeout: 5, total_timeout: nil) }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
it 'does not relabel an enclosing block’s expiry as its own' do
|
|
38
|
+
assert_raises(ArgumentError) do
|
|
39
|
+
Timeout.timeout(0.2, ArgumentError) do
|
|
40
|
+
Micky.get(server.url('/drip'), timeout: 5, total_timeout: 30)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
data/test/uri_test.rb
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
require 'test_helper'
|
|
2
|
+
|
|
3
|
+
describe 'Micky.URI' do
|
|
4
|
+
# Ruby only prints the deprecation under $VERBOSE, so turn it on to see it
|
|
5
|
+
def without_warnings
|
|
6
|
+
verbose, $VERBOSE = $VERBOSE, true
|
|
7
|
+
_, err = capture_io { yield }
|
|
8
|
+
err
|
|
9
|
+
ensure
|
|
10
|
+
$VERBOSE = verbose
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it 'escapes a URL that ::URI.parse rejects' do
|
|
14
|
+
uri = nil
|
|
15
|
+
err = without_warnings { uri = Micky.URI('http://example.com/a b?x=1|2') }
|
|
16
|
+
assert_equal 'http://example.com/a%20b?x=1%7C2', uri.to_s
|
|
17
|
+
assert_empty err
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it 'extracts http URLs from text' do
|
|
21
|
+
urls = nil
|
|
22
|
+
err = without_warnings { urls = Micky::URI.extract('see http://a.com and ftp://c.com') }
|
|
23
|
+
assert_equal ['http://a.com'], urls
|
|
24
|
+
assert_empty err
|
|
25
|
+
end
|
|
26
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,56 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: micky
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- Rafael Masson
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: logger
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: base64
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '0'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '0'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: uri
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 0.13.1
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 0.13.1
|
|
12
54
|
- !ruby/object:Gem::Dependency
|
|
13
55
|
name: rake
|
|
14
56
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -23,6 +65,20 @@ dependencies:
|
|
|
23
65
|
- - ">="
|
|
24
66
|
- !ruby/object:Gem::Version
|
|
25
67
|
version: '0'
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: minitest
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - ">="
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: '0'
|
|
75
|
+
type: :development
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - ">="
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: '0'
|
|
26
82
|
description: Micky makes simple HTTP requests (GET/HEAD), follows redirects, handles
|
|
27
83
|
exceptions (invalid hosts/URIs, server errors, timeouts, redirect loops), automatically
|
|
28
84
|
parses responses (JSON, etc.), is very lightweight, and has no dependency.
|
|
@@ -32,8 +88,7 @@ executables: []
|
|
|
32
88
|
extensions: []
|
|
33
89
|
extra_rdoc_files: []
|
|
34
90
|
files:
|
|
35
|
-
- ".
|
|
36
|
-
- Gemfile
|
|
91
|
+
- ".github/workflows/ci.yml"
|
|
37
92
|
- LICENSE.txt
|
|
38
93
|
- README.md
|
|
39
94
|
- Rakefile
|
|
@@ -43,11 +98,18 @@ files:
|
|
|
43
98
|
- lib/micky/response.rb
|
|
44
99
|
- lib/micky/uri.rb
|
|
45
100
|
- lib/micky/version.rb
|
|
46
|
-
-
|
|
47
|
-
|
|
101
|
+
- test/max_response_size_test.rb
|
|
102
|
+
- test/redirect_test.rb
|
|
103
|
+
- test/test_helper.rb
|
|
104
|
+
- test/total_timeout_test.rb
|
|
105
|
+
- test/uri_test.rb
|
|
106
|
+
homepage: https://github.com/rafbm/micky
|
|
48
107
|
licenses:
|
|
49
108
|
- MIT
|
|
50
|
-
metadata:
|
|
109
|
+
metadata:
|
|
110
|
+
homepage_uri: https://github.com/rafbm/micky
|
|
111
|
+
source_code_uri: https://github.com/rafbm/micky
|
|
112
|
+
rubygems_mfa_required: 'true'
|
|
51
113
|
rdoc_options: []
|
|
52
114
|
require_paths:
|
|
53
115
|
- lib
|
|
@@ -55,14 +117,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
55
117
|
requirements:
|
|
56
118
|
- - ">="
|
|
57
119
|
- !ruby/object:Gem::Version
|
|
58
|
-
version: '
|
|
120
|
+
version: '3.3'
|
|
59
121
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
60
122
|
requirements:
|
|
61
123
|
- - ">="
|
|
62
124
|
- !ruby/object:Gem::Version
|
|
63
125
|
version: '0'
|
|
64
126
|
requirements: []
|
|
65
|
-
rubygems_version:
|
|
127
|
+
rubygems_version: 4.0.20
|
|
66
128
|
specification_version: 4
|
|
67
129
|
summary: Lightweight and worry-free HTTP client
|
|
68
130
|
test_files: []
|
data/.gitignore
DELETED
data/Gemfile
DELETED
data/micky.gemspec
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
# coding: utf-8
|
|
2
|
-
lib = File.expand_path('../lib', __FILE__)
|
|
3
|
-
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
-
require 'micky/version'
|
|
5
|
-
|
|
6
|
-
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name = 'micky'
|
|
8
|
-
spec.version = Micky::VERSION
|
|
9
|
-
spec.authors = ['Rafaël Blais Masson']
|
|
10
|
-
spec.email = ['rafbmasson@gmail.com']
|
|
11
|
-
spec.description =
|
|
12
|
-
'Micky makes simple HTTP requests (GET/HEAD), follows redirects, handles ' \
|
|
13
|
-
'exceptions (invalid hosts/URIs, server errors, timeouts, redirect loops), ' \
|
|
14
|
-
'automatically parses responses (JSON, etc.), is very lightweight, and has no ' \
|
|
15
|
-
'dependency.'
|
|
16
|
-
spec.summary = 'Lightweight and worry-free HTTP client'
|
|
17
|
-
spec.homepage = 'http://github.com/rafBM/micky'
|
|
18
|
-
spec.license = 'MIT'
|
|
19
|
-
|
|
20
|
-
spec.files = `git ls-files`.split($/)
|
|
21
|
-
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
22
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
23
|
-
spec.require_paths = ['lib']
|
|
24
|
-
|
|
25
|
-
spec.add_development_dependency 'rake'
|
|
26
|
-
end
|