twingly-http 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +39 -4
- data/lib/twingly/http.rb +29 -14
- data/lib/twingly/version.rb +1 -1
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 21eb122f9e909d09327bb318f00259ed67b5f3a95f7bb7a9963992376730bd4d
|
4
|
+
data.tar.gz: 742b3386edb9e6cb72d84a9d01f1e52ed597446eb67cbd050e800a0842158bf8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4bf39f3fd4f2939a2648d8dd7c117647b41f561c97ecc879c1735da5f02c1870d702f924a8ed96020e8178a3f4382fb650359873c392a08a994067ae81e7de2d
|
7
|
+
data.tar.gz: 7f31cd2df3b531607987d6915f40d6074f4f4c65cfd0ad6b0fb92897deae3310cb3dd1dc3fb759b1dd667ab14a6d59839c28e5600bea2937b2d5e739677daed8
|
data/README.md
CHANGED
@@ -1,7 +1,45 @@
|
|
1
1
|
# Twingly::HTTP
|
2
2
|
|
3
|
-
|
3
|
+
[](https://travis-ci.com/twingly/twingly-http)
|
4
4
|
|
5
|
+
Robust HTTP client, tailored by Twingly.
|
6
|
+
|
7
|
+
## Getting Started
|
8
|
+
|
9
|
+
Install the gem:
|
10
|
+
|
11
|
+
gem install twingly-http
|
12
|
+
|
13
|
+
Example "one-liner" usage:
|
14
|
+
|
15
|
+
```
|
16
|
+
ruby -rlogger -rtwingly/http -e '\
|
17
|
+
logger = Logger.new(STDOUT); logger.level = :INFO; \
|
18
|
+
puts Twingly::HTTP::Client.new(logger: logger, \
|
19
|
+
base_user_agent: "").get("http://example.org").status'
|
20
|
+
```
|
21
|
+
|
22
|
+
Example `irb` usage:
|
23
|
+
|
24
|
+
```
|
25
|
+
irb -rlogger -rtwingly/http
|
26
|
+
```
|
27
|
+
```ruby
|
28
|
+
logger = Logger.new(STDOUT); logger.level = :INFO
|
29
|
+
client = Twingly::HTTP::Client.new(logger: logger, base_user_agent: "")
|
30
|
+
client.get("http://example.org").status
|
31
|
+
```
|
32
|
+
|
33
|
+
## Tests
|
34
|
+
|
35
|
+
The tests require [Toxiproxy](https://github.com/Shopify/toxiproxy#1-installing-toxiproxy) to be installed and running. On macOS you can install it with Homebrew:
|
36
|
+
|
37
|
+
brew tap shopify/shopify
|
38
|
+
brew install toxiproxy
|
39
|
+
|
40
|
+
Run tests with
|
41
|
+
|
42
|
+
bundle exec rake
|
5
43
|
|
6
44
|
## Release workflow
|
7
45
|
|
@@ -18,6 +56,3 @@ Robust HTTP client
|
|
18
56
|
github_changelog_generator
|
19
57
|
|
20
58
|
[twingly-rubygems]: https://rubygems.org/profiles/twingly
|
21
|
-
[ruby-prof]: http://ruby-prof.rubyforge.org/
|
22
|
-
[memory_profiler]: https://github.com/SamSaffron/memory_profiler
|
23
|
-
[examples]: examples/url.rb
|
data/lib/twingly/http.rb
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
require "net/http"
|
4
4
|
require "faraday"
|
5
|
+
require "faraday_middleware/response/follow_redirects"
|
5
6
|
|
6
7
|
require_relative "../faraday/logfmt_logger"
|
7
8
|
require_relative "../faraday/url_size_limit"
|
@@ -12,6 +13,7 @@ module Twingly
|
|
12
13
|
module HTTP
|
13
14
|
class ConnectionError < StandardError; end
|
14
15
|
class UrlSizeLimitExceededError < StandardError; end
|
16
|
+
class RedirectLimitReachedError < StandardError; end
|
15
17
|
class Client # rubocop:disable Metrics/ClassLength
|
16
18
|
DEFAULT_RETRYABLE_EXCEPTIONS = [
|
17
19
|
Faraday::ConnectionFailed,
|
@@ -28,6 +30,7 @@ module Twingly
|
|
28
30
|
DEFAULT_NUMBER_OF_RETRIES = 0
|
29
31
|
DEFAULT_RETRY_INTERVAL = 1
|
30
32
|
DEFAULT_MAX_URL_SIZE_BYTES = Float::INFINITY
|
33
|
+
DEFAULT_FOLLOW_REDIRECTS_LIMIT = 3
|
31
34
|
|
32
35
|
attr_writer :http_timeout
|
33
36
|
attr_writer :http_open_timeout
|
@@ -36,27 +39,20 @@ module Twingly
|
|
36
39
|
attr_writer :on_retry_callback
|
37
40
|
attr_writer :max_url_size_bytes
|
38
41
|
attr_writer :request_id
|
42
|
+
attr_writer :follow_redirects
|
39
43
|
|
44
|
+
attr_accessor :follow_redirects_limit
|
40
45
|
attr_accessor :retryable_exceptions
|
41
46
|
|
42
47
|
def initialize(logger:, base_user_agent:)
|
43
48
|
@logger = logger
|
44
49
|
@base_user_agent = base_user_agent
|
45
|
-
@request_id = nil
|
46
50
|
|
47
|
-
|
48
|
-
@http_open_timeout = DEFAULT_HTTP_OPEN_TIMEOUT
|
49
|
-
|
50
|
-
@retryable_exceptions = DEFAULT_RETRYABLE_EXCEPTIONS
|
51
|
-
@number_of_retries = DEFAULT_NUMBER_OF_RETRIES
|
52
|
-
@retry_interval = DEFAULT_RETRY_INTERVAL
|
53
|
-
@on_retry_callback = nil
|
54
|
-
|
55
|
-
@max_url_size_bytes = DEFAULT_MAX_URL_SIZE_BYTES
|
51
|
+
initialize_defaults
|
56
52
|
end
|
57
53
|
|
58
|
-
def get(url, params: {})
|
59
|
-
http_response_for(:get, url: url, params: params)
|
54
|
+
def get(url, params: {}, headers: {})
|
55
|
+
http_response_for(:get, url: url, params: params, headers: headers)
|
60
56
|
end
|
61
57
|
|
62
58
|
def post(url, body:, headers: {})
|
@@ -65,6 +61,19 @@ module Twingly
|
|
65
61
|
|
66
62
|
private
|
67
63
|
|
64
|
+
def initialize_defaults
|
65
|
+
@request_id = nil
|
66
|
+
@http_timeout = DEFAULT_HTTP_TIMEOUT
|
67
|
+
@http_open_timeout = DEFAULT_HTTP_OPEN_TIMEOUT
|
68
|
+
@retryable_exceptions = DEFAULT_RETRYABLE_EXCEPTIONS
|
69
|
+
@number_of_retries = DEFAULT_NUMBER_OF_RETRIES
|
70
|
+
@retry_interval = DEFAULT_RETRY_INTERVAL
|
71
|
+
@on_retry_callback = nil
|
72
|
+
@follow_redirects = false
|
73
|
+
@follow_redirects_limit = DEFAULT_FOLLOW_REDIRECTS_LIMIT
|
74
|
+
@max_url_size_bytes = DEFAULT_MAX_URL_SIZE_BYTES
|
75
|
+
end
|
76
|
+
|
68
77
|
# rubocop:disable Metrics/MethodLength
|
69
78
|
def http_response_for(method, *args)
|
70
79
|
response = case method
|
@@ -81,14 +90,16 @@ module Twingly
|
|
81
90
|
raise ConnectionError
|
82
91
|
rescue Faraday::UrlSizeLimit::LimitExceededError => error
|
83
92
|
raise UrlSizeLimitExceededError, error.message
|
93
|
+
rescue FaradayMiddleware::RedirectLimitReached => error
|
94
|
+
raise RedirectLimitReachedError, error.message
|
84
95
|
end
|
85
96
|
# rubocop:enable all
|
86
97
|
|
87
|
-
def http_get_response(url:, params:
|
98
|
+
def http_get_response(url:, params:, headers:)
|
88
99
|
binary_url = url.dup.force_encoding(Encoding::BINARY)
|
89
100
|
http_client = create_http_client
|
90
101
|
|
91
|
-
headers = default_headers
|
102
|
+
headers = default_headers.merge(headers)
|
92
103
|
|
93
104
|
http_client.get do |request|
|
94
105
|
request.url(binary_url)
|
@@ -128,6 +139,10 @@ module Twingly
|
|
128
139
|
headers: true,
|
129
140
|
bodies: true,
|
130
141
|
request_id: @request_id
|
142
|
+
if @follow_redirects
|
143
|
+
faraday.use FaradayMiddleware::FollowRedirects,
|
144
|
+
limit: @follow_redirects_limit
|
145
|
+
end
|
131
146
|
faraday.adapter Faraday.default_adapter
|
132
147
|
faraday.headers[:user_agent] = user_agent
|
133
148
|
end
|
data/lib/twingly/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twingly-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Twingly AB
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-11-
|
11
|
+
date: 2019-11-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0.15'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: faraday_middleware
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.13'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.13'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: climate_control
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|