twingly-http 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +38 -4
- data/lib/twingly/http.rb +96 -25
- data/lib/twingly/version.rb +1 -1
- metadata +31 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 573e3beb22fd03c1961fe5b6264b3a2b1ef9f0a5a89b95e0094e1693e76dff7f
|
4
|
+
data.tar.gz: 30a0dda321f1a3804492fd91e49907456cd3e45494f79ff571c054b648f7725c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d209c54bbd796f27d38796d6a5449cf961e7de9a351700098ba92f022502a34950dc7652c4c9d16f80d9ae7474fe65bd6d3a60976cf43db65f767d5fb97277e
|
7
|
+
data.tar.gz: 6e4db092c8893e0539b17af90233d9ac86f2bbb2f51a2ab3b29fbd7c671211861168e30b434b4abdbf209c332861292a71bee37cb47fe4a91f3623b14f67a21f
|
data/README.md
CHANGED
@@ -1,7 +1,44 @@
|
|
1
1
|
# Twingly::HTTP
|
2
2
|
|
3
|
-
|
3
|
+
[](https://github.com/twingly/twingly-http/actions)
|
4
4
|
|
5
|
+
Robust HTTP client, tailored by Twingly.
|
6
|
+
|
7
|
+
## Getting Started
|
8
|
+
|
9
|
+
Install the gem:
|
10
|
+
|
11
|
+
gem install twingly-http
|
12
|
+
|
13
|
+
Example "one-liner" usage:
|
14
|
+
|
15
|
+
```
|
16
|
+
ruby -rlogger -rtwingly/http -e '\
|
17
|
+
logger = Logger.new(STDOUT); logger.level = :INFO; \
|
18
|
+
puts Twingly::HTTP::Client.new(logger: logger, \
|
19
|
+
base_user_agent: "").get("http://example.org").status'
|
20
|
+
```
|
21
|
+
|
22
|
+
Example `irb` usage:
|
23
|
+
|
24
|
+
```
|
25
|
+
irb -rlogger -rtwingly/http
|
26
|
+
```
|
27
|
+
```ruby
|
28
|
+
logger = Logger.new(STDOUT); logger.level = :INFO
|
29
|
+
client = Twingly::HTTP::Client.new(logger: logger, base_user_agent: "")
|
30
|
+
client.get("http://example.org").status
|
31
|
+
```
|
32
|
+
|
33
|
+
## Tests
|
34
|
+
|
35
|
+
The tests require [Toxiproxy](https://github.com/Shopify/toxiproxy)
|
36
|
+
|
37
|
+
docker-compose up
|
38
|
+
|
39
|
+
Run tests with
|
40
|
+
|
41
|
+
bundle exec rake
|
5
42
|
|
6
43
|
## Release workflow
|
7
44
|
|
@@ -18,6 +55,3 @@ Robust HTTP client
|
|
18
55
|
github_changelog_generator
|
19
56
|
|
20
57
|
[twingly-rubygems]: https://rubygems.org/profiles/twingly
|
21
|
-
[ruby-prof]: http://ruby-prof.rubyforge.org/
|
22
|
-
[memory_profiler]: https://github.com/SamSaffron/memory_profiler
|
23
|
-
[examples]: examples/url.rb
|
data/lib/twingly/http.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "logger"
|
3
4
|
require "net/http"
|
4
5
|
require "faraday"
|
6
|
+
require "faraday_middleware"
|
5
7
|
|
6
8
|
require_relative "../faraday/logfmt_logger"
|
7
9
|
require_relative "../faraday/url_size_limit"
|
@@ -12,6 +14,7 @@ module Twingly
|
|
12
14
|
module HTTP
|
13
15
|
class ConnectionError < StandardError; end
|
14
16
|
class UrlSizeLimitExceededError < StandardError; end
|
17
|
+
class RedirectLimitReachedError < StandardError; end
|
15
18
|
class Client # rubocop:disable Metrics/ClassLength
|
16
19
|
DEFAULT_RETRYABLE_EXCEPTIONS = [
|
17
20
|
Faraday::ConnectionFailed,
|
@@ -28,6 +31,7 @@ module Twingly
|
|
28
31
|
DEFAULT_NUMBER_OF_RETRIES = 0
|
29
32
|
DEFAULT_RETRY_INTERVAL = 1
|
30
33
|
DEFAULT_MAX_URL_SIZE_BYTES = Float::INFINITY
|
34
|
+
DEFAULT_FOLLOW_REDIRECTS_LIMIT = 3
|
31
35
|
|
32
36
|
attr_writer :http_timeout
|
33
37
|
attr_writer :http_open_timeout
|
@@ -36,43 +40,60 @@ module Twingly
|
|
36
40
|
attr_writer :on_retry_callback
|
37
41
|
attr_writer :max_url_size_bytes
|
38
42
|
attr_writer :request_id
|
43
|
+
attr_writer :follow_redirects
|
39
44
|
|
45
|
+
attr_accessor :follow_redirects_limit
|
46
|
+
attr_accessor :logger
|
40
47
|
attr_accessor :retryable_exceptions
|
41
48
|
|
42
|
-
def initialize(
|
43
|
-
@logger = logger
|
49
|
+
def initialize(base_user_agent:, logger: default_logger)
|
44
50
|
@base_user_agent = base_user_agent
|
45
|
-
@
|
46
|
-
|
47
|
-
@http_timeout = DEFAULT_HTTP_TIMEOUT
|
48
|
-
@http_open_timeout = DEFAULT_HTTP_OPEN_TIMEOUT
|
49
|
-
|
50
|
-
@retryable_exceptions = DEFAULT_RETRYABLE_EXCEPTIONS
|
51
|
-
@number_of_retries = DEFAULT_NUMBER_OF_RETRIES
|
52
|
-
@retry_interval = DEFAULT_RETRY_INTERVAL
|
53
|
-
@on_retry_callback = nil
|
51
|
+
@logger = logger
|
54
52
|
|
55
|
-
|
53
|
+
initialize_defaults
|
56
54
|
end
|
57
55
|
|
58
|
-
def get(url, params: {})
|
59
|
-
http_response_for(:get, url: url, params: params)
|
56
|
+
def get(url, params: {}, headers: {})
|
57
|
+
http_response_for(:get, url: url, params: params, headers: headers)
|
60
58
|
end
|
61
59
|
|
62
60
|
def post(url, body:, headers: {})
|
63
61
|
http_response_for(:post, url: url, body: body, headers: headers)
|
64
62
|
end
|
65
63
|
|
64
|
+
def put(url, body:, headers: {})
|
65
|
+
http_response_for(:put, url: url, body: body, headers: headers)
|
66
|
+
end
|
67
|
+
|
68
|
+
def patch(url, body:, headers: {})
|
69
|
+
http_response_for(:patch, url: url, body: body, headers: headers)
|
70
|
+
end
|
71
|
+
|
72
|
+
def delete(url, params: {}, headers: {})
|
73
|
+
http_response_for(:delete, url: url, params: params, headers: headers)
|
74
|
+
end
|
75
|
+
|
66
76
|
private
|
67
77
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
78
|
+
def default_logger
|
79
|
+
Logger.new(File::NULL)
|
80
|
+
end
|
81
|
+
|
82
|
+
def initialize_defaults
|
83
|
+
@request_id = nil
|
84
|
+
@http_timeout = DEFAULT_HTTP_TIMEOUT
|
85
|
+
@http_open_timeout = DEFAULT_HTTP_OPEN_TIMEOUT
|
86
|
+
@retryable_exceptions = DEFAULT_RETRYABLE_EXCEPTIONS
|
87
|
+
@number_of_retries = DEFAULT_NUMBER_OF_RETRIES
|
88
|
+
@retry_interval = DEFAULT_RETRY_INTERVAL
|
89
|
+
@on_retry_callback = nil
|
90
|
+
@follow_redirects = false
|
91
|
+
@follow_redirects_limit = DEFAULT_FOLLOW_REDIRECTS_LIMIT
|
92
|
+
@max_url_size_bytes = DEFAULT_MAX_URL_SIZE_BYTES
|
93
|
+
end
|
94
|
+
|
95
|
+
def http_response_for(method, **args)
|
96
|
+
response = send("http_#{method}_response", **args)
|
76
97
|
|
77
98
|
Response.new(headers: response.headers.to_h,
|
78
99
|
status: response.status,
|
@@ -81,14 +102,15 @@ module Twingly
|
|
81
102
|
raise ConnectionError
|
82
103
|
rescue Faraday::UrlSizeLimit::LimitExceededError => error
|
83
104
|
raise UrlSizeLimitExceededError, error.message
|
105
|
+
rescue FaradayMiddleware::RedirectLimitReached => error
|
106
|
+
raise RedirectLimitReachedError, error.message
|
84
107
|
end
|
85
|
-
# rubocop:enable all
|
86
108
|
|
87
|
-
def http_get_response(url:, params:
|
109
|
+
def http_get_response(url:, params:, headers:)
|
88
110
|
binary_url = url.dup.force_encoding(Encoding::BINARY)
|
89
111
|
http_client = create_http_client
|
90
112
|
|
91
|
-
headers = default_headers
|
113
|
+
headers = default_headers.merge(headers)
|
92
114
|
|
93
115
|
http_client.get do |request|
|
94
116
|
request.url(binary_url)
|
@@ -114,6 +136,51 @@ module Twingly
|
|
114
136
|
end
|
115
137
|
end
|
116
138
|
|
139
|
+
def http_put_response(url:, body:, headers:)
|
140
|
+
binary_url = url.dup.force_encoding(Encoding::BINARY)
|
141
|
+
http_client = create_http_client
|
142
|
+
|
143
|
+
headers = default_headers.merge(headers)
|
144
|
+
|
145
|
+
http_client.put do |request|
|
146
|
+
request.url(binary_url)
|
147
|
+
request.headers.merge!(headers)
|
148
|
+
request.body = body
|
149
|
+
request.options.timeout = @http_timeout
|
150
|
+
request.options.open_timeout = @http_open_timeout
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def http_patch_response(url:, body:, headers:)
|
155
|
+
binary_url = url.dup.force_encoding(Encoding::BINARY)
|
156
|
+
http_client = create_http_client
|
157
|
+
|
158
|
+
headers = default_headers.merge(headers)
|
159
|
+
|
160
|
+
http_client.patch do |request|
|
161
|
+
request.url(binary_url)
|
162
|
+
request.headers.merge!(headers)
|
163
|
+
request.body = body
|
164
|
+
request.options.timeout = @http_timeout
|
165
|
+
request.options.open_timeout = @http_open_timeout
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def http_delete_response(url:, params:, headers:)
|
170
|
+
binary_url = url.dup.force_encoding(Encoding::BINARY)
|
171
|
+
http_client = create_http_client
|
172
|
+
|
173
|
+
headers = default_headers.merge(headers)
|
174
|
+
|
175
|
+
http_client.delete do |request|
|
176
|
+
request.url(binary_url)
|
177
|
+
request.params.merge!(params)
|
178
|
+
request.headers.merge!(headers)
|
179
|
+
request.options.timeout = @http_timeout
|
180
|
+
request.options.open_timeout = @http_open_timeout
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
117
184
|
def create_http_client # rubocop:disable Metrics/MethodLength
|
118
185
|
Faraday.new do |faraday|
|
119
186
|
faraday.request :url_size_limit,
|
@@ -128,6 +195,10 @@ module Twingly
|
|
128
195
|
headers: true,
|
129
196
|
bodies: true,
|
130
197
|
request_id: @request_id
|
198
|
+
if @follow_redirects
|
199
|
+
faraday.use FaradayMiddleware::FollowRedirects,
|
200
|
+
limit: @follow_redirects_limit
|
201
|
+
end
|
131
202
|
faraday.adapter Faraday.default_adapter
|
132
203
|
faraday.headers[:user_agent] = user_agent
|
133
204
|
end
|
data/lib/twingly/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twingly-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Twingly AB
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-11-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -16,14 +16,34 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '1'
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 1.0.1
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - "~>"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1'
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 1.0.1
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: faraday_middleware
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 1.0.0
|
20
40
|
type: :runtime
|
21
41
|
prerelease: false
|
22
42
|
version_requirements: !ruby/object:Gem::Requirement
|
23
43
|
requirements:
|
24
44
|
- - "~>"
|
25
45
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
46
|
+
version: 1.0.0
|
27
47
|
- !ruby/object:Gem::Dependency
|
28
48
|
name: climate_control
|
29
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -72,14 +92,14 @@ dependencies:
|
|
72
92
|
requirements:
|
73
93
|
- - "~>"
|
74
94
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
95
|
+
version: 0.77.0
|
76
96
|
type: :development
|
77
97
|
prerelease: false
|
78
98
|
version_requirements: !ruby/object:Gem::Requirement
|
79
99
|
requirements:
|
80
100
|
- - "~>"
|
81
101
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
102
|
+
version: 0.77.0
|
83
103
|
- !ruby/object:Gem::Dependency
|
84
104
|
name: rubocop-rspec
|
85
105
|
requirement: !ruby/object:Gem::Requirement
|
@@ -158,13 +178,13 @@ homepage: http://github.com/twingly/twingly-http
|
|
158
178
|
licenses:
|
159
179
|
- MIT
|
160
180
|
metadata: {}
|
161
|
-
post_install_message:
|
181
|
+
post_install_message:
|
162
182
|
rdoc_options: []
|
163
183
|
require_paths:
|
164
184
|
- lib
|
165
185
|
required_ruby_version: !ruby/object:Gem::Requirement
|
166
186
|
requirements:
|
167
|
-
- - "
|
187
|
+
- - ">="
|
168
188
|
- !ruby/object:Gem::Version
|
169
189
|
version: '2.5'
|
170
190
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
@@ -173,8 +193,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
173
193
|
- !ruby/object:Gem::Version
|
174
194
|
version: '0'
|
175
195
|
requirements: []
|
176
|
-
rubygems_version: 3.
|
177
|
-
signing_key:
|
196
|
+
rubygems_version: 3.1.6
|
197
|
+
signing_key:
|
178
198
|
specification_version: 4
|
179
199
|
summary: Robust HTTP client
|
180
200
|
test_files: []
|