maremma 1.0.4 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +13 -5
- data/CHANGELOG.md +6 -0
- data/Gemfile.lock +7 -10
- data/lib/maremma.rb +13 -0
- data/lib/maremma/version.rb +1 -1
- data/maremma.gemspec +2 -2
- data/spec/maremma_spec.rb +30 -0
- metadata +50 -46
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NDU5MzBlM2Q3ZGNiZThlMTFhZWQyMTlkOGY0YjY1OGZhNjJiODU3MQ==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
MTE5ODlmMjM3NDQ5NWVhYjliNGVmNGQzMDBkOWIxMDJmZjA0MmRkMw==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YmQ4YTA2YjZmNTE2OTBkZTE3MjJhZDk3MTI1ZWFlNTExMGIyODc2MjY5OTJm
|
10
|
+
YTdhOWJhMDBjMTkwYmIyMjQ3MzhkYThmMGUzNTFiZTU0MmE5NGM1NDk0ZjI2
|
11
|
+
MDJmMDcwNDk3ZGIxNjY4ZTE1MGM5YWIwMzUxZTBkYjhiNzZiNWI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
MjdlOTJmNzNiNzkzMGYyY2Q3YjU1Mzc1ZTY4YWMwZDYyN2ViZTMyODM4NzNj
|
14
|
+
N2YwZWE0NDIxOTY1Yjk5MTJkYzM5YWUwYmM1ZDQ4YWZhM2I4NWFiODZjMDVl
|
15
|
+
ZTI4MjEyOTliY2E3YmE5ZWU4YzVmYTYxZTcyMTI3YmUwZGFkMGE=
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## v.1.1.0 (January 5, 2016)
|
2
|
+
|
3
|
+
[maremma 1.1](https://github.com/datacite/maremma/releases/tag/v.1.1) was released on January 5, 2015:
|
4
|
+
|
5
|
+
* added support for rate-limiting headers (Twitter, Github)
|
6
|
+
|
1
7
|
## v.1.0 (November 28, 2015)
|
2
8
|
|
3
9
|
[maremma 1.0](https://github.com/datacite/maremma/releases/tag/v.1.0) was released on November 28, 2015:
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
maremma (1.0
|
4
|
+
maremma (1.1.0)
|
5
5
|
activesupport (~> 4.2, >= 4.2.5)
|
6
6
|
builder (~> 3.2, >= 3.2.2)
|
7
7
|
excon (~> 0.45.0)
|
@@ -23,7 +23,7 @@ GEM
|
|
23
23
|
tzinfo (~> 1.1)
|
24
24
|
addressable (2.3.8)
|
25
25
|
builder (3.2.2)
|
26
|
-
crack (0.4.
|
26
|
+
crack (0.4.3)
|
27
27
|
safe_yaml (~> 1.0.0)
|
28
28
|
diff-lcs (1.2.5)
|
29
29
|
excon (0.45.4)
|
@@ -64,9 +64,9 @@ GEM
|
|
64
64
|
thread_safe (0.3.5)
|
65
65
|
tzinfo (1.2.2)
|
66
66
|
thread_safe (~> 0.1)
|
67
|
-
vcr (3.0.
|
68
|
-
webmock (1.22.
|
69
|
-
addressable (
|
67
|
+
vcr (3.0.1)
|
68
|
+
webmock (1.22.5)
|
69
|
+
addressable (< 2.4.0)
|
70
70
|
crack (>= 0.3.2)
|
71
71
|
hashdiff
|
72
72
|
|
@@ -76,11 +76,8 @@ PLATFORMS
|
|
76
76
|
DEPENDENCIES
|
77
77
|
bundler (~> 1.0)
|
78
78
|
maremma!
|
79
|
-
rack-test
|
79
|
+
rack-test (~> 0)
|
80
80
|
rake
|
81
81
|
rspec (~> 3.4)
|
82
82
|
vcr
|
83
|
-
webmock
|
84
|
-
|
85
|
-
BUNDLED WITH
|
86
|
-
1.10.6
|
83
|
+
webmock (~> 1.22, >= 1.22.3)
|
data/lib/maremma.rb
CHANGED
@@ -49,6 +49,12 @@ module Maremma
|
|
49
49
|
conn.options[:timeout] = options[:timeout] || DEFAULT_TIMEOUT
|
50
50
|
|
51
51
|
response = conn.get url, {}, options[:headers]
|
52
|
+
|
53
|
+
# return error if we are close to the rate limit, if supported in headers
|
54
|
+
if get_rate_limit_remaining(response.headers) < 10
|
55
|
+
return { error: "Rate limit soon exceeded", status: 429 }
|
56
|
+
end
|
57
|
+
|
52
58
|
parse_response(response.body)
|
53
59
|
rescue *NETWORKABLE_EXCEPTIONS => error
|
54
60
|
rescue_faraday_error(error)
|
@@ -113,6 +119,13 @@ module Maremma
|
|
113
119
|
|
114
120
|
protected
|
115
121
|
|
122
|
+
# currently supported by Twitter and Github
|
123
|
+
# with slightly different header names
|
124
|
+
# use arbitrary high value if not supported
|
125
|
+
def self.get_rate_limit_remaining(headers)
|
126
|
+
(headers["X-Rate-Limit-Remaining"] || headers["X-RateLimit-Remaining"] || 100).to_i
|
127
|
+
end
|
128
|
+
|
116
129
|
def self.from_xml(string)
|
117
130
|
if Nokogiri::XML(string).errors.empty?
|
118
131
|
Hash.from_xml(string)
|
data/lib/maremma/version.rb
CHANGED
data/maremma.gemspec
CHANGED
@@ -31,7 +31,7 @@ Gem::Specification.new do |s|
|
|
31
31
|
s.add_development_dependency 'bundler', '~> 1.0'
|
32
32
|
s.add_development_dependency 'rspec', '~> 3.4'
|
33
33
|
s.add_development_dependency 'rake'
|
34
|
-
s.add_development_dependency 'rack-test'
|
34
|
+
s.add_development_dependency 'rack-test', '~> 0'
|
35
35
|
s.add_development_dependency 'vcr'
|
36
|
-
s.add_development_dependency 'webmock'
|
36
|
+
s.add_development_dependency 'webmock', '~> 1.22', '>= 1.22.3'
|
37
37
|
end
|
data/spec/maremma_spec.rb
CHANGED
@@ -150,6 +150,36 @@ describe Maremma do
|
|
150
150
|
end
|
151
151
|
end
|
152
152
|
|
153
|
+
context "rate limit exceeded" do
|
154
|
+
it "get json" do
|
155
|
+
stub = stub_request(:get, url).to_return(status: 200, headers: { 'X-Rate-Limit-Remaining' => 3 })
|
156
|
+
response = subject.get(url)
|
157
|
+
expect(response).to eq(error: "Rate limit soon exceeded", status: 429)
|
158
|
+
expect(stub).to have_been_requested
|
159
|
+
end
|
160
|
+
|
161
|
+
it "get xml" do
|
162
|
+
stub = stub_request(:get, url).to_return(status: 200, headers: { 'X-Rate-Limit-Remaining' => 3 })
|
163
|
+
response = subject.get(url, content_type: 'xml')
|
164
|
+
expect(response).to eq(error: "Rate limit soon exceeded", status: 429)
|
165
|
+
expect(stub).to have_been_requested
|
166
|
+
end
|
167
|
+
|
168
|
+
it "get html" do
|
169
|
+
stub = stub_request(:get, url).to_return(status: 200, headers: { 'X-Rate-Limit-Remaining' => 3 })
|
170
|
+
response = subject.get(url, content_type: 'html')
|
171
|
+
expect(response).to eq(error: "Rate limit soon exceeded", status: 429)
|
172
|
+
expect(stub).to have_been_requested
|
173
|
+
end
|
174
|
+
|
175
|
+
it "post xml" do
|
176
|
+
stub = stub_request(:post, url).with(:body => post_data.to_xml)
|
177
|
+
.to_return(status: 200, headers: { 'X-Rate-Limit-Remaining' => 3 })
|
178
|
+
subject.post(url, content_type: 'xml', data: post_data.to_xml) { |response| expect(response).to be_nil }
|
179
|
+
expect(stub).to have_been_requested
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
153
183
|
context "redirect requests" do
|
154
184
|
let(:redirect_url) { "http://www.example.org/redirect" }
|
155
185
|
|
metadata
CHANGED
@@ -1,237 +1,243 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maremma
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Martin Fenner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ~>
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 0.9.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ~>
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 0.9.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: faraday_middleware
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ~>
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: 0.10.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: 0.10.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: faraday-encoding
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: 0.0.1
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 0.0.1
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: excon
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 0.45.0
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - ~>
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 0.45.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: nokogiri
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ~>
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: 1.6.6.4
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ~>
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: 1.6.6.4
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: builder
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - ~>
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '3.2'
|
90
|
-
- -
|
90
|
+
- - ! '>='
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: 3.2.2
|
93
93
|
type: :runtime
|
94
94
|
prerelease: false
|
95
95
|
version_requirements: !ruby/object:Gem::Requirement
|
96
96
|
requirements:
|
97
|
-
- -
|
97
|
+
- - ~>
|
98
98
|
- !ruby/object:Gem::Version
|
99
99
|
version: '3.2'
|
100
|
-
- -
|
100
|
+
- - ! '>='
|
101
101
|
- !ruby/object:Gem::Version
|
102
102
|
version: 3.2.2
|
103
103
|
- !ruby/object:Gem::Dependency
|
104
104
|
name: multi_json
|
105
105
|
requirement: !ruby/object:Gem::Requirement
|
106
106
|
requirements:
|
107
|
-
- -
|
107
|
+
- - ~>
|
108
108
|
- !ruby/object:Gem::Version
|
109
109
|
version: 1.11.2
|
110
110
|
type: :runtime
|
111
111
|
prerelease: false
|
112
112
|
version_requirements: !ruby/object:Gem::Requirement
|
113
113
|
requirements:
|
114
|
-
- -
|
114
|
+
- - ~>
|
115
115
|
- !ruby/object:Gem::Version
|
116
116
|
version: 1.11.2
|
117
117
|
- !ruby/object:Gem::Dependency
|
118
118
|
name: oj
|
119
119
|
requirement: !ruby/object:Gem::Requirement
|
120
120
|
requirements:
|
121
|
-
- -
|
121
|
+
- - ~>
|
122
122
|
- !ruby/object:Gem::Version
|
123
123
|
version: 2.13.1
|
124
124
|
type: :runtime
|
125
125
|
prerelease: false
|
126
126
|
version_requirements: !ruby/object:Gem::Requirement
|
127
127
|
requirements:
|
128
|
-
- -
|
128
|
+
- - ~>
|
129
129
|
- !ruby/object:Gem::Version
|
130
130
|
version: 2.13.1
|
131
131
|
- !ruby/object:Gem::Dependency
|
132
132
|
name: activesupport
|
133
133
|
requirement: !ruby/object:Gem::Requirement
|
134
134
|
requirements:
|
135
|
-
- -
|
135
|
+
- - ~>
|
136
136
|
- !ruby/object:Gem::Version
|
137
137
|
version: '4.2'
|
138
|
-
- -
|
138
|
+
- - ! '>='
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: 4.2.5
|
141
141
|
type: :runtime
|
142
142
|
prerelease: false
|
143
143
|
version_requirements: !ruby/object:Gem::Requirement
|
144
144
|
requirements:
|
145
|
-
- -
|
145
|
+
- - ~>
|
146
146
|
- !ruby/object:Gem::Version
|
147
147
|
version: '4.2'
|
148
|
-
- -
|
148
|
+
- - ! '>='
|
149
149
|
- !ruby/object:Gem::Version
|
150
150
|
version: 4.2.5
|
151
151
|
- !ruby/object:Gem::Dependency
|
152
152
|
name: bundler
|
153
153
|
requirement: !ruby/object:Gem::Requirement
|
154
154
|
requirements:
|
155
|
-
- -
|
155
|
+
- - ~>
|
156
156
|
- !ruby/object:Gem::Version
|
157
157
|
version: '1.0'
|
158
158
|
type: :development
|
159
159
|
prerelease: false
|
160
160
|
version_requirements: !ruby/object:Gem::Requirement
|
161
161
|
requirements:
|
162
|
-
- -
|
162
|
+
- - ~>
|
163
163
|
- !ruby/object:Gem::Version
|
164
164
|
version: '1.0'
|
165
165
|
- !ruby/object:Gem::Dependency
|
166
166
|
name: rspec
|
167
167
|
requirement: !ruby/object:Gem::Requirement
|
168
168
|
requirements:
|
169
|
-
- -
|
169
|
+
- - ~>
|
170
170
|
- !ruby/object:Gem::Version
|
171
171
|
version: '3.4'
|
172
172
|
type: :development
|
173
173
|
prerelease: false
|
174
174
|
version_requirements: !ruby/object:Gem::Requirement
|
175
175
|
requirements:
|
176
|
-
- -
|
176
|
+
- - ~>
|
177
177
|
- !ruby/object:Gem::Version
|
178
178
|
version: '3.4'
|
179
179
|
- !ruby/object:Gem::Dependency
|
180
180
|
name: rake
|
181
181
|
requirement: !ruby/object:Gem::Requirement
|
182
182
|
requirements:
|
183
|
-
- -
|
183
|
+
- - ! '>='
|
184
184
|
- !ruby/object:Gem::Version
|
185
185
|
version: '0'
|
186
186
|
type: :development
|
187
187
|
prerelease: false
|
188
188
|
version_requirements: !ruby/object:Gem::Requirement
|
189
189
|
requirements:
|
190
|
-
- -
|
190
|
+
- - ! '>='
|
191
191
|
- !ruby/object:Gem::Version
|
192
192
|
version: '0'
|
193
193
|
- !ruby/object:Gem::Dependency
|
194
194
|
name: rack-test
|
195
195
|
requirement: !ruby/object:Gem::Requirement
|
196
196
|
requirements:
|
197
|
-
- -
|
197
|
+
- - ~>
|
198
198
|
- !ruby/object:Gem::Version
|
199
199
|
version: '0'
|
200
200
|
type: :development
|
201
201
|
prerelease: false
|
202
202
|
version_requirements: !ruby/object:Gem::Requirement
|
203
203
|
requirements:
|
204
|
-
- -
|
204
|
+
- - ~>
|
205
205
|
- !ruby/object:Gem::Version
|
206
206
|
version: '0'
|
207
207
|
- !ruby/object:Gem::Dependency
|
208
208
|
name: vcr
|
209
209
|
requirement: !ruby/object:Gem::Requirement
|
210
210
|
requirements:
|
211
|
-
- -
|
211
|
+
- - ! '>='
|
212
212
|
- !ruby/object:Gem::Version
|
213
213
|
version: '0'
|
214
214
|
type: :development
|
215
215
|
prerelease: false
|
216
216
|
version_requirements: !ruby/object:Gem::Requirement
|
217
217
|
requirements:
|
218
|
-
- -
|
218
|
+
- - ! '>='
|
219
219
|
- !ruby/object:Gem::Version
|
220
220
|
version: '0'
|
221
221
|
- !ruby/object:Gem::Dependency
|
222
222
|
name: webmock
|
223
223
|
requirement: !ruby/object:Gem::Requirement
|
224
224
|
requirements:
|
225
|
-
- -
|
225
|
+
- - ~>
|
226
226
|
- !ruby/object:Gem::Version
|
227
|
-
version: '
|
227
|
+
version: '1.22'
|
228
|
+
- - ! '>='
|
229
|
+
- !ruby/object:Gem::Version
|
230
|
+
version: 1.22.3
|
228
231
|
type: :development
|
229
232
|
prerelease: false
|
230
233
|
version_requirements: !ruby/object:Gem::Requirement
|
231
234
|
requirements:
|
232
|
-
- -
|
235
|
+
- - ~>
|
233
236
|
- !ruby/object:Gem::Version
|
234
|
-
version: '
|
237
|
+
version: '1.22'
|
238
|
+
- - ! '>='
|
239
|
+
- !ruby/object:Gem::Version
|
240
|
+
version: 1.22.3
|
235
241
|
description: Simplifies network calls, including json/xml parsing and error handling.
|
236
242
|
Based on Faraday.
|
237
243
|
email: mfenner@datacite.org
|
@@ -240,8 +246,8 @@ extensions: []
|
|
240
246
|
extra_rdoc_files:
|
241
247
|
- README.md
|
242
248
|
files:
|
243
|
-
-
|
244
|
-
-
|
249
|
+
- .gitignore
|
250
|
+
- .travis.yml
|
245
251
|
- CHANGELOG.md
|
246
252
|
- Gemfile
|
247
253
|
- Gemfile.lock
|
@@ -263,20 +269,18 @@ require_paths:
|
|
263
269
|
- lib
|
264
270
|
required_ruby_version: !ruby/object:Gem::Requirement
|
265
271
|
requirements:
|
266
|
-
- -
|
272
|
+
- - ! '>='
|
267
273
|
- !ruby/object:Gem::Version
|
268
274
|
version: '0'
|
269
275
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
270
276
|
requirements:
|
271
|
-
- -
|
277
|
+
- - ! '>='
|
272
278
|
- !ruby/object:Gem::Version
|
273
279
|
version: '0'
|
274
280
|
requirements: []
|
275
281
|
rubyforge_project:
|
276
|
-
rubygems_version: 2.4.5
|
282
|
+
rubygems_version: 2.4.5
|
277
283
|
signing_key:
|
278
284
|
specification_version: 4
|
279
285
|
summary: Simplified network calls
|
280
|
-
test_files:
|
281
|
-
- spec/maremma_spec.rb
|
282
|
-
- spec/spec_helper.rb
|
286
|
+
test_files: []
|