wgit 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/wgit/crawler.rb +11 -4
- data/lib/wgit/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 70e6ec83e53550bcfe180b66248747763314c33738ecd0fabddca65dbb3918b0
|
|
4
|
+
data.tar.gz: a1c3d1e8bb6d078731876093cb2beed0ea4da65cb03dff1ead975f714bd3d9b5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ab519107506ec0798995cb52f986040da12d1a5c59c0c36f84bf8e09d847fd5ab83b3bd7f47ff95b6e474a35d855b176fdc9d245b1cef356781eadb21a4a84f7
|
|
7
|
+
data.tar.gz: '010748005ded444f44812c8b6022d258b60a3485dcef8b78c562012428e3955a7fbfe80f53a570cb9f6b524042388949cf2cf08d6a1b27581f2cfd9b424603b0'
|
data/lib/wgit/crawler.rb
CHANGED
|
@@ -221,9 +221,10 @@ module Wgit
|
|
|
221
221
|
yield(url, response, location) if block_given?
|
|
222
222
|
|
|
223
223
|
# Handle redirect logic.
|
|
224
|
-
raise "External redirect not allowed - Redirected to: \
|
|
225
|
-
'#{location}', which is outside of host: '#{host}'" \
|
|
226
224
|
if !follow_external_redirects && !location.relative?(host: host)
|
|
225
|
+
raise "External redirect not allowed - Redirected to: \
|
|
226
|
+
'#{location}', which is outside of host: '#{host}'"
|
|
227
|
+
end
|
|
227
228
|
|
|
228
229
|
raise "Too many redirects, exceeded: #{redirect_count}" \
|
|
229
230
|
if redirect_count >= @redirect_limit
|
|
@@ -250,14 +251,20 @@ module Wgit
|
|
|
250
251
|
url = url.normalize if url.respond_to?(:normalize)
|
|
251
252
|
|
|
252
253
|
opts = {
|
|
253
|
-
followlocation: false,
|
|
254
|
+
followlocation: false,
|
|
255
|
+
timeout: @time_out,
|
|
256
|
+
accept_encoding: 'gzip',
|
|
257
|
+
headers: {
|
|
258
|
+
'User-Agent' => "wgit/#{Wgit::VERSION}",
|
|
259
|
+
'Accept' => 'text/html'
|
|
260
|
+
}
|
|
254
261
|
}
|
|
255
262
|
|
|
256
263
|
response = Typhoeus.get(url, opts)
|
|
257
264
|
|
|
258
265
|
# Handle response status code.
|
|
259
266
|
raise "No response (within timeout: #{@time_out} second(s))" \
|
|
260
|
-
if response.code
|
|
267
|
+
if response.code.zero?
|
|
261
268
|
|
|
262
269
|
response
|
|
263
270
|
end
|
data/lib/wgit/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wgit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michael Telford
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-10-
|
|
11
|
+
date: 2019-10-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: addressable
|