digger 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/digger/http.rb +9 -2
- data/lib/digger/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 586e17ec8338bf102a91c8ee241632619a55b6b7
|
4
|
+
data.tar.gz: d93040c059eaf4c4e34cc329c4938a324feea142
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 089a7c6b6b1f8904202cfa8f362cc30d39b1dbfaaff5be721ded38f9448fe03aed23c4986ff587ab60499bc661a7abe104d3096f9eb9bbd0fa91dae18abcd933
|
7
|
+
data.tar.gz: 7ed6a2c324d55e634d86d1a450c12af045cba6049905e3a97d4eca3fe025fb11f92812c0e88d9c41cf5fa00eb44f454efb7efcfc598ad0b05257a6b0c6328a19
|
data/lib/digger/http.rb
CHANGED
@@ -49,7 +49,7 @@ module Digger
|
|
49
49
|
url = URI(url)
|
50
50
|
pages = []
|
51
51
|
get(url, referer) do |response, code, location, redirect_to, response_time|
|
52
|
-
handle_compression response
|
52
|
+
handle_compression response if handle_compression?
|
53
53
|
pages << Page.new(location, body: response.body,
|
54
54
|
code: code,
|
55
55
|
headers: response.to_hash,
|
@@ -70,6 +70,13 @@ module Digger
|
|
70
70
|
[Page.new(url, error: e, referer: referer, depth: depth)]
|
71
71
|
end
|
72
72
|
|
73
|
+
#
|
74
|
+
# Accept response compression, may bring encoding error if true
|
75
|
+
#
|
76
|
+
def handle_compression?
|
77
|
+
@opts[:handle_compression]
|
78
|
+
end
|
79
|
+
|
73
80
|
#
|
74
81
|
# The maximum number of redirects to follow
|
75
82
|
#
|
@@ -185,7 +192,7 @@ module Digger
|
|
185
192
|
opts['User-Agent'] = user_agent if user_agent
|
186
193
|
opts['Referer'] = referer.to_s if referer
|
187
194
|
opts['Cookie'] = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
|
188
|
-
opts['Accept-Encoding'] = 'gzip,deflate'
|
195
|
+
opts['Accept-Encoding'] = 'gzip,deflate' if handle_compression?
|
189
196
|
|
190
197
|
retries = 0
|
191
198
|
begin
|
data/lib/digger/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: digger
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- binz
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-06-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|