digger 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/digger/http.rb +9 -2
- data/lib/digger/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 586e17ec8338bf102a91c8ee241632619a55b6b7
|
|
4
|
+
data.tar.gz: d93040c059eaf4c4e34cc329c4938a324feea142
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 089a7c6b6b1f8904202cfa8f362cc30d39b1dbfaaff5be721ded38f9448fe03aed23c4986ff587ab60499bc661a7abe104d3096f9eb9bbd0fa91dae18abcd933
|
|
7
|
+
data.tar.gz: 7ed6a2c324d55e634d86d1a450c12af045cba6049905e3a97d4eca3fe025fb11f92812c0e88d9c41cf5fa00eb44f454efb7efcfc598ad0b05257a6b0c6328a19
|
data/lib/digger/http.rb
CHANGED
|
@@ -49,7 +49,7 @@ module Digger
|
|
|
49
49
|
url = URI(url)
|
|
50
50
|
pages = []
|
|
51
51
|
get(url, referer) do |response, code, location, redirect_to, response_time|
|
|
52
|
-
handle_compression response
|
|
52
|
+
handle_compression response if handle_compression?
|
|
53
53
|
pages << Page.new(location, body: response.body,
|
|
54
54
|
code: code,
|
|
55
55
|
headers: response.to_hash,
|
|
@@ -70,6 +70,13 @@ module Digger
|
|
|
70
70
|
[Page.new(url, error: e, referer: referer, depth: depth)]
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
+
#
|
|
74
|
+
# Accept response compression, may bring encoding error if true
|
|
75
|
+
#
|
|
76
|
+
def handle_compression?
|
|
77
|
+
@opts[:handle_compression]
|
|
78
|
+
end
|
|
79
|
+
|
|
73
80
|
#
|
|
74
81
|
# The maximum number of redirects to follow
|
|
75
82
|
#
|
|
@@ -185,7 +192,7 @@ module Digger
|
|
|
185
192
|
opts['User-Agent'] = user_agent if user_agent
|
|
186
193
|
opts['Referer'] = referer.to_s if referer
|
|
187
194
|
opts['Cookie'] = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
|
|
188
|
-
opts['Accept-Encoding'] = 'gzip,deflate'
|
|
195
|
+
opts['Accept-Encoding'] = 'gzip,deflate' if handle_compression?
|
|
189
196
|
|
|
190
197
|
retries = 0
|
|
191
198
|
begin
|
data/lib/digger/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: digger
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- binz
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2015-
|
|
11
|
+
date: 2015-06-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|