digger 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 26deb68788670976da4b7edc2170236d35e695b5
4
- data.tar.gz: 8e1c83697d912e8b401c4933c8d7e437024559c8
3
+ metadata.gz: 586e17ec8338bf102a91c8ee241632619a55b6b7
4
+ data.tar.gz: d93040c059eaf4c4e34cc329c4938a324feea142
5
5
  SHA512:
6
- metadata.gz: 47d08104b079873296e1c94cdb6ecd384be7e02e38343eba91c25dbfa27be067bd62084d7347b4c9dd0662ea17cbbacfa3196c6f2bcbf2a18e4919996b613e91
7
- data.tar.gz: b29784d26b7044e6201ad9322fa932d7226ac68bffc2bcf6f802dee98f79097ed88c2e0c4dbe0bc4e99fd5e4dc4c7bf0a6562d25687b616477973f90bd91f514
6
+ metadata.gz: 089a7c6b6b1f8904202cfa8f362cc30d39b1dbfaaff5be721ded38f9448fe03aed23c4986ff587ab60499bc661a7abe104d3096f9eb9bbd0fa91dae18abcd933
7
+ data.tar.gz: 7ed6a2c324d55e634d86d1a450c12af045cba6049905e3a97d4eca3fe025fb11f92812c0e88d9c41cf5fa00eb44f454efb7efcfc598ad0b05257a6b0c6328a19
@@ -49,7 +49,7 @@ module Digger
49
49
  url = URI(url)
50
50
  pages = []
51
51
  get(url, referer) do |response, code, location, redirect_to, response_time|
52
- handle_compression response
52
+ handle_compression response if handle_compression?
53
53
  pages << Page.new(location, body: response.body,
54
54
  code: code,
55
55
  headers: response.to_hash,
@@ -70,6 +70,13 @@ module Digger
70
70
  [Page.new(url, error: e, referer: referer, depth: depth)]
71
71
  end
72
72
 
73
+ #
74
+ # Accept response compression, may bring encoding error if true
75
+ #
76
+ def handle_compression?
77
+ @opts[:handle_compression]
78
+ end
79
+
73
80
  #
74
81
  # The maximum number of redirects to follow
75
82
  #
@@ -185,7 +192,7 @@ module Digger
185
192
  opts['User-Agent'] = user_agent if user_agent
186
193
  opts['Referer'] = referer.to_s if referer
187
194
  opts['Cookie'] = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
188
- opts['Accept-Encoding'] = 'gzip,deflate'
195
+ opts['Accept-Encoding'] = 'gzip,deflate' if handle_compression?
189
196
 
190
197
  retries = 0
191
198
  begin
@@ -1,3 +1,3 @@
1
1
  module Digger
2
- VERSION = "0.1.3"
2
+ VERSION = "0.1.4"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: digger
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - binz
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-22 00:00:00.000000000 Z
11
+ date: 2015-06-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler