cobweb 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/cobweb.rb +12 -7
- metadata +4 -4
data/lib/cobweb.rb
CHANGED
|
@@ -62,6 +62,9 @@ class CobWeb
|
|
|
62
62
|
uri = Addressable::URI.parse(url.strip)
|
|
63
63
|
|
|
64
64
|
# retrieve data
|
|
65
|
+
if uri.scheme == "https"
|
|
66
|
+
uri.port = 443
|
|
67
|
+
end
|
|
65
68
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
66
69
|
if uri.scheme == "https"
|
|
67
70
|
http.use_ssl = true
|
|
@@ -79,7 +82,6 @@ class CobWeb
|
|
|
79
82
|
if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
|
|
80
83
|
puts "redirected... " unless @options[:quiet]
|
|
81
84
|
url = absolutize.url(response['location']).to_s
|
|
82
|
-
ap redirect_limit
|
|
83
85
|
redirect_limit = redirect_limit - 1
|
|
84
86
|
content = get(url, redirect_limit)
|
|
85
87
|
content[:url] = uri.to_s
|
|
@@ -111,12 +113,12 @@ class CobWeb
|
|
|
111
113
|
link_parser = ContentLinkParser.new(content[:url], content[:body])
|
|
112
114
|
content[:links] = link_parser.link_data
|
|
113
115
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
116
|
+
end
|
|
117
|
+
# add content to cache if required
|
|
118
|
+
if @options[:cache]
|
|
119
|
+
content[:body] = Base64.encode64(content[:body]) unless content[:body].nil? or content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
|
|
120
|
+
redis.set(unique_id, content.to_json)
|
|
121
|
+
redis.expire unique_id, @options[:cache].to_i
|
|
120
122
|
end
|
|
121
123
|
rescue SocketError => e
|
|
122
124
|
puts "ERROR: #{e.message}"
|
|
@@ -179,6 +181,9 @@ class CobWeb
|
|
|
179
181
|
uri = Addressable::URI.parse(url.strip)
|
|
180
182
|
|
|
181
183
|
# retrieve data
|
|
184
|
+
if uri.scheme == "https"
|
|
185
|
+
uri.port = 443
|
|
186
|
+
end
|
|
182
187
|
http = Net::HTTP.new(uri.host, uri.port)
|
|
183
188
|
if uri.scheme == "https"
|
|
184
189
|
http.use_ssl = true
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cobweb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
4
|
+
hash: 17
|
|
5
5
|
prerelease: false
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
8
|
- 0
|
|
9
|
-
-
|
|
10
|
-
version: 0.0.
|
|
9
|
+
- 7
|
|
10
|
+
version: 0.0.7
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Stewart McKee
|
|
@@ -15,7 +15,7 @@ autorequire:
|
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
17
|
|
|
18
|
-
date: 2011-01
|
|
18
|
+
date: 2011-03-01 00:00:00 +00:00
|
|
19
19
|
default_executable:
|
|
20
20
|
dependencies:
|
|
21
21
|
- !ruby/object:Gem::Dependency
|