caboodle 0.2.26 → 0.2.27
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/lib/caboodle/kits/posterous/posterous.rb +4 -5
- data/lib/caboodle/scrape.rb +19 -6
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.27
|
@@ -65,7 +65,7 @@ module Caboodle
|
|
65
65
|
def self.from_slug(slug)
|
66
66
|
puts "Looking for http://#{Site.posterous_sitename}.posterous.com/#{slug}"
|
67
67
|
doc = Caboodle.scrape("http://#{Site.posterous_sitename}.posterous.com/#{slug}")
|
68
|
-
puts doc
|
68
|
+
#puts doc
|
69
69
|
opts = {}
|
70
70
|
opts["body"] = doc.css('div.bodytext').inner_html
|
71
71
|
opts["title"] = doc.css('title').inner_html.split(" - ").first
|
@@ -78,7 +78,7 @@ module Caboodle
|
|
78
78
|
rescue
|
79
79
|
opts["date"] = Date.parse(Time.now)
|
80
80
|
end
|
81
|
-
puts opts["date"]
|
81
|
+
#puts opts["date"]
|
82
82
|
PosterousPost.new(opts)
|
83
83
|
end
|
84
84
|
|
@@ -92,7 +92,7 @@ module Caboodle
|
|
92
92
|
r = []
|
93
93
|
p = PosterousAPI.new
|
94
94
|
opts[:site_id] = Site.posterous_site_id
|
95
|
-
rsp = p.all(opts).
|
95
|
+
rsp = p.all(opts).perform_sleepily.parse["rsp"]
|
96
96
|
posts = rsp["post"]
|
97
97
|
posts = [posts] if posts.class == Hash
|
98
98
|
posts.each{|a| r << PosterousPost.new(a) } if posts
|
@@ -149,8 +149,7 @@ module Caboodle
|
|
149
149
|
end
|
150
150
|
|
151
151
|
def date
|
152
|
-
|
153
|
-
@date ||= attributes["date"]
|
152
|
+
@date ||= Date.parse(attributes["date"]) rescue nil
|
154
153
|
end
|
155
154
|
|
156
155
|
def full_url
|
data/lib/caboodle/scrape.rb
CHANGED
@@ -5,6 +5,14 @@ require 'open-uri'
|
|
5
5
|
|
6
6
|
module Caboodle
|
7
7
|
|
8
|
+
begin
|
9
|
+
require 'memcached'
|
10
|
+
CACHE = Memcached.new
|
11
|
+
puts "Running with memcache"
|
12
|
+
rescue
|
13
|
+
puts "Running without memcache"
|
14
|
+
end
|
15
|
+
|
8
16
|
def self.round_time(integer, factor)
|
9
17
|
return integer if(integer % factor == 0)
|
10
18
|
return integer - (integer % factor)
|
@@ -12,20 +20,25 @@ module Caboodle
|
|
12
20
|
|
13
21
|
def self.scrape url
|
14
22
|
begin
|
15
|
-
if
|
23
|
+
if defined?(CACHE)
|
24
|
+
puts "Scraping with cache optimisation"
|
16
25
|
timeout = 60*60*1000
|
17
|
-
|
18
|
-
response = sleepy.get("#{round_time(Time.new.to_i, timeout)}:#{url}") rescue nil
|
26
|
+
response = CACHE.get("#{round_time(Time.new.to_i, timeout)}:#{url}") rescue nil
|
19
27
|
response ||= open(url).read
|
20
|
-
|
21
|
-
|
28
|
+
CACHE.set("#{round_time(Time.new.to_i, timeout)}:#{url}", response)
|
29
|
+
CACHE.set("0:#{url}", response)
|
22
30
|
else
|
31
|
+
puts "Scraping without cache optimisation"
|
23
32
|
response = open(url).read
|
24
33
|
end
|
25
34
|
::Nokogiri::HTML(response)
|
26
35
|
rescue Exception => e
|
27
36
|
puts e.inspect
|
28
|
-
|
37
|
+
if defined?(CACHE)
|
38
|
+
response = CACHE.get("0:#{url}")
|
39
|
+
end
|
40
|
+
response ||= ""
|
41
|
+
::Nokogiri::HTML(response)
|
29
42
|
end
|
30
43
|
end
|
31
44
|
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: caboodle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 33
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 27
|
10
|
+
version: 0.2.27
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Stef Lewandowski
|