gandalf 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/gandalf.rb +1 -1
- data/lib/gandalf/models.rb +3 -1
- data/lib/gandalf/scheduler.rb +7 -2
- data/lib/gandalf/worker.rb +9 -1
- data/lib/redis_ext/dist_redis.rb +2 -2
- data/lib/redis_ext/pipeline.rb +1 -1
- data/lib/redis_ext/redis.rb +1 -1
- metadata +1 -1
data/lib/gandalf.rb
CHANGED
data/lib/gandalf/models.rb
CHANGED
@@ -31,12 +31,14 @@ module Gandalf
|
|
31
31
|
belongs_to :seed, :child_key => [:channel_id]
|
32
32
|
|
33
33
|
def clean!
|
34
|
-
self.title = self.title[0,255]
|
34
|
+
self.title = self.title[0,255] if self.title
|
35
35
|
if self.description
|
36
36
|
self.description.gsub!(/\<[^\>]+\>|\n| /,' ')
|
37
37
|
self.description.gsub!(/>/,'<')
|
38
38
|
self.description.gsub!(/</,'>')
|
39
39
|
self.description.gsub!(/\s{2,}/,' ')
|
40
|
+
self.description.strip!
|
41
|
+
self.description = nil if self.description = ""
|
40
42
|
end
|
41
43
|
end
|
42
44
|
|
data/lib/gandalf/scheduler.rb
CHANGED
@@ -4,7 +4,7 @@ module Gandalf
|
|
4
4
|
include DataMapper::Resource
|
5
5
|
|
6
6
|
property :id, Serial
|
7
|
-
property :redis_host, String
|
7
|
+
property :redis_host, String
|
8
8
|
property :redis_db_id, Integer, :default => 0
|
9
9
|
property :seed_table, String
|
10
10
|
property :seed_count, Integer, :default => 0
|
@@ -36,9 +36,14 @@ module Gandalf
|
|
36
36
|
|
37
37
|
scheduler.every 10*interval do
|
38
38
|
# TODO Use dm-aggregates when the bug gets fixed.
|
39
|
-
self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1")
|
39
|
+
self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1").first.to_i
|
40
40
|
save
|
41
41
|
end
|
42
|
+
|
43
|
+
def scheduler.handle_exception(job, exception)
|
44
|
+
puts exception
|
45
|
+
raise exception
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def execute
|
data/lib/gandalf/worker.rb
CHANGED
@@ -13,7 +13,9 @@ module Gandalf
|
|
13
13
|
belongs_to :scheduler
|
14
14
|
|
15
15
|
def setup(options = {})
|
16
|
-
@queue = RedisQueue.new(:key => self.id,
|
16
|
+
@queue = RedisQueue.new(:key => self.id,
|
17
|
+
:redis => options[:redis],
|
18
|
+
:host => self.scheduler.redis_host)
|
17
19
|
if options[:post_class]
|
18
20
|
@Post = options[:post_class]
|
19
21
|
else
|
@@ -26,6 +28,12 @@ module Gandalf
|
|
26
28
|
@crawl_scheduler.every interval do
|
27
29
|
crawl new_jobs(max_jobs) if jobs_to_do > 0
|
28
30
|
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
def @crawl_scheduler.handle_exception(job, exception)
|
35
|
+
puts exception
|
36
|
+
raise exception
|
29
37
|
end
|
30
38
|
|
31
39
|
def stop
|
data/lib/redis_ext/dist_redis.rb
CHANGED
data/lib/redis_ext/pipeline.rb
CHANGED
data/lib/redis_ext/redis.rb
CHANGED