gandalf 0.0.3 → 0.0.4

Sign up to get free protection for your applications and to get access to all the features.
@@ -21,5 +21,5 @@ require 'gandalf/worker'
21
21
  require 'gandalf/models'
22
22
 
23
23
  module Gandalf
24
- VERSION = "0.0.1"
24
+ VERSION = "0.0.4"
25
25
  end
@@ -31,12 +31,14 @@ module Gandalf
31
31
  belongs_to :seed, :child_key => [:channel_id]
32
32
 
33
33
  def clean!
34
- self.title = self.title[0,255]
34
+ self.title = self.title[0,255] if self.title
35
35
  if self.description
36
36
  self.description.gsub!(/\<[^\>]+\>|\n|&nbsp;/,' ')
37
37
  self.description.gsub!(/&gt;/,'<')
38
38
  self.description.gsub!(/&lt;/,'>')
39
39
  self.description.gsub!(/\s{2,}/,' ')
40
+ self.description.strip!
41
+ self.description = nil if self.description = ""
40
42
  end
41
43
  end
42
44
 
@@ -4,7 +4,7 @@ module Gandalf
4
4
  include DataMapper::Resource
5
5
 
6
6
  property :id, Serial
7
- property :redis_host, String, :default => nil
7
+ property :redis_host, String
8
8
  property :redis_db_id, Integer, :default => 0
9
9
  property :seed_table, String
10
10
  property :seed_count, Integer, :default => 0
@@ -36,9 +36,14 @@ module Gandalf
36
36
 
37
37
  scheduler.every 10*interval do
38
38
  # TODO Use dm-aggregates when the bug gets fixed.
39
- self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1")
39
+ self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1").first.to_i
40
40
  save
41
41
  end
42
+
43
+ def scheduler.handle_exception(job, exception)
44
+ puts exception
45
+ raise exception
46
+ end
42
47
  end
43
48
 
44
49
  def execute
@@ -13,7 +13,9 @@ module Gandalf
13
13
  belongs_to :scheduler
14
14
 
15
15
  def setup(options = {})
16
- @queue = RedisQueue.new(:key => self.id, :redis => options[:redis]) unless @queue
16
+ @queue = RedisQueue.new(:key => self.id,
17
+ :redis => options[:redis],
18
+ :host => self.scheduler.redis_host)
17
19
  if options[:post_class]
18
20
  @Post = options[:post_class]
19
21
  else
@@ -26,6 +28,12 @@ module Gandalf
26
28
  @crawl_scheduler.every interval do
27
29
  crawl new_jobs(max_jobs) if jobs_to_do > 0
28
30
  end
31
+
32
+ end
33
+
34
+ def @crawl_scheduler.handle_exception(job, exception)
35
+ puts exception
36
+ raise exception
29
37
  end
30
38
 
31
39
  def stop
@@ -1,5 +1,5 @@
1
- require 'redis'
2
- require 'hash_ring'
1
+ #require 'redis'
2
+ #require 'hash_ring'
3
3
  class DistRedis
4
4
  attr_reader :ring
5
5
  def initialize(opts={})
@@ -1,4 +1,4 @@
1
- require "redis"
1
+ #require "redis"
2
2
 
3
3
  class Redis
4
4
  class Pipeline < Redis
@@ -1,5 +1,5 @@
1
1
  require 'socket'
2
- require File.join(File.dirname(__FILE__),'pipeline')
2
+ #require File.join(File.dirname(__FILE__),'pipeline')
3
3
 
4
4
  begin
5
5
  if RUBY_VERSION >= '1.9'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gandalf
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kijun Seo