gandalf 0.0.3 → 0.0.4

data/lib/gandalf.rb

@@ -21,5 +21,5 @@ require 'gandalf/worker'
 require 'gandalf/models'
 
 module Gandalf
-  VERSION = "0.0.1"
+  VERSION = "0.0.4"
 end

data/lib/gandalf/models.rb

@@ -31,12 +31,14 @@ module Gandalf
     belongs_to :seed, :child_key => [:channel_id]
 
     def clean!
-      self.title = self.title[0,255]
+      self.title = self.title[0,255] if self.title
       if self.description
         self.description.gsub!(/\<[^\>]+\>|\n|&nbsp;/,' ')
         self.description.gsub!(/&gt;/,'<')
         self.description.gsub!(/&lt;/,'>')
         self.description.gsub!(/\s{2,}/,' ')
+        self.description.strip!
+        self.description = nil if self.description = ""
       end
     end
 

data/lib/gandalf/scheduler.rb

@@ -4,7 +4,7 @@ module Gandalf
     include DataMapper::Resource
 
     property :id, Serial
-    property :redis_host, String, :default => nil
+    property :redis_host, String
     property :redis_db_id, Integer, :default => 0
     property :seed_table, String
     property :seed_count, Integer, :default => 0
@@ -36,9 +36,14 @@ module Gandalf
 
       scheduler.every 10*interval do
         # TODO Use dm-aggregates when the bug gets fixed.
-        self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1")
+        self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1").first.to_i
         save
       end
+
+      def scheduler.handle_exception(job, exception)
+        puts exception
+        raise exception
+      end
     end
 
     def execute

data/lib/gandalf/worker.rb

@@ -13,7 +13,9 @@ module Gandalf
     belongs_to :scheduler
 
     def setup(options = {})
-      @queue = RedisQueue.new(:key => self.id, :redis => options[:redis]) unless @queue
+      @queue = RedisQueue.new(:key => self.id, 
+                              :redis => options[:redis],
+                              :host => self.scheduler.redis_host)
       if options[:post_class]
         @Post = options[:post_class]
       else
@@ -26,6 +28,12 @@ module Gandalf
       @crawl_scheduler.every interval do
         crawl new_jobs(max_jobs) if jobs_to_do > 0
       end
+
+    end
+
+    def @crawl_scheduler.handle_exception(job, exception)
+      puts exception
+      raise exception
     end
 
     def stop

data/lib/redis_ext/dist_redis.rb

@@ -1,5 +1,5 @@
-require 'redis'
-require 'hash_ring'
+#require 'redis'
+#require 'hash_ring'
 class DistRedis
   attr_reader :ring
   def initialize(opts={})

data/lib/redis_ext/pipeline.rb

@@ -1,4 +1,4 @@
-require "redis"
+#require "redis"
 
 class Redis
   class Pipeline < Redis

data/lib/redis_ext/redis.rb

@@ -1,5 +1,5 @@
 require 'socket'
-require File.join(File.dirname(__FILE__),'pipeline')
+#require File.join(File.dirname(__FILE__),'pipeline')
 
 begin
   if RUBY_VERSION >= '1.9'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: gandalf
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors: 
 - Kijun Seo