gandalf 0.0.1

data/lib/gandalf/models.rb

@@ -0,0 +1,55 @@
+module Gandalf
+  class Seed
+    include DataMapper::Resource
+
+    property :id, Serial
+    property :include_update, Boolean, :index => true
+    property :interval_update, Integer
+    property :url, String, :length => 255, :unique_index => true
+    
+    def to_json
+      {:id => id,
+       :include_update => include_update,
+       :interval_update => interval_update,
+       :url => url}.to_json
+    end
+  end
+
+  class Post
+    include DataMapper::Resource
+
+    storage_names[:default] = 'posts'
+    property :id, Serial, :field => 'psid'
+    property :channel_id, Integer, :length => 11, :index => true
+    property :link, Text, :length => 255
+    property :title, String, :length => 255
+    property :author, String, :lazy => true
+    property :pub_date, DateTime, :field => 'pubDate'
+    property :update_date, DateTime, :field => 'updateDate', :default => DateTime.now
+    property :description, Text
+    property :cache_link, String, :length => 32, :unique_index => true
+
+    belongs_to :seed, :child_key => [:channel_id]
+
+    def clean!
+      self.title = self.title[0,255]
+      if self.description
+        self.description.gsub!(/\<[^\>]+\>|\n|&nbsp;/,' ')
+        self.description.gsub!(/\s{2,}/,' ')
+      end
+    end
+
+    def Post.parse(feed)
+      feed.entries.map do |entry|
+        post = self.new({
+          :title => entry.title,
+          :link => entry.url,
+          :author => entry.author,
+          :description => entry.summary,
+          :pub_date => entry.published.to_datetime,
+          :cache_link => Digest::MD5.hexdigest(entry.url)
+        })
+      end
+    end
+  end
+end

data/lib/gandalf/scheduler.rb

@@ -0,0 +1,89 @@
+module Gandalf
+  # A magical scheduler
+  class Scheduler
+    include DataMapper::Resource
+
+    property :id, Serial
+    property :redis_host, String, :default => nil
+    property :redis_db_id, Integer, :default => 0
+    property :seed_table, String
+    property :seed_count, Integer, :default => 0
+    property :last_job_id, Integer, :default => 0
+    # Sleep length in seconds, initialized to 60
+    property :interval, Integer, :length => 5, :default => 60
+
+    has n, :workers
+
+    attr_accessor :redis
+
+    # Sets workers' queues with Redis connection object.
+    def setup(options = {:seed_class => Seed})
+      @redis = Redis.new(:host => self.redis_host, :db => self.redis_db_id)
+      @Seed = options[:seed_class]
+      workers.each { |worker| worker.setup(:redis => @redis) }
+    end
+
+    # Executes jobs using a scheduler
+    def run
+      scheduler = Rufus::Scheduler.start_new
+      scheduler.every interval do
+        execute
+      end
+
+      scheduler.every 10*interval do
+        # TODO Use dm-aggregates when the bug gets fixed.
+        self.seed_count = repository.adapter.query("SELECT COUNT(*) FROM #{seed_table} WHERE include_update = 1")
+        save
+      end
+    end
+
+    def execute
+      jobs = new_jobs
+      self.last_job_id = jobs.last.id
+      save
+
+      new_loads = job_distribution(current_workload, jobs.count)
+      push_jobs(jobs, new_loads)
+    end
+
+    def jobs_per_interval
+      seed_count / (1440 * 60 / interval)
+    end
+
+    def new_jobs
+      jobs = @Seed.all(:id.gte => last_job_id,
+                       :include_update => true,
+                       :limit => jobs_per_interval)
+      if jobs.length < jobs_per_interval
+        jobs += @Seed.all(:limit => jobs_per_interval - jobs.length)
+      end
+      jobs
+    end
+
+    def current_workload
+      workload = {}
+      workers.each { |worker| workload[worker.id] = worker.jobs_to_do }
+      workload
+    end
+
+    def push_jobs(jobs, workload)
+      workload.each do |worker_id, wload|
+        worker = workers.get(worker_id)
+        worker.push(jobs.slice!(0,wload))
+      end
+    end
+
+    # Recursively calculates next job distribution
+    # TODO Find a formula
+    def job_distribution(workload, jobs)
+      workload = workload.clone
+      distribution = Hash.new(0)
+      jobs.times do
+        min_index = workload.min_by{|k, v| v}.first
+        workload[min_index] += 1
+        distribution[min_index] += 1
+      end
+      distribution
+    end
+  end
+end

data/lib/gandalf/worker.rb

@@ -0,0 +1,93 @@
+module Gandalf
+  # A magical slave
+  class Worker
+    include DataMapper::Resource
+
+    property :id, Serial
+    # Maximum number of jobs per interval
+    property :max_jobs, Integer, :default => 30
+    # Sleep length in seconds, initialized to 15
+    property :interval, Integer, :default => 15
+    property :max_errors, Integer, :length => 1, :default => 2
+
+    belongs_to :scheduler
+
+    def setup(options = {:post_class => Post})
+      @queue = RedisQueue.new(:key => self.id, :redis => options[:redis]) unless @queue
+      @Post = (options[:post_class])
+    end
+
+    def run
+      @crawl_scheduler = Rufus::Scheduler.start_new unless @crawl_scheduler
+      @crawl_scheduler.every interval do
+        crawl new_jobs(max_jobs)
+      end
+    end
+
+    def stop
+      @crawl_scheduler.stop
+    end
+
+    def stop
+      @crawl_scheduler.start
+    end
+
+    def crawl(jobs)
+      urls = jobs.keys
+      feeds = Feedzirra::Feed.fetch_and_parse(urls)
+      jobs.each do |url, job|
+        if feeds[url].is_a? Feedzirra::Parser::RSS 
+          save_posts(feeds[url], job[:id])
+        else
+          handle_error(job)
+        end
+      end
+    end
+
+    def save_posts(feed, channel_id)
+      posts = @Post.parse(feed)
+      posts.each do |p|
+        p.channel_id = channel_id
+        p.clean!
+        begin
+          p.save
+        rescue MysqlError => err
+          break
+        end
+      end
+    end
+
+    def handle_error(job)
+      if job[:errors].is_a? Fixnum
+        job[:errors] += 1
+      else
+        job[:errors] = 1
+      end
+
+      if job[:errors] >= max_errors
+        puts job
+      else
+        @queue.push(job)
+      end
+    end
+
+    def jobs_to_do
+      @queue.length
+    end
+
+    def push(jobs)
+      jobs.each do |job|
+        @queue.push(job)
+      end
+    end
+
+    def new_jobs(count)
+      jobs = @queue.pop_first(count)
+      hash = {}
+      jobs.each do |job|
+        hash[job[:url]] = job
+      end
+      hash
+    end
+  end
+end

data/lib/gandalf.rb

@@ -0,0 +1,21 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+require 'feedzirra'
+require 'activesupport'
+require 'rufus/scheduler'
+require 'dm-core'
+#has a bug
+#require 'dm-aggregates'
+
+require 'digest/md5'
+require 'json'
+
+require 'redis_ext/redis_queue'
+
+require 'gandalf/scheduler'
+require 'gandalf/worker'
+require 'gandalf/models'
+
+module Gandalf
+  VERSION = "0.0.1"
+end

data/lib/redis_ext/redis_queue.rb

@@ -0,0 +1,42 @@
+require 'redis'
+require 'json'
+
+class RedisQueue
+
+  attr_accessor :key, :redis
+
+  def initialize(options = {})
+    @key = options[:key]
+    @redis = (options[:redis] || Redis.new(options))
+  end
+
+  def push(value)
+    @redis.rpush(@key, value.to_json)
+  end
+
+  def pop
+    begin
+      value = JSON.parse(@redis.lpop(@key))
+      new_hash = {}
+      value.each{|k, v| new_hash[k.to_sym] = v}
+      return new_hash 
+    rescue Exception => ex
+      puts ex
+      return nil
+    end
+  end
+
+  def length
+    @redis.llen(@key)
+  end
+
+  def pop_first(length)
+    list = [] 
+    length.times do
+      element = self.pop
+      break unless element
+      list << element
+    end
+    list
+  end
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification 
+name: gandalf
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Kijun Seo
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-09-30 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: feedzirra
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rufus-scheduler
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: dm-core
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Scheduler and Worker makes a unit. You can also make your own worker as well.
+email: kijun@kijunseo.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/redis_ext/redis_queue.rb
+- lib/gandalf.rb
+- lib/gandalf/models.rb
+- lib/gandalf/scheduler.rb
+- lib/gandalf/worker.rb
+has_rdoc: true
+homepage: http://github.com/kijun/gandalf
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A magical distributed web crawler
+test_files: []
+