bluth 0.5.2

data/CHANGES.txt

@@ -0,0 +1,6 @@
+BLUTH, CHANGES
+
+#### 0.5.2 (2010-12-10) ###############################
+
+Initial public release
+

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2010-2011 Solutious Inc, Delano Mandelbaum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,14 @@
+# Bluth - 0.8 BETA
+
+**A Redis queuing system built on top of Familia**
+
+
+## More Information
+
+* [Codes](http://github.com/delano/bluth)
+* [RDocs](http://delano.github.com/bluth)
+
+## Credits
+
+* [Delano Mandelbaum](http://solutious.com)
+* Bluth lego by [Baby Elephant](http://www.flickr.com/photos/baby_elephant/454780652/) 

data/Rakefile

@@ -0,0 +1,68 @@
+require "rubygems"
+require "rake"
+require "rake/clean"
+require 'yaml'
+
+begin
+  require 'hanna/rdoctask'
+rescue LoadError
+  require 'rake/rdoctask'
+end
+ 
+config = YAML.load_file("VERSION.yml")
+task :default => ["build"]
+CLEAN.include [ 'pkg', 'doc' ]
+name = "bluth"
+
+begin
+  require "jeweler"
+  Jeweler::Tasks.new do |gem|
+    gem.version = "#{config[:MAJOR]}.#{config[:MINOR]}.#{config[:PATCH]}"
+    gem.name = name
+    gem.rubyforge_project = gem.name
+    gem.summary = "A Redis queuing system built on top of Familia"
+    gem.description = "A Redis queuing system built on top of Familia"
+    gem.email = "delano@solutious.com"
+    gem.homepage = "http://github.com/delano/bluth"
+    gem.authors = ["Delano Mandelbaum"]
+    gem.add_dependency("familia",    ">= 0.5.3")
+    gem.add_dependency('sysinfo',    '>= 0.7.3')
+    
+    #gem.add_development_dependency("rspec", ">= 1.2.9")
+    #gem.add_development_dependency("mocha", ">= 0.9.8")
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+
+Rake::RDocTask.new do |rdoc|
+  version = "#{config[:MAJOR]}.#{config[:MINOR]}.#{config[:PATCH]}.#{config[:BUILD]}"
+  rdoc.rdoc_dir = "doc"
+  rdoc.title = "#{name} #{version}"
+  rdoc.rdoc_files.include("README*")
+  rdoc.rdoc_files.include("LICENSE.txt")
+  rdoc.rdoc_files.include("bin/*.rb")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+
+# Rubyforge Release / Publish Tasks ==================================
+
+#about 'Publish website to rubyforge'
+task 'publish:rdoc' => 'doc/index.html' do
+  #sh "scp -rp doc/* rubyforge.org:/var/www/gforge-projects/#{name}/"
+end
+
+#about 'Public release to rubyforge'
+task 'publish:gem' => [:package] do |t|
+  sh <<-end
+    rubyforge add_release -o Any -a CHANGES.txt -f -n README.md #{name} #{name} #{@spec.version} pkg/#{name}-#{@spec.version}.gem &&
+    rubyforge add_file -o Any -a CHANGES.txt -f -n README.md #{name} #{name} #{@spec.version} pkg/#{name}-#{@spec.version}.tgz 
+  end
+end
+
+
+
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:MAJOR: 0
+:MINOR: 5
+:PATCH: 2

data/bluth.gemspec

@@ -0,0 +1,54 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bluth}
+  s.version = "0.5.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Delano Mandelbaum"]
+  s.date = %q{2010-12-10}
+  s.description = %q{A Redis queuing system built on top of Familia}
+  s.email = %q{delano@solutious.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+     "README.rdoc"
+  ]
+  s.files = [
+    "CHANGES.txt",
+     "LICENSE.txt",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "bluth.gemspec",
+     "lib/bluth.rb",
+     "lib/bluth/gob.rb",
+     "lib/bluth/worker.rb",
+     "lib/daemonizing.rb"
+  ]
+  s.homepage = %q{http://github.com/delano/bluth}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{bluth}
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{A Redis queuing system built on top of Familia}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<familia>, [">= 0.5.3"])
+      s.add_runtime_dependency(%q<sysinfo>, [">= 0.7.3"])
+    else
+      s.add_dependency(%q<familia>, [">= 0.5.3"])
+      s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
+    end
+  else
+    s.add_dependency(%q<familia>, [">= 0.5.3"])
+    s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
+  end
+end
+

data/lib/bluth.rb

@@ -0,0 +1,211 @@
+# encoding: utf-8
+BLUTH_LIB_HOME = File.expand_path File.dirname(__FILE__) unless defined?(BLUTH_LIB_HOME)
+
+require 'sysinfo'
+require 'familia'
+
+
+module Bluth
+  module VERSION
+    def self.to_s
+      load_config
+      [@version[:MAJOR], @version[:MINOR], @version[:PATCH]].join('.')
+    end
+    alias_method :inspect, :to_s
+    def self.load_config
+      require 'yaml'
+      @version ||= YAML.load_file(File.join(BLUTH_LIB_HOME, '..', 'VERSION.yml'))
+    end
+  end
+end
+
+module Bluth   
+  # A fatal error. Gob fails.
+  class Buster < Familia::Problem; end
+  # A non-fatal error. Gob succeeds.
+  class Maeby < Familia::Problem; end
+  # A shutdown request. We burn down the banana stand.
+  class Shutdown < Familia::Problem; end
+  
+  @db = 15
+  @queues = {}
+  @poptimeout = 60.seconds
+  @handlers = []
+  @locks = []
+  @sysinfo = nil
+  @priority = []
+  @scheduler = nil 
+  class << self
+    attr_reader :queues, :handlers, :db, :conf, :locks
+    attr_accessor :redis, :uri, :priority, :scheduler, :poptimeout
+    def sysinfo
+      @sysinfo ||= SysInfo.new.freeze
+      @sysinfo 
+    end
+  end
+  
+  def Bluth.clear_locks
+    @locks.each { |lock| 
+      Familia.info "Removing lock #{lock}"
+      Bluth.redis.del lock 
+    }
+  end
+  
+  def Bluth.queue?(n)
+    @queues.has_key?(n.to_sym)
+  end
+  def Bluth.queue(n)
+    @queues[n.to_sym]
+  end
+  
+  def Bluth.conf=(conf={})
+    @conf = conf.clone
+    @conf[:db] = @db
+    connect!
+    @conf
+  end
+  
+  def Bluth.connect!
+    @uri = Redis.uri(@conf).freeze
+    @redis = Familia.connect @uri
+  end
+  
+  def Bluth.find_locks
+    @locks = Bluth.redis.keys(Familia.key('*', :lock))
+  end
+  
+  class Queue
+    include Familia
+    prefix :queue
+    def self.rangeraw(count=100)
+      gobids = Queue.redis.lrange(key, 0, count-1) || []
+    end
+    def self.range(count=100)
+      gobids = rangeraw count
+      gobids.collect { |gobid| 
+        gob = Gob.from_redis gobid 
+        next if gob.nil?
+        gob.current_queue = self
+        gob
+      }.compact
+    end
+    def self.dequeue(gobid)
+      Queue.redis.lrem key, 0, gobid
+    end
+    def self.inherited(obj)
+      obj.prefix self.prefix
+      obj.suffix obj.to_s.split('::').last.downcase.to_sym
+      raise Buster.new("Duplicate queue: #{obj.suffix}") if Bluth.queue?(obj.suffix)
+      Bluth.queues[obj.suffix] = obj
+      super(obj)
+    end
+    def self.key(pref=nil,suff=nil)
+      Familia.key( pref || prefix, suff || suffix)
+    end
+    def self.report
+      Bluth.queues.keys.collect { |q| 
+        klass = Bluth.queue(q)
+        ("%10s: %4d" % [q, klass.size]) 
+      }.join($/)
+    end
+    def self.from_string(str)
+      raise Buster, "Unknown queue: #{str}" unless Bluth.queue?(str)
+      Bluth.queue(str)
+    end
+    def self.any?
+      size > 0
+    end
+
+    def self.empty?
+      size == 0
+    end
+
+    def self.size
+      begin
+        Queue.redis.llen key
+      rescue => ex
+        STDERR.puts ex.message, ex.backtrace
+        0
+      end
+    end 
+    def self.push(gobid)
+      Queue.redis.lpush self.key, gobid
+    end
+    
+    def self.pop
+      gobid = Queue.redis.rpoplpush key, Bluth::Running.key
+      return if gobid.nil?
+      Familia.ld "FOUND gob #{gobid} from #{self.key}"
+      gob = Gob.from_redis gobid
+      if gob.nil?
+        Familia.info "No such gob object: #{gobid}" 
+        Bluth::Running.dequeue gobid
+        return
+      end
+      gob.current_queue = Bluth::Running
+      gob.save
+      gob
+    end
+  end
+  
+  # Workers use a blocking pop and will wait for up to 
+  # Bluth.poptimeout (seconds) before returnning nil. 
+  # Note that the queues are still processed in order. 
+  # If all queues are empty, the first one to return a
+  # value is use. See: 
+  #
+  # http://code.google.com/p/redis/wiki/BlpopCommand
+  def Bluth.pop
+    #Bluth.priority.each { |queue| 
+    #  ret = queue.pop
+    #  return ret unless ret.nil?
+    #}
+    begin
+      #Familia.ld :BRPOP, Queue.redis, self, caller[1] if Familia.debug?
+      order = Bluth.priority.collect { |queue| queue.key }
+      order << Bluth.poptimeout  # We do it this way to support Ruby 1.8
+      gobinfo = Bluth::Queue.redis.brpop *order
+      unless gobinfo.nil?
+        Familia.info "FOUND #{gobinfo.inspect}" if Familia.debug?
+        gob = Gob.from_redis gobinfo[1]
+        raise Bluth::Buster, "No such gob object: #{gobinfo[1]}" if gob.nil?
+        Bluth::Running.push gob.id
+        gob.current_queue = Bluth::Running
+        gob.save
+      end
+    rescue => ex
+      if gobinfo.nil?
+        Familia.info "ERROR: #{ex.message}"
+      else
+        Familia.info "ERROR (#{ex.message}); putting #{gobinfo[1]} back on queue"
+        Bluth::Orphaned.push gobinfo[1]
+      end
+    end
+    gob
+  end
+  
+  class Critical < Queue
+  end
+  class High < Queue
+  end
+  class Low < Queue
+  end
+  class Running < Queue
+  end
+  class Failed < Queue
+  end
+  class Successful < Queue
+  end
+  class Scheduled < Queue
+  end
+  class Orphaned < Queue
+  end
+    
+  require 'bluth/gob'
+  require 'bluth/worker'
+  
+  Bluth.priority = [Bluth::Critical, Bluth::High, Bluth::Low]
+  Bluth.scheduler = ScheduleWorker
+  
+end
+

data/lib/bluth/gob.rb

@@ -0,0 +1,180 @@
+
+
+module Bluth
+  
+  class Gob < Storable
+    MAX_ATTEMPTS = 3.freeze unless defined?(Gob::MAX_ATTEMPTS)
+    include Familia
+    prefix :gob
+    ttl 1.hour
+    field :id => Gibbler::Digest
+    field :kind => String
+    field :data => Hash
+    field :messages => Array
+    field :attempts => Integer
+    field :create_time => Float
+    field :stime => Float
+    field :etime => Float
+    field :current_queue => String
+    field :thread_id => Integer
+    field :cpu => Array
+    field :wid => Gibbler::Digest
+    
+    def self.inherited(obj)
+      obj.extend Bluth::Gob::ClassMethods
+      obj.prefix [:job, obj.to_s.split('::').last.downcase].join(':')
+      Bluth.handlers << obj
+    end
+    
+    module ClassMethods
+      def clear
+        keys.each do |key|
+          Gob.redis.del key
+        end
+      end
+      def enqueue(data={},q=nil)
+        q ||= self.queue
+        job = Gob.create generate_id(data), self, data
+        job.current_queue = q
+        Familia.ld "ENQUEUING: #{self} #{job.id.short} to #{q}"
+        Bluth::Queue.redis.lpush q.key, job.id
+        job.create_time = Time.now.utc.to_f
+        job.attempts = 0
+        job
+      end
+      def queue(name=nil)
+        @queue = name if name
+        @queue || Bluth::High
+      end
+      def generate_id(*args)
+        a = [self, Process.pid, Bluth.sysinfo.hostname, Time.now.to_f, *args]
+        a.gibbler
+      end
+      def all
+        Bluth::Gob.all.select do |job|
+          job.kind == self
+        end
+      end
+      def size
+        all.size
+      end
+      def lock_key
+        Familia.key(prefix, :lock)
+      end
+      def lock!
+        raise Bluth::Buster, "#{self} is already locked!" if locked?
+        Familia.info "Locking #{self}"
+        ret = Bluth::Gob.redis.set lock_key, 1
+        Bluth.locks << lock_key
+        ret == 'OK'
+      end
+      def unlock!
+        Familia.info "Unlocking #{self}"
+        ret = Bluth::Gob.redis.del lock_key
+        Bluth.locks.delete lock_key
+        ret
+      end
+      def locked?
+        Bluth::Gob.redis.exists lock_key
+      end
+      def prepare
+      end
+      
+      [:success, :failure, :running].each do |w|
+        define_method "#{w}_key" do                # success_key
+          Familia.key(self.prefix, w)
+        end
+        define_method "#{w}!" do |*args|           # success!(1)
+          by = args.first || 1
+          Bluth::Gob.redis.incrby send("#{w}_key"), by    
+        end
+        define_method "#{w}" do                    # success
+          Bluth::Gob.redis.get(send("#{w}_key")).to_i 
+        end
+      end
+    end
+    
+    def id
+      @id = Gibbler::Digest.new(@id) if String === @id
+    end
+    def clear!
+      @attempts = 0
+      @messages = []
+      save
+    end 
+    def preprocess
+      @attempts ||= 0
+      @messages ||= []
+      @create_time ||= Time.now.utc.to_f
+    end
+    def attempt?
+      attempts < MAX_ATTEMPTS
+    end
+    def attempt!
+      @attempts = attempts + 1
+    end
+    def current_queue
+      @current_queue
+    end
+    def kind
+      @kind = eval "::#{@kind}" rescue @kind if @kind.is_a?(String)
+      @kind
+    end
+    def kind=(v)
+      @kind = v
+    end
+    def perform
+      @attempts += 1
+      Familia.ld "PERFORM: #{self.to_hash.inspect}"
+      @stime = Time.now.utc.to_f
+      save # update the time
+      self.kind.prepare if self.class.respond_to?(:prepare)
+      self.kind.perform @data
+      @etime = Time.now.utc.to_f
+      save # update the time
+    end
+    def delayed?
+      start = @stime || 0
+      start > Time.now.utc.to_f
+    end
+    def retry!(msg=nil) 
+      move! Bluth::High, msg
+    end
+    def failure!(msg=nil)
+      @etime = Time.now.utc.to_i
+      self.kind.failure!
+      move! Bluth::Failed, msg
+    end
+    def success!(msg=nil)
+      @etime = Time.now.utc.to_i
+      self.kind.success!
+      move! Bluth::Successful, msg
+    end
+    def duration
+      return 0 if @stime.nil?
+      et = @etime || Time.now.utc.to_i
+      et - @stime
+    end
+    def dequeue!
+      Familia.ld "Deleting #{self.id} from #{current_queue.key}"
+      Bluth::Queue.redis.lrem current_queue.key, 0, self.id
+    end
+    private
+    def move!(to, msg=nil)
+      @thread_id = $$
+      if to.to_s == current_queue.to_s
+        raise Bluth::Buster, "Cannot move job to the queue it's in: #{to}"
+      end
+      Familia.ld "Moving #{self.id.short} from #{current_queue.key} to #{to.key}"
+      @messages << msg unless msg.nil? || msg.empty?
+      # We push first to make sure we never lose a Gob ID. Instead
+      # there's the small chance of a job ID being in two queues. 
+      Bluth::Queue.redis.lpush to.key, @id
+      dequeue!
+      save # update messages
+      @current_queue = to
+    end
+  end
+  
+end
+

data/lib/bluth/worker.rb

@@ -0,0 +1,356 @@
+require 'eventmachine'
+require 'rufus/scheduler'
+require 'daemonizing'
+require 'timeout'
+
+module Bluth
+  @salt = rand.gibbler.shorten(10).freeze
+  class << self
+    attr_reader :salt
+  end
+  
+  module WorkerBase
+
+    def id
+      @id ||= [host, user, rand, Time.now].gibbler.short
+    end
+    
+    def longid
+      [host, user, id].join('-')
+    end
+    
+    # Used by daemonize as the process name (linux only)
+    def name
+      "bs-#{self.class.prefix}-#{id}"
+    end
+    
+    def key(suffix=nil)
+      self.class.key longid, suffix
+    end
+    
+    def initialize
+      @host, @user = Bluth.sysinfo.hostname, Bluth.sysinfo.user
+      @pid_file ||= "/tmp/#{self.class.prefix}-#{id}.pid"
+      @log_file ||= "/tmp/#{self.class.prefix}-#{id}.log"
+      @success, @failure, @problem = 0, 0, 0
+    end
+    
+    def current_job 
+      Gibbler::Digest.new(@current_job || '')
+    end
+    
+    def kill(force=false)
+      if force || host == Bluth.sysinfo.hostname
+        STDERR.puts "Destroying #{self.index} (this machine is: #{Bluth.sysinfo.hostname}; worker is: #{host})"
+        Worker.kill self.pid_file if File.exists?(self.pid_file) rescue Errno::ESRCH
+        File.delete self.log_file if File.exists?(self.log_file)
+        destroy!
+      else
+        STDERR.puts "Worker #{self.index} not running on #{Bluth.sysinfo.hostname}"
+      end
+    end
+    
+    def working! gobid
+      @current_job = gobid
+      update_time
+      save
+    end
+    
+    def self.included(obj)
+      obj.extend WorkerBase::ClassMethods
+    end
+    
+    module ClassMethods
+      def from_redis(wid)
+        me = new 
+        me.id = wid
+        super(me.longid)
+      end
+
+      def run!(*args)
+        me = new
+        Familia.info "Created: #{me.key}"
+        me.run!
+        me
+      end
+
+      def run(*args)
+        me = new
+        Familia.info "Created: #{me.key}"
+        me.run
+        me
+      end
+      
+      def kill(pid_file)
+        pid = read_pid_file pid_file
+        super(pid_file, 10)
+      end
+      
+      
+    end
+    
+  end
+  
+  class Worker < Storable
+    include WorkerBase
+    @interval = 2.seconds
+    class << self
+      attr_accessor :interval
+    end
+    include Familia
+    include Logging
+    include Daemonizable
+    prefix :worker
+    index :id
+    field :host
+    field :user
+    field :id
+    field :process_id => Integer
+    field :pid_file
+    field :log_file
+    field :current_job
+    field :success => Integer
+    field :failure => Integer
+    field :problem => Integer
+    include Familia::Stamps
+    def success!
+      @success += 1
+      @current_job = ""
+      update_time
+      save
+    end
+    def failure!
+      @failure += 1
+      @current_job = ""
+      update_time
+      save
+    end
+    def problem!
+      @problem += 1
+      @current_job = ""
+      update_time
+      save
+    end
+    
+    def run!
+      begin
+        find_gob
+      rescue => ex
+        msg = "#{ex.class}: #{ex.message}"
+        STDERR.puts msg
+        Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
+        destroy!
+      rescue Interrupt => ex
+        puts $/, "Exiting..."
+        destroy!
+      end
+    end
+    
+    def run
+      begin
+        @process_id = $$
+        save
+      
+        scheduler = Rufus::Scheduler.start_new
+        Familia.info "Setting interval: #{Worker.interval} sec (poptimeout: #{Bluth.poptimeout})"
+        Familia.reconnect_all! # Need to reconnect after daemonize
+        ## TODO: Works but needs to restart scheduler
+        ##Signal.trap("USR1") do
+        ##  Worker.interval += 1
+        ##  Familia.info "Setting interval: #{Worker.interval} sec"
+        ##end
+        ##Signal.trap("USR2") do
+        ##  Worker.interval -= 1
+        ##  Familia.info "Setting interval: #{Worker.interval}"
+        ##end
+        scheduler.every Worker.interval, :blocking => true do |task|
+          Familia.ld "#{$$} TICK @ #{Time.now.utc}"
+          sleep rand
+          find_gob task
+        end
+        scheduler.join
+            
+      rescue => ex
+        msg = "#{ex.class}: #{ex.message}"
+        STDERR.puts msg
+        Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
+        destroy!
+      rescue Interrupt => ex
+        puts <<-EOS.gsub(/(?:^|\n)\s*/, "\n")
+          Exiting...
+          (You may need to wait up to #{Bluth.poptimeout} seconds
+          for this worker to exit cleanly.)
+        EOS
+        # We reconnect to the queue in case we're currently
+        # waiting on a brpop (blocking pop) timeout.
+        destroy!
+      end
+      
+    end
+    
+    
+    private 
+    require 'benchmark'
+    # DO NOT return from this method
+    def find_gob(task=nil)
+      begin
+        job = Bluth.pop
+        unless job.nil?
+          job.wid = self.id
+          if job.delayed?
+            job.attempts = 0
+            job.retry!
+          elsif !job.attempt?
+            job.failure! "Too many attempts"
+          else
+            job.stime = Time.now.utc.to_i
+            self.working! job.id
+            tms = Benchmark.measure do
+              job.perform
+            end
+            job.cpu = [tms.utime.fineround(3),tms.stime.fineround(3),tms.real.fineround(3)]
+            job.save
+            job.success!
+            self.success!
+          end
+        end   
+      rescue Bluth::Shutdown => ex
+        msg = "Shutdown requested: #{ex.message}"
+        job.success! msg
+        Familia.info msg
+        task.unschedule
+        destroy!
+        exit
+      rescue Bluth::Maeby => ex
+        Familia.info ex.message
+        job.success! ex.message
+        self.success!
+      rescue Bluth::Buster => ex  
+        Familia.info ex.message
+        job.failure! ex.message
+        self.failure!
+      rescue => ex
+        Familia.info ex.message
+        Familia.info ex.backtrace
+        job.retry! "#{ex.class}: #{ex.message}" if job
+        problem!
+        #if problem > 5
+        #  ## TODO: SEND EMAIL
+        #  task.unschedule unless task.nil? # Kill this worker b/c something is clearly wrong
+        #  destroy!
+        #  EM.stop
+        #  exit 1
+        #end
+      end
+    end
+
+  end
+  
+  class ScheduleWorker < Storable
+    include WorkerBase
+    @interval = 20
+    @timeout = 60 #not working
+    class << self
+      attr_accessor :interval, :timeout
+      def interval(v=nil)
+        @interval = v unless v.nil?
+        @interval
+      end
+    end
+    include Familia
+    include Logging
+    include Daemonizable
+    prefix :scheduler
+    index :id
+    field :host
+    field :user
+    field :id
+    field :process_id => Integer
+    field :pid_file
+    field :log_file
+    field :scheduled => Integer
+    field :monitored => Integer
+    field :timeouts => Integer
+    include Familia::Stamps
+    attr_reader :schedule
+    attr_reader :monitors
+
+    def scheduled!(count=1)
+      @scheduled ||= 0
+      @scheduled += count
+      update_time
+      save
+    end
+    def monitored!(count=1)
+      @monitored ||= 0
+      @monitored += count
+      update_time
+      save
+    end
+    def timeout!(count=1)
+      @timeouts ||= 0
+      @timeouts += count
+      update_time
+      save
+    end
+    def run!
+      run
+    end
+    def run
+      begin
+        raise Familia::Problem, "Only 1 scheduler at a time" if ScheduleWorker.any?
+        
+        EM.run {
+          @process_id = $$
+          srand(Bluth.salt.to_i(16) ** @process_id)
+          @schedule = Rufus::Scheduler::EmScheduler.start_new
+          save # persist and make note the scheduler is running
+          prepare
+          @schedule.every self.class.interval, :tags => :keeper do |keeper_task|
+            begin
+              scheduled_work(keeper_task)
+            rescue => ex
+              msg = "#{ex.class}: #{ex.message}"
+              STDERR.puts msg
+              STDERR.puts ex.backtrace
+              Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
+            end
+            sleep rand  # prevent thrashing
+          end
+        }
+      rescue => ex
+        msg = "#{ex.class}: #{ex.message}"
+        puts msg
+        STDERR.puts ex.backtrace
+        Familia.ld :EXCEPTION, msg, caller[1] if Familia.debug?
+        destroy!
+      rescue Interrupt => ex
+        puts $/, "Exiting..."
+        destroy!
+      end
+    end
+    
+    protected
+  
+    def prepare
+    end
+    
+    def scheduled_work(keeper)
+      STDOUT.puts "Come on!"
+    end
+    
+  end
+  
+end
+
+class Rufus::Scheduler::SchedulerCore
+  # See lib/rufus/sc/scheduler.rb
+  def handle_exception(job, exception)
+    case exception
+    when SystemExit
+      exit
+    else
+      super
+    end
+  end
+end

data/lib/daemonizing.rb

@@ -0,0 +1,221 @@
+require 'etc'
+require 'daemons'
+
+module Process
+  # Returns +true+ if the process identifed by +pid+ is running.
+  def running?(pid)
+    Process.getpgid(pid) != -1
+  rescue Errno::ESRCH
+    false
+  end
+  module_function :running?
+end
+
+  # Raised when the pid file already exist starting as a daemon.
+  class PidFileExist < RuntimeError; end
+  
+  # Module included in classes that can be turned into a daemon.
+  # Handle stuff like:
+  # * storing the PID in a file
+  # * redirecting output to the log file
+  # * changing processs privileges
+  # * killing the process gracefully
+  module Daemonizable
+    attr_accessor :pid_file, :log_file
+    
+    def self.included(base)
+      base.extend ClassMethods
+    end
+    
+    def pid
+      File.exist?(pid_file) ? open(pid_file).read.to_i : nil
+    end
+    
+    # Turns the current script into a daemon process that detaches from the console.
+    def daemonize
+      raise ArgumentError,        'You must specify a pid_file to daemonize' unless @pid_file
+      
+      remove_stale_pid_file
+      
+      pwd = Dir.pwd # Current directory is changed during daemonization, so store it
+      
+      # HACK we need to create the directory before daemonization to prevent a bug under 1.9
+      #      ignoring all signals when the directory is created after daemonization.
+      FileUtils.mkdir_p File.dirname(@pid_file)
+      
+      Daemonize.daemonize(File.expand_path(@log_file), name)
+      
+      Dir.chdir(pwd)
+      
+      write_pid_file
+      
+      at_exit do
+        log ">> Exiting!"
+        remove_pid_file
+      end
+    end
+
+    # Change privileges of the process
+    # to the specified user and group.
+    def change_privilege(user, group=user)
+      log ">> Changing process privilege to #{user}:#{group}"
+      
+      uid, gid = Process.euid, Process.egid
+      target_uid = Etc.getpwnam(user).uid
+      target_gid = Etc.getgrnam(group).gid
+
+      if uid != target_uid || gid != target_gid
+        # Change process ownership
+        Process.initgroups(user, target_gid)
+        Process::GID.change_privilege(target_gid)
+        Process::UID.change_privilege(target_uid)
+      end
+    rescue Errno::EPERM => e
+      log "Couldn't change user and group to #{user}:#{group}: #{e}"
+    end
+    
+    # Register a proc to be called to restart the server.
+    def on_restart(&block)
+      @on_restart = block
+    end
+    
+    # Restart the server.
+    def restart
+      if @on_restart
+        log '>> Restarting ...'
+        stop
+        remove_pid_file
+        @on_restart.call
+        exit!
+      end
+    end
+    
+    module ClassMethods
+      # Send a QUIT or INT (if timeout is +0+) signal the process which
+      # PID is stored in +pid_file+.
+      # If the process is still running after +timeout+, KILL signal is
+      # sent.
+      def kill(pid_file, timeout=60)
+        if timeout == 0
+          send_signal('INT', pid_file, timeout)
+        else
+          send_signal('QUIT', pid_file, timeout)
+        end
+      end
+      
+      # Restart the server by sending HUP signal.
+      def restart(pid_file)
+        send_signal('HUP', pid_file)
+      end
+      
+      # Send a +signal+ to the process which PID is stored in +pid_file+.
+      def send_signal(signal, pid_file, timeout=60)
+        if pid = read_pid_file(pid_file)
+          Logging.log "Sending #{signal} signal to process #{pid} ... "
+          Process.kill(signal, pid)
+          Timeout.timeout(timeout) do
+            sleep 0.1 while Process.running?(pid)
+          end
+        else
+          Logging.log "Can't stop process, no PID found in #{pid_file}"
+        end
+      rescue Timeout::Error
+        Logging.log "Timeout!"
+        force_kill pid_file
+      rescue Interrupt
+        force_kill pid_file
+      rescue Errno::ESRCH # No such process
+        Logging.log "process not found!"
+        force_kill pid_file
+      end
+      
+      def force_kill(pid_file)
+        if pid = read_pid_file(pid_file)
+          Logging.log "Sending KILL signal to process #{pid} ... "
+          Process.kill("KILL", pid)
+          File.delete(pid_file) if File.exist?(pid_file)
+        else
+          Logging.log "Can't stop process, no PID found in #{pid_file}"
+        end
+      end
+      
+      def read_pid_file(file)
+        if File.file?(file) && pid = File.read(file)
+          pid.to_i
+        else
+          nil
+        end
+      end
+    end
+    
+    protected
+      def remove_pid_file
+        File.delete(@pid_file) if @pid_file && File.exists?(@pid_file)
+      end
+    
+      def write_pid_file
+        log ">> Writing PID to #{@pid_file}"
+        open(@pid_file,"w") { |f| f.write(Process.pid) }
+        File.chmod(0644, @pid_file)
+      end
+      
+      # If PID file is stale, remove it.
+      def remove_stale_pid_file
+        if File.exist?(@pid_file)
+          if pid && Process.running?(pid)
+            raise PidFileExist, "#{@pid_file} already exists, seems like it's already running (process ID: #{pid}). " +
+                                "Stop the process or delete #{@pid_file}."
+          else
+            log ">> Deleting stale PID file #{@pid_file}"
+            remove_pid_file
+          end
+        end
+      end
+  end
+
+
+  module Logging
+    class << self
+      attr_writer :trace, :debug, :silent
+      
+      def trace?;  !@silent && @trace  end
+      def debug?;  !@silent && @debug  end
+      def silent?;  @silent            end
+    end
+    
+    # Global silencer methods
+    def silent
+      Logging.silent?
+    end
+    def silent=(value)
+      Logging.silent = value
+    end
+    
+    # Log a message to the console
+    def log(msg)
+      puts msg unless Logging.silent?
+    end
+    module_function :log
+    public :log
+    
+    # Log a message to the console if tracing is activated
+    def trace(msg=nil)
+      log msg || yield if Logging.trace?
+    end
+    module_function :trace
+    public :trace
+    
+    # Log a message to the console if debugging is activated
+    def debug(msg=nil)
+      log msg || yield if Logging.debug?
+    end
+    module_function :debug
+    public :debug
+    
+    # Log an error backtrace if debugging is activated
+    def log_error(e=$!)
+      debug "#{e}\n\t" + e.backtrace.join("\n\t")
+    end
+    module_function :log_error
+    public :log_error
+  end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification 
+name: bluth
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 2
+  version: 0.5.2
+platform: ruby
+authors: 
+- Delano Mandelbaum
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-10 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: familia
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 0
+        - 5
+        - 3
+        version: 0.5.3
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sysinfo
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 0
+        - 7
+        - 3
+        version: 0.7.3
+  type: :runtime
+  version_requirements: *id002
+description: A Redis queuing system built on top of Familia
+email: delano@solutious.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- CHANGES.txt
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- bluth.gemspec
+- lib/bluth.rb
+- lib/bluth/gob.rb
+- lib/bluth/worker.rb
+- lib/daemonizing.rb
+has_rdoc: true
+homepage: http://github.com/delano/bluth
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: bluth
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A Redis queuing system built on top of Familia
+test_files: []
+