bluth 0.5.3 → 0.6.0

data/CHANGES.txt

@@ -1,5 +1,18 @@
 BLUTH, CHANGES
 
+#### 0.6.0 (2010-12-30) ###############################
+
+* CHANGE: Now depends on Familia 0.6
+* CHANGE: Worker process name now starts with 'bluth'
+* CHANGE: Bluth.pop is now correctly named Bluth.shift (b/c it's returning
+          the first value in the lists)
+* ADDED: Bluth::Handler
+
+
+#### 0.5.3 (2010-12-17) ###############################
+
+* CHANGE: requires Familia 0.5.3
+
 #### 0.5.2 (2010-12-10) ###############################
 
 Initial public release

data/Rakefile

@@ -25,8 +25,9 @@ begin
     gem.email = "delano@solutious.com"
     gem.homepage = "http://github.com/delano/bluth"
     gem.authors = ["Delano Mandelbaum"]
-    gem.add_dependency("familia",    "= 0.5.3")
+    gem.add_dependency("familia",    "= 0.6.3")
     gem.add_dependency('sysinfo',    '>= 0.7.3')
+    gem.add_dependency('annoy')
     
     #gem.add_development_dependency("rspec", ">= 1.2.9")
     #gem.add_development_dependency("mocha", ">= 0.9.8")

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :MAJOR: 0
-:MINOR: 5
-:PATCH: 3
+:MINOR: 6
+:PATCH: 0

data/bin/bluth

@@ -0,0 +1,99 @@
+#!/usr/bin/ruby
+
+# = Bluth: the queuing system.
+# 
+#
+# Usage:
+# 
+#     $ bluth -h
+#     $ bluth -Ipath/2/inc -ryourlib start
+#     $ bluth -Ipath/2/inc -ryourlib stop
+#
+#--
+
+# Put our local lib in first place
+BASE_PATH = File.expand_path File.join(File.dirname(__FILE__), '..')
+$:.unshift File.join(BASE_PATH, 'lib')
+
+require 'bluth'
+require 'bluth/cli'
+
+autoload :Rye, 'rye'
+autoload :Annoy, 'annoy'
+
+# Command-line interface for bin/bluth
+class Bluth::CLI::Definition
+  extend Drydock
+
+  debug :on
+  
+  global :Y, :auto, "Auto confirm"
+  global :d, :database, Integer, "Redis database"
+  global :e, :environment, String
+  global :d, :daemon, "Start as a daemon"
+  global :D, :debug do
+    Familia.debug = true
+  end
+  global :U, :uri, String, "Redis connection URI" do |v|
+    uri = URI.parse v
+    Bluth.uri = uri
+  end
+  global :v, :verbose, "Increase output" do 
+    @verbose ||= 0
+    @verbose += 1
+  end
+  global :I, String, "Add a load path" do |v|
+    $LOAD_PATH << v
+  end
+  global :r, String, "Require a library" do |v|
+    $LOAD_PATH << 'lib' if File.exists? 'lib'
+    begin
+      require v
+    rescue LoadError => ex
+      puts ex.message
+      exit
+    end
+  end
+  global :V, :version, "Display version" do 
+    puts "Version: #{Bluth::VERSION.to_s}"
+    exit 0
+  end
+  
+  before do |obj|
+    obj.global.environment ||= 'dev'
+  end
+    
+  #TODO: command :enqueue
+  
+  command :start_worker => Bluth::CLI
+  command :start_scheduler => Bluth::CLI
+
+  option :f, :force
+  command :stop_workers => Bluth::CLI
+  option :f, :force
+  command :stop_worker => Bluth::CLI
+  option :f, :force
+  command :stop_scheduler => Bluth::CLI
+
+  command :workers => Bluth::CLI  
+  command :schedulers => Bluth::CLI
+  
+  #command :flush_workers => Bluth::CLI  
+end
+
+begin
+  Drydock.run!(ARGV, STDIN) if Drydock.run? && !Drydock.has_run?
+rescue Drydock::ArgError, Drydock::OptError => ex
+  STDERR.puts ex.message
+  STDERR.puts ex.usage
+rescue Drydock::InvalidArgument => ex
+  STDERR.puts ex.message
+rescue Drydock::UnknownCommand => ex
+  STDERR.puts "Unknown command: %s" % ex.name
+rescue Interrupt
+  puts $/, "Exiting..."
+  exit 1
+rescue => ex
+  STDERR.puts "ERROR (#{ex.class.to_s}): #{ex.message}"
+  STDERR.puts ex.backtrace #if Familia.debug
+end

data/bluth.gemspec

@@ -5,13 +5,15 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluth}
-  s.version = "0.5.3"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Delano Mandelbaum"]
-  s.date = %q{2010-12-17}
+  s.date = %q{2010-12-30}
+  s.default_executable = %q{bluth}
   s.description = %q{A Redis queuing system built on top of Familia}
   s.email = %q{delano@solutious.com}
+  s.executables = ["bluth"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
      "README.rdoc"
@@ -22,11 +24,19 @@ Gem::Specification.new do |s|
      "README.rdoc",
      "Rakefile",
      "VERSION.yml",
+     "bin/bluth",
      "bluth.gemspec",
      "lib/bluth.rb",
+     "lib/bluth/cli.rb",
      "lib/bluth/gob.rb",
+     "lib/bluth/test_helpers.rb",
      "lib/bluth/worker.rb",
-     "lib/daemonizing.rb"
+     "lib/daemonizing.rb",
+     "try/15_queue_try.rb",
+     "try/16_worker_try.rb",
+     "try/17_gob_try.rb",
+     "try/18_handler_try.rb",
+     "try/19_bluth_try.rb"
   ]
   s.homepage = %q{http://github.com/delano/bluth}
   s.rdoc_options = ["--charset=UTF-8"]
@@ -40,15 +50,18 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<familia>, ["= 0.5.3"])
+      s.add_runtime_dependency(%q<familia>, ["= 0.6.3"])
       s.add_runtime_dependency(%q<sysinfo>, [">= 0.7.3"])
+      s.add_runtime_dependency(%q<annoy>, [">= 0"])
     else
-      s.add_dependency(%q<familia>, ["= 0.5.3"])
+      s.add_dependency(%q<familia>, ["= 0.6.3"])
       s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
+      s.add_dependency(%q<annoy>, [">= 0"])
     end
   else
-    s.add_dependency(%q<familia>, ["= 0.5.3"])
+    s.add_dependency(%q<familia>, ["= 0.6.3"])
     s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
+    s.add_dependency(%q<annoy>, [">= 0"])
   end
 end
 

data/lib/bluth/cli.rb

@@ -0,0 +1,72 @@
+require 'drydock'
+
+module Bluth
+  class CLI < Drydock::Command
+    
+    
+    def start_workers
+      start_worker
+    end
+    
+    def start_scheduler
+      start_worker Bluth.scheduler
+    end
+    
+    def stop_scheduler
+      stop_workers Bluth.scheduler
+    end
+    
+    def schedulers
+      workers Bluth.scheduler
+    end
+    
+    #def flush_workers
+    #  if @global.auto || Annoy.are_you_sure?
+    #    Bluth::Worker.prefix
+    #  end
+    #end
+    
+    def start_worker worker_class=Bluth::Worker
+      if @global.daemon
+        worker = worker_class.new
+        Familia.info "Created: #{worker.rediskey}"
+        worker.daemonize
+        worker.run
+      else
+        Bluth.poptimeout = 3.seconds
+        worker_class.run
+      end
+    end
+    
+    def stop_workers worker_class=Bluth::Worker
+      worker_class.instances.each do |worker|
+        kill_worker worker, worker_class
+      end
+    end
+    
+    def stop_worker wid=nil,worker_class=Bluth::Worker
+      wids = wid ? [wid] : @argv
+      wids.each do |wid|
+        worker = worker_class.from_redis wid
+        kill_worker worker, worker_class
+      end
+    end
+    
+    def workers worker_class=Bluth::Worker
+      Familia.info worker_class.all.collect &:key
+    end
+    
+    private 
+    
+    def kill_worker worker, worker_class=Bluth::Worker
+      if worker.nil?
+        Familia.info "No such worker"
+        exit 1
+      else
+        Familia.info "Killing #{worker.rediskey}"
+        worker.kill @global.auto
+      end
+    end
+    
+  end
+end

data/lib/bluth/gob.rb

@@ -1,180 +1 @@
 
-
-module Bluth
-  
-  class Gob < Storable
-    MAX_ATTEMPTS = 3.freeze unless defined?(Gob::MAX_ATTEMPTS)
-    include Familia
-    prefix :gob
-    ttl 1.hour
-    field :id => Gibbler::Digest
-    field :kind => String
-    field :data => Hash
-    field :messages => Array
-    field :attempts => Integer
-    field :create_time => Float
-    field :stime => Float
-    field :etime => Float
-    field :current_queue => String
-    field :thread_id => Integer
-    field :cpu => Array
-    field :wid => Gibbler::Digest
-    
-    def self.inherited(obj)
-      obj.extend Bluth::Gob::ClassMethods
-      obj.prefix [:job, obj.to_s.split('::').last.downcase].join(':')
-      Bluth.handlers << obj
-    end
-    
-    module ClassMethods
-      def clear
-        keys.each do |key|
-          Gob.redis.del key
-        end
-      end
-      def enqueue(data={},q=nil)
-        q ||= self.queue
-        job = Gob.create generate_id(data), self, data
-        job.current_queue = q
-        Familia.ld "ENQUEUING: #{self} #{job.id.short} to #{q}"
-        Bluth::Queue.redis.lpush q.key, job.id
-        job.create_time = Time.now.utc.to_f
-        job.attempts = 0
-        job
-      end
-      def queue(name=nil)
-        @queue = name if name
-        @queue || Bluth::High
-      end
-      def generate_id(*args)
-        a = [self, Process.pid, Bluth.sysinfo.hostname, Time.now.to_f, *args]
-        a.gibbler
-      end
-      def all
-        Bluth::Gob.all.select do |job|
-          job.kind == self
-        end
-      end
-      def size
-        all.size
-      end
-      def lock_key
-        Familia.key(prefix, :lock)
-      end
-      def lock!
-        raise Bluth::Buster, "#{self} is already locked!" if locked?
-        Familia.info "Locking #{self}"
-        ret = Bluth::Gob.redis.set lock_key, 1
-        Bluth.locks << lock_key
-        ret == 'OK'
-      end
-      def unlock!
-        Familia.info "Unlocking #{self}"
-        ret = Bluth::Gob.redis.del lock_key
-        Bluth.locks.delete lock_key
-        ret
-      end
-      def locked?
-        Bluth::Gob.redis.exists lock_key
-      end
-      def prepare
-      end
-      
-      [:success, :failure, :running].each do |w|
-        define_method "#{w}_key" do                # success_key
-          Familia.key(self.prefix, w)
-        end
-        define_method "#{w}!" do |*args|           # success!(1)
-          by = args.first || 1
-          Bluth::Gob.redis.incrby send("#{w}_key"), by    
-        end
-        define_method "#{w}" do                    # success
-          Bluth::Gob.redis.get(send("#{w}_key")).to_i 
-        end
-      end
-    end
-    
-    def id
-      @id = Gibbler::Digest.new(@id) if String === @id
-    end
-    def clear!
-      @attempts = 0
-      @messages = []
-      save
-    end 
-    def preprocess
-      @attempts ||= 0
-      @messages ||= []
-      @create_time ||= Time.now.utc.to_f
-    end
-    def attempt?
-      attempts < MAX_ATTEMPTS
-    end
-    def attempt!
-      @attempts = attempts + 1
-    end
-    def current_queue
-      @current_queue
-    end
-    def kind
-      @kind = eval "::#{@kind}" rescue @kind if @kind.is_a?(String)
-      @kind
-    end
-    def kind=(v)
-      @kind = v
-    end
-    def perform
-      @attempts += 1
-      Familia.ld "PERFORM: #{self.to_hash.inspect}"
-      @stime = Time.now.utc.to_f
-      save # update the time
-      self.kind.prepare if self.class.respond_to?(:prepare)
-      self.kind.perform @data
-      @etime = Time.now.utc.to_f
-      save # update the time
-    end
-    def delayed?
-      start = @stime || 0
-      start > Time.now.utc.to_f
-    end
-    def retry!(msg=nil) 
-      move! Bluth::High, msg
-    end
-    def failure!(msg=nil)
-      @etime = Time.now.utc.to_i
-      self.kind.failure!
-      move! Bluth::Failed, msg
-    end
-    def success!(msg=nil)
-      @etime = Time.now.utc.to_i
-      self.kind.success!
-      move! Bluth::Successful, msg
-    end
-    def duration
-      return 0 if @stime.nil?
-      et = @etime || Time.now.utc.to_i
-      et - @stime
-    end
-    def dequeue!
-      Familia.ld "Deleting #{self.id} from #{current_queue.key}"
-      Bluth::Queue.redis.lrem current_queue.key, 0, self.id
-    end
-    private
-    def move!(to, msg=nil)
-      @thread_id = $$
-      if to.to_s == current_queue.to_s
-        raise Bluth::Buster, "Cannot move job to the queue it's in: #{to}"
-      end
-      Familia.ld "Moving #{self.id.short} from #{current_queue.key} to #{to.key}"
-      @messages << msg unless msg.nil? || msg.empty?
-      # We push first to make sure we never lose a Gob ID. Instead
-      # there's the small chance of a job ID being in two queues. 
-      Bluth::Queue.redis.lpush to.key, @id
-      dequeue!
-      save # update messages
-      @current_queue = to
-    end
-  end
-  
-end
-

data/lib/bluth/test_helpers.rb

@@ -0,0 +1,33 @@
+
+
+module ExampleHandler
+  extend Bluth::Handler
+  queue :critical
+  
+  # :hostid => String
+  # :force => Boolean (update all monitors)
+  def self.enqueue(opts={})
+    super opts, opts.delete(:queue)
+  end
+  
+  def self.perform(data={})
+    begin
+      
+    rescue => ex
+      
+    end
+  end  
+end
+
+class ExampleWorker < Bluth::Worker
+  
+end
+
+
+Bluth::Worker.onstart do
+  puts "onstart called"
+end
+
+Bluth::Worker.onexit do
+  puts "onexit called"
+end