bluth 0.6.8 → 0.7.0

data/CHANGES.txt

@@ -1,5 +1,14 @@
 BLUTH, CHANGES
 
+#### 0.7.0 (2011-03-04) ###############################
+
+* ADDED: backtrace field to Bluth::Gob
+* ADDED: Bluth::TimingBelt
+* ADDED: Bluth::Queue.queues/entry_queues include TimingBelt queues 
+  when Bluth::TimingBelt is defined.
+* ADDED: replace-worker command
+
+
 #### 0.6.8 (2011-02-07) ###############################
 
 * FIXED: Remove use of calls to fineround in Worker

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :MAJOR: 0
-:MINOR: 6
-:PATCH: 8
+:MINOR: 7
+:PATCH: 0

data/bin/bluth

@@ -68,14 +68,17 @@ class Bluth::CLI::Definition
   
   command :start_worker => Bluth::CLI
   command :start_scheduler => Bluth::CLI
-
+  
   option :f, :force
   command :stop_workers => Bluth::CLI
   option :f, :force
   command :stop_worker => Bluth::CLI
   option :f, :force
   command :stop_scheduler => Bluth::CLI
-
+  
+  about "Stop the oldest worker and start a new instance in its place."
+  command :replace_worker => Bluth::CLI
+  
   command :workers => Bluth::CLI  
   command :schedulers => Bluth::CLI
   

data/bluth.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluth}
-  s.version = "0.6.8"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Delano Mandelbaum"]
-  s.date = %q{2011-02-08}
+  s.date = %q{2011-03-04}
   s.default_executable = %q{bluth}
   s.description = %q{A Redis queuing system built on top of Familia}
   s.email = %q{delano@solutious.com}
@@ -29,23 +29,24 @@ Gem::Specification.new do |s|
      "lib/bluth.rb",
      "lib/bluth/cli.rb",
      "lib/bluth/test_helpers.rb",
+     "lib/bluth/timingbelt.rb",
      "lib/bluth/worker.rb",
      "lib/daemonizing.rb",
      "try/15_queue_try.rb",
      "try/16_worker_try.rb",
      "try/17_gob_try.rb",
      "try/18_handler_try.rb",
-     "try/19_bluth_try.rb"
+     "try/19_bluth_try.rb",
+     "try/30_timingbelt_try.rb"
   ]
   s.homepage = %q{http://github.com/delano/bluth}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{bluth}
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{A Redis queuing system built on top of Familia (w/ daemons!)}
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/bluth.rb

@@ -88,7 +88,7 @@ module Bluth
   
   require 'bluth/worker'
   
-  module Queue  # if this is a module the 
+  module Queue 
     include Familia
     prefix [:bluth, :queue]
     class_list :critical #, :class => Bluth::Gob
@@ -98,16 +98,41 @@ module Bluth
     class_list :successful
     class_list :failed
     class_list :orphaned
+    def self.create_queue name
+      unless queue(name)
+        q = Familia::List.new name, :parent => self
+        @queuecache[name.to_s.to_sym] = q
+      end
+      queue(name)
+    end
     class << self
       # The complete list of queues in the order they were defined
       def queues
-        Bluth::Queue.class_lists.collect(&:name).collect do |qname|
+        qs = Bluth::Queue.class_lists.collect(&:name).collect do |qname|
           self.send qname
         end
+        if defined?(Bluth::TimingBelt)
+          notch_queues = Bluth::TimingBelt.priority.collect { |notch| notch.queue }
+          qs.insert 1, *notch_queues
+        end
+        qs
       end
       # The subset of queues that new jobs arrive in, in order of priority
       def entry_queues
-        Bluth.priority.collect { |qname| self.send qname }
+        qs = Bluth.priority.collect { |qname| self.send qname }
+        if defined?(Bluth::TimingBelt)
+          notch_queues = Bluth::TimingBelt.priority.collect { |notch| notch.queue }
+          qs.insert 1, *notch_queues
+        end
+        qs
+      end
+      def queue name
+        if class_list? name.to_s.to_sym
+          self.send(name)
+        else
+          @queuecache ||= {}
+          @queuecache[name.to_s.to_sym] 
+        end
       end
     end
     
@@ -115,9 +140,9 @@ module Bluth
     Bluth.priority = [:critical, :high, :low]
   end
   
-  # Workers use a blocking pop and will wait for up to 
-  # Bluth.queuetimeout (seconds) before returnning nil. 
-  # Note that the queues are still processed in order. 
+  # Workers use a blocking pop and will wait for up to
+  # Bluth.queuetimeout (seconds) before returnning nil.
+  # Note that the queues are still processed in order.
   # If all queues are empty, the first one to return a
   # value is use. See: 
   #
@@ -137,6 +162,7 @@ module Bluth
     gob = nil
     begin
       order = Bluth::Queue.entry_queues.collect(&:rediskey)
+      Familia.ld " QUEUE ORDER: #{order.join(', ')}"
       order << Bluth.queuetimeout  # We do it this way to support Ruby 1.8
       queue, gobid = *(Bluth::Queue.redis.send(meth, *order) || [])
       unless queue.nil?
@@ -179,12 +205,27 @@ module Bluth
       end
     end
     
-    def enqueue(data={},q=nil)
-      q = self.queue(q)
+    def engauge(data={}, notch=nil)
+      notch ||= Bluth::TimingBelt.notch 1
+      gob = create_job data
+      gob.notch = notch.name
+      gob.save
+      Familia.ld "ENNOTCHING: #{self} #{gob.jobid.short} to #{notch.rediskey}" if Familia.debug?
+      notch.add gob.jobid
+      gob
+    end
+    
+    def create_job data={}
       gob = Gob.create generate_id(data), self, data
-      gob.current_queue = q.name
       gob.created
       gob.attempts = 0
+      gob
+    end
+    
+    def enqueue(data={}, q=nil)
+      q = self.queue(q) if q.nil? || Symbol === q
+      gob = create_job data
+      gob.current_queue = q.name
       gob.save
       Familia.ld "ENQUEUING: #{self} #{gob.jobid.short} to #{q}" if Familia.debug?
       q << gob.jobid
@@ -192,7 +233,7 @@ module Bluth
     end
     def queue(name=nil)
       @queue = name if name
-      Bluth::Queue.send(@queue || :high)
+      Bluth::Queue.queue(@queue || :high)
     end
     def generate_id(*args)
       [self, Process.pid, Bluth.sysinfo.hostname, Time.now.to_f, *args].gibbler
@@ -222,6 +263,8 @@ module Bluth
     field :messages => Array
     field :attempts => Integer
     field :create_time => Float
+    field :backtrace
+    field :notch     # populated only via TimingBelt
     field :stime => Float
     field :etime => Float
     field :current_queue => Symbol

data/lib/bluth/cli.rb

@@ -45,17 +45,25 @@ module Bluth
       end
     end
     
-    def stop_worker wid=nil,worker_class=Bluth::Worker
-      wids = wid ? [wid] : @argv
+    def stop_worker wid=nil, worker_class=Bluth::Worker
       Bluth.connect
+      wids = wid ? [wid] : @argv
       wids.each do |wid|
         worker = worker_class.from_redis wid
         kill_worker worker, worker_class
       end
     end
     
+    def replace_worker worker_class=Bluth::Worker
+      Bluth.connect
+      @global.daemon = true
+      worker = worker_class.instances.first  # grabs the oldest worker
+      kill_worker worker, worker_class
+      start_worker worker_class
+    end
+    
     def workers worker_class=Bluth::Worker
-      Familia.info worker_class.all.collect &:key
+      Familia.info worker_class.all.collect &:rediskey
     end
     
     private 

data/lib/bluth/timingbelt.rb

@@ -0,0 +1,118 @@
+require 'time'
+
+module Bluth 
+  
+  module TimingBelt 
+    include Familia
+    prefix [:bluth, :timingbelt]
+    # This module extends the Familia::Set that represents 
+    # a notch. IOW, these are instance methods for notch objs.
+    module Notch
+      attr_accessor :stamp, :filter, :time
+      def next
+        skip
+      end
+      def prev
+        skip -1
+      end
+      def skip mins=1
+        time = Time.parse(stamp || '')
+        Bluth::TimingBelt.notch mins, filter, time
+      end
+      def queue
+        Bluth::Queue.create_queue stamp
+      end
+      def -(other)
+        ((self.time - other.time)/60).to_i
+      end
+    end
+    @length = 60 # minutes
+    class << self
+      attr_reader :notchcache, :length
+      def find v, mins=length, filter=nil, time=now
+        raise ArgumentError, "value cannot be nil" if v.nil?
+        select(mins, filter, time) do |notch| 
+          notch.member?(v)
+        end
+      end
+      def range rng, filter=nil, time=now, &blk
+        rng.to_a.each { |idx|
+          notch = Bluth::TimingBelt.notch idx, filter, time
+          blk.call notch
+        }
+      end
+      # mins: the number of minutes to look ahead. 
+      def each mins=length, filter=nil, time=now, &blk
+        mins.times { |idx|
+          notch = Bluth::TimingBelt.notch idx, filter, time
+          blk.call notch
+        }
+      end
+      def select mins=length, filter=nil, time=now, &blk
+        ret = []
+        each(mins, filter, time) { |notch| ret << notch if blk.call(notch) }
+        ret
+      end
+      def collect mins=length, filter=nil, time=now, &blk
+        ret = []
+        each(mins, filter, time) { |notch| ret << blk.call(notch) }
+        ret
+      end
+      def now mins=0, time=Time.now.utc
+        time + (mins*60)  # time wants it in seconds
+      end
+      def stamp mins=0, time=now
+        (time + (mins*60)).strftime('%H:%M')
+      end
+      def notch mins=0, filter=nil, time=now
+        key = rediskey(stamp(mins, time), filter)
+        @notchcache ||= {}
+        if @notchcache[key].nil?
+          @notchcache[key] ||= Familia::Set.new key, 
+              :ttl => 2*60*60, # 2 hours
+              :extend => Bluth::TimingBelt::Notch, 
+              :db => Bluth::TimingBelt.db
+          @notchcache[key].stamp = stamp(mins, time) 
+          @notchcache[key].filter = filter
+          @notchcache[key].time = now(mins, time)
+        end
+        @notchcache[key]
+      end
+      def priority minutes=2, filter=nil, time=now
+        (0..minutes).to_a.reverse.collect { |min| notch(min*-1, filter, time) }
+      end
+      def next_empty_notch filter=nil, time=now
+        length.times { |min| 
+          possible = notch min+1, filter, time  # add 1 so we don't start at 0
+          return possible if possible.empty?
+        }
+        nil
+      end
+      def add data, notch=nil
+        notch ||= Bluth::TimingBelt.notch 1
+        notch.add data
+      end
+      def pop minutes=2, filter=nil, time=now
+        gob = nil
+        priority = Bluth::TimingBelt.priority minutes, filter, time
+        begin
+          gobid, notch = nil, nil
+          priority.each { |n| gobid, notch = n.pop, n.name; break unless gobid.nil? }
+          unless gobid.nil?
+            Familia.ld "FOUND #{gobid} id #{notch}" if Familia.debug?
+            gob = Bluth::Gob.from_redis gobid
+            raise Bluth::Buster, "No such gob object: #{gobid}" if gob.nil?
+            Bluth::Queue.running << gob.jobid
+            gob.current_queue = :running
+            gob.save
+          end
+        rescue => ex
+          Familia.info ex.message
+          Familia.ld ex.backtrace if Familia.debug?
+        end
+        gob
+      end
+    end
+    
+  end
+end

data/lib/bluth/worker.rb

@@ -200,7 +200,7 @@ module Bluth
     
     private 
     
-    # DO NOT return from this method
+    # DO NOT call return from this method
     def find_gob(task=nil)
       begin
         job = Bluth.pop

data/try/15_queue_try.rb

@@ -27,4 +27,20 @@ Bluth::Queue.critical.size
 job = Bluth::Queue.critical.shift
 #=> 'job1'
 
+
+## Can create a queue on the fly
+q = Bluth::Queue.create_queue :anything
+q.rediskey
+#=> "bluth:queue:anything"
+
+## And that new queue has a method
+q = Bluth::Queue.queue :anything
+q.class
+#=> Familia::List
+
+## We can get a list of queues by priority
+Bluth::Queue.entry_queues.collect { |q| q.name }
+#=> [:critical, :high, :low]
+
+
 Bluth::Queue.critical.clear

data/try/17_gob_try.rb

@@ -1,7 +1,7 @@
 require 'bluth'
 require 'bluth/test_helpers'
 
-Familia.debug = true
+#Familia.debug = true
 
 ## Can enqueue a job
 @job = ExampleHandler.enqueue :arg1 => :val1

data/try/19_bluth_try.rb

@@ -35,6 +35,7 @@ Bluth.queuetimeout = 2
 Bluth.pop
 #=> nil
 
+
 Bluth::Queue.critical.clear
 @job1.destroy! if @job1
 @job2.destroy! if @job2

data/try/30_timingbelt_try.rb

@@ -0,0 +1,120 @@
+require 'bluth'
+require 'bluth/timingbelt'
+require 'bluth/test_helpers'
+
+#Familia.debug = true
+
+@now = Time.at(1297641600).utc # 2011-02-14 20:00:00
+Bluth::TimingBelt.redis.flushdb
+
+## Knows now
+Bluth::TimingBelt.now(0, @now).to_s
+#=> '2011-02-14 00:00:00 UTC'
+
+## Now can have an offset
+Bluth::TimingBelt.now(5, @now).to_s
+#=> '2011-02-14 00:05:00 UTC'
+
+## Can create a timestamp
+Bluth::TimingBelt.stamp 0, @now
+#=> '00:00'
+
+## Knows the current key
+Bluth::TimingBelt.rediskey '00:00', nil
+#=> 'bluth:timingbelt:00:00'
+
+## Creates a Set object for the current time
+Bluth::TimingBelt.notch(0, nil, @now).class
+#=> Familia::Set
+
+## A notch knows its stamp
+Bluth::TimingBelt.notch(0, nil, @now).stamp
+#=> '00:00'
+
+## A notch knows the next stamp
+Bluth::TimingBelt.notch(0, nil, @now).next.stamp
+#=> '00:01'
+
+## A notch knows the previous stamp
+Bluth::TimingBelt.notch(0, nil, @now).prev.stamp
+#=> '23:59'
+
+## A notch can skip to arbitrary number ahead
+Bluth::TimingBelt.notch(0, nil, @now).skip(15).stamp
+#=> '00:15'
+
+## Set for the current time doesn't exist
+Bluth::TimingBelt.notch(0, nil, @now).exists?
+#=> false
+
+## Set for the current time is empty
+Bluth::TimingBelt.notch(0, nil, @now).empty?
+#=> true
+
+## Knows the current set priority
+Bluth::TimingBelt.priority(2, nil, @now).collect { |q| q.name }
+#=> ["bluth:timingbelt:23:58", "bluth:timingbelt:23:59", "bluth:timingbelt:00:00"]
+
+## Handler can engauge right now
+notch = Bluth::TimingBelt.notch(0, nil, @now)
+ExampleHandler.engauge({}, notch).notch
+#=> 'bluth:timingbelt:00:00'
+
+## Handler can engauge 1 minute ago
+notch = Bluth::TimingBelt.notch(-1, nil, @now)
+ExampleHandler.engauge({}, notch).notch
+#=> 'bluth:timingbelt:23:59'
+
+## Handler can engauge 10 minutes from now
+notch = Bluth::TimingBelt.notch(10, nil, @now)
+@gob3 = ExampleHandler.engauge({}, notch)
+@gob3.notch
+#=> 'bluth:timingbelt:00:10'
+
+## Will get a job from the highest priority notch
+@gob1 = Bluth::TimingBelt.pop(2, nil, @now)
+@gob1.notch
+#=> 'bluth:timingbelt:23:59'
+
+## Will get a job from the next priority notch
+@gob2 = Bluth::TimingBelt.pop(2, nil, @now)
+@gob2.notch
+#=> 'bluth:timingbelt:00:00'
+
+## Knows next available notch
+@next_notch = Bluth::TimingBelt.next_empty_notch(nil, @now)
+@next_notch.name unless @next_notch.nil?
+#=> 'bluth:timingbelt:00:01'
+
+## Knows next available notch
+notches = Bluth::TimingBelt.find(@gob3.jobid, 60, nil, @now)
+notches.first.name unless notches.first.nil?
+#=> 'bluth:timingbelt:00:10'
+
+## Can calculate the difference between two notches
+notch1 = Bluth::TimingBelt.notch
+notch2 = Bluth::TimingBelt.notch 67
+puts notch2.name
+notch2 - notch1
+#=> 67
+
+## A notch has an associated queue
+notch = Bluth::TimingBelt.notch(0, nil, @now)
+notch.queue.class
+#=> Familia::List
+
+## And that queue has the same timestamp
+notch = Bluth::TimingBelt.notch(0, nil, @now)
+notch.queue.rediskey
+#=> 'bluth:queue:00:00'
+
+## We can get a list of queues by priority
+@current_notch = Bluth::TimingBelt.notch 
+Bluth::Queue.entry_queues.collect { |q| q.name }
+#=> [:critical, @current_notch.prev.prev.queue.name, @current_notch.prev.queue.name, @current_notch.queue.name, :high, :low]
+
+## Just a test
+Bluth.pop
+##=> true
+
+

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: bluth
 version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
-  segments: 
-  - 0
-  - 6
-  - 8
-  version: 0.6.8
+  prerelease: 
+  version: 0.7.0
 platform: ruby
 authors: 
 - Delano Mandelbaum
@@ -15,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-08 00:00:00 -05:00
+date: 2011-03-04 00:00:00 -05:00
 default_executable: bluth
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,11 +21,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 13
-        segments: 
-        - 0
-        - 6
-        - 5
         version: 0.6.5
   type: :runtime
   version_requirements: *id001
@@ -42,11 +32,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 0
-        - 7
-        - 3
         version: 0.7.3
   type: :runtime
   version_requirements: *id002
@@ -58,9 +43,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id003
@@ -84,6 +66,7 @@ files:
 - lib/bluth.rb
 - lib/bluth/cli.rb
 - lib/bluth/test_helpers.rb
+- lib/bluth/timingbelt.rb
 - lib/bluth/worker.rb
 - lib/daemonizing.rb
 - try/15_queue_try.rb
@@ -91,6 +74,7 @@ files:
 - try/17_gob_try.rb
 - try/18_handler_try.rb
 - try/19_bluth_try.rb
+- try/30_timingbelt_try.rb
 has_rdoc: true
 homepage: http://github.com/delano/bluth
 licenses: []
@@ -105,23 +89,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: bluth
-rubygems_version: 1.3.7
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: A Redis queuing system built on top of Familia (w/ daemons!)