matthewrudy-rudeq 0.1 → 2.0

data/README

@@ -13,7 +13,6 @@ RudeQ
 INSTALL
 ============
 This plugin requires Rails 2.* currently, and has only been tested on MySQL.
-It definitely does not work on SQLite (as it requires :limit and :order options for a SQL UPDATE command)
 
 On rails 2.1 you can install straight from github:
   ruby script/plugin install git://github.com/matthewrudy/rudeq.git
@@ -44,6 +43,9 @@ After you've installed it just run
 
       RudeQueue.set(:queue_name, RandomObject)
       RudeQueue.get(:queue_name)
+      RudeQueue.fetch(:queue_name) do |data|
+        process(data)
+      end
 
   And, to keep the queue running fast,
   set up a cron job to run

data/lib/rude_q.rb

@@ -1,8 +1,16 @@
 # RudeQ
 require 'digest/sha1'
+
+# simply doing;
+#   class RudeQueue < ActiveRecord::Base
+#     include RudeQ
+#   end
+# will include RudeQ::ClassMethods
+#   :get
+#   :set
 module RudeQ
 
-  def self.included(mod)
+  def self.included(mod) # :nodoc:
     mod.extend(ClassMethods)
     mod.serialize(:data)
   end
@@ -16,17 +24,17 @@ module RudeQ
       self.delete_all(["processed = ? AND updated_at < ?", true, expiry.to_i.ago])
     end
   
-    # Add any serialize-able *data* to the queue *queue_name* (strings and symbols are treated the same)
+    # Add any serialize-able +data+ to the queue +queue_name+ (strings and symbols are treated the same)
     #
     #   RudeQueue.set(:sausage_queue, Sausage.new(:sauce => "yummy"))
     #   RudeQueue.set("sausage_queue", Sausage.new(:other => true))
     #
-    #     RudeQueue.get("sausage_queue")
-    #     -> *yummy sausage*
-    #     RudeQueue.get(:sausage_queue)
-    #     -> *other_sausage*
-    #     RudeQueue.get(:sausage_queue)
-    #     -> nil
+    #   >> RudeQueue.get("sausage_queue")
+    #   -> *yummy sausage*
+    #   >> RudeQueue.get(:sausage_queue)
+    #   -> *other_sausage*
+    #   >> RudeQueue.get(:sausage_queue)
+    #   -> nil
     def set(queue_name, data)
       queue_name = sanitize_queue_name(queue_name)
       self.create!(:queue_name => queue_name, :data => data)
@@ -34,51 +42,151 @@ module RudeQ
     end
 
     # Grab the first item from the queue *queue_name* (strings and symbols are treated the same)
-    # - it should always come out the same as it went in
-    # - they should always come out in the same order they went in
-    # - it will return a nil if there is no unprocessed entry in the queue
+    #   - it should always come out the same as it went in
+    #   - they should always come out in the same order they went in
+    #   - it will return a nil if there is no unprocessed entry in the queue
     #
-    #  RudeQueue.get(21)
-    #  -> {:a => "hash"}
-    #  RudeQueue.get(:a_symbol)
-    #  -> 255
-    #  RudeQueue.get("a string")
-    #  -> nil
+    #   >> RudeQueue.get(21)
+    #   -> {:a => "hash"}
+    #   >> RudeQueue.get(:a_symbol)
+    #   -> 255
+    #   >> RudeQueue.get("a string")
+    #   -> nil
     def get(queue_name)
       qname = sanitize_queue_name(queue_name)
-      token = get_unique_token
-    
-      self.update_all(["token = ?", token], ["queue_name = ? AND processed = ? AND token IS NULL", qname, false], :limit => 1, :order => "id ASC")
-      queued = self.find_by_queue_name_and_token_and_processed(qname, token, false)
-      if queued
-        queued.update_attribute(:processed, true)
-        return queued.data
-      else
-        return nil # in line with Starling
+      
+      fetch_with_lock(qname) do |record|
+        if record
+          processed!(record)
+          return record.data
+        else
+          return nil # Starling waits indefinitely for a corresponding queue item
+        end
+      end
+    end
+
+    # Grab the first item from the queue, and execute the supplied block if there is one
+    #   - it will return the value of the block
+    #
+    #   >> RudeQueue.fetch(:my_queue) do |data|
+    #   >>   Monster.devour(data)
+    #   >> end
+    #   -> nil
+    #
+    #   >> status = RudeQueue.fetch(:my_queue) do |data|
+    #   >>   process(data) # returns the value :update in this case
+    #   >> end
+    #   -> :update
+    #   >> status
+    #   -> :update
+    def fetch(queue_name, &block)
+      if data = get(queue_name)
+        return block.call(data)
       end
     end
-  
-    def get_unique_token # :nodoc:
 
-      digest = Digest::SHA1.new
-      digest << Time.now.to_s
-      digest << Process.pid.to_s
-      digest << Socket.gethostname
-      digest << self.token_count!.to_s # multiple requests from the same pid in the same second get different token
+    # A snapshot count of unprocessed items for the given +queue_name+
+    def backlog(queue_name)
+      qname = sanitize_queue_name(queue_name)
+      self.count(:conditions => {:queue_name => qname, :processed => false})
+    end
     
-      return digest.hexdigest
+    def fetch_with_lock(qname, &block) # :nodoc:
+      lock = case queue_options[:lock]
+      when :pessimistic : RudeQ::PessimisticLock
+      when :token       : RudeQ::TokenLock
+      else
+        raise(ArgumentError, "bad queue_option for :lock - #{queue_options[:lock].inspect}")
+      end
+      lock.fetch_with_lock(self, qname, &block)
     end
     
-    protected
+    # class method to make it more easily stubbed
+    def processed!(record) # :nodoc:
+      case queue_options[:processed]
+      when :set_flag
+        record.update_attribute(:processed, true)
+      when :destroy
+        record.destroy
+      else
+        raise(ArgumentError, "bad queue_option for :processed - #{queue_options[:processed].inspect}")
+      end
+    end
+    protected :processed!
     
-    def token_count!
-      @token_count ||= 0
-      @token_count += 1
-      return @token_count
+    # configure your RudeQ
+    # ==== :processed - what do we do after retrieving a queue item?
+    # * <tt>:set_flag</tt> - set the +processed+ flag to +true+ (keep data in the db) [*default*]
+    # * <tt>:destroy</tt>  - destroy the processed item (keep our queue as lean as possible
+    #
+    # ==== :lock - what locking method should we use?
+    # * <tt>:pessimistic</tt> - RudeQ::PessimisticLock [*default*]
+    # * <tt>:token</tt>       - RudeQ::TokenLock
+    def queue_options
+      @queue_options ||= {:processed => :set_flag, :lock => :pessimistic}
     end
+
+    private
     
-    def sanitize_queue_name(queue_name)
+    def sanitize_queue_name(queue_name) # :nodoc:
       queue_name.to_s
     end
   end
-end
+  
+  # uses standard ActiveRecord :lock => true
+  # this will invoke a lock on the particular queue
+  #   eg. daemon1: RudeQueue.get(:abc)
+  #       daemon2: RudeQueue.get(:abc) - will have to wait for daemon1 to finish
+  #       daemon3: RudeQueue.get(:def) - will avoid the lock
+  module PessimisticLock
+    class << self
+      
+      def fetch_with_lock(klass, qname) # :nodoc:
+        klass.transaction do
+          record = klass.find(:first,
+            :conditions => {:queue_name => qname, :processed => false},
+            :lock => true, :order => "id ASC", :limit => 1)
+      
+          return yield(record)
+        end
+      end
+    
+    end
+  end
+  
+  # a crazy hack around database locking
+  # that I thought was a good idea
+  # turns out we can't make it use transactions properly
+  # without creating a whole table lock
+  # which misses the point
+  #
+  # also, it doesn't work on SQLite as it requires "UPDATE ... LIMIT 1 ORDER id ASC"
+  module TokenLock
+    class << self
+      
+      def fetch_with_lock(klass, qname) # :nodoc:
+        token = get_unique_token
+        klass.update_all(["token = ?", token], ["queue_name = ? AND processed = ? AND token IS NULL", qname, false], :limit => 1, :order => "id ASC")
+        record = klass.find_by_queue_name_and_token_and_processed(qname, token, false)
+      
+        return yield(record)
+      end
+
+      def token_count! # :nodoc:
+        @token_count ||= 0
+        @token_count += 1
+        return @token_count
+      end
+
+      def get_unique_token # :nodoc:
+        digest = Digest::SHA1.new
+        digest << Time.now.to_s
+        digest << Process.pid.to_s
+        digest << Socket.gethostname
+        digest << self.token_count!.to_s # multiple requests from the same pid in the same second get different token
+        return digest.hexdigest
+      end
+    end
+  end
+
+end

data/spec/process_queue.rb

@@ -1,3 +1,13 @@
 class ProcessQueue < ActiveRecord::Base
   include RudeQ
+  
+  class << self
+    def processed_with_raise_hack!(*args)
+      processed_without_raise_hack!(*args)
+      raise RuntimeError if raise_on_processed # want to be able to raise afterwards to check transactions
+    end
+    alias_method_chain :processed!, :raise_hack
+    attr_accessor :raise_on_processed
+  end
+    
 end

data/spec/rude_q_spec.rb

@@ -1,15 +1,15 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe ProcessQueue do
+describe RudeQ::ClassMethods do # ProcessQueue extends ClassMethods
   before(:each) do
     ProcessQueue.delete_all
+    ProcessQueue.raise_on_processed = false
     create_some_noise
   end
   
   def create_some_noise
     ProcessQueue.create!(:queue_name => "doNT use this in Specs", :data => {:not => "to be messed with"})
     ProcessQueue.create!(:queue_name => "abcde", :data => {:same_as => "the specs but already processed"}, :processed => true)
-    ProcessQueue.create!(:queue_name => "abcde", :data => {:same_as => "the specs but with token"}, :token => "    unlikely    ")
   end
   
   describe "get and set" do
@@ -31,6 +31,17 @@ describe ProcessQueue do
       ProcessQueue.set('abcde', hash)
       ProcessQueue.get('abcde').should == hash
     end
+    it "should work with integers" do
+      ProcessQueue.set('abcde', 7816327370)
+      ProcessQueue.get('abcde').should == 7816327370
+    end
+    it "unfortunately doesnt resolve booleans correctly" do
+      ProcessQueue.set('abcde', true)
+      ProcessQueue.get('abcde').should == 1
+      
+      ProcessQueue.set('abcde', false)
+      ProcessQueue.get('abcde').should == 0
+    end
     
     it "should :get in the same order they are :set" do
       ProcessQueue.set('abcde', :first)
@@ -82,38 +93,125 @@ describe ProcessQueue do
     end
   end
   
-  describe ".get" do
-    it "should not return a processed item with the same token" do
-      @token = "tokEEEannn"
-      ProcessQueue.should_receive(:get_unique_token).exactly(3).times.and_return(@token)
-      @existing = ProcessQueue.create!(:queue_name => 'abcde', :data => :old_data, :token => @token, :processed => true)
-
-      ProcessQueue.get('abcde').should be(nil)
-        
-      ProcessQueue.set('abcde', :new_data)
-      ProcessQueue.get('abcde').should == :new_data
-      ProcessQueue.get('abcde').should be(nil)
+  describe ".get" do    
+    it "should revert a record if something goes wrong before it finishes" do
+      ProcessQueue.raise_on_processed = true
+      ProcessQueue.set('abcde', :this_will_remain_unprocessed)
+      
+      # confirm the object is in the db
+      record = ProcessQueue.find(:first, :order => "id DESC")
+      record.queue_name.should == 'abcde'
+      record.data.should == :this_will_remain_unprocessed
+      record.processed?.should == false
+      record.token.should == nil
+      
+      lambda {ProcessQueue.get('abcde')}.should raise_error(RuntimeError)
+      
+      record.reload
+      record.queue_name.should == 'abcde'
+      record.data.should == :this_will_remain_unprocessed
+      record.processed?.should == false
+      record.token.should == nil
     end
   end
+
+  describe "fetch" do
+    describe "with data" do
+      
+      before(:each) do
+        ProcessQueue.set(:fetch_queue, "some data")
+      end
   
-  describe ".get_unique_token" do
-    it "should create a unique token" do
-      lots_of_tokens = Array.new(50) do
-        ProcessQueue.get_unique_token
+      it "should return the value of the block" do
+        rtn = ProcessQueue.fetch(:fetch_queue) do |data|
+          data.should == "some data"
+          :the_return
+        end
+        rtn.should == :the_return
       end
-      lots_of_tokens.uniq.should == lots_of_tokens
+
+      it "should execute the block with the data" do
+        self.should_receive(:something)
+        ProcessQueue.fetch(:fetch_queue) do |data|
+          self.something
+          data.should == "some data"
+        end
+      end
+
     end
-    
-    it "should create a unique token even if time stands still" do
-      time_now = Time.now
-      Time.should_receive(:now).at_least(50).times.and_return(time_now)
-      lots_of_tokens = Array.new(50) do
-        ProcessQueue.get_unique_token
+
+    describe "without data" do
+
+      it "should not execute the block" do
+        self.should_not_receive(:something)
+        ProcessQueue.fetch(:fetch_queue) do |data|
+          raise(Exception, "this should never get here")
+        end
+      end
+
+      it "should return nil" do
+        rtn = ProcessQueue.fetch(:fetch_queue) do |data|
+          raise(Exception, "again this shouldnt happen")
+        end
+        rtn.should be_nil
+      end
+
+    end
+  end
+
+  describe "queue_options" do
+    describe :processed do
+      describe "set to :destroy" do
+        before(:each) do
+          @old_processed = ProcessQueue.queue_options[:processed]
+          ProcessQueue.queue_options[:processed] = :destroy
+        end
+        after(:each) do
+          ProcessQueue.queue_options[:processed] = @old_processed
+        end
+        it "should delete processed items" do
+          count = ProcessQueue.count
+          
+          ProcessQueue.set(:abcde, "some value")
+          ProcessQueue.count.should == (count + 1)
+          
+          ProcessQueue.get(:abcde).should == "some value"
+          ProcessQueue.count.should == count
+        end
+      end
+      describe "set to something crazy" do
+        before(:each) do
+          @old_processed = ProcessQueue.queue_options[:processed]
+          ProcessQueue.queue_options[:processed] = :something_crazy
+        end
+        after(:each) do
+          ProcessQueue.queue_options[:processed] = @old_processed
+        end
+        it "should raise an exception" do
+          ProcessQueue.set(:abcde, "some value")
+          lambda {ProcessQueue.get(:abcde)}.should raise_error(ArgumentError)
+        end
       end
-      lots_of_tokens.uniq.should == lots_of_tokens
     end
   end
   
+  describe ".backlog" do
+    it "should count the unprocessed items for the provided queue_name" do
+      ProcessQueue.backlog(:abcde).should == 0
+      
+      ProcessQueue.set(:abcde, "a value")
+      ProcessQueue.backlog(:abcde).should == 1
+      
+      ProcessQueue.set(:something_else, "another value")
+      3.times { ProcessQueue.set(:abcde, :add_three_more)}
+      
+      ProcessQueue.backlog(:abcde).should == 4
+      
+      ProcessQueue.get(:abcde).should == "a value"
+      ProcessQueue.backlog(:abcde).should == 3
+    end
+  end
+
   describe ".cleanup!" do
     it "should use :delete_all" do
       ProcessQueue.should_receive(:delete_all) # not :destroy_all
@@ -223,3 +321,40 @@ describe ProcessQueue do
     end
   end
 end
+
+describe RudeQ::TokenLock do
+
+  describe ".get_unique_token" do
+    it "should create a unique token" do
+      lots_of_tokens = Array.new(50) do
+        RudeQ::TokenLock.get_unique_token
+      end
+      lots_of_tokens.uniq.should == lots_of_tokens
+    end
+    
+    it "should create a unique token even if time stands still" do
+      time_now = Time.now
+      Time.should_receive(:now).at_least(50).times.and_return(time_now)
+      lots_of_tokens = Array.new(50) do
+        RudeQ::TokenLock.get_unique_token
+      end
+      lots_of_tokens.uniq.should == lots_of_tokens
+    end
+  end
+  
+  # it "should not return a processed item with the same token" do
+  #   @token = "tokEEEannn"
+  # 
+  #   RudeQ::TokenLock.should respond_to(:get_unique_token) # ensure our stub is safe
+  #   RudeQ::TokenLock.should_receive(:get_unique_token).exactly(3).times.and_return(@token)
+  # 
+  #   @existing = ProcessQueue.create!(:queue_name => 'abcde', :data => :old_data, :token => @token, :processed => true)
+  # 
+  #   ProcessQueue.get('abcde').should be(nil)
+  #     
+  #   ProcessQueue.set('abcde', :new_data)
+  #   ProcessQueue.get('abcde').should == :new_data
+  #   ProcessQueue.get('abcde').should be(nil)
+  # end
+  
+end

data/spec/spec.opts

@@ -1,6 +1,6 @@
 --colour
 --format
-progress
+specdoc
 --loadby
 mtime
 --reverse

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: matthewrudy-rudeq
 version: !ruby/object:Gem::Version 
-  version: "0.1"
+  version: "2.0"
 platform: ruby
 authors: 
 - Matthew Rudy Jacobs
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-06-21 00:00:00 -07:00
+date: 2009-02-12 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,7 +21,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: "0"
     version: 
-description: A simple DB queueing library built on top of ActiveRecord, and designed for use with MySQL.
+description: A simple DB queueing library built on top of ActiveRecord.
 email: MatthewRudyJacobs@gmail.com
 executables: []
 
@@ -62,7 +62,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.0.1
+rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
 summary: ActiveRecord-based DB-queue