smq 0.2.2 → 0.3.0

data/README.markdown

@@ -81,6 +81,15 @@ This process, although heavy on `SELECT`s, results in a minimum of table locking
 
 These limitations are deemed acceptable due to the simple nature of this queueing system.
 
+### Workaround to Locking Limitations
+
+There is a workaround to the implicit limit of 5 workers per queue, which is to use the "batching" facility.  This works by splitting up the messages by the [modulo](http://en.wikipedia.org/wiki/Modulo_operation) of the ID; in effect this means you can then run up to 5 workers per batch:
+
+	SMQ::Worker.new("queue_name", total_batches, this_batch).work do |msg|
+	  puts msg.data.inspect
+	  msg.ack!
+	end
+
 ## Licensing and Attribution
 
 SMQ is released under the MIT license as detailed in the LICENSE file that should be distributed with this library; the source code is [freely available](http://github.com/timblair/smq).

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

data/lib/smq.rb

@@ -43,9 +43,23 @@ module SMQ
   end
 
   def self.flush_all_queues!
-    # yes, not the most efficient, but *should* only be called during testing
-    # so it's not that much of a concern
-    SMQ::Message.delete_all
+    # we're totally resetting everything here, including any auto-increment
+    # keys: this should only ever be called for testing.
+    # adapted from: http://www.manu-j.com/blog/post/221/
+    case ActiveRecord::Base.connection.instance_variable_get(:@config)[:adapter]
+      when "mysql"
+        ActiveRecord::Base.connection.tables.each do |table|
+          ActiveRecord::Base.connection.execute("TRUNCATE #{SMQ::Message.table_name}")
+        end
+      when "sqlite", "sqlite3"
+        ActiveRecord::Base.connection.tables.each do |table|
+          ActiveRecord::Base.connection.execute("DELETE FROM #{SMQ::Message.table_name}")
+          ActiveRecord::Base.connection.execute("DELETE FROM sqlite_sequence where name='#{SMQ::Message.table_name}'")
+        end
+        ActiveRecord::Base.connection.execute("VACUUM")
+      else
+        raise "Unsupported connection type: #{ActiveRecord::Base.connection.instance_variable_get(:@config)[:adapter]}."
+    end
   end
 
 end

data/lib/smq/queue.rb

@@ -25,15 +25,23 @@ module SMQ
     end
 
     def reserve(worker)
-      find_available.each do |msg|
+      find_available(worker.batches, worker.batch).each do |msg|
         m = msg.lock!(worker)
         return m unless m == nil
       end
       nil
     end
 
-    def find_available
-      SMQ::Message.find(:all, :select => 'id, updated_at', :conditions => ["queue = ? AND completed_at IS NULL AND locked_by IS NULL", @name], :order => "id ASC", :limit => @batch_size).sort_by { rand() }
+    def find_available(batches=1, batch=1)
+      SMQ::Message.find(
+        :all,
+        :select => 'id, updated_at',
+        :conditions => [
+          "queue = ? AND (id % ?) = ? AND completed_at IS NULL AND locked_by IS NULL",
+          @name, batches, batch-1
+        ],
+        :order => "id ASC", :limit => @batch_size
+      ).sort_by { rand() }
     end
 
     def length
@@ -57,12 +65,16 @@ module SMQ
     private
 
     def delete_queue_items(where = nil, limit = nil)
-      # delete_all doesn't support a :limit clause, so we have to fake it
-      msg = SMQ::Message.find(:first, :select => 'id', :conditions => ["queue = ?", @name], :offset => limit) if !limit.nil?
-      if (msg.nil?)
+      if (limit.nil?)
         SMQ::Message.delete_all(["queue = ? #{where ? 'AND ' + where : ''}", @name])
       else
-        SMQ::Message.delete_all(["queue = ? AND id < ? #{where ? 'AND ' + where : ''}", @name, msg.id])
+        # delete_all doesn't support a :limit clause, so we have to fake it
+        msg = SMQ::Message.find(
+          :first, :select => 'id',
+          :conditions => ["queue = ? #{where ? 'AND ' + where : ''}", @name],
+          :order => "id ASC", :offset => limit
+        )
+        SMQ::Message.delete_all(["queue = ? AND id < ? #{where ? 'AND ' + where : ''}", @name, msg.id]) unless msg.nil?
       end
     end
 

data/lib/smq/worker.rb

@@ -4,15 +4,16 @@ module SMQ
 
   class Worker
 
-    attr_accessor :name
-    attr_accessor :queue
+    attr_accessor :name, :queue, :batches, :batch
 
     @working = false
     @stopping = false
 
-    def initialize(queue)
+    def initialize(queue, batches=1, batch=1)
       self.name = "#{Socket.gethostname}:#{Process.pid}" rescue "pid:#{Process.pid}"
       self.queue = queue.instance_of?(SMQ::Queue) ? queue : SMQ::Queue.new(queue)
+      self.batches = batches
+      self.batch = batch
     end
 
     def to_s

data/test/test_queue.rb

@@ -89,15 +89,24 @@ class QueueTest < Test::Unit::TestCase
     assert_equal 5, @queue.length
   end
 
+  def test_should_return_only_correctly_batched_messages
+    populate_queue(@queue, 5)
+    assert_equal 1, @queue.find_available(3,1).length
+    assert_equal 2, @queue.find_available(3,2).length
+    assert_equal 2, @queue.find_available(3,3).length
+  end
+
   private
 
   def populate_and_ack_all_but_fail_one(queue = nil)
     queue ||= @queue
     populate_queue(queue, 5)
-    msg = SMQ::Message.find(:first)
-    msg.fail
-    msg.save!
-    @worker.work(true) { |m| m.ack! }
+    SMQ::Message.find(:all, :conditions => ["queue = ?", queue.name]).each do |msg|
+      msg.ack!
+    end
+    failure = SMQ::Message.find(:first)
+    failure.fail
+    failure.save!
   end
 
 end

data/test/test_worker.rb

@@ -57,4 +57,12 @@ class WorkerTest < Test::Unit::TestCase
     assert !@worker.is_working?, "Working and should have stopped"
   end
 
+  def test_working_a_populated_queue_in_batches_should_work_all_batch_jobs
+    @worker.batches = 2
+    populate_queue(@worker.queue.name, 5)
+    block_calls = 0
+    @worker.work(true) { block_calls += 1 }
+    assert_equal 2, block_calls
+  end
+
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: smq
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 19
+  prerelease: 
   segments: 
   - 0
-  - 2
-  - 2
-  version: 0.2.2
+  - 3
+  - 0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Tim Blair
@@ -14,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-26 00:00:00 +01:00
+date: 2011-05-16 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activerecord
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -33,9 +36,11 @@ dependencies:
   name: yajl-ruby
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -45,9 +50,11 @@ dependencies:
   name: sqlite3-ruby
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -63,7 +70,6 @@ extra_rdoc_files:
 - LICENSE
 - README.markdown
 files: 
-- .gitignore
 - LICENSE
 - README.markdown
 - Rakefile
@@ -84,34 +90,34 @@ homepage: http://github.com/timblair/smq
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.5.0
 signing_key: 
 specification_version: 3
 summary: Simple Message Queue
-test_files: 
-- test/helper.rb
-- test/test_message.rb
-- test/test_queue.rb
-- test/test_worker.rb
-- examples/worker.rb
+test_files: []
+

data/.gitignore

@@ -1 +0,0 @@
-pkg