mongo-dequeue 0.1.0

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+gem 'json'
+gem 'mongo'
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.2"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,26 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    bson (1.3.1)
+    git (1.2.5)
+    jeweler (1.6.2)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.5.3)
+    mongo (1.3.1)
+      bson (>= 1.3.1)
+    rake (0.9.2)
+    rcov (0.9.9)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.2)
+  json
+  mongo
+  rcov
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 TelegramSam
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,84 @@
+= Mongo::Dequeue
+
+A de-duplicating priority queue that uses mongodb as the storage engine.
+
+Heavily inspired by Mongo::Queue https://github.com/skiz/mongo_queue
+
+== Features
+* Atomic Locking (provided by Mongo)
+* Thread Safe
+* Priority Support
+* Fully Tested
+* Simple API
+
+== Examples
+
+=== Setting up a queue
+Mongo::Dequeue requires a Mongo::Connection to your Mongo Server, and allows you to configure the database, 
+collection, and the default read timeout of items in your queue. Items popped from the queue must be confirmed
+complete, or they will be reissued after the timeout specified.
+
+    db = Mongo::Connection.new
+
+    options   = {
+      :database   => 'whateverdb',
+      :collection => 'my_queue',
+      :timeout    => 60
+    }
+
+    queue = Mongo::Dequeue.new(db, options)
+
+=== Pushing Items
+Items have a body, passed as the first argument to push(). A body can be a string, number, hash, or array. These 
+values are preserved in the Mongo collection used to store the queue, allowing you to inspect the queue and see these values.
+The optional options argument is a hash of values. you can set a custom duplicate_key that will be used to de-duplicate 
+queue items. There are no keys or reserved values in the body of an item. Pass in anything you like.
+
+
+    options = {
+        :duplicate_key => 'match'
+    }
+    
+    body = "foo"
+    body = 5
+    body = {
+        :stuff => "here"
+    }
+    body = ["a", "b", "c"]
+
+    queue.push(body,options)
+
+=== Popping Items
+Pop items with the pop() method. Specifying a timeout (in seconds) will override the default timeout set in 
+the queue options. After the timeout, the popped item will be available in another pop() call.
+    
+    item = queue.pop(:timeout => 1)
+    
+The pop call will return a hash, with an :id and a :body. After the queue item is done, be sure to call the 
+complete() method, with the id of the message.
+
+    queue.complete(item[:id])
+    
+Note that completed items are not removed from the database immediately, allowing inspection of completion times, 
+duplication numbers, etc. Calling the cleanup() method removes completed items.
+
+    queue.cleanup()
+  
+=== Queue Status
+You can call the +stats+ method to take a quick peek at your queue. Locked items are items that have been popped, 
+but not confirmed.
+
+   queue.stats
+   # => {:available => 3, :locked => 2, :complete => 5, :total => 10}
+
+=== Mongo::Queue Comparison
+
+* Items are not deleted immediately after completion.
+* Item locks auto-expire
+* Items are de-duplicated
+* No reserved words in item body
+
+=== TODO
+
+* Adjust auto generated duplication_key to be more accurate.
+* Full examples

data/Rakefile

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "mongo-dequeue"
+  gem.homepage = "http://github.com/TelegramSam/Dequeue"
+  gem.license = "MIT"
+  gem.summary = %Q{Mongo based de-duplicating pritority queue}
+  gem.description = %Q{A de-duplicating priority queue that uses mongodb as the storage engine.}
+  gem.email = "telegramsam@gmail.com"
+  gem.authors = ["TelegramSam"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "mongo-dequeue #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/mongo-dequeue.rb

@@ -0,0 +1,149 @@
+require 'digest/md5'
+require 'json'
+require 'mongo'
+
+# heavily inspired by https://github.com/skiz/mongo_queue
+
+class Mongo::Dequeue
+	attr_reader :connection, :config
+	
+	DEFAULT_CONFIG = {
+		:database   => 'mongo_dequeue',
+		:collection => 'mongo_dequeue',
+		:timeout    => 300,
+		:default_priority => 3
+	}.freeze
+	
+	# Create a new instance of MongoDequeue with the provided mongodb connection and optional configuration.
+	# See +DEFAULT_CONFIG+ for default configuration and possible configuration options.
+	#
+	# Example:
+	#    db = Mongo::Connection.new('localhost')
+	#    config = {:timeout => 90, :attempts => 2}
+	#    queue = Mongo::Queue.new(db, config)
+	#
+	def initialize(connection, opts={})
+		@connection = connection
+		@config = DEFAULT_CONFIG.merge(opts)
+	end
+ 
+	# Remove all items from the queue. Use with caution!
+	def flush!
+		collection.drop
+	end
+	
+	# Insert a new item into the queue.
+	#
+	# Example:
+	#    queue.insert(:name => 'Billy', :email => 'billy@example.com', :message => 'Here is the thing you asked for')
+	def push(body, item_opts = {})
+		dup_key = item_opts[:duplicate_key] || Mongo::Dequeue.generate_duplicate_key(body)
+		
+		selector = {
+			:duplicate_key => dup_key,
+			:complete => false,
+			:locked_at => nil
+		}
+		item = {
+			'$set' => {
+				:body => body,
+				:inserted_at => Time.now.utc,
+				:complete => false,
+				:locked_at => nil,
+				:completed_at => nil,
+				:priority => item_opts[:priority] || @config[:default_priority],
+				:duplicate_key => dup_key
+			},
+			'$inc' => {:count => 1 }
+		}
+		
+		id = collection.update(selector, item, :upsert => true)
+	end
+	
+	# Lock and return the next queue message if one is available. Returns nil if none are available. Be sure to
+	# review the README.rdoc regarding proper usage of the locking process identifier (locked_by).
+	# Example:
+	#    doc = queue.pop()
+	
+	# {:body=>"foo", :id=>"4e039c372b70275e345206e4"}
+
+	def pop(opts = {})
+		begin
+			timeout = opts[:timeout] || @config[:timeout]
+			cmd = BSON::OrderedHash.new
+		    cmd['findandmodify'] = @config[:collection]
+		    cmd['update']        = {'$set' => {:locked_till => Time.now.utc+timeout}} 
+		    cmd['query']         = {:complete => false, '$or'=>[{:locked_till=> nil},{:locked_till=>{'$lt'=>Time.now.utc}}] }
+		    cmd['sort']          = {:priority=>-1,:inserted_at=>1}
+		    cmd['limit']         = 1
+		    cmd['new']           = true
+		    result = collection.db.command(cmd)
+		rescue Mongo::OperationFailure => of
+			return nil
+		end
+	    return {
+	    	:body => result['value']['body'],
+	    	:id => result['value']['_id'].to_s
+	    }
+	end
+	
+	
+	# Remove the document from the queue. This should be called when the work is done and the document is no longer needed.
+	# You must provide the process identifier that the document was locked with to complete it.
+	def complete(id)
+		cmd = BSON::OrderedHash.new
+		cmd['findandmodify'] = @config[:collection]
+		cmd['query']         = {:_id => BSON::ObjectId.from_string(id), :complete => false}
+		cmd['update']        = {:completed_at => Time.now.utc, :complete => true}
+		cmd['limit']         = 1
+		collection.db.command(cmd)
+	end
+	
+	# Removes completed job history
+	def cleanup()
+		collection.remove({:complete=>true});
+	end
+  	
+	# Provides some information about what is in the queue. We are using an eval to ensure that a
+	# lock is obtained during the execution of this query so that the results are not skewed.
+	# please be aware that it will lock the database during the execution, so avoid using it too
+	# often, even though it it very tiny and should be relatively fast.
+	def stats
+    	js = "function queue_stat(){
+			      return db.eval(
+			      function(){
+			      	var nowutc = new Date();
+			      	var a = db.#{config[:collection]}.count({'complete': false, '$or':[{'locked_till':null},{'locked_till':{'$lt':nowutc}}] });
+			        var c = db.#{config[:collection]}.count({'complete': true});
+			        var t = db.#{config[:collection]}.count();
+			        var l = db.#{config[:collection]}.count({'complete': false, 'locked_till': {'$gte':nowutc} });
+			        return [a, c, t, l];
+			      }
+			    );
+			  }"
+		available, complete, total, locked = collection.db.eval(js) 
+		{ :locked    => locked.to_i,
+		  :complete => complete.to_i,
+		  :available => available.to_i,
+		  :total     => total.to_i }
+	end
+	
+	def self.generate_duplicate_key(body)
+		return Digest::MD5.hexdigest(body) if body.class == "String"
+		return Digest::MD5.hexdigest(body) if body.class == "Fixnum"
+		#else	
+		return Digest::MD5.hexdigest(body.to_json) #won't ever match a duplicate. Need a better way to handle hashes and arrays.
+	end
+	
+	
+	protected
+	
+	def value_of(result) #:nodoc:
+		result['okay'] == 0 ? nil : result['value']
+	end
+	
+	def collection #:nodoc:
+		@connection.db(@config[:database]).collection(@config[:collection])
+	end
+	
+end

data/spec/mongo_dequeue_spec.rb

@@ -0,0 +1,256 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'pp'
+
+describe Mongo::Dequeue do
+  
+  def insert_and_inspect(body, options={})
+  	 @queue.push(body,options)
+     @queue.send(:collection).find_one  
+  end
+  
+  
+  before(:all) do
+    opts   = {
+      :database   => 'mongo_queue_spec',
+      :collection => 'spec',
+      :timeout    => 60}
+    @db = Mongo::Connection.new('localhost', nil, :pool_size => 4)
+    @queue = Mongo::Dequeue.new(@db, opts)
+  end
+  
+  before(:each) do
+    @queue.flush!
+  end
+  
+  describe "Configuration" do
+
+    it "should set the connection" do
+      @queue.connection.should be(@db)
+    end
+
+    it "should allow database option" do
+      @queue.config[:database].should eql('mongo_queue_spec')
+    end
+    
+    it "should allow collection option" do
+      @queue.config[:collection].should eql('spec')
+    end
+
+    it "should allow timeout option" do
+      @queue.config[:timeout].should eql(60)
+    end
+  
+    it "should have a sane set of defaults" do
+      q = Mongo::Dequeue.new(nil)
+      q.config[:collection].should eql 'mongo_dequeue'
+      q.config[:timeout].should    eql 300
+    end
+  end
+  
+  describe "Inserting a standard Job" do
+    before(:each) do
+       @item = insert_and_inspect({:message => 'MongoQueueSpec', :foo => 5})
+    end
+    
+    it "should set priority to 3 by default" do
+      @item['priority'].should be(3)
+    end
+    
+    it "should not be complete" do
+      @item['complete'].should be false
+    end
+    
+    it "should have a null completed_at" do
+    	@item['completed_at'].should be nil
+    end
+    
+    it "should set a null locked_at" do
+      @item['locked_at'].should be nil
+    end
+    
+    it "should have no duplicates" do
+      @item['count'].should be 1
+    end
+    
+    it "should have a duplicate_key" do
+      @item['duplicate_key'].should_not be nil 
+    end
+    
+    it "should return struct body properly" do
+      @item['body']['message'].should eql('MongoQueueSpec')
+      @item['body']['foo'].should be(5)
+    end
+  end
+
+	describe "Inserting different body types" do
+	    before(:each) do
+	       @queue.flush!
+	    end
+	    
+	    it "should handle a struct" do
+	    	i = insert_and_inspect({:message => 'MongoQueueSpec', :foo => 5})
+		    i['body']['message'].should eql('MongoQueueSpec')
+	     	i['body']['foo'].should be(5)
+	    end
+
+	    it "should handle a string" do
+	    	i = insert_and_inspect("foobarbaz")
+		    i['body'].should eql "foobarbaz"
+	    end
+	    
+	    it "should handle a number" do
+	    	i = insert_and_inspect(42)
+		    i['body'].should be 42
+	    end
+	end
+
+	describe "Deduplicating messages" do
+	    before(:each) do
+	       @queue.flush!
+	    end
+	    
+	    it "should combine identical bodies of type string" do
+	    	a = insert_and_inspect("foo")
+		    b = insert_and_inspect("foo")
+		    @queue.send(:collection).count.should be 1  
+	    end
+
+	    it "should not combine different bodies of type string" do
+	    	a = insert_and_inspect("foo")
+		    b = insert_and_inspect("bar")
+		    @queue.send(:collection).count.should be 2
+		    b['count'].should be 1
+	    end
+	    
+	    it "should combine identical bodies of type struct" do
+	    	pending "Test after we have a better way of handling structs"
+	    	a = insert_and_inspect({:a=>'a',:b=>'b'})
+		    b = insert_and_inspect({:a=>'a',:b=>'b'})
+		    c = insert_and_inspect({:b=>'b',:a=>'a'})
+		    @queue.send(:collection).count.should be 1
+	    end
+
+	    it "should not combine different bodies of type struct" do
+	    	a = insert_and_inspect({:a=>'a',:b=>'b'})
+		    b = insert_and_inspect({:a=>'a',:c=>'c'})
+		    @queue.send(:collection).count.should be 2
+		    b['count'].should be 1
+	    end
+
+	    it "should combine based on duplication_key" do
+	    	a = insert_and_inspect({:a=>'a',:b=>'b'}, :duplicate_key => 'match')
+		    b = insert_and_inspect({:a=>'a',:c=>'c'}, :duplicate_key => 'match')
+		    @queue.send(:collection).count.should be 1
+		    b['count'].should be 2
+	    end
+	    
+	    it "should not combine based on duplication_key" do
+	    	a = insert_and_inspect("foo", :duplicate_key => 'match')
+		    b = insert_and_inspect("foo", :duplicate_key => 'nomatch')
+		    @queue.send(:collection).count.should be 2
+		    b['count'].should be 1
+	    end
+	    
+	end
+
+	describe "Popping messages" do
+	    before(:each) do
+	       @queue.flush!
+	    end
+	    
+	    it "should return message" do
+	    	a = insert_and_inspect("foo")
+		    m = @queue.pop
+		    m[:body].should eq "foo"
+		    @queue.send(:collection).count.should be 1  
+	    end
+	    
+	    it "should return an id" do
+	    	a = insert_and_inspect("foo")
+		    m = @queue.pop
+		    m[:id].should_not be nil
+		    @queue.send(:collection).count.should be 1  
+	    end
+
+	    it "should return nil when queue is empty" do
+	    	m = @queue.pop
+		    m.should be nil
+		    @queue.send(:collection).count.should be 0 
+	    end
+	    
+	    it "should complete ok" do
+	    	a = insert_and_inspect("foo")
+		    m = @queue.pop
+		    @queue.complete(m[:id])
+		    m2 = @queue.pop
+		    m2.should be nil
+		    @queue.send(:collection).count.should be 1
+	    end
+	    
+	    it "should pop again after timeout" do
+	    	a = insert_and_inspect("foo")
+		    m = @queue.pop(:timeout => 1)
+		    sleep(2)
+		    m2 = @queue.pop
+		    m2[:id].should eq m[:id]
+		    @queue.send(:collection).count.should be 1
+	    end
+	    
+	    it "should pop in order" do
+	       @a = insert_and_inspect("a")
+	       @b = insert_and_inspect("b")
+	       @c = insert_and_inspect("c")
+	       @d = insert_and_inspect("d")
+	       
+	       @ap = @queue.pop
+	       @bp = @queue.pop
+	       @cp = @queue.pop
+	       @dp = @queue.pop
+	       
+	       @ap[:body].should eq "a"
+	       @bp[:body].should eq "b"
+	       @cp[:body].should eq "c"
+	       @dp[:body].should eq "d"
+	    end
+	    
+	end
+
+	describe "Stats" do
+	    before(:all) do
+	       @queue.flush!
+	       @a = insert_and_inspect("a")
+
+	       @b = insert_and_inspect("b")
+	       @c = insert_and_inspect("c")
+	       @d = insert_and_inspect("d")
+	       @e = insert_and_inspect("e")
+	       
+	       @ap = @queue.pop(:timeout => 1)
+	       @bp = @queue.pop
+	       @cp = @queue.pop
+	       
+	       sleep(2)
+	       
+	       @queue.complete(@bp[:id])
+	       
+	       @stats = @queue.stats
+	    end
+	    #locked, complete, available, total
+	    it "should count complete" do
+	    	@stats[:complete].should == 1
+	    end
+	    it "should count total" do
+	    	@stats[:total].should == 5
+	    end
+	    it "should count available" do
+	    	@stats[:available].should == 3
+	    end
+	    it "should count locked" do
+	    	@stats[:locked].should == 1
+	    end
+
+	    
+	end
+
+    
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'rubygems'
+require 'mongo'
+require 'mongo-dequeue'

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification 
+name: mongo-dequeue
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- TelegramSam
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-29 00:00:00 -06:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.6.2
+  type: :development
+  prerelease: false
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id006
+description: A de-duplicating priority queue that uses mongodb as the storage engine.
+email: telegramsam@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/mongo-dequeue.rb
+- spec/mongo_dequeue_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/TelegramSam/Dequeue
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 4221506242229050956
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.3
+signing_key: 
+specification_version: 3
+summary: Mongo based de-duplicating pritority queue
+test_files: []
+