resque-loner 1.0.1 → 1.2.0

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+**.DS_Store
+**.swp
+Gemfile.lock
+.rvmrc

data/CHANGELOG.markdown

@@ -0,0 +1,22 @@
+1.2.0
+--------------------------------
+Thanks @unclebilly for your pull request. Resque-loner now supports
+a maximum time for which a job should be unique. Just define @loner_ttl
+in your job (or leave it at -1 to never expire) and after @loner_ttl
+seconds your job can be enqueued again, even if an older one is still
+marked as running.
+
+1.1.0
+--------------------------------
+Merged in @ryansch's pull requests to clean up things a bit.
+This removed the `enqueue_to` and `dequeue_from` methods from
+resque-loner because it caused troubles with resque's own 
+`enqueue_to`.
+
+1.0.1
+--------------------------------
+Pulled in #8 and #9 so that removing empty queues
+does not fail
+
+1.0
+---------------------------------

data/Gemfile

@@ -0,0 +1,8 @@
+source 'http://rubygems.org'
+gemspec
+
+group :development do
+  gem 'gemcutter'
+  gem 'ruby-debug', :platform => :mri_18
+  gem 'ruby-debug19', :platform => :mri_19
+end

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2010 Moviepilot GmbH http://moviepilot.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -1,4 +1,3 @@
-
 Resque-Loner
 ======
 
@@ -38,12 +37,13 @@ Your queue is really full, so the job does not get executed right away. But the
 
 At this point you will have two jobs in the queue, the second of which has no effect: You don't have to run it, once the cache has been updated for the first time. This is where resque-loner's UniqueJobs come in. If you define CacheSweeper like this:
 
-    class CacheSweeper < Resque::Plugins::Loner::UniqueJob
-       @queue = :cache_sweeps
+    class CacheSweeper
+      include Resque::Plugins::UniqueJob
+      @queue = :cache_sweeps
 
-       def self.perform(article_id)
-         # Cache Me If You Can...
-       end
+      def self.perform(article_id)
+        # Cache Me If You Can...
+      end
     end
 
 Just like that you've assured that on the :cache_sweeps queue, there can only be one CacheSweeper job for each article. Let's see what happens when you try to enqueue a couple of these jobs now:
@@ -68,13 +68,6 @@ Since resque-loner keeps track of which jobs are queued in a way that allows for
     >> Resque.enqueued_in? :another_queue, CacheSweeper, 1
     => false
 
-If you want the same type of job in different queues, resque-loner lets you enqueue/dequeue to a queue of your choice:
-
-    >> Resque.enqueue_to :another_queue, CacheSweeper, 1
-    => "OK"
-    >> Resqueue.dequeue_from :another_queue, CacheSweeper, 1
-    => 1
-
 How it works
 --------
 

data/Rakefile

@@ -0,0 +1,83 @@
+#
+# Setup
+#
+
+$LOAD_PATH.unshift 'lib'
+
+require "rubygems"
+require "bundler"
+Bundler.setup
+
+require 'rspec/core/rake_task'
+
+load 'tasks/redis.rake'
+require 'rake/testtask'
+
+require 'resque/tasks'
+
+require 'bundler/gem_tasks'
+
+def command?(command)
+  system("type #{command} > /dev/null 2>&1")
+end
+
+
+#
+# Tests
+#
+
+task :default => :spec
+
+desc "Run specs for resque-loner"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = "spec/**/*_spec.rb"
+  t.rspec_opts = %w(-fd -c)
+end
+
+# desc "Run resque's test suite to make sure we did not break anything"
+# task :test do
+#   rg = command?(:rg)
+#   Dir['test/**/*_test.rb'].each do |f|
+#     rg ? sh("rg #{f}") : ruby(f)
+#   end
+# end
+
+if command?(:rg)
+  desc "Run the test suite with rg"
+  task :test do
+    Dir['test/**/*_test.rb'].each do |f|
+      sh("rg #{f}")
+    end
+  end
+else
+  Rake::TestTask.new do |test|
+    test.libs << "test"
+    test.test_files = FileList['test/**/*_test.rb']
+  end
+end
+
+if command? :kicker
+  desc "Launch Kicker (like autotest)"
+  task :kicker do
+    puts "Kicking... (ctrl+c to cancel)"
+    exec "kicker -e rake test lib examples"
+  end
+end
+
+
+#
+# Install
+#
+
+task :install => [ 'redis:install', 'dtach:install' ]
+
+
+#
+# Documentation
+#
+
+begin
+  require 'sdoc_helpers'
+rescue LoadError
+end
+

data/init.rb

@@ -0,0 +1 @@
+require 'rails/init.rb'

data/lib/resque-ext/job.rb

@@ -12,6 +12,7 @@ module Resque
     #  after Resque::Job.create has called Resque.push
     #
     def self.create_with_loner(queue, klass, *args)
+      return create_without_loner(queue, klass, *args) if Resque.inline?
       item = { :class => klass.to_s, :args => args }
       return "EXISTED" if Resque::Plugins::Loner::Helpers.loner_queued?(queue, item)
       job = create_without_loner(queue, klass, *args)
@@ -24,7 +25,7 @@ module Resque
     #
     def self.reserve_with_loner(queue)
       item = reserve_without_loner(queue)
-      Resque::Plugins::Loner::Helpers.mark_loner_as_unqueued( queue, item ) if item
+      Resque::Plugins::Loner::Helpers.mark_loner_as_unqueued( queue, item ) if item && !Resque.inline?
       item
     end
 
@@ -35,7 +36,7 @@ module Resque
     #  as the original method Resque::Job.destroy. Couldn't make it any dry'er.
     #
     def self.destroy_with_loner(queue, klass, *args)
-      Resque::Plugins::Loner::Helpers.job_destroy(queue, klass, *args)
+      Resque::Plugins::Loner::Helpers.job_destroy(queue, klass, *args) unless Resque.inline?
       destroy_without_loner(queue, klass, *args)
     end
 

data/lib/resque-ext/resque.rb

@@ -1,16 +1,5 @@
 module Resque
 
-  #
-  #  Why force one job type into one queue?
-  #
-  def self.enqueue_to( queue, klass, *args )
-    Job.create(queue, klass, *args)
-  end
-
-  def self.dequeue_from( queue, klass, *args)
-    Job.destroy(queue, klass, *args)
-  end
-
   def self.enqueued?( klass, *args)
     enqueued_in?(queue_from_class(klass), klass, *args )
   end

data/lib/resque-loner/helpers.rb

@@ -11,7 +11,11 @@ module Resque
 
         def self.mark_loner_as_queued(queue, item)
           return unless item_is_a_unique_job?(item)
-          redis.set(unique_job_queue_key(queue, item), 1)
+          key = unique_job_queue_key(queue, item)
+          redis.set(key, 1)
+          unless(ttl=item_ttl(item)) == -1 # no need to incur overhead for default value 
+            redis.expire(key, ttl)
+          end
         end
 
         def self.mark_loner_as_unqueued(queue, job)
@@ -34,6 +38,14 @@ module Resque
           end     # so resque-loner should not start throwing up when that happens.
         end
 
+        def self.item_ttl(item)
+          begin
+            constantize(item[:class] || item["class"]).loner_ttl
+          rescue
+            -1
+          end
+        end
+
         def self.job_destroy(queue, klass, *args)
           klass = klass.to_s
           redis_queue = "queue:#{queue}"

data/lib/resque-loner/unique_job.rb

@@ -33,6 +33,22 @@ module Resque
           digest = Digest::MD5.hexdigest encode(:class => job, :args => args)
           digest
         end
+
+        #
+        # The default ttl of a locking key is -1, i.e. forever.  If for some reason you only
+        # want the lock to be in place after a certain amount of time, just set a ttl for
+        # for your job.  For example:
+        #
+        # class FooJob
+        #   include Resque::Plugins::UniqueJob
+        #   @loner_ttl = 40
+        #   end
+        # end
+        #
+        def loner_ttl
+          @loner_ttl || -1
+        end
+
       end # ClassMethods
 
 

data/lib/resque-loner/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module Loner
-      VERSION = "1.0.1"
+      VERSION = "1.2.0"
     end
   end
 end

data/rails/init.rb

@@ -0,0 +1 @@
+require 'resque-loner'

data/resque-loner.gemspec

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'resque-loner/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'resque-loner'
+  s.version     = Resque::Plugins::Loner::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Jannis Hermanns']
+  s.email       = ['jannis@moviepilot.com']
+  s.homepage    = 'http://github.com/jayniz/resque-loner'
+  s.summary     = 'Adds unique jobs to resque'
+  s.has_rdoc    = false
+
+  s.rubyforge_project = 'resque-loner'
+
+  s.add_dependency 'resque', '~>1.0'
+  {
+    'rake'                => '> 0.8.7',
+    'rack-test'           => '~> 0.5.7',
+    'rspec'               => '~> 2.5.0',
+    'mock_redis'          => '~> 0.2.0',
+    'yajl-ruby'           => '~> 0.8.2'
+  }.each do |lib, version|
+    s.add_development_dependency lib, version
+  end
+
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ["lib"]
+
+  s.description = <<desc
+Makes sure that for special jobs, there can be only one job with the same workload in one queue.
+
+Example:
+    class CacheSweeper 
+
+       include Resque::Plugins::UniqueJob
+
+       @queue = :cache_sweeps
+
+       def self.perform(article_id)
+         # Cache Me If You Can...
+       end
+    end
+desc
+end

data/spec/loner_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+
+#
+#  Resque-loner specific specs. I'm shooting right through the stack here and just
+#  test the outcomes, because the implementation will change soon and the tests run
+#  quite quickly.
+#
+
+class SomeJob
+  @queue = :some_queue
+end
+
+class SomeUniqueJob 
+
+  include Resque::Plugins::UniqueJob
+
+  @queue = :other_queue
+  def self.perform(foo); end
+end
+
+class FailingUniqueJob
+  include Resque::Plugins::UniqueJob
+  @queue = :other_queue
+  def self.perform(foo)
+    raise "I beg to differ"
+  end
+end
+
+class DeprecatedUniqueJob < Resque::Plugins::Loner::UniqueJob
+
+  @queue = :other_queue
+  def self.perform(foo); end
+end
+
+class UniqueJobWithTtl
+  include Resque::Plugins::UniqueJob
+  @queue = :unique_with_ttl
+  @loner_ttl = 300
+
+  def self.perform(*args); end
+end
+
+describe "Resque" do
+
+  before(:each) do
+    Resque.redis.flushall
+    Resque.size(:other_queue).should == 0
+    Resque.size(:some_queue).should == 0
+  end
+
+  describe "Jobs" do
+    it "can put multiple normal jobs on a queue" do
+      Resque.enqueue SomeJob, "foo"
+      Resque.enqueue SomeJob, "foo"
+      Resque.size(:some_queue).should == 2
+    end
+
+    it "should allow only one of the same job to sit in a queue" do
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should support deprecated Resque::Plugins::Loner::UniqueJob class" do
+      Resque.enqueue DeprecatedUniqueJob, "foo"
+      Resque.enqueue DeprecatedUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should allow the same jobs to be executed one after the other" do
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+
+      Resque.reserve(:other_queue)
+      Resque.size(:other_queue).should == 0
+
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should be robust regarding hash attributes" do
+      Resque.enqueue SomeUniqueJob, :bar => 1, :foo => 2
+      Resque.enqueue SomeUniqueJob, :foo => 2, :bar => 1
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should be robust regarding hash attributes (JSON does not distinguish between string and symbol)" do
+      Resque.enqueue SomeUniqueJob, :bar => 1, :foo  => 1
+      Resque.enqueue SomeUniqueJob, :bar => 1, "foo" => 1
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should mark jobs as unqueued, when Job.destroy is killing them" do
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+
+      Resque::Job.destroy(:other_queue, SomeUniqueJob)
+      Resque.size(:other_queue).should == 0
+
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should mark jobs as unqueued, when they raise an exception during #perform" do
+      2.times { Resque.enqueue( FailingUniqueJob, "foo" ) }
+      Resque.size(:other_queue).should == 1
+
+      worker = Resque::Worker.new(:other_queue)
+      worker.work 0
+      Resque.size(:other_queue).should == 0
+
+      2.times { Resque.enqueue( FailingUniqueJob, "foo" ) }
+      Resque.size(:other_queue).should == 1
+    end
+
+    it "should report if a job is queued or not" do
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueued?(SomeUniqueJob, "foo").should be_true
+      Resque.enqueued?(SomeUniqueJob, "bar").should be_false
+    end
+
+    it "should report if a job is in a special queue or not" do
+      default_queue = SomeUniqueJob.instance_variable_get(:@queue)
+      SomeUniqueJob.instance_variable_set(:@queue, :special_queue)
+
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueued_in?( :special_queue, SomeUniqueJob, "foo").should be_true
+
+      SomeUniqueJob.instance_variable_set(:@queue, default_queue)
+
+      Resque.enqueued?( SomeUniqueJob, "foo").should be_false
+    end
+
+    it "should not be able to report if a non-unique job was enqueued" do
+      Resque.enqueued?(SomeJob).should be_nil
+    end
+
+    it "should cleanup all loners when a queue is destroyed" do
+      Resque.enqueue SomeUniqueJob, "foo"
+      Resque.enqueue FailingUniqueJob, "foo"
+
+      Resque.remove_queue(:other_queue)
+
+      Resque.enqueue(SomeUniqueJob, "foo")
+      Resque.size(:other_queue).should == 1
+    end
+    
+    it 'should not raise an error when deleting an already empty queue' do
+      expect { Resque.remove_queue(:other_queue) }.to_not raise_error
+    end
+
+    it 'should honor loner_ttl in the redis key' do
+      Resque.enqueue UniqueJobWithTtl
+      Resque.enqueued?(UniqueJobWithTtl).should be_true
+      k=Resque.redis.keys "loners:queue:unique_with_ttl:job:*"
+      k.length.should == 1
+      Resque.redis.ttl(k[0]).should be_within(2).of(UniqueJobWithTtl.loner_ttl)
+    end
+  end
+end