resque-unique-job 0.0.1

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Engine Yard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,15 @@
+resque-unique-job
+===============
+Depends on [Resque](http://github.com/defunkt/resque/) 1.8
+
+About
+-----
+
+
+Examples
+--------
+
+
+Requirements
+------------
+* [resque](http://github.com/defunkt/resque/) 1.8

data/Rakefile

@@ -0,0 +1 @@
+#TODO

data/lib/resque/plugins/resque_ext/destroy.rb

@@ -0,0 +1,15 @@
+Resque.class_eval do
+
+  def self.dequeue_with_unique_job(klass, *args)
+    klass.destroy_matching_keys(queue_from_class(klass), args)
+    dequeue_without_unique_job(klass, *args)
+  end
+
+
+  class << self
+    alias_method :dequeue_without_unique_job, :dequeue
+    alias_method :dequeue, :dequeue_with_unique_job
+  end
+
+
+end

data/lib/resque/plugins/unique_job.rb

@@ -0,0 +1,255 @@
+require 'resque/plugins/resque_ext/destroy'
+
+module Resque
+  module Plugins
+    module UniqueJob
+      include Resque::Helpers
+
+      def enqueue(*args)
+        key = unique_redis_key(args)
+        ttl = Resque.redis.ttl(key)
+        # if it's already enQ'd
+        if Resque.redis.getset(key, "1")
+          if expire = unique_redis_expiration
+            # reset existing expiration
+            Resque.redis.expire(key, ttl)
+          end
+        else
+          # set new expiration
+          if expire = unique_redis_expiration
+            Resque.redis.expire(key, expire)
+          end
+          super
+        end
+      end
+
+      def before_perform_unique_job(*args)
+        Resque.redis.del(unique_redis_key(args))
+      end
+
+      def unique_key(args)
+        Digest::MD5.hexdigest(encode(:class => self.to_s, :args => args))
+      end
+
+      def destroy_matching_keys(queue, args)
+        queue = "queue:#{queue}"
+
+        klass = self.to_s
+        if args.empty?
+          redis.lrange(queue, 0, -1).each do |string|
+            if decode(string)['class'] == klass
+              key = unique_redis_key(decode(string)['args'])
+              Resque.redis.del(key)
+            end
+          end
+        else
+          Resque.redis.del(unique_redis_key(args))
+        end
+      end
+
+      private
+
+      def unique_redis_expiration
+        600
+      end
+
+      def unique_redis_key(args)
+        job_unique_key = unique_key(args)
+        "plugin:unique_job:#{job_unique_key}"
+      end
+      # def self.extended(mod)
+      #   mod.extend(Resque::Plugins::Meta)
+      #   mod.extend(Resque::Plugins::Lock)
+      # end
+      #
+      # class Step
+      #   def initialize(args, run_last = false, &block)
+      #     @run_last = run_last
+      #     @signature = args.map{|a| a.to_s}.join(" ")
+      #     if args.size == 1
+      #       #no inputs or output
+      #       @inputs = []
+      #     elsif args.size >= 2
+      #       unless @run_last
+      #         @output = args.pop.to_s
+      #       end
+      #       @inputs = []
+      #       args.each_with_index do |a, index|
+      #         if index % 2 == 1
+      #           @inputs << a.to_s
+      #         end
+      #       end
+      #     else
+      #       raise ArgumentError, "invalid arguments #{args.inspect}"
+      #     end
+      #     @block = block
+      #   end
+      #   attr_reader :block, :signature, :inputs, :output, :run_last
+      #   # attr_accessor :run_last
+      #   # attr_reader :step_name, :what_it_makes, :block
+      #   def run(available_inputs)
+      #     begin
+      #       block_args = @inputs.map do |input_name|
+      #         available_inputs[input_name]
+      #       end
+      #       @block.call(*block_args)
+      #     rescue Exception => e
+      #       puts e.inspect
+      #       puts e.backtrace.join("\n")
+      #       raise e
+      #     end
+      #   end
+      # end
+      #
+      # #TODO:
+      # #
+      # # Need a way to edit meta data BEFORE a job is enQ'd
+      # # so that we can we sure that data is available on dQ
+      # #
+      # # Need a way to "Lock" editing of the meta data on a job
+      # # so other editors of job data must wait before editing job
+      # # (2 jobs marking a step done and then, therefore re-enQ-ing the parent)
+      # #
+      # # need a way to mark a job as already enQ'd (and not yet running)
+      # # so that if the tomato just re-enQ'd the sandwich, the cheese will see it on the Q and let it be
+      # #
+      # # basically, there should be a lock such that if there are any child jobs still enQ'd for a job
+      # # it should not run but prioritize the child jobs first
+      #
+      # class StepDependency
+      #   def initialize(job_class, args)
+      #     @job_class = job_class
+      #     @job_args = args
+      #   end
+      #   attr_reader :job_class, :job_args
+      # end
+      #
+      # class Retry
+      #   def initialize(seconds)
+      #     @seconds = seconds
+      #   end
+      #   attr_reader :seconds
+      # end
+      #
+      # def run_steps(meta_id, *args)
+      #   @step_list = []
+      #   @meta = self.get_jobdata(meta_id)
+      #   # puts "I have meta of: " + @meta.inspect
+      #
+      #   #implicitly builds the @step_list
+      #   steps(*args)
+      #
+      #   #TODO: raise error if there are duplicate steps defined?
+      #
+      #   # require 'pp'
+      #   # pp @step_list
+      #
+      #   #figure out which step we are on from meta data
+      #   @meta["step_count"] ||= @step_list.size
+      #   steps_ran = @meta["steps_ran"] ||= []
+      #   steps_running = @meta["steps_running"] ||= []
+      #   # puts "my steps_ran are #{steps_ran.inspect}"
+      #   # @step_list.map{ |step| step.signature }
+      #   available_inputs = @meta["available_inputs"] ||= {}
+      #   # puts "my available_inputs are #{available_inputs.inspect}"
+      #
+      #   #run last step if no more steps are needed
+      #   @step_list.each do |step|
+      #     if steps_ran.include?(step.signature)
+      #       #already ran
+      #     elsif steps_running.include?(step.signature)
+      #       #already Q'd
+      #     elsif step.run_last
+      #       # puts "Can't run last step #{step.signature} yet"
+      #       #this is the last step, only run if all other steps are run
+      #     elsif (step.inputs - available_inputs.keys).empty?
+      #       #all of the steps needed inputs are available
+      #       #run!
+      #       result = step.run(available_inputs)
+      #       puts "running step #{step.signature}"
+      #       if result.is_a?(Retry)
+      #         puts "enqueue #{self} in #{result.seconds}"
+      #         Resque.enqueue_in(result.seconds, self, meta_id, *args)
+      #       elsif result.is_a?(StepDependency)
+      #         #TODO: what if the child job is dQ'd before caller has a chance to set parent_job
+      #         # don't re-enQ a child that's already enQ'd!
+      #         # it might not be in steps ran but it doesn't need to be duplicated!
+      #         puts "enqueue #{result.job_class}"
+      #         child_job = result.job_class.enqueue(*result.job_args)
+      #         @meta["steps_running"] << step.signature
+      #         child_job["parent_job"] = [self, meta_id, args]
+      #         child_job["expected_output"] = step.output
+      #         child_job["signature_from_parent"] = step.signature
+      #         child_job.save
+      #       else
+      #         if step.output
+      #           available_inputs[step.output] = result
+      #         end
+      #         # puts "available_inputs are now #{available_inputs.inspect}"
+      #         if @meta["steps_ran"].include?(step.signature)
+      #           raise "WHAT? ran #{step.signature} twice!"
+      #         end
+      #         @meta["steps_ran"] << step.signature
+      #       end
+      #     else
+      #       # puts "waiting before we can run step #{step.signature} -- need #{step.inputs}"
+      #     end
+      #   end
+      #
+      #   if steps_ran.size + 1 == @step_list.size
+      #     puts "now running last step of #{self} -- already ran #{steps_ran.inspect}"
+      #
+      #     step = @step_list.last
+      #     result = step.run(available_inputs)
+      #     if @meta["parent_job"]
+      #       # puts "#{meta_id} has parent"
+      #       parent_job_class_name, parent_meta_id, parent_args = @meta["parent_job"]
+      #       parent_job_class = const_get(parent_job_class_name)
+      #       parent_meta = parent_job_class.get_jobdata(parent_meta_id)
+      #       if expected_output = @meta["expected_output"]
+      #         parent_meta["available_inputs"][expected_output] = result
+      #         parent_meta.save
+      #       end
+      #       if @meta["signature_from_parent"]
+      #         if parent_meta["steps_ran"].include?(@meta["signature_from_parent"])
+      #           raise "WHAT? ran #{@meta["signature_from_parent"]} twice!"
+      #         end
+      #         parent_meta["steps_ran"] << @meta["signature_from_parent"]
+      #         parent_meta.save
+      #       end
+      #       puts "enqueue #{parent_job_class}"
+      #       Resque.enqueue(parent_job_class, parent_meta_id, *parent_args)
+      #     end
+      #     if @meta["steps_ran"].include?(step.signature)
+      #       raise "WHAT? ran #{step.signature} twice!"
+      #     end
+      #     @meta["steps_ran"] << step.signature
+      #   end
+      #   @meta.save
+      #
+      #   puts "End of #{self}"
+      # end
+      #
+      # def step(*args, &block)
+      #   @step_list << Step.new(args, &block)
+      # end
+      #
+      # def last_step(*args, &block)
+      #   @step_list << Step.new(args, true, &block)
+      # end
+      #
+      # def depend_on(job_class, *args)
+      #   StepDependency.new(job_class, args)
+      # end
+      #
+      # def retry_in(seconds)
+      #   Retry.new(seconds)
+      # end
+      #
+      # def get_jobdata(meta_id)
+      #   get_meta(meta_id)
+      # end
+
+    end
+  end
+end

data/lib/resque/plugins/unique_job/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module UniqueJob
+      Version = '0.0.1'
+    end
+  end
+end

data/spec/unique_job_spec.rb

@@ -0,0 +1,149 @@
+require 'resque'
+require 'resque/plugins/unique_job'
+
+class WhatHappened
+  require 'tempfile'  
+  def self.reset!
+    @what_happened = Tempfile.new("what_happened")
+  end
+  def self.what_happened
+    File.read(@what_happened.path)
+  end
+  def self.record(*event)
+    @what_happened.write(event.to_s)
+    @what_happened.flush
+  end
+end
+
+class BaseJob
+
+  def self.enqueue(*args)
+    Resque.enqueue(self, *args)
+  end
+
+  def self.perform(*args)
+    begin
+      WhatHappened.record(self, args)
+    rescue => e
+      puts e.inspect
+      puts e.backtrace.join("\n")
+    end
+  end
+
+end
+
+class BasicJob < BaseJob
+  extend Resque::Plugins::UniqueJob
+  @queue = :test
+
+end
+
+class DifferentJob < BaseJob
+  extend Resque::Plugins::UniqueJob
+  @queue = :test
+
+end
+
+class NotUniqueJob < BaseJob
+  @queue = :test
+end
+
+class ShortExpiringJob < BaseJob
+  extend Resque::Plugins::UniqueJob
+  @queue = :test
+  
+  def self.unique_redis_expiration
+    1
+  end
+end
+
+
+describe Resque::Plugins::UniqueJob do
+  before(:each) do
+    WhatHappened.reset!
+    Resque.redis.flushall
+  end
+  
+  it "works for 1 job" do
+    BasicJob.enqueue("foo", "bar")
+    worker = Resque::Worker.new(:test)
+    worker.work(0)
+    WhatHappened.what_happened.should == "BasicJobfoobar"
+  end
+
+  describe "uniqueness" do
+
+    it "cares about job class" do
+      BasicJob.enqueue("foo", "bar")
+      DifferentJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "BasicJobfoobarDifferentJobfoobar"
+    end
+    
+    it "only enqueues the job once" do
+      BasicJob.enqueue("foo", "bar")
+      BasicJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "BasicJobfoobar"
+    end
+    
+    it "re-enqueues after the job has been processed" do
+      worker = Resque::Worker.new(:test)
+      BasicJob.enqueue("foo", "bar")
+      worker.work(0)
+      BasicJob.enqueue("foo", "bar")
+      worker.work(0)
+      WhatHappened.what_happened.should == "BasicJobfoobarBasicJobfoobar"
+    end
+
+    it "doesn't make non unique jobs unique" do
+      NotUniqueJob.enqueue("foo", "bar")
+      NotUniqueJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "NotUniqueJobfoobarNotUniqueJobfoobar"
+    end
+    
+    it "respects dequeue" do
+      BasicJob.enqueue("foo", "bar")
+      Resque.dequeue(BasicJob, "foo", "bar")
+      BasicJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "BasicJobfoobar"
+    end
+
+    it "respects destroy dequeue all" do
+      BasicJob.enqueue("foo", "bar")
+      BasicJob.enqueue("bar", "baz")
+      Resque.dequeue(BasicJob)
+      BasicJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "BasicJobfoobar"
+    end
+
+    it "doesn't destroy too much" do
+      BasicJob.enqueue("foo", "bar")
+      DifferentJob.enqueue("foo", "bar")
+      Resque.dequeue(BasicJob, "foo", "bar")
+      # DifferentJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "DifferentJobfoobar"
+    end
+     
+    it "expires the uniqueness key" do
+      ShortExpiringJob.enqueue("foo", "bar")
+      sleep 2
+      ShortExpiringJob.enqueue("foo", "bar")
+      worker = Resque::Worker.new(:test)
+      worker.work(0)
+      WhatHappened.what_happened.should == "ShortExpiringJobfoobarShortExpiringJobfoobar"
+    end
+    
+  end
+
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification 
+name: resque-unique-job
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Andy Delcambre
+- Jacob Burkhart
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-04 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 1
+        - 8
+        - 0
+        version: 1.8.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: A Resque plugin for unique jobs
+email: cloud@engineyard.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.markdown
+- Rakefile
+- LICENSE
+- lib/resque/plugins/resque_ext/destroy.rb
+- lib/resque/plugins/unique_job/version.rb
+- lib/resque/plugins/unique_job.rb
+- spec/unique_job_spec.rb
+has_rdoc: true
+homepage: http://github.com/engineyard/resque-unique-job
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: A Resque plugin for unique jobs
+test_files: []
+