tresque 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e49c5387c1d0949971273fc5c215714def53530d
+  data.tar.gz: 51efde231efbe4a817705db15e6cb98d673e3045
+SHA512:
+  metadata.gz: 4b402c7bab3c066d4a7d3de8feb58beb1d2be6d897975fdacbdbd925a198623a353e424bf69ddd56ca3b02cf142a40351af78eeb29b33fbf6850725cc5aa38bc
+  data.tar.gz: 6fe1ec450d11825103f6d1b2904df3d20ba5b40d3205fdbff73607ecd9a6e526af5a26a398b6167c4b88847dc16bd5e6f861eea20f0b2dad137bd2c14b88e200

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tresque.gemspec
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 TaskRabbit
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,186 @@
+# TResque
+
+Patterns for Resque usage at TaskRabbit
+
+* So work in instance method `work` instead of class method `perform`
+* Enqueue hash instead of struct for more flexibility and change management
+* Locking based on queue or timing
+* Easy way to delay method calls
+* Method for registering queue names to centralize ops
+* Abstraction so we can move to Sidekiq even easier when that is the goal
+
+## Worker
+
+```ruby
+require 'tresque'
+
+module MyEngine
+  class ImageProcessor
+    include ::TResque::Worker
+
+    inputs :user_id, :size
+
+    def work
+      User.find(user_id).upload_image!(size)
+    end
+  end
+end
+```
+
+#### Example Usage
+
+```ruby
+  MyEngine::ImageProcessor.enqueue(size: "small", user_id: 255)
+```
+
+## Queue Management
+
+Say what queues you process
+
+```ruby
+require 'tresque'
+
+TResque.register("account") do
+  queue :default, 100
+  queue :refresh, -5000
+end
+```
+
+Can put workers in those queues
+
+```
+module Account
+  class RegularWorker
+    include ::TResque::Worker
+    # defaults to account_default queue
+  end
+end
+
+module Account
+  class RegularWorker
+    include ::TResque::Worker
+    queue :refresh # lower priority account_refresh queue
+  end
+end
+```
+
+#### Rake setup
+
+```ruby
+require 'resque/tasks'
+require 'resque_scheduler/tasks'
+require "resque_bus/tasks"
+
+namespace :resque do
+  task :setup => [:environment] do
+    require 'resque_scheduler'
+    require 'resque/scheduler'
+    require 'tresque'
+  end
+  
+  task :queues => [:setup] do
+    queues = ::TResque::Registry.queues
+    ENV["QUEUES"] = queues.join(",")
+    puts "TResque: #{ENV["QUEUES"]}"
+  end
+end
+
+```
+
+Work those queues by priority
+```
+  $ bundle exec rake resque:queues resque:work
+  TResque: account_default, account_refresh
+```
+
+## Locking
+
+```ruby
+module MyEngine
+  class SingletonWorker
+    include ::TResque::Worker
+    inputs :user_id
+
+    # does not enqueue another worker if this worker with same user_id waiting to be processed
+    queue_lock :user_id
+  end
+end
+
+module MyEngine
+  class MutexWorker
+    include ::TResque::Worker
+    inputs :user_id, :any_other_input
+
+    # does work two of these workers at the same time for the same user_id
+    worker_lock :user_id
+  end
+end
+```
+
+Those locks are for the same worker. You can also coordinate across workers using a namespace. Or, in other words, the default namespace is the worker class name but can be overridden. The keys need the same name.
+
+```ruby
+module MyEngine
+  class FirstWorker
+    include ::TResque::Worker
+    inputs :user_id
+
+    lock_namespace :user_calculations
+    worker_lock :user_id
+  end
+
+  class SecondWorker
+    include ::TResque::Worker
+    inputs :user_id, :other_input
+
+    lock_namespace :user_calculations
+    worker_lock :user_id
+  end
+
+  class ThirdWorker
+    include ::TResque::Worker
+    inputs :other_key
+
+    lock_namespace :user_calculations
+    worker_lock :user_id
+
+    def user_id
+      # can be a method too
+      User.find_by_other_key(other_key).id
+    end
+  end
+end
+```
+
+## Delay
+
+```ruby
+  class User < ::ActiveRecord::Base
+    include ::TResque::Delay
+
+    def heavy_lifting
+      # stuff in background
+    end
+    async :heavy_lifting, queue: 'my_queue'
+
+    def other_stuff
+
+    end
+  end
+```
+#### Example Usage
+
+```ruby
+  user = User.find(1)
+
+  # Always in the background
+  user.heavy_lifting
+
+  # optionally in the background
+  user.delay.other_stuff
+```
+
+## Notes
+
+Generally based on [qe](https://github.com/fnando/qe).
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/tresque.rb

@@ -0,0 +1,24 @@
+require "tresque/version"
+require 'resque'
+
+module TResque
+  autoload :Util,                     'tresque/util'
+  autoload :Worker,                   'tresque/worker'
+  autoload :Delay,                    'tresque/delay'
+  autoload :DelayExecutionWorker,     'tresque/delay_execution_worker'
+  autoload :WorkerLock,               'tresque/worker_lock'
+  autoload :QueueLock,                'tresque/queue_lock'
+  autoload :Registry,                 'tresque/registry'
+  
+  module Spec
+    autoload :Delay,                  'tresque/spec/delay'
+  end
+  
+  class << self
+    def register(app_key, &block)
+      registry = ::TResque::Registry.new(app_key)
+      registry.instance_eval(&block)
+      registry
+    end
+  end
+end

data/lib/tresque/delay.rb

@@ -0,0 +1,115 @@
+module TResque
+  module Delay
+    extend ::ActiveSupport::Concern
+
+    module ClassMethods
+
+      def delay(options = {})
+        InvocationProxy.new(self, options)
+      end
+
+
+      def async(method_name, options = {})
+        raise "Attempted to handle #{self.name}##{method_name} asyncronously but #{method_name} was not yet defined" unless (self.instance_methods + self.private_instance_methods).include?(method_name)
+
+        class_eval <<-EV, __FILE__, __LINE__ + 1
+          def #{method_name}_with_delay(*args)
+            self.delay(#{options.inspect}).#{method_name}_without_delay(*args)
+          end
+          alias_method_chain :#{method_name}, :delay
+        EV
+      end
+
+    end
+
+    def delay(options = {})
+      InvocationProxy.new(self, options)
+    end
+
+
+    def method_missing(method_name, *args)
+      if method_name.to_s =~ /^delay__(.+)$/
+        self.delay.send($1, *args)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      method_name.to_s =~ /^delay__(.+)/ || super
+    end
+
+
+    class InvocationProxy
+
+      def initialize(object, options = {})
+
+        @object = object
+        @run_at = options[:run_at]
+        @run_at ||= (!!options[:force] ? true : nil)
+        @synchronous = !!options[:synchronous]
+
+        @queue_namespace  = options[:queue_namespace] || Util.calculate_namespace_from_class(object)
+        @queue_name       = options[:queue] || 'default'
+
+        @lock_namespace   = options[:lock_namespace]
+        @queue_lock_key   = options[:queue_lock_key]
+        @worker_lock_key  = options[:worker_lock_key]
+      end
+
+      def method_missing(method_name, *args)
+        if !@synchronous && (!in_resque? || @run_at == true || @run_at.to_i > Time.now.to_i)
+          @method_name = method_name.to_s
+          @args = args
+          queue_delayed_invocation!
+        else
+          @object.send(method_name, *args)
+        end
+      end
+
+      def respond_to?(*args)
+        return true unless in_resque?
+        @object.respond_to?(*args)
+      end
+
+      protected
+
+      def in_resque?
+        !!(ENV['QUEUE'] || ENV['QUEUES'])
+      end
+
+      def queue_delayed_invocation!
+        push = {}
+
+        if @object.is_a?(Class)
+          push["class_name"] = @object.name
+        else
+          push["class_name"] = @object.class.name
+          push["id"] = @object.respond_to?(:delay_id) ? @object.delay_id : @object.id
+        end
+
+        push["method_name"]     = @method_name
+        push["args"]            = @args
+        push["queue_namespace"] = @queue_namespace
+        push["queue"]           = @queue_name
+        push["run_at"]          = @run_at             if @run_at && @run_at != true
+        push["lock_namespace"]  = @lock_namespace     if @lock_namespace
+
+        if @queue_lock_key
+          push["queue_lock"]          = @queue_lock_key.to_s
+          push[@queue_lock_key.to_s]  = push["id"]
+        end
+
+        if @worker_lock_key
+          push["worker_lock"]         = @worker_lock_key.to_s
+          push[@worker_lock_key.to_s] = push["id"]
+        end
+
+        TResque::DelayExecutionWorker.enqueue(push)
+      end
+
+    end
+
+
+  end
+end

data/lib/tresque/delay_execution_worker.rb

@@ -0,0 +1,58 @@
+module TResque
+  class DelayExecutionWorker
+    include ::TResque::Worker
+    inputs :class_name, :id, :method_name, :args
+
+    # enabled both of these by option
+    extend ::TResque::WorkerLock
+    extend ::TResque::QueueLock
+
+    class << self
+      # handle dynamic locks
+      def get_lock_namespace(options)
+        options["lock_namespace"] || options["class_name"]
+      end
+
+      def get_queue_lock_attributes(options)
+        return nil unless options["queue_lock"]
+        [options["queue_lock"]].flatten
+      end
+
+      def get_worker_lock_attributes(options)
+        return nil unless options["worker_lock"]
+        [options["worker_lock"]].flatten
+      end
+    end
+
+    def work
+      return unless record
+      if args.nil? || args.empty?
+        record.send(self.method_name)
+      else
+        record.send(self.method_name, *self.args)
+      end
+    end
+
+    protected
+
+    def klass
+      @klass ||= class_name.constantize
+    end
+
+    def record
+      @record ||= if id.nil?
+        klass
+      else
+        if klass.respond_to?(:find_by)
+          klass.find_by(id: id)
+        elsif klass.respond_to?(:find_by_id)
+          klass.find_by_id(id)
+        else
+          klass.find(id)
+        end
+      end
+    end
+
+
+  end
+end

data/lib/tresque/queue_lock.rb

@@ -0,0 +1,70 @@
+module TResque
+  # If you want only one instance of your job queued at a time,
+  # extend it with this module.
+  module QueueLock
+
+    # Override in your job to control the lock experiation time. This is the
+    # time in seconds that the lock should be considered valid. The default
+    # is one hour (3600 seconds).
+    def queue_lock_timeout
+      3600
+    end
+
+    # Override in your job to control the lock key. It is
+    # passed the same arguments as `perform`, that is, your job's
+    # payload.
+    #def queue_lock_key(options)
+    #  "#{name}-#{options.to_s}"
+    #end    
+
+    # See the documentation for SETNX http://redis.io/commands/setnx for an
+    # explanation of this deadlock free locking pattern
+    def before_enqueue_queue_lock(options)
+      val = queue_lock_key(options)
+      if val
+        key = "lock:#{val}"
+        now = Time.now.to_i
+        from_now = queue_lock_timeout + 1
+        key_expire = from_now + 600 # some exra time
+        timeout = now + from_now
+
+        # return true if we successfully acquired the lock
+        if Resque.redis.setnx(key, timeout)
+          # expire in case of error to make sure it goes away
+          Resque.redis.expire(key, key_expire)
+          return true
+        end
+
+        # see if the existing timeout is still valid and return false if it is
+        # (we cannot acquire the lock during the timeout period)
+        return false if now <= Resque.redis.get(key).to_i
+
+        # otherwise set the timeout and ensure that no other worker has
+        # acquired the lock
+        if now > Resque.redis.getset(key, timeout).to_i
+          # expire in case of error to make sure it goes away
+          Resque.redis.expire(key, key_expire)
+          return true
+        else
+          return false
+        end
+
+      end
+    end
+
+    def clear_queue_lock(options)
+      val = queue_lock_key(options)
+      if val
+        Resque.redis.del("lock:#{val}")
+      end
+    end
+
+    def before_perform_queue_lock(options)
+      clear_queue_lock(options)
+    end
+
+    def after_dequeue_queue_lock(options)
+      clear_queue_lock(options)
+    end
+  end
+end