dmantilla-delayed_job_sqs 0.1.4

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Daniel Mantilla
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND
+NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.textile

@@ -0,0 +1,6 @@
+h1. Delayed Job for Amazon Simple Queue
+
+h3. Versions
+
+* 0.1.0: Initial release, based on delayed_job version 1.7.0 
+* 0.1.1, 0.1.3: Minor changes here and there

data/delayed_job_sqs.gemspec

@@ -0,0 +1,28 @@
+#version = File.read('README.textile').scan(/^\*\s+([\d\.]+)/).flatten
+
+Gem::Specification.new do |s|
+  s.name     = "delayed_job_sqs"
+  s.version  = "0.1.4"
+  s.date     = "2009-07-17"
+  s.summary  = "Asynchronous queue execution using Amazon SQS -- Most of the code was extracted from the delayed_job gem"
+  s.email    = "daniel@celect.org"
+  s.homepage = "http://github.com/dmantilla/delayed_job_sqs/tree/master"
+  s.description = "delayed_job_sqs reuses most of the code from the delayed_job gem by Tobias Lütke. ActiveRecord was replaced by Amazon Simple Queue and other minor changes"
+  s.authors  = ["Daniel Mantilla"]
+
+  s.has_rdoc = false
+  s.rdoc_options = ["--main", "README.textile"]
+  s.extra_rdoc_files = ["README.textile"]
+
+  # run git ls-files to get an updated list
+  s.files = %w[
+    MIT-LICENSE
+    README.textile
+    delayed_job_sqs.gemspec
+    init.rb
+    lib/delayed/job.rb
+    lib/delayed/message_sending.rb
+    lib/delayed/performable_method.rb
+    lib/delayed_job_sqs.rb
+  ]
+end

data/init.rb

@@ -0,0 +1,3 @@
+require 'right_aws'
+require File.dirname(__FILE__) + '/lib/delayed_job_sqs'
+

data/lib/delayed/job.rb

@@ -0,0 +1,137 @@
+module Delayed
+
+  class DeserializationError < StandardError
+  end
+
+  class Job
+    MAX_RUN_TIME = 4.hours
+
+    ParseObjectFromYaml = /\!ruby\/\w+\:([^\s]+)/
+
+    def initialize(message)
+      @message = message
+      @logger = defined?(RAILS_DEFAULT_LOGGER) ? RAILS_DEFAULT_LOGGER : Logger.new(STDOUT)
+    end
+
+    def payload_object
+      @payload_object ||= deserialize(@message.to_s)
+    end
+
+    def name
+      @name ||= begin
+        payload = payload_object
+        if payload.respond_to?(:display_name)
+          payload.display_name
+        else
+          payload.class.name
+        end
+      end
+    end
+
+    def run(max_run_time = MAX_RUN_TIME)
+      begin
+        runtime = Benchmark.realtime do
+          invoke_job # TODO: raise error if takes longer than max_run_time
+        end
+        # TODO : warn if runtime > max_run_time
+        @logger.info "* [JOB] #{name} completed after %.4f" % runtime
+        # The message is finally delete from the queue
+        @message.delete
+        true
+      rescue Exception => e
+        log_error(e)
+        false # work failed
+      end
+
+    end
+
+    # This is a good hook if you need to report job processing errors in additional or different ways
+    def log_error(error)
+      @logger.error "* [JOB] #{name} failed with #{error.class.name}: #{error.message}"
+      @logger.error error.backtrace.inspect
+    end
+
+    # Moved into its own method so that new_relic can trace it.
+    def invoke_job
+      payload_object.perform
+    end
+
+    # Add a job to the queue
+    def self.enqueue(*args, &block)
+      sqs_queue = args.shift
+      raise ArgumentError, 'SQS Queue was not provided' unless sqs_queue.is_a? RightAws::SqsGen2::Queue
+
+      object = block_given? ? EvaledJob.new(&block) : args.shift
+
+      unless object.respond_to?(:perform) || block_given?
+        raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
+      end
+
+      sqs_queue.send_message object.to_yaml
+    end
+
+    # Do num jobs and return stats on success/failure.
+    # Exit early if interrupted.
+    def self.work_off(sqs_queue)
+      exit_flag = false
+      until($exit)
+        success, failure = 0, 0
+
+        while(message = sqs_queue.receive)
+          if message.to_s == 'stop'
+            exit_flag = true
+            break
+          end
+          
+          case Job.new(message).run
+          when true
+              success += 1
+          when false
+              failure += 1
+          else
+            break  # leave if no work could be done
+          end
+          break if $exit # leave if we're exiting
+        end
+        
+        break if exit_flag
+        
+        puts "Success: #{success}, Failures: #{failure}"
+        sleep(60)
+      end
+    end
+
+  private
+
+    def deserialize(source)
+      handler = YAML.load(source) rescue nil
+
+      unless handler.respond_to?(:perform)
+        if handler.nil? && source =~ ParseObjectFromYaml
+          handler_class = $1
+        end
+        attempt_to_load(handler_class || handler.class)
+        handler = YAML.load(source)
+      end
+
+      return handler if handler.respond_to?(:perform)
+
+      raise DeserializationError,
+        'Job failed to load: Unknown handler. Try to manually require the appropiate file.'
+    rescue TypeError, LoadError, NameError => e
+      raise DeserializationError,
+        "Job failed to load: #{e.message}. Try to manually require the required file."
+    end
+
+  end
+
+  class EvaledJob
+    def initialize
+      @job = yield
+    end
+
+    def perform
+      eval(@job)
+    end
+  end
+end

data/lib/delayed/message_sending.rb

@@ -0,0 +1,25 @@
+module Delayed
+  module MessageSending
+    def send_later(sqs_queue, method, *args)
+      delayed_method = Delayed::PerformableMethod.new(self, method.to_sym, args)
+
+      # If an actual queue was provided and the message size is less than 8K, it's appended to the queue
+      if sqs_queue.is_a?(RightAws::SqsGen2::Queue) && delayed_method.to_yaml.size > 8192
+        Delayed::Job.enqueue(sqs_queue, delayed_method)
+      else
+        # else the method is executed, no queueing
+        self.send method.to_sym, *args #.map{|a| a}
+      end
+    end
+
+    module ClassMethods
+      def handle_asynchronously(method)
+        without_name = "#{method}_without_send_later"
+        define_method("#{method}_with_send_later") do |*args|
+          send_later(without_name, *args)
+        end
+        alias_method_chain method, :send_later
+      end
+    end
+  end
+end

data/lib/delayed/performable_method.rb

@@ -0,0 +1,55 @@
+module Delayed
+  class PerformableMethod < Struct.new(:object, :method, :args)
+    CLASS_STRING_FORMAT = /^CLASS\:([A-Z][\w\:]+)$/
+    AR_STRING_FORMAT    = /^AR\:([A-Z][\w\:]+)\:(\d+)$/
+
+    def initialize(object, method, args)
+      raise NoMethodError, "undefined method `#{method}' for #{self.inspect}" unless object.respond_to?(method)
+
+      self.object = dump(object)
+      self.args   = args.map { |a| dump(a) }
+      self.method = method.to_sym
+    end
+    
+    def display_name  
+      case self.object
+      when CLASS_STRING_FORMAT then "#{$1}.#{method}"
+      when AR_STRING_FORMAT    then "#{$1}##{method}"
+      else "Unknown##{method}"
+      end      
+    end    
+
+    def perform
+      load(object).send(method, *args.map{|a| load(a)})
+    #rescue ActiveRecord::RecordNotFound
+      # We cannot do anything about objects which were deleted in the meantime
+      true
+    end
+
+    private
+
+    def load(arg)
+      case arg
+      when CLASS_STRING_FORMAT then $1.constantize
+      when AR_STRING_FORMAT    then $1.constantize.find($2)
+      else arg
+      end
+    end
+
+    def dump(arg)
+      case arg
+      when Class              then class_to_string(arg)
+      when ActiveRecord::Base then ar_to_string(arg)
+      else arg
+      end
+    end
+
+    def ar_to_string(obj)
+      "AR:#{obj.class}:#{obj.id}"
+    end
+
+    def class_to_string(obj)
+      "CLASS:#{obj.name}"
+    end
+  end
+end

data/lib/delayed_job_sqs.rb

@@ -0,0 +1,6 @@
+require File.dirname(__FILE__) + '/delayed/message_sending'
+require File.dirname(__FILE__) + '/delayed/performable_method'
+require File.dirname(__FILE__) + '/delayed/job'
+
+Object.send(:include, Delayed::MessageSending)
+Module.send(:include, Delayed::MessageSending::ClassMethods)

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification 
+name: dmantilla-delayed_job_sqs
+version: !ruby/object:Gem::Version 
+  version: 0.1.4
+platform: ruby
+authors: 
+- Daniel Mantilla
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-07-17 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: "delayed_job_sqs reuses most of the code from the delayed_job gem by Tobias L\xC3\xBCtke. ActiveRecord was replaced by Amazon Simple Queue and other minor changes"
+email: daniel@celect.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.textile
+files: 
+- MIT-LICENSE
+- README.textile
+- delayed_job_sqs.gemspec
+- init.rb
+- lib/delayed/job.rb
+- lib/delayed/message_sending.rb
+- lib/delayed/performable_method.rb
+- lib/delayed_job_sqs.rb
+has_rdoc: false
+homepage: http://github.com/dmantilla/delayed_job_sqs/tree/master
+post_install_message: 
+rdoc_options: 
+- --main
+- README.textile
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Asynchronous queue execution using Amazon SQS -- Most of the code was extracted from the delayed_job gem
+test_files: []
+