simple_job 0.0.0

data/CHANGELOG.rdoc

@@ -0,0 +1,5 @@
+= Simple Job
+
+== Version 0.0.0
+* Supports declaring queues, easy declaration of job definitions, and enqueuing/polling
+* Contains a queue implementation backed by Amazon SQS

data/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright (c) 2012 RevPAR Collective, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,101 @@
+= Simple Job
+
+A gem containing libraries that support running background jobs or tasks. It's designed to make it easy to:
+
+* Define a job
+* Enqueue a job
+* Poll for and execute jobs
+
+It is architected to support multiple types of queue implementations, but currently, it only includes an implementation using AWS SQS (http://aws.amazon.com/sqs/). Alternate queue implementations could be plugged in by the client using lib/simple_job/sqs_job_queue.rb as an example.
+
+The AWS SQS queue implementation requires the aws-sdk gem, which must be initialized (by calling AWS.config) for this API to be capable of enqueuing or polling for jobs.
+
+
+== Queue Configuration
+
+Queue configuration must be done by both the client and the server.
+
+Only the queues that will be used by each must be defined (a client may configure a subset of the queues used by the server, so long as it configures all the queues that it uses).
+
+SQSJobQueue is the queue implementation used by default. This may be overridden by calling SimpleJob::JobQueue.config :implementation => 'my_queue_type'
+
+=== Minimal configuration - specify queue prefix and define one default queue
+
+ SimpleJob::SQSJobQueue.config :queue_prefix => 'my-job'
+ SimpleJob::SQSJobQueue.define_queue 'normal', :default => true
+
+=== Complex configuration with explicit queue implementation, non-rails-defined environment, and multiple queues
+
+ SimpleJob::JobQueue.config :implementation => 'sqs'
+ SimpleJob::SQSJobQueue.config :queue_prefix => 'my-job', :environment => 'production'
+ SimpleJob::SQSJobQueue.define_queue 'normal', :visibility_timeout => 60, :default => true
+ SimpleJob::SQSJobQueue.define_queue 'high-priority', :visibility_timeout => 10
+ SimpleJob::SQSJobQueue.define_queue 'long-running', :visibility_timeout => 3600
+
+
+== Job Definition
+
+ class FooSender
+   include SimpleJob::JobDefinition
+
+   simple_job_attribute :target, :foo_content  # defines getters/setters for each, and
+                                               # adds them to serialized message
+
+   validates :target, :presence => true  # standard ActiveModel validation
+
+   def execute
+     puts "#{foo_content} -> #{target}"
+   end
+ end
+
+
+== Job Client Usage
+
+=== Typical usage of default queue
+
+You may call #enqueue with no arguments, in which case JobQueue.default will be used.
+
+ f = FooSender.new(:target => 'joe', :foo_content => 'foo!')  # can also assign attributes with f.target=, f.foo_content=
+ if f.enqueue
+   puts 'i just sent some foo to joe!'
+ else
+   puts "the following errors occurred: #{f.errors.full_messages.join('; ')}"
+ end
+
+ json = f.to_json # { "type": "foo_sender", "version": "1", "data": { "target": "joe", "foo_content": "foo!" } }
+ f_copy = FooSender.new.from_json(json)
+
+=== Simple usage with explicit queue
+
+To explicitly specify the queue to use, simply specify its type identifier (defined in SQSJobQueue.define_queue declaration) when calling enqueue.
+
+ f = FooSender.new
+ f.target = 'bob'
+ f.foo_content = 'foo!'
+ f.enqueue!('normal')  # raises exception if operation fails
+
+=== Queue configuration for multiple enqueue operations
+
+Alternatively, the queue may be attached to the job upfront for multiple enqueue operations:
+
+ FooSender.job_queue('high-priority')
+ f1 = FooSender.new(:target => 'cookie monster', :foo_content => 'cookies and milk')
+ f1.enqueue  # high-priority queue will be used
+ f2 = FooSender.new(:target => 'oscar the grouch', :foo_content => 'pizza')
+ f2.enqueue  # high-priority queue will be used
+
+
+== Job Server Usage
+
+Calling #poll on a queue instance will, by default, dispatch each job to the proper registered class based on type and version. Its options are passed through to the underlying implementation. In the case of the SQSJobQueue, poll accepts the options documented for the aws-sdk AWS::SQS::Queue#poll method.
+
+ JobQueue.default.poll(:poll_interval => 1, :idle_timeout => 60)
+
+You can override the default behavior by passing a block to #poll that accepts both a job definition instance and an SQS message.
+
+ JobQueue.default.poll(:poll_interval => 1, :idle_timeout => 60) do |definition, message|
+   logger.debug "received message: #{message.inspect}"
+   logger.debug "dispatched to definition #{definition.inspect}; executing..."
+   definition.execute
+ end
+

data/lib/simple_job/job_definition.rb

@@ -0,0 +1,180 @@
+require 'active_model'
+require 'active_support/inflector'
+
+module SimpleJob
+module JobDefinition
+
+  RESERVED_ATTRIBUTES = [ :type, :version, :data ]
+
+  def self.included(klass)
+
+    klass.extend(ClassMethods)
+
+    klass.class_eval do
+      include ::ActiveModel::Validations
+      include ::ActiveModel::Serializers::JSON
+    end
+
+    klass.include_root_in_json = false
+    klass.register_simple_job
+
+  end
+
+  # should be overridden by including classes
+  if !method_defined?(:execute)
+    def execute
+    end
+  end
+
+  def attributes
+    {
+      'type' => type,
+      'version' => version,
+      'data' => data,
+    }
+  end
+
+  def attributes=(attributes)
+    attributes.each do |key, value|
+      send("#{key}=", value)
+    end
+  end
+
+  def data
+    @data ||= {}
+    self.class.simple_job_attributes.each do |attribute|
+      @data[attribute.to_s] ||= nil
+    end
+    @data
+  end
+
+  def data=(data)
+    self.attributes = data
+  end
+
+  def type
+    self.class.definition[:type]
+  end
+
+  def type=(type)
+    if type.to_sym != self.type
+      raise "tried to deserialize object with type #{type}, but this object only " +
+        "supports type: #{self.type}"
+    end
+  end
+
+  def versions
+    self.class.definition[:versions]
+  end
+
+  def version
+    versions.first
+  end
+
+  def version=(version)
+    if !versions.include?(version.to_s)
+      raise "tried to deserialize object with version #{version}, but this object " +
+        "only supports versions: #{versions.join(", ")}"
+    end
+  end
+
+  def enqueue(queue_type = nil)
+    if valid?
+      queue = (queue_type && JobQueue[queue_type]) || self.class.job_queue || JobQueue.default
+      queue.enqueue(self.to_json)
+    else
+      false
+    end
+  end
+
+  def enqueue!(queue_type = nil)
+    enqueue(queue_type) || raise("object is not valid: #{errors.full_messages.join('; ')}")
+  end
+
+  def read_simple_job_attribute(attribute)
+    data[attribute.to_s]
+  end
+
+  def write_simple_job_attribute(attribute, value)
+    data[attribute.to_s] = value
+  end
+
+  def initialize(attributes = {})
+    attributes.each do |key, value|
+      send("#{key}=", value)
+    end
+  end
+
+  def self.job_definition_class_for(type, version)
+    @job_definitions.each do |definition|
+      if (definition[:type] == type.to_sym) && (definition[:versions].include?(version))
+        return definition[:class]
+      end
+    end
+    nil
+  end
+
+  def self.job_definitions
+    @job_definitions ||= []
+  end
+
+  private
+
+  module ClassMethods
+
+    def definition
+      @definition
+    end
+
+    def register_simple_job(options = {})
+      default_type = self.name.underscore.to_sym
+
+      @definition = {
+        :class => self,
+        :type => default_type,
+        :versions => [ '1' ],
+      }.merge(options)
+
+      @definition[:type] = @definition[:type].to_sym
+      @definition[:versions] = Array(@definition[:versions])
+      @definition[:versions].collect! { |value| value.to_s }
+
+      ::SimpleJob::JobDefinition.job_definitions.delete_if { |item| item[:type] == default_type }
+      ::SimpleJob::JobDefinition.job_definitions << @definition
+    end
+
+    def job_queue(queue_type = nil)
+      @job_queue = JobQueue[queue_type] if queue_type
+      @job_queue
+    end
+
+    def simple_job_attributes
+      @simple_job_attributes ||= []
+    end
+
+    def simple_job_attribute(*attributes)
+      attributes.each do |attribute|
+        attribute = attribute.to_sym
+
+        if RESERVED_ATTRIBUTES.include?(attribute)
+          raise "attempted to declare reserved attribute: #{attribute}"
+        end
+
+        simple_job_attributes << attribute
+
+        class_eval <<-__EOF__
+          def #{attribute}
+            read_simple_job_attribute(:#{attribute})
+          end
+
+          def #{attribute}=(value)
+            write_simple_job_attribute(:#{attribute}, value)
+          end
+        __EOF__
+      end
+    end
+
+  end
+
+end
+end

data/lib/simple_job/job_queue.rb

@@ -0,0 +1,111 @@
+require 'logger'
+require 'aws-sdk'
+
+# Requires the aws-sdk gem, which must be initialized for this API to be capable of queuing requests.
+#
+# Synopsis:
+#
+#  include SimpleJob
+#
+#  JobQueue::DEFAULT.poll do |message|
+#    puts message
+#  end
+#
+# == Creating a queue implementation
+#
+# To create a new queue implementation, just extend the SimpleJob::JobQueue
+# class, and call the register_job_queue declaration with an identifier. The
+# class must implement the get_queue class method and the enqueue/poll instance
+# methods to fulfill the interface. The default_queue method may be overridden
+# to set a default queue.
+#
+# Example:
+# 
+#  class ArrayQueue < SimpleJob::JobQueue
+#    register_job_queue 'array', self
+#
+#    include Singleton
+#    default self.instance
+#  
+#    def self.get_queue(type, options = {})
+#      instance
+#    end
+#    def enqueue(message, options = {})
+#      queue << message
+#    end
+#    def poll(options = {}, &block)
+#      options = {
+#        :interval => 1
+#      }.merge(options)
+#      loop do
+#        message = queue.shift
+#        yield(message) if message
+#        Kernel.sleep(options[:interval])
+#      end
+#    end
+#    private
+#    def queue
+#      @queue ||= []
+#    end
+#  end
+#
+# Then you can use the new queue implementation by passing its identifier to
+# JobQueue.config:
+#
+#  SimpleJob::JobQueue.config :implementation => 'array'
+module SimpleJob
+class JobQueue
+
+  def self.register_job_queue(identifier, klass)
+    @@registered_job_queues ||= {}
+    @@registered_job_queues[identifier.to_s] = klass
+  end
+
+  def self.config(options = {})
+    @config ||= {
+      :implementation => 'sqs',
+      :logger => default_logger,
+    }
+    @config.merge!(options) if options
+    @config
+  end
+
+  def self.[](type, options = {})
+    queue_class.get_queue(type, options)
+  end
+
+  def self.default_queue
+    raise "default queue not defined"
+  end
+
+  def self.default
+    queue_class.default_queue
+  end
+
+  def self.get_queue(type, options = {})
+    raise "queue with type #{type} not defined"
+  end
+
+  def self.queue_class
+    @@registered_job_queues[config[:implementation].to_s]
+  end
+
+  def enqueue(message, options = {})
+    raise NotImplementedError
+  end
+
+  def poll(options = {}, &block)
+    raise NotImplementedError
+  end
+
+  private
+
+  def self.default_logger
+    return Rails.logger if defined?(Rails)
+    logger = Logger.new(STDERR)
+    logger.level = Logger::INFO
+    logger
+  end
+
+end
+end

data/lib/simple_job/sqs_job_queue.rb

@@ -0,0 +1,97 @@
+module SimpleJob
+class SQSJobQueue < JobQueue
+
+  # Registers this queue implementation with SimpleJob::JobQueue with identifier "sqs".
+  register_job_queue 'sqs', self
+
+  def self.config(options = {})
+    @config ||= {
+      :queue_prefix => ENV['SIMPLE_JOB_SQS_JOB_QUEUE_PREFIX'],
+      :default_visibility_timeout => 60,
+      :environment => (defined?(Rails) && Rails.env) || 'development',
+    }
+
+    @config.merge!(options) if options
+
+    raise 'must configure :queue_prefix using SQSJobQueue.config' if !@config[:queue_prefix]
+
+    @config
+  end
+
+  def self.default_queue
+    @default_queue || super
+  end
+
+  def self.define_queue(type, options = {})
+    type = type.to_s
+
+    options = {
+      :visibility_timeout => config[:default_visibility_timeout],
+      :default => false,
+    }.merge(options)
+
+    queue = self.new(type, options[:visibility_timeout])
+    self.queues ||= {}
+    self.queues[type] = queue
+
+    @default_queue = queue if options[:default]
+    
+    queue
+  end
+
+  def self.get_queue(type, options = {})
+    type = type.to_s
+    (self.queues || {})[type] || super
+  end
+
+  def enqueue(message, options = {})
+    raise("enqueue expects a raw string") unless message.is_a?(String)
+    sqs_queue.send_message(message)
+  end
+
+  def poll(options = {}, &block)
+    options = {
+      :visibility_timeout => visibility_timeout,
+      :attributes => [ :sent_at, :receive_count, :first_received_at ],
+      :raise_exceptions => false,
+    }.merge(options)
+
+    message_handler = block || lambda do |definition, message|
+      definition.execute
+    end
+
+    sqs_queue.poll(options) do |message|
+      begin
+        raw_message = JSON.parse(message.body)
+        definition_class = JobDefinition.job_definition_class_for(raw_message['type'], raw_message['version'])
+        raise('no definition found') if !definition_class
+        definition = definition_class.new.from_json(message.body)
+        message_handler.call(definition, message)
+      rescue Exception => e
+        if options[:raise_exceptions]
+          raise e
+        else
+          JobQueue.config[:logger].error("unable to process message: #{e.message}")
+          JobQueue.config[:logger].error("message body: #{message.body}")
+          JobQueue.config[:logger].error(e.backtrace.join("\n  "))
+        end
+      end
+    end
+  end
+
+  private
+
+  class << self
+    attr_accessor :queues
+  end
+
+  attr_accessor :sqs_queue, :visibility_timeout
+
+  def initialize(type, visibility_timeout)
+    sqs = ::AWS::SQS.new
+    self.sqs_queue = sqs.queues.create "#{self.class.config[:queue_prefix]}-#{type}-#{self.class.config[:environment]}"
+    self.visibility_timeout = visibility_timeout
+  end
+
+end
+end

data/lib/simple_job/version.rb

@@ -0,0 +1,3 @@
+module SimpleJob
+  VERSION = '0.0.0'
+end

data/lib/simple_job.rb

@@ -0,0 +1,7 @@
+module SimpleJob
+
+  autoload :JobDefinition, 'simple_job/job_definition'
+  autoload :JobQueue, 'simple_job/job_queue'
+  autoload :SQSJobQueue, 'simple_job/sqs_job_queue'
+
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: simple_job
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- David Dawson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-10 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: &70245578980360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70245578980360
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &70245578978120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70245578978120
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &70245578976320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70245578976320
+description: Contains libraries that support defining, queueing, and executing jobs.
+email: daws23@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+- CHANGELOG.rdoc
+- LICENSE.txt
+files:
+- lib/simple_job/job_definition.rb
+- lib/simple_job/job_queue.rb
+- lib/simple_job/sqs_job_queue.rb
+- lib/simple_job/version.rb
+- lib/simple_job.rb
+- README.rdoc
+- CHANGELOG.rdoc
+- LICENSE.txt
+homepage: https://github.com/daws/simple_job
+licenses: []
+post_install_message: 
+rdoc_options:
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2274642139086505235
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Support classes and modules for executable jobs.
+test_files: []