sqs-job 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e08b70c3678927599b80b3e4ac69146633dbdcef
+  data.tar.gz: 8e460f4f8bf5627d36c65a52d7a8469d5c6d89f2
+SHA512:
+  metadata.gz: 394347d447b86f505368070c0e1824c21114d9d47a044e1d6ac681210a2132720578645a079cb762ecd73f7b2689f6f8f9c23fbbfd24f6a48593de8e9b5a4608
+  data.tar.gz: 2531d606b29ad4a615561b1f34f0c0fe5bd2aac5c9683e0ab9ebb787e41089e8eb506a562e250baa97c150ebcfc2b8497ba348d37e9309b21444f7dd4b997191

data/.gitignore

@@ -0,0 +1,25 @@
+features/reports
+spec/reports
+policy.json
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.project

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>sqs-job</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>com.aptana.ide.core.unifiedBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>com.aptana.ruby.core.rubynature</nature>
+		<nature>com.aptana.projects.webnature</nature>
+	</natures>
+</projectDescription>

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+#ruby-gemset=sqs-job
+
+# Specify your gem's dependencies in sqs-job.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Kevin Gilpin
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+# SQS::Job
+
+Simple job processor which uses SQS.
+
+Here's an interesting description of job processing using SQS, which isn't actually a spec of this code,
+but is nicely related:
+
+http://mauricio.github.io/2014/09/01/make-the-most-of-sqs.html
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sqs-job'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sqs-job
+
+# Cucumber
+
+Populate aws.secrets with `aws_access_key_id` and `aws_secret_access_key`. Then:
+
+    $ conjur env run -c aws.secrets -- conjur policy load -c policy.json cucumber-policy.rb
+    $ conjur env run -c aws.secrets -- env POLICY_FILE=policy.json rake provision
+    $ cucumber
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/sqs-job/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,71 @@
+require "bundler/gem_tasks"
+
+$LOAD_PATH.unshift 'lib'
+
+require 'ci/reporter/rake/rspec'
+require 'ci/reporter/rake/cucumber'
+require 'cucumber'
+require 'cucumber/rake/task'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new :spec
+Cucumber::Rake::Task.new :features
+
+task :jenkins => ['ci:setup:rspec', :spec, 'ci:setup:cucumber_report_cleanup'] do
+  Cucumber::Rake::Task.new do |t|
+    t.cucumber_opts = "--format CI::Reporter::Cucumber"
+  end.runner.run
+  File.write('build_number', ENV['BUILD_NUMBER']) if ENV['BUILD_NUMBER']
+end
+
+desc 'Creates resources and loads access credentials into Conjur variables. Pre-requisite to cucumber tests.'
+task :provision do
+  [ 'POLICY_FILE', 'AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY' ].each do |v|
+    ENV[v] or raise "#{v} is a required environment variable"
+  end
+
+  require 'json'
+
+  policy = JSON.parse(File.read(ENV['POLICY_FILE']))
+  ENV['CONJUR_POLICY_ID'] = policy_id = policy['policy']
+  
+  require 'conjur/cli'
+  require 'sqs/job'
+  require 'aws-sdk'
+  
+  Conjur::Config.load
+  Conjur::Config.apply
+  
+  conjur = Conjur::Authn.connect
+  iam = AWS::IAM.new
+  sqs = AWS::SQS.new
+
+  queue_name = [ policy_id.gsub(/[^a-zA-Z0-9-]/, '-'), 'job-queue' ].join('-')
+  user_name  = [ 'sys', policy_id.gsub(/[^a-zA-Z0-9-]/, '_'), 'alice' ].join('_')
+  
+  job_provisioner = SQS::Job::Provisoner.new(conjur, policy_id)
+  
+  $stderr.puts "Creating signing key"
+  job_provisioner.create_signing_key 'jobs'
+  
+  sqs_queue = begin
+    sqs.queues.named(queue_name)
+  rescue AWS::SQS::Errors::NonExistentQueue
+    $stderr.puts "Creating SQS queue #{queue_name}"
+    sqs.queues.create(queue_name)
+  end
+  
+  user = iam.users[user_name]
+  unless user.exists?
+    $stderr.puts "Creating IAM user #{user_name}"
+    user = iam.users.create(user_name)
+  end
+  
+  job_provisioner.permit_queue_send    user, sqs_queue
+  job_provisioner.permit_queue_receive user, sqs_queue
+    
+  access_key = user.access_keys.create
+  $stderr.puts "Saving access_key_id and secret_access_key"
+  conjur.variable([ policy_id, 'aws/access_key_id'].join('/')).add_value     access_key.id
+  conjur.variable([ policy_id, 'aws/secret_access_key'].join('/')).add_value access_key.secret
+end

data/cucumber-policy.rb

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift 'lib'
+require 'sqs/job'
+
+policy "sqs-job-cucumber-1.0" do
+  create_signing_key_variables 'jobs'
+
+  aws_credentials = [ 
+    variable("aws/access_key_id"), 
+    variable("aws/secret_access_key")
+  ]
+  
+  user "alice" do
+    can_submit_job  'jobs', aws_credentials
+    can_process_job 'jobs', aws_credentials
+  end
+end

data/features/send_message.feature

@@ -0,0 +1,5 @@
+Feature: Successfully send a message
+
+  Scenario: A sent message can be received and processed
+    When I send a message
+    Then the message is processed

data/features/sqs/job/message/test_message.rb

@@ -0,0 +1,7 @@
+module SQS::Job::Message
+  class TestMessage < Base
+    def invoke!
+      $messages_received << TestMessage
+    end
+  end
+end

data/features/step_definitions/job_steps.rb

@@ -0,0 +1,8 @@
+When(/^I send a message$/) do
+  SQS::Job.send_message $queue, "test_message", {}
+  sleep 2
+end
+
+Then(/^the message is processed$/) do
+  expect($messages_received).to eq([ SQS::Job::Message::TestMessage ])
+end

data/features/support/env.rb

@@ -0,0 +1,43 @@
+$LOAD_PATH.unshift 'features'
+
+require 'conjur/cli'
+require 'sqs/job'
+require 'slosilo'
+
+Conjur::Config.load
+Conjur::Config.apply
+conjur = Conjur::Authn.connect
+
+policy = JSON.parse(File.read('policy.json'))
+policy_id = policy['policy']
+
+ENV['AWS_ACCESS_KEY_ID'] = conjur.variable([ policy_id, 'aws/access_key_id' ].join('/')).value
+ENV['AWS_SECRET_ACCESS_KEY'] = conjur.variable([ policy_id, 'aws/secret_access_key' ].join('/')).value
+  
+signing_key_id = SQS::Job::Policy.private_variable_name([ policy_id, 'jobs' ].join('/'))
+queue_name = [ policy_id.gsub(/[^a-zA-Z0-9-]/, '-'), 'job-queue' ].join('-')
+
+require 'aws-sdk'
+sqs ||= AWS::SQS::new
+$queue = sqs.queues.named(queue_name)
+
+SQS::Job.signing_keys = [ Slosilo::Key.new(conjur.variable(signing_key_id).value) ]
+
+$messages_received = []
+$worker_thread = nil
+  
+Before do
+  $messages_received.clear
+  $worker_thread = Thread.new do
+    begin
+      SQS::Job::Worker.new($queue).run
+    rescue
+      $stderr.puts $!
+      raise $!
+    end
+  end
+end
+
+After do
+  $worker_thread.kill
+end

data/lib/sqs/job.rb

@@ -0,0 +1,65 @@
+require "sqs/job/version"
+require 'logger'
+require 'active_support'
+require 'active_support/core_ext'
+require 'json'
+
+module SQS
+  module Job
+    class << self
+      def send_message queue, type, parameters
+        require 'base64'
+
+        message = {
+          "type" => type,
+          "params" => parameters
+        }.to_json
+        signature   = signing_keys.last.sign message
+        fingerprint = signing_keys.last.fingerprint
+        message_attributes = {
+          "signature" => {
+            "string_value" => Base64.strict_encode64(signature),
+            "data_type" => "String",
+          },
+          "key_fingerprint" => {
+            "string_value" => fingerprint,
+            "data_type" => "String"
+          }
+        }
+        queue.send_message message, message_attributes: message_attributes
+      end
+      
+      def signature_valid? message, fingerprint, signature
+        !signing_keys.find do |k|
+          k.fingerprint == fingerprint && k.verify_signature(message, signature)
+        end.nil?
+      end
+      
+      def min_threads
+        ENV['SQS_JOB_MIN_THREADS'] || 1
+      end
+      
+      def max_threads
+        ENV['SQS_JOB_MAX_THREADS'] || 10
+      end
+      
+      def signing_keys=(keys); @signing_keys = keys; end
+      def signing_keys; @signing_keys or raise "No signing keys are configured"; end
+      
+      def logger=(logger); @logger = logger; end
+      def logger
+        @logger ||= Logger.new(STDERR)
+      end
+    end
+  end
+end
+
+require 'sqs/job/exceptions'
+require 'sqs/job/worker'
+require 'sqs/job/handler'
+require 'sqs/job/message/base'
+require 'sqs/job/provisioner'
+
+if defined?(Conjur)
+  require 'sqs/job/policy'
+end

data/lib/sqs/job/exceptions.rb

@@ -0,0 +1,17 @@
+class UnrecoverableException < RuntimeError
+end
+
+class InvalidMessageException < UnrecoverableException
+  def initialize(message)
+    super message.errors.full_messages.join(', ')
+  end
+end
+
+class MissingTypeException < UnrecoverableException
+end
+
+class SignatureInvalidException < UnrecoverableException
+end
+
+class UnrecognizedMessageTypeException < UnrecoverableException
+end

data/lib/sqs/job/handler.rb

@@ -0,0 +1,63 @@
+module SQS::Job
+  # One Handler instance is created per SQS message 
+  # received, and is responsible for processing it 
+  # by instantiating a Message::Base subclass and 
+  # calling its #invoke! method.  The "main" method
+  # is #run!.
+  #
+  # Messages have the following structure:
+  # {type: 'create', params: { ... }}
+  # Where params is optional (for example, keepalive messages
+  # might not have params).
+  #
+  # The message class is loaded by requiring 'vm2/message/#{type}'
+  # constantizing it in the normal way, and passing the params hash
+  # to #new.
+  class Handler
+    def initialize sqs_message
+      @sqs_message = sqs_message
+    end
+    
+    # Run this handler
+    def run!
+      require 'base64'
+      
+      fingerprint = @sqs_message.message_attributes['key_fingerprint'][:string_value]
+      signature   = Base64.strict_decode64(@sqs_message.message_attributes['signature'][:string_value])
+        
+      raise SignatureInvalidException unless SQS::Job.signature_valid? @sqs_message.body, fingerprint, signature
+        
+      msg = JSON.parse(@sqs_message.body)
+      raise MissingTypeException unless (type = msg['type'])
+        
+      klass = message_class type
+      SQS::Job.logger.info "Received message #{klass}"
+      
+      message = klass.new(msg['params'] || {})
+      raise InvalidMessageException, message unless message.valid?
+      message.invoke!
+    end
+    
+    private
+    
+    def message_class type
+      # This seems simpler than trying to fake abstract classes in ruby
+      raise UnrecognizedMessageTypeException, type if type == 'base'
+      
+      # This would be the place to implement a message whitelist
+      message_file = "sqs/job/message/#{type}"
+      
+      # We might do some caching here at some point...the only reason
+      # I'm not adding it now is that the multithreaded environment makes
+      # it slightly less than a freebie
+      begin
+        require message_file
+      rescue LoadError
+        SQS::Job.logger.info $!
+        raise UnrecognizedMessageTypeException, type
+      end
+      
+      message_file.gsub(/^sqs/, 'SQS').classify.constantize
+    end
+  end
+end

data/lib/sqs/job/message/base.rb

@@ -0,0 +1,35 @@
+require 'validatable'
+
+module SQS::Job
+  module Message
+    # Base class for messages.  A Message::Base subclass
+    # must implement an #invoke! method which performs the
+    # appropriate operation.
+    class Base 
+      include Validatable
+      
+      def initialize params
+        # We allow messages with no params field
+        @params = (params || {}).symbolize_keys.freeze
+      end
+      
+      # Parameters for the message.  This Hash is frozen.
+      attr_reader :params
+      
+      # Id of this message.  Used to send replies.
+      attr_reader :message_id
+      
+      # Get a parameter or raise an exception if it's not present
+      # @param name [Symbol,String] the param name
+      def param! name
+        params[name.to_sym] or raise "Missing parameter #{name}"
+      end
+      
+      # Type field for this message.  Used primarily when sending 
+      # replies.
+      def type
+        self.class.name.split('::')[-1].underscore
+      end
+    end
+  end
+end

data/lib/sqs/job/policy.rb

@@ -0,0 +1,51 @@
+module SQS::Job
+  module Policy
+    def create_signing_key_variables queue_name
+      options = {
+        'mime_type' => 'application/x-pem-file'
+      }
+      public_key =  variable(SQS::Job::Policy.public_variable_name(queue_name),  options).tap do |v|
+        v.resource.annotations['kind'] = "RSA public key"
+      end
+      private_key = variable(SQS::Job::Policy.private_variable_name(queue_name), options).tap do |v|
+        v.resource.annotations['kind'] = "RSA private key"
+      end
+      [ public_key, private_key ].tap do |vars|
+        vars.each do |var|
+          options.each do |k,v|
+            var.resource.annotations[k] = v
+          end
+          var.resource.annotations['facility'] = 'sqs/job'
+        end
+      end
+    end
+    
+    def can_submit_job queue_name, aws_credentials
+      can "execute", variable(SQS::Job::Policy.public_variable_name(queue_name))
+      aws_credentials.each do |var|
+        can "execute", var
+      end
+    end
+    
+    def can_process_job queue_name, aws_credentials
+      can "execute", variable(SQS::Job::Policy.private_variable_name(queue_name))
+      aws_credentials.each do |var|
+        can "execute", var
+      end
+    end
+    
+    class << self
+      def public_variable_name queue_name
+        [ queue_name, "signing-key/public" ].join('/')
+      end
+      
+      def private_variable_name queue_name
+        [ queue_name, "signing-key/private" ].join('/')
+      end
+    end
+  end
+end
+
+Conjur::DSL::Runner.module_eval do
+  include SQS::Job::Policy
+end

data/lib/sqs/job/provisioner.rb

@@ -0,0 +1,46 @@
+module SQS::Job
+  Provisoner = Struct.new(:conjur, :policy_id) do
+    # Create a signing key (Slosilo::Key) and store it in +variable_name+/public and +variable_name+/private.
+    def create_signing_key queue_name
+      require 'slosilo'
+      signing_key = Slosilo::Key.new
+      conjur.variable([ policy_id, SQS::Job::Policy.public_variable_name(queue_name)  ].join('/')).add_value signing_key.key.public_key.to_pem
+      conjur.variable([ policy_id, SQS::Job::Policy.private_variable_name(queue_name) ].join('/')).add_value signing_key.key.to_pem
+      nil
+    end
+    
+    # Configure a user policy to send to the sqs_queue.
+    def permit_queue_send user, sqs_queue
+      user.policies['send_to_queue'] = JSON.pretty_generate({
+        "Statement" => [
+          "Effect" => "Allow",
+          "Action" => [ "sqs:SendMessage" ],
+          "Resource" => [ sqs_queue.arn ]
+        ]})
+      user.policies['info'] = info_policy
+    end
+    
+    # Configure a user policy to receive from the sqs_queue.
+    def permit_queue_receive user, sqs_queue
+      user.policies['receive_from_queue'] = JSON.pretty_generate({
+        "Statement" => [
+          "Effect" => "Allow",
+          "Action" => [ "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:ChangeMessageVisibility" ],
+          "Resource" => [ sqs_queue.arn ]
+        ]})
+      user.policies['info'] = info_policy
+    end
+    
+    protected
+    
+    def info_policy
+      JSON.pretty_generate({
+      "Statement" => [
+        "Effect" => "Allow",
+        "Action" => [ "sqs:ListQueues", "sqs:GetQueueUrl" ],
+        "Resource" => [ "*" ]
+      ]
+      })
+    end
+  end
+end

data/lib/sqs/job/thread_pool.rb

@@ -0,0 +1,187 @@
+require 'thread'
+
+# Shamelessly stolen from puma ;-)
+
+# A simple thread pool management object.
+#
+module SQS::Job
+  class ThreadPool
+
+    # Maintain a minimum of +min+ and maximum of +max+ threads
+    # in the pool.
+    #
+    # The block passed is the work that will be performed in each
+    # thread.
+    #
+    def initialize(min, max, *extra, &block)
+      @cond  = ConditionVariable.new
+      @mutex = Mutex.new
+
+      @todo = []
+
+      @spawned = 0
+      @waiting = 0
+
+      @min   = Integer(min)
+      @max   = Integer(max)
+      @block = block
+      @extra = extra
+
+      @shutdown = false
+
+      @trim_requested = 0
+
+      @workers = []
+
+      @auto_trim = nil
+
+      @mutex.synchronize do
+        @min.times { spawn_thread }
+      end
+    end
+
+    attr_reader :spawned, :trim_requested
+
+    # How many objects have yet to be processed by the pool?
+    #
+    def backlog
+      @mutex.synchronize { @todo.size }
+    end
+
+    # :nodoc:
+    #
+    # Must be called with @mutex held!
+    #
+    def spawn_thread
+      @spawned += 1
+
+      th = Thread.new do
+        todo  = @todo
+        block = @block
+        mutex = @mutex
+        cond  = @cond
+
+        extra = @extra.map { |i| i.new }
+
+        while true
+          work = nil
+
+          continue = true
+
+          mutex.synchronize do
+            while todo.empty?
+              if @trim_requested > 0
+                @trim_requested -= 1
+                continue        = false
+                break
+              end
+
+              if @shutdown
+                continue = false
+                break
+              end
+
+              @waiting += 1
+              cond.wait mutex
+              @waiting -= 1
+            end
+
+            work = todo.pop if continue
+          end
+
+          break unless continue
+
+          block.call(work, *extra)
+        end
+
+        mutex.synchronize do
+          @spawned -= 1
+          @workers.delete th
+        end
+      end
+
+      @workers << th
+
+      th
+    end
+
+    private :spawn_thread
+
+    # Add +work+ to the todo list for a Thread to pickup and process.
+    def <<(work)
+      @mutex.synchronize do
+        if @shutdown
+          raise "Unable to add work while shutting down"
+        end
+
+        @todo << work
+
+        if @waiting == 0 and @spawned < @max
+          spawn_thread
+        end
+
+        @cond.signal
+      end
+    end
+
+    # If too many threads are in the pool, tell one to finish go ahead
+    # and exit. If +force+ is true, then a trim request is requested
+    # even if all threads are being utilized.
+    #
+    def trim(force=false)
+      @mutex.synchronize do
+        if (force or @waiting > 0) and @spawned - @trim_requested > @min
+          @trim_requested += 1
+          @cond.signal
+        end
+      end
+    end
+
+    class AutoTrim
+      def initialize(pool, timeout)
+        @pool    = pool
+        @timeout = timeout
+        @running = false
+      end
+
+      def start!
+        @running = true
+
+        @thread = Thread.new do
+          while @running
+            @pool.trim
+            sleep @timeout
+          end
+        end
+      end
+
+      def stop
+        @running = false
+        @thread.wakeup
+      end
+    end
+
+    def auto_trim!(timeout=5)
+      @auto_trim = AutoTrim.new(self, timeout)
+      @auto_trim.start!
+    end
+
+    # Tell all threads in the pool to exit and wait for them to finish.
+    #
+    def shutdown
+      @mutex.synchronize do
+        @shutdown = true
+        @cond.broadcast
+
+        @auto_trim.stop if @auto_trim
+      end
+
+      # Use this instead of #each so that we don't stop in the middle
+      # of each and see a mutated object mid #each
+      @workers.first.join until @workers.empty?
+
+      @spawned = 0
+      @workers = []
+    end
+  end
+end

data/lib/sqs/job/version.rb

@@ -0,0 +1,5 @@
+module SQS
+  module Job
+    VERSION = "1.0.0"
+  end
+end

data/lib/sqs/job/worker.rb

@@ -0,0 +1,39 @@
+module SQS::Job
+  # A Worker maintains a SQS::Job::ThreadPool and AWS::SQS::Queue
+  # and creates SQS::Job::Handler instances to process each message
+  # received.  It is also responsible for boot/configuration 
+  # stuff.  There should only be one worker per process.
+  class Worker
+    attr_reader :queue
+    
+    def initialize(queue)
+      @queue = queue
+    end
+    
+    def run options = {}
+      require 'sqs/job/thread_pool'
+
+      min_threads = options[:min_threads] || SQS::Job.min_threads 
+      max_threads = options[:max_threads] || SQS::Job.max_threads
+      @pool = SQS::Job::ThreadPool.new min_threads, max_threads do |msg|
+        log_exceptions{ Handler.new(msg).run! }
+      end
+      
+      while true
+        # KEG: it's not clear if queue.poll accepts message_attribute_names
+        queue.receive_messages(wait_time_seconds: 10, batch_size: 10, message_attribute_names: [ 'signature', 'key_fingerprint' ]) do |msg|
+          @pool << msg
+        end
+      end
+    end
+    
+    def log_exceptions &block
+      begin
+        block.call
+      rescue => ex
+        SQS::Job.logger.error "Error processing message: #{ex.class.name} #{ex}\n\t#{ex.backtrace.join("\t\n")}"
+        raise ex unless ex.is_a?(UnrecoverableException)
+      end
+    end
+  end
+end

data/spec/handler_spec.rb

@@ -0,0 +1,92 @@
+require 'spec_helper'
+require 'base64'
+
+describe SQS::Job::Handler do
+  let(:message_type){ 'dummy' }
+  let(:message_params){ {'foo' => 'bar'} }
+  let(:message_hash){ { type: message_type, params: message_params } }
+  let(:message_body){ JSON.generate(message_hash) }
+  let(:key) { KEY }
+  let(:message_attributes) {
+    {
+      'key_fingerprint' => {
+        string_value: key.fingerprint
+      },
+      'signature' => {
+        string_value: Base64.strict_encode64(key.sign(message_body))
+      }
+    }
+  }
+  let(:sqs_message){ double('AWS::SQS::Message', body: message_body, message_attributes: message_attributes) }
+  let(:message_instance){ double('message instance', :"invoke!" => 'invoked', :"valid?" => true) }
+  let(:message_class){ double('message class', new: message_instance) }
+  let(:handler){ SQS::Job::Handler.new sqs_message }
+  subject { handler }
+
+  before do
+    allow(handler).to receive(:require).and_call_original
+    allow(handler).to receive(:require).with("sqs/job/message/dummy")
+    stub_const("SQS::Job::Message::Dummy", message_class)
+    allow(SQS::Job).to receive(:signing_keys).and_return [ key ]
+  end
+  
+  describe "#run!" do
+    context "when message does not contain 'type'" do
+      let(:message_hash){ {} }
+      it "fails the message permanently" do
+        expect{ handler.run! }.to raise_error(MissingTypeException)
+      end
+    end
+    
+    context "when type is 'base'" do
+      let(:message_type){ 'base' }
+      it "fails the message permanently" do
+        expect{ handler.run! }.to raise_error(UnrecognizedMessageTypeException, 'base')
+      end
+    end
+    
+    context "when type is present" do
+      context "with invalid message signature" do
+        before do
+          expect(SQS::Job).to receive(:signing_keys).and_return [ ]
+        end
+        it "fails the message permanently" do
+          expect{ subject.run! }.to raise_error(SignatureInvalidException)
+        end
+      end
+      
+      context "with an unknown type" do
+        let(:message_type){ 'foobar' }
+        it "fails the message permanently" do
+          expect{ handler.run! }.to raise_error(UnrecognizedMessageTypeException, 'foobar')
+        end
+      end
+      
+      context "with an invalid message" do
+        it "fails the message permanently" do
+          allow(message_instance).to receive(:valid?).and_return false
+          expect(message_instance).to receive(:errors).and_return double('errors', full_messages: ['full-messages'])
+          expect(message_instance).not_to receive(:invoke!)
+          expect{ subject.run! }.to raise_error(InvalidMessageException, "full-messages")
+        end
+      end
+      
+      context "with a valid message" do
+        it "requires 'sqs/job/message/dummy'" do
+          expect(handler).to receive(:require).with('sqs/job/message/dummy').and_return true
+          subject.run!
+        end
+        
+        it "creates a message instance message body's params" do
+          expect(message_class).to receive(:new).with(message_params)
+          subject.run!
+        end
+        
+        it "calls invoke! on the message instance" do
+          expect(message_instance).to receive(:invoke!)
+          subject.run!
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,61 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter "/spec/"
+end
+
+require 'rubygems'
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+$:.unshift File.join(File.dirname(__FILE__), "lib")
+
+# Allows loading of an environment config based on the environment
+require 'rspec'
+
+# Uncomment the next line to use webrat's matchers
+#require 'webrat/integrations/rspec-rails'
+
+RSpec.configure do |config|
+  # If you're not using ActiveRecord you should remove these
+  # lines, delete config/database.yml and disable :active_record
+  # in your config/boot.rb
+  #config.use_transactional_fixtures = true
+  #config.use_instantiated_fixtures  = false
+  #config.fixture_path = File.join(redmine_root, 'test', 'fixtures')
+
+  # == Fixtures
+  #
+  # You can declare fixtures for each example_group like this:
+  #   describe "...." do
+  #     fixtures :table_a, :table_b
+  #
+  # Alternatively, if you prefer to declare them only once, you can
+  # do so right here. Just uncomment the next line and replace the fixture
+  # names with your fixtures.
+  #
+  #
+  # If you declare global fixtures, be aware that they will be declared
+  # for all of your examples, even those that don't use them.
+  #
+  # You can also declare which fixtures to use (for example fixtures for test/fixtures):
+  #
+  # config.fixture_path = RAILS_ROOT + '/spec/fixtures/'
+  #
+  # == Mock Framework
+  #
+  # RSpec uses its own mocking framework by default. If you prefer to
+  # use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  #
+  # == Notes
+  #
+  # For more information take a look at Spec::Runner::Configuration and Spec::Runner
+end
+
+Dir[File.expand_path(File.join(File.dirname(__FILE__),'support','**','*.rb'))].each {|f| require f}
+
+require 'sqs/job'
+require 'slosilo'
+
+KEY = Slosilo::Key.new

data/sqs-job.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sqs/job/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sqs-job"
+  spec.version       = SQS::Job::VERSION
+  spec.authors       = ["Jon Mason", "Kevin Gilpin"]
+  spec.email         = ["jonathan.j.mason@gmail.com", "kgilpin@gmail.com"]
+  spec.summary       = %q{Simple job processing library which uses SQS.}
+  spec.homepage      = "https://github.com/conjurinc/sqs-job"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "validatable"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "aws-sdk"
+  spec.add_dependency "slosilo"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "conjur-api"
+  spec.add_development_dependency "conjur-cli"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "cucumber"
+  spec.add_development_dependency "ci_reporter_rspec"
+  spec.add_development_dependency 'ci_reporter_cucumber'
+  spec.add_development_dependency "simplecov"
+end

metadata

@@ -0,0 +1,257 @@
+--- !ruby/object:Gem::Specification
+name: sqs-job
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jon Mason
+- Kevin Gilpin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: validatable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slosilo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: conjur-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: conjur-cli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ci_reporter_rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ci_reporter_cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- jonathan.j.mason@gmail.com
+- kgilpin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .project
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- cucumber-policy.rb
+- features/send_message.feature
+- features/sqs/job/message/test_message.rb
+- features/step_definitions/job_steps.rb
+- features/support/env.rb
+- lib/sqs/job.rb
+- lib/sqs/job/exceptions.rb
+- lib/sqs/job/handler.rb
+- lib/sqs/job/message/base.rb
+- lib/sqs/job/policy.rb
+- lib/sqs/job/provisioner.rb
+- lib/sqs/job/thread_pool.rb
+- lib/sqs/job/version.rb
+- lib/sqs/job/worker.rb
+- spec/handler_spec.rb
+- spec/spec_helper.rb
+- sqs-job.gemspec
+homepage: https://github.com/conjurinc/sqs-job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Simple job processing library which uses SQS.
+test_files:
+- features/send_message.feature
+- features/sqs/job/message/test_message.rb
+- features/step_definitions/job_steps.rb
+- features/support/env.rb
+- spec/handler_spec.rb
+- spec/spec_helper.rb