active_elastic_job 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a15dee97e68451ab5d4e68fdc849c8898c40ebd6
-  data.tar.gz: 76a7503a7a8d3385b71675587a7094224025ade3
+  metadata.gz: be0c3eef5f0c1b0aa66524d644f7f855f8c3a99d
+  data.tar.gz: af9a0f1e16545725b593d01a42743769f3f9d621
 SHA512:
-  metadata.gz: fa4289336749473efac64d3f6bb09751ddad6d7eee12fe977ee0d9bb84c888caed6711ba0cc01e5af8040943bcabaa32ef6b4b2a39c383df90a93f3a2e5003dd
-  data.tar.gz: 397a332e277690241346c84c3fda6fc13d3010d902c6b7406b16b2841139f99e727c5baeb42d54808dde0506bc9ea5e930ddffa02ea6bb2002eac0ea70ffeab3
+  metadata.gz: 82c4cda1a5c365a803ee5e3d0948a22d9a372d77f354671bfd2e50197db807ee424f27bb5fb1474c6a6130c5a1b50279dd1e7d60fc5b8c47ee101f326ca933ed
+  data.tar.gz: 39aee6ee9c19d0fac3b263045211fb420643c93fb7bb65f249113c5d53d89fc1611ea171d717a9b7e288346fa9a1a8efa6f8c8fcc9d40fa1edb7437e3ee013d5

data/active-elastic-job.gemspec

@@ -7,11 +7,11 @@ require 'active_elastic_job/version'
 Gem::Specification.new do |spec|
   spec.platform      = Gem::Platform::RUBY
   spec.name          = 'active_elastic_job'
-  spec.version       = ActiveElasticJob::VERSION
+  spec.version       = ActiveElasticJob.version
   spec.authors       = ['Tawan Sierek']
   spec.email         = ['tawan@sierek.com']
-  spec.description   = 'Active Elastic Job is a simple to use Active Job backend for Rails applications deployed on the Amazon Elastic Beanstalk platform.'
-  spec.summary       = spec.description
+  spec.summary       = 'Active Elastic Job is a simple to use Active Job backend for Rails applications deployed on the Amazon Elastic Beanstalk platform.'
+  spec.description   = 'Run background jobs / tasks of Rails applications deployed in Amazon Elastic Beanstalk environments. Active Elastic Job is an Active Job backend which is easy to setup. No need for customised container commands or other workarounds.'
   spec.license       = 'MIT'
   spec.homepage      = 'https://github.com/tawan/active-elastic-job'
 

data/lib/active_elastic_job.rb

@@ -1,5 +1,6 @@
 require 'aws-sdk-core'
 require 'active_elastic_job/version'
+require 'active_elastic_job/md5_message_digest_calculation'
 require 'active_job/queue_adapters/active_elastic_job_adapter'
 require 'active_elastic_job/rack/sqs_message_consumer'
 require 'active_elastic_job/message_verifier'

data/lib/active_elastic_job/md5_message_digest_calculation.rb

@@ -0,0 +1,79 @@
+require 'digest'
+module ActiveElasticJob
+  # This module provides methods that calculate the MD5 digest for Amazon
+  # SQS message bodies and message attributes.
+  # The digest can be used to verify that Amazon SQS received the message
+  # correctly.
+  #
+  # Example:
+  #
+  #   extend ActiveElasticJob::MD5MessageDigestCalculation
+  #
+  #   resp = Aws::SQS::Client.new.send_message(
+  #     queue_url: queue_url,
+  #     message_body: body,
+  #     message_attributes: attributes
+  #   )
+  #
+  #   if resp.md5_of_message_body != md5_of_message_body(body)
+  #     raise "Returned digest of message body is invalid!"
+  #   end
+  #
+  #   if resp.md5_of_message_attributes != md5_of_message_attributes(attributes)
+  #     raise "Returned digest of message attributes is invalid!"
+  #   end
+  module MD5MessageDigestCalculation
+    TRANSPORT_TYPE_ENCODINGS = {
+      'String' => 1,
+      'Binary' => 2,
+      'Number' => 1
+    }
+
+    CHARSET_ENCODING = Encoding::UTF_8
+
+    # Returns MD5 digest of +message_body+.
+    def md5_of_message_body(message_body)
+      OpenSSL::Digest::MD5.hexdigest(message_body)
+    end
+
+    # Returns MD5 digest of +message_attributes+.
+    #
+    # The calculation follows the official algorithm which
+    # is specified by Amazon.
+    def md5_of_message_attributes(message_attributes)
+      encoded = message_attributes.each.with_object({ }) do |(name, v), hash|
+        hash[name.to_s] = ""
+        data_type = v['data_type'] || v[:data_type]
+
+        hash[name.to_s] << encode_length_and_bytes(name.to_s) <<
+          encode_length_and_bytes(data_type) <<
+          [ TRANSPORT_TYPE_ENCODINGS[data_type] ].pack('C')
+
+        if string_value = v['string_value'] || v[:string_value]
+          hash[name.to_s] << encode_length_and_string(string_value)
+        elsif binary_value = v['binary_value'] || v[:binary_value]
+          hash[name.to_s] << encode_length_and_bytes(binary_value)
+        end
+      end
+
+      buffer = encoded.keys.sort.reduce("") do |b, name|
+        b << encoded[name]
+      end
+      OpenSSL::Digest::MD5.hexdigest(buffer)
+    end
+
+    private
+
+    def encode_length_and_string(string)
+      return '' if string.nil?
+      string = String.new(string)
+      string.encode!(CHARSET_ENCODING)
+      encode_length_and_bytes(string)
+    end
+
+    def encode_length_and_bytes(bytes)
+      return '' if bytes.nil?
+      [ bytes.bytesize, bytes ].pack("L>a#{bytes.bytesize}")
+    end
+  end
+end

data/lib/active_elastic_job/message_verifier.rb

@@ -13,7 +13,7 @@ module ActiveElasticJob
       @secret = secret
     end
 
-    def verify(message, digest)
+    def verify!(message, digest)
       if message.nil? || message.blank? || digest.nil? || digest.blank?
         raise InvalidDigest
       end

data/lib/active_elastic_job/rack/sqs_message_consumer.rb

@@ -21,9 +21,9 @@ module ActiveElasticJob
 
       def call(env) #:nodoc:
         request = ActionDispatch::Request.new env
-        if aws_sqsd?(request)
+        if aws_sqsd?(request) && originates_from_gem?(request)
           begin
-            verify(request)
+            verify!(request)
             job = JSON.load(request.body)
             ActiveJob::Base.execute(job)
           rescue ActiveElasticJob::MessageVerifier::InvalidDigest => e
@@ -38,11 +38,11 @@ module ActiveElasticJob
 
       private
 
-      def verify(request)
+      def verify!(request)
         secret_key_base = Rails.application.secrets[:secret_key_base]
         @verifier ||= ActiveElasticJob::MessageVerifier.new(secret_key_base)
         digest = request.headers['HTTP_X_AWS_SQSD_ATTR_MESSAGE_DIGEST']
-        @verifier.verify(request.body.string, digest)
+        @verifier.verify!(request.body.string, digest)
       end
 
       def aws_sqsd?(request)
@@ -56,6 +56,10 @@ module ActiveElasticJob
           current_user_agent.size >= USER_AGENT_PREFIX.size &&
           current_user_agent[0..(USER_AGENT_PREFIX.size - 1)] == USER_AGENT_PREFIX)
       end
+
+      def originates_from_gem?(request)
+        request.headers['HTTP_X_AWS_SQSD_ATTR_ORIGIN'] == 'AEJ'
+      end
     end
   end
 end

data/lib/active_elastic_job/version.rb

@@ -1,3 +1,18 @@
 module ActiveElasticJob
-  VERSION = '1.2.1'
+  module VERSION
+    MAJOR = 1
+    MINOR = 3
+    TINY  = 0
+    PRE   = nil
+
+    STRING = [MAJOR, MINOR, TINY, PRE].compact.join('.')
+
+    def self.to_s
+      STRING
+    end
+  end
+
+  def self.version
+    VERSION::STRING
+  end
 end

data/lib/active_job/queue_adapters/active_elastic_job_adapter.rb

@@ -16,7 +16,9 @@ module ActiveJob
     class ActiveElasticJobAdapter
       MAX_MESSAGE_SIZE = (256 * 1024)
 
-      class Error < StandardError; end;
+      extend ActiveElasticJob::MD5MessageDigestCalculation
+
+      class Error < RuntimeError; end;
 
       # Raised when job exceeds 256 KB in its serialized form. The limit is
       # imposed by Amazon SQS.
@@ -45,48 +47,79 @@ which exceeds the allowed maximum of #{MAX_MESSAGE_SIZE} bytes imposed by Amazon
       #    #..
       #  end
       class NonExistentQueue < Error
-        def initialize(job)
+        def initialize(queue_name)
           msg = <<-MSG
-The job is bound to queue at #{job.queue_name}. Unfortunately a queue
+The job is bound to queue at #{queue_name}. Unfortunately a queue
 with this name does not exist in this region. Either create an Amazon SQS queue
-named #{job.queue_name} - you can do this in AWS console, make sure to select
-region '#{ENV['AWS_REGION']}' - or you select another queue for
-#{job.class.name} jobs.
+named #{queue_name} - you can do this in AWS console, make sure to select
+region '#{ENV['AWS_REGION']}' - or you select another queue for your jobs.
           MSG
           super msg
         end
       end
 
+      # Raised when calculated MD5 digest does not match the MD5 Digest
+      # of the response from Amazon SQS.
+      class MD5MismatchError < Error
+        def initialize( message_id)
+          msg = <<-MSG
+MD5 returned by Amazon SQS does not match the calculation on the original request.
+The message with Message ID #{message_id} sent to SQS might be corrupted.
+          MSG
+        end
+      end
+
       class << self
         def enqueue(job) #:nodoc:
           enqueue_at(job, Time.now)
         end
 
         def enqueue_at(job, timestamp) #:nodoc:
-          queue_url = nil
-          begin
-            resp = aws_sqs_client.get_queue_url(queue_name: job.queue_name.to_s)
-            queue_url = resp.queue_url
-          rescue Aws::SQS::Errors::NonExistentQueue => e
-            raise NonExistentQueue, job
-          end
           serialized_job = JSON.dump(job.serialize)
           check_job_size!(serialized_job)
-          aws_sqs_client.send_message(
-            queue_url: queue_url,
+          message = {
+            queue_url: queue_url(job.queue_name),
             message_body: serialized_job,
             delay_seconds: calculate_delay(timestamp),
             message_attributes: {
               "message_digest" => {
                 string_value: message_digest(serialized_job),
                 data_type: "String"
+              },
+              "origin" => {
+                string_value: "AEJ",
+                data_type: "String"
               }
             }
+          }
+          resp = aws_sqs_client.send_message(message)
+          verify_md5_digests!(
+          resp,
+            message[:message_body],
+            message[:message_attributes]
           )
+        rescue Aws::SQS::Errors::NonExistentQueue => e
+          unless @queue_urls[job.queue_name.to_s].nil?
+            @queue_urls[job.queue_name.to_s] = nil
+            retry
+          end
+          raise NonExistentQueue, job
+        rescue Aws::Errors::ServiceError => e
+          raise Error, "Could not enqueue job, #{e.message}"
         end
 
         private
 
+        def queue_url(queue_name)
+          cache_key = queue_name.to_s
+          @queue_urls ||= { }
+          return @queue_urls[cache_key] if @queue_urls[cache_key]
+          resp = aws_sqs_client.get_queue_url(queue_name: queue_name.to_s)
+          @queue_urls[cache_key] = resp.queue_url
+        rescue Aws::SQS::Errors::NonExistentQueue => e
+          raise NonExistentQueue, queue_name
+        end
+
         def calculate_delay(timestamp)
           delay = (timestamp - Time.current.to_f).to_i + 1
           if delay > 15.minutes
@@ -119,6 +152,19 @@ for further details!
           verifier = ActiveElasticJob::MessageVerifier.new(secret_key_base)
           verifier.generate_digest(messsage_body)
         end
+
+        def verify_md5_digests!(response, messsage_body, message_attributes = nil)
+          if md5_of_message_body(messsage_body) != response.md5_of_message_body
+            raise MD5MismatchError, response.message_id
+          end
+
+          if message_attributes
+            if md5_of_message_attributes(message_attributes) !=
+              response.md5_of_message_attributes
+              raise MD5MismatchError, response.message_id
+            end
+          end
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_elastic_job
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Tawan Sierek
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-20 00:00:00.000000000 Z
+date: 2016-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -38,8 +38,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
-description: Active Elastic Job is a simple to use Active Job backend for Rails applications
-  deployed on the Amazon Elastic Beanstalk platform.
+description: Run background jobs / tasks of Rails applications deployed in Amazon
+  Elastic Beanstalk environments. Active Elastic Job is an Active Job backend which
+  is easy to setup. No need for customised container commands or other workarounds.
 email:
 - tawan@sierek.com
 executables: []
@@ -48,6 +49,7 @@ extra_rdoc_files: []
 files:
 - active-elastic-job.gemspec
 - lib/active_elastic_job.rb
+- lib/active_elastic_job/md5_message_digest_calculation.rb
 - lib/active_elastic_job/message_verifier.rb
 - lib/active_elastic_job/rack/sqs_message_consumer.rb
 - lib/active_elastic_job/railtie.rb