active_elastic_job2 2.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c07318b8577b4c3f4dd20abd95d232c734c09b46a772ecb828f3a5b8b80f4426
+  data.tar.gz: 334a44c79aeca8b35dbd87a79c0323605dc8eb0d3f8ed84bb74ef6ee8a490bc3
+SHA512:
+  metadata.gz: 10d52fb153a3cc695affc5275fc3d52ff08f200a0677dd93d8a779e3501132154df59de6c7ec7e846721abbaabc1562da4ab32321f8abaa26e28b57f8e13393a
+  data.tar.gz: 1ff0bb9431ffe9fe46196358ef0ed60bac37c38b583b57cbbe95b09626b1000cb259af94c2e556b0d9441ca8edfc4fd79ac04c6d9f631c2e37602fd826b88afe

data/active-elastic-job.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'active_elastic_job/version'
+
+Gem::Specification.new do |spec|
+  spec.platform      = Gem::Platform::RUBY
+  spec.name          = 'active_elastic_job2'
+  spec.version       = ActiveElasticJob.version
+  spec.authors       = ['unchi']
+  spec.email         = ['developer.japan@gmail.com']
+  spec.summary       = 'Active Elastic Job is a simple to use Active Job backend for Rails applications deployed on the Amazon Elastic Beanstalk platform.'
+  spec.description   = 'Run background jobs / tasks of Rails applications deployed in Amazon Elastic Beanstalk environments. Active Elastic Job is an Active Job backend which is easy to setup. No need for customised container commands or other workarounds.'
+  spec.license       = 'MIT'
+  spec.homepage      = 'https://github.com/unchi/active-elastic-job'
+
+  spec.files         = Dir.glob('lib/**/*') + [ 'active-elastic-job.gemspec' ]
+  spec.executables   = []
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 1.9.3'
+
+  spec.add_dependency 'aws-sdk-sqs', '~> 1'
+  spec.add_dependency 'rails', '>= 4.2'
+end

data/lib/active_elastic_job.rb

@@ -0,0 +1,12 @@
+require 'aws-sdk-core'
+require 'active_elastic_job/version'
+require 'active_elastic_job/md5_message_digest_calculation'
+require 'active_job/queue_adapters/active_elastic_job_adapter'
+require 'active_elastic_job/rack/sqs_message_consumer'
+require 'active_elastic_job/message_verifier'
+
+module ActiveElasticJob
+  ACRONYM = 'AEJ'.freeze
+end
+
+require "active_elastic_job/railtie" if defined? Rails

data/lib/active_elastic_job/md5_message_digest_calculation.rb

@@ -0,0 +1,72 @@
+require 'openssl'
+module ActiveElasticJob
+  # This module provides methods that calculate the MD5 digest for Amazon
+  # SQS message bodies and message attributes.
+  # The digest can be used to verify that Amazon SQS received the message
+  # correctly.
+  #
+  # Example:
+  #
+  #   extend ActiveElasticJob::MD5MessageDigestCalculation
+  #
+  #   resp = Aws::SQS::Client.new.send_message(
+  #     queue_url: queue_url,
+  #     message_body: body,
+  #     message_attributes: attributes
+  #   )
+  #
+  #   if resp.md5_of_message_body != md5_of_message_body(body)
+  #     raise "Returned digest of message body is invalid!"
+  #   end
+  #
+  #   if resp.md5_of_message_attributes != md5_of_message_attributes(attributes)
+  #     raise "Returned digest of message attributes is invalid!"
+  #   end
+  module MD5MessageDigestCalculation
+    TRANSPORT_TYPE_ENCODINGS = {
+      'String' => 1,
+      'Binary' => 2,
+      'Number' => 1
+    }
+
+    NORMALIZED_ENCODING = Encoding::UTF_8
+
+    def md5_of_message_body(message_body)
+      OpenSSL::Digest::MD5.hexdigest(message_body)
+    end
+
+    def md5_of_message_attributes(message_attributes)
+      encoded = { }
+      message_attributes.each do |name, attribute|
+        name = name.to_s
+        encoded[name] = String.new
+        encoded[name] << encode_length_and_bytes(name) <<
+        encode_length_and_bytes(attribute[:data_type]) <<
+        [TRANSPORT_TYPE_ENCODINGS[attribute[:data_type]]].pack('C'.freeze)
+
+        if attribute[:string_value] != nil
+          encoded[name] << encode_length_and_string(attribute[:string_value])
+        elsif attribute[:binary_value] != nil
+          encoded[name] << encode_length_and_bytes(attribute[:binary_value])
+        end
+      end
+
+      buffer = encoded.keys.sort.reduce(String.new) do |string, name|
+        string << encoded[name]
+      end
+      OpenSSL::Digest::MD5.hexdigest(buffer)
+    end
+
+    private
+
+    def encode_length_and_string(string)
+      string = String.new(string)
+      string.encode!(NORMALIZED_ENCODING)
+      encode_length_and_bytes(string)
+    end
+
+    def encode_length_and_bytes(bytes)
+      [bytes.bytesize, bytes].pack('L>a*'.freeze)
+    end
+  end
+end

data/lib/active_elastic_job/message_verifier.rb

@@ -0,0 +1,34 @@
+require 'active_support/security_utils'
+
+module ActiveElasticJob
+  class MessageVerifier #:nodoc:
+
+    # Raised when digest generated by
+    # <tt>ActiveJob::QueueAdapters::ActiveElasticJobAdapter</tt> could not
+    # be verified.
+    class InvalidDigest < StandardError
+    end
+
+    def initialize(secret)
+      @secret = secret
+    end
+
+    def verify(message, digest)
+      if message.nil? || message.blank? || digest.nil? || digest.blank?
+        return false
+      end
+
+      return ActiveSupport::SecurityUtils.secure_compare(
+        digest, generate_digest(message))
+    end
+
+    def verify!(message, digest)
+      raise InvalidDigest unless verify(message, digest)
+    end
+
+    def generate_digest(message)
+      require 'openssl' unless defined?(OpenSSL)
+      OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA1.new, @secret, message)
+    end
+  end
+end

data/lib/active_elastic_job/rack/sqs_message_consumer.rb

@@ -0,0 +1,129 @@
+require "action_dispatch"
+
+module ActiveElasticJob
+  module Rack
+    # This middleware intercepts requests which are sent by the SQS daemon
+    # running in {Amazon Elastic Beanstalk worker environments}[http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html].
+    # It does this by looking at the +User-Agent+ header.
+    # Requesets from the SQS daemon are handled in two alternative cases:
+    #
+    # (1) the processed SQS message was originally triggered by a periodic task
+    # supported by Elastic Beanstalk's Periodic Task feature
+    #
+    # (2) the processed SQS message was queued by this gem representing an active job.
+    # In this case it verifies the digest which is sent along with a legit SQS
+    # message, and passed as an HTTP header in the resulting request.
+    # The digest is based on Rails' +secrets.secret_key_base+.
+    # Therefore, the application running in the web environment, which generates
+    # the digest, and the application running in the worker
+    # environment, which verifies the digest, have to use the *same*
+    # +secrets.secret_key_base+ setting.
+    class SqsMessageConsumer
+      OK_RESPONSE = [ '200'.freeze, { 'Content-Type'.freeze => 'text/plain'.freeze }, [ 'OK'.freeze ] ]
+      FORBIDDEN_RESPONSE = [
+        '403'.freeze,
+        { 'Content-Type'.freeze => 'text/plain'.freeze },
+        [ 'Request forbidden!'.freeze ]
+      ]
+      DOCKER_HOST_IP = '172.17.0.1'.freeze
+
+      def initialize(app) #:nodoc:
+        @app = app
+      end
+
+      def call(env) #:nodoc:
+        request = ActionDispatch::Request.new env
+        if enabled? && aws_sqsd?(request)
+          unless request.local? || sent_from_docker_host?(request)
+            return FORBIDDEN_RESPONSE
+          end
+
+          if periodic_task?(request)
+            execute_periodic_task(request)
+            return OK_RESPONSE
+          elsif originates_from_gem?(request)
+            begin
+              execute_job(request)
+            rescue ActiveElasticJob::MessageVerifier::InvalidDigest => e
+              return FORBIDDEN_RESPONSE
+            end
+            return OK_RESPONSE 
+          end
+        end
+        @app.call(env)
+      end
+
+      private
+
+      def enabled?
+        Rails.application.config.active_elastic_job.process_jobs == true
+      end
+
+      def verify!(request)
+        @verifier ||= ActiveElasticJob::MessageVerifier.new(secret_key_base)
+        digest = request.headers['HTTP_X_AWS_SQSD_ATTR_MESSAGE_DIGEST'.freeze]
+        message = request.body_stream.read
+        request.body_stream.rewind
+        @verifier.verify!(message, digest)
+      end
+
+      def secret_key_base
+        config.secret_key_base
+      end
+
+      def config
+        Rails.application.config.active_elastic_job
+      end
+
+      def aws_sqsd?(request)
+        # Does not match against a Regexp
+        # in order to avoid performance penalties.
+        # Instead performs a simple string comparison.
+        # Benchmark runs showed an performance increase of
+        # up to 40%
+        current_user_agent = request.headers['User-Agent'.freeze]
+        return (current_user_agent.present? &&
+          current_user_agent.size >= 'aws-sqsd'.freeze.size &&
+          current_user_agent[0..('aws-sqsd'.freeze.size - 1)] == 'aws-sqsd'.freeze)
+      end
+
+      def periodic_tasks_route
+        @periodic_tasks_route ||= config.periodic_tasks_route
+      end
+
+      def periodic_task?(request)
+        !request.fullpath.nil? && request.fullpath[0..(periodic_tasks_route.size - 1)] == periodic_tasks_route
+      end
+
+      def execute_job(request)
+        verify!(request)
+        job = JSON.load(request.body)
+        ActiveJob::Base.execute(job)
+      end
+
+      def execute_periodic_task(request)
+        job_name = request.headers['X-Aws-Sqsd-Taskname']
+        job = job_name.constantize.new
+        job.perform_now
+      end
+
+      def originates_from_gem?(request)
+        if request.headers['HTTP_X_AWS_SQSD_ATTR_ORIGIN'.freeze] == ActiveElasticJob::ACRONYM
+          return true
+        elsif request.headers['HTTP_X_AWS_SQSD_ATTR_MESSAGE_DIGEST'.freeze] != nil
+          return true
+        else
+          return false
+        end
+      end
+
+      def sent_from_docker_host?(request)
+        app_runs_in_docker_container? && request.remote_ip == DOCKER_HOST_IP
+      end
+
+      def app_runs_in_docker_container?
+        @app_in_docker_container ||= `[ -f /proc/1/cgroup ] && cat /proc/1/cgroup` =~ /docker/
+      end
+    end
+  end
+end

data/lib/active_elastic_job/railtie.rb

@@ -0,0 +1,23 @@
+module ActiveElasticJob
+  class Railtie < Rails::Railtie
+    config.active_elastic_job = ActiveSupport::OrderedOptions.new
+    process_active_elastic_jobs = ENV['PROCESS_ACTIVE_ELASTIC_JOBS']
+    config.active_elastic_job.process_jobs = !process_active_elastic_jobs.nil? && process_active_elastic_jobs.downcase == 'true'
+    config.active_elastic_job.aws_credentials = lambda { Aws::InstanceProfileCredentials.new }
+    config.active_elastic_job.periodic_tasks_route = '/periodic_tasks'.freeze
+
+    initializer "active_elastic_job.insert_middleware" do |app|
+      if app.config.active_elastic_job.secret_key_base.blank?
+        app.config.active_elastic_job.secret_key_base = app.secrets[:secret_key_base]
+      end
+
+      if app.config.active_elastic_job.process_jobs == true
+        if app.config.force_ssl
+          app.config.middleware.insert_before(ActionDispatch::SSL,ActiveElasticJob::Rack::SqsMessageConsumer)
+        else
+          app.config.middleware.use(ActiveElasticJob::Rack::SqsMessageConsumer)
+        end
+      end
+    end
+  end
+end

data/lib/active_elastic_job/version.rb

@@ -0,0 +1,18 @@
+module ActiveElasticJob
+  module VERSION
+    MAJOR = 2
+    MINOR = 0
+    TINY  = 2
+    PRE   = nil
+
+    STRING = [MAJOR, MINOR, TINY, PRE].compact.join('.')
+
+    def self.to_s
+      STRING
+    end
+  end
+
+  def self.version
+    VERSION::STRING
+  end
+end

data/lib/active_job/queue_adapters/active_elastic_job_adapter.rb

@@ -0,0 +1,221 @@
+module ActiveJob
+  module QueueAdapters
+    # == Active Elastic Job adapter for Active Job
+    #
+    # Active Elastic Job provides (1) an adapter (this class) for Rails'
+    # Active Job framework and (2) a Rack middleware to process job requests,
+    # which are sent by the SQS daemon running in {Amazon Elastic Beanstalk worker
+    # environments}[http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/using-features-managing-env-tiers.html].
+    #
+    # This adapter serializes job objects and sends them as a message to an
+    # Amazon SQS queue specified by the job's queue name,
+    # see <tt>ActiveJob::Base.queue_as</tt>
+    #
+    # To use Active Elastic Job, set the queue_adapter config
+    # to +:active_elastic_job+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :active_elastic_job
+    class ActiveElasticJobAdapter
+      MAX_MESSAGE_SIZE = (256 * 1024)
+      MAX_DELAY_IN_MINUTES = 15
+
+      extend ActiveElasticJob::MD5MessageDigestCalculation
+
+      class Error < RuntimeError; end;
+
+      # Raised when job exceeds 256 KB in its serialized form. The limit is
+      # imposed by Amazon SQS.
+      class SerializedJobTooBig < Error
+        def initialize(serialized_job)
+          super(<<-MSG)
+            The job contains #{serialized_job.bytesize} bytes in its serialzed form,
+            which exceeds the allowed maximum of #{MAX_MESSAGE_SIZE} bytes imposed by Amazon SQS.
+          MSG
+        end
+      end
+
+      # Raised when job queue does not exist. The job queue is determined by
+      # <tt>ActiveJob::Base.queue_as</tt>. You can either: (1) create a new
+      # Amazon SQS queue and attach a worker environment to it, or (2) select a
+      # different queue for your jobs.
+      #
+      # Example:
+      # * Open your AWS console and create an SQS queue named +high_priority+ in
+      #   the same AWS region of your Elastic Beanstalk environments.
+      # * Queue your jobs accordingly:
+      #
+      #  class MyJob < ActiveJob::Base
+      #    queue_as :high_priority
+      #    #..
+      #  end
+      class NonExistentQueue < Error
+        def initialize(queue_name, aws_region)
+
+          super(<<-MSG)
+            The job is bound to queue at #{queue_name}.
+            Unfortunately a queue with this name does not exist in this
+            region. Either create an Amazon SQS queue named #{queue_name} -
+            you can do this in AWS console, make sure to select region
+            '#{aws_region}' - or you
+            select another queue for your jobs.
+          MSG
+        end
+      end
+
+      # Raised when calculated MD5 digest does not match the MD5 Digest
+      # of the response from Amazon SQS.
+      class MD5MismatchError < Error
+        def initialize(message_id, calculated, returned)
+
+          super(<<-MSG)
+            MD5 '#{returned}' returned by Amazon SQS does not match the
+            calculation on the original request which was '#{calculated}'.
+            The message with Message ID #{message_id} sent to SQS might be
+            corrupted.
+          MSG
+        end
+      end
+
+      # Raised when the delay is longer than the MAX_DELAY_IN_MINUTES
+      class DelayTooLong < RangeError
+        def initialize()
+          super(<<-MSG)
+            Jobs cannot be scheduled more than #{MAX_DELAY_IN_MINUTES} minutes
+            into the future.
+            See http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html
+            for further details!
+          MSG
+        end
+      end
+
+      def enqueue(job) #:nodoc:
+        self.class.enqueue job
+      end
+
+      def enqueue_at(job, timestamp) #:nodoc:
+        self.class.enqueue_at(job, timestamp)
+      end
+
+      class << self
+        def enqueue(job) #:nodoc:
+          enqueue_at(job, Time.now)
+        end
+
+        def enqueue_at(job, timestamp) #:nodoc:
+          serialized_job = JSON.dump(job.serialize)
+          check_job_size!(serialized_job)
+          message = build_message(job.queue_name, serialized_job, timestamp)
+          resp = aws_sqs_client.send_message(message)
+          unless aws_client_verifies_md5_digests?
+            verify_md5_digests!(
+              resp,
+              message[:message_body],
+              message[:message_attributes])
+          end
+        rescue Aws::SQS::Errors::NonExistentQueue => e
+          unless @queue_urls[job.queue_name.to_s].nil?
+            @queue_urls[job.queue_name.to_s] = nil
+            retry
+          end
+          raise NonExistentQueue.new(job, aws_region)
+        rescue Aws::Errors::ServiceError => e
+          raise Error, "Could not enqueue job, #{e.message}"
+        end
+
+        private
+
+        def aws_client_verifies_md5_digests?
+          Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new('2.2.19'.freeze)
+        end
+
+        def build_message(queue_name, serialized_job, timestamp)
+          {
+            queue_url: queue_url(queue_name),
+            message_body: serialized_job,
+            delay_seconds: calculate_delay(timestamp),
+            message_attributes: {
+              "message-digest".freeze => {
+                string_value: message_digest(serialized_job),
+                data_type: "String".freeze
+              },
+              origin: {
+                string_value: ActiveElasticJob::ACRONYM,
+                data_type: "String".freeze
+              }
+            }
+          }
+        end
+
+        def queue_url(queue_name)
+          cache_key = queue_name.to_s
+          @queue_urls ||= { }
+          return @queue_urls[cache_key] if @queue_urls[cache_key]
+          resp = aws_sqs_client.get_queue_url(queue_name: queue_name.to_s)
+          @queue_urls[cache_key] = resp.queue_url
+        rescue Aws::SQS::Errors::NonExistentQueue => e
+          raise NonExistentQueue.new(queue_name, aws_region)
+        end
+
+        def calculate_delay(timestamp)
+          delay = (timestamp - Time.current.to_f).to_i + 1
+          if delay > MAX_DELAY_IN_MINUTES.minutes
+            raise DelayTooLong.new
+          end
+          delay = 0 if delay < 0
+          delay
+        end
+
+        def check_job_size!(serialized_job)
+          if serialized_job.bytesize > MAX_MESSAGE_SIZE
+            raise SerializedJobTooBig, serialized_job
+          end
+        end
+
+        def aws_sqs_client
+          @aws_sqs_client ||= Aws::SQS::Client.new(region: aws_region, credentials: aws_sqs_client_credentials )
+        end
+
+        def aws_sqs_client_credentials
+          @aws_credentials ||= if config.aws_credentials.kind_of?(Proc)
+                                 config.aws_credentials.call
+                               else
+                                 config.aws_credentials
+                               end
+        end
+
+        def aws_region
+          config.aws_region
+        end
+
+        def config
+          Rails.application.config.active_elastic_job
+        end
+
+        def message_digest(messsage_body)
+          @verifier ||= ActiveElasticJob::MessageVerifier.new(secret_key_base)
+          @verifier.generate_digest(messsage_body)
+        end
+
+        def verify_md5_digests!(response, messsage_body, message_attributes)
+          calculated = md5_of_message_body(messsage_body)
+          returned = response.md5_of_message_body
+          if calculated != returned
+            raise MD5MismatchError.new response.message_id, calculated, returned
+          end
+
+          if message_attributes
+            calculated = md5_of_message_attributes(message_attributes)
+            returned = response.md5_of_message_attributes
+            if  calculated != returned
+              raise MD5MismatchError.new response.message_id, calculated, returned
+            end
+          end
+        end
+
+        def secret_key_base
+          config.secret_key_base
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: active_elastic_job2
+version: !ruby/object:Gem::Version
+  version: 2.0.2
+platform: ruby
+authors:
+- unchi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-12-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: Run background jobs / tasks of Rails applications deployed in Amazon
+  Elastic Beanstalk environments. Active Elastic Job is an Active Job backend which
+  is easy to setup. No need for customised container commands or other workarounds.
+email:
+- developer.japan@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- active-elastic-job.gemspec
+- lib/active_elastic_job.rb
+- lib/active_elastic_job/md5_message_digest_calculation.rb
+- lib/active_elastic_job/message_verifier.rb
+- lib/active_elastic_job/rack/sqs_message_consumer.rb
+- lib/active_elastic_job/railtie.rb
+- lib/active_elastic_job/version.rb
+- lib/active_job/queue_adapters/active_elastic_job_adapter.rb
+homepage: https://github.com/unchi/active-elastic-job
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: Active Elastic Job is a simple to use Active Job backend for Rails applications
+  deployed on the Amazon Elastic Beanstalk platform.
+test_files: []