aws-sdk-rails 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d4664fce310e44e2613d3caf795eead1a767194d48c7b407854a4a3dbe342f4
-  data.tar.gz: c49b2bd9ec55f6b1fc63260c0704bc3e2594d4084742a774a527cbec81dfbad9
+  metadata.gz: b252fc5ed5514c1ab48750fc65acde4008f1f6a8b685bae42903708f7b987a14
+  data.tar.gz: da794888829a46dfeec19350997e753b9858bf07a0668ef603594d9db46bdf9a
 SHA512:
-  metadata.gz: 8b38a5870854a7be9867aeb95e1ae2f3fd0a332289ae1f8124306811e42aab5d0274c2abdb7914ff33d9ccb162c334e06034376d4c4a702ab4b39c56b2667b7e
-  data.tar.gz: d82eb9df09e04c9426c3191d994568f3565574c7c2f1ed24d70a0ecd8e288179fb9462480f9795409ac857a5e07839096f4b6fd02ab5545d1926e0fc10aa2964
+  metadata.gz: 5c5a5f74b8faab027f2d8b3b64e32a4feb2e85c29207d051a66e93424b6b71d8dbdf0dd80ba1baac1b9be66489759c8efba1860fdba7d41990fbd8279d173b08
+  data.tar.gz: 227f06499dbd9a1705eae1ffb00777502cbfa3f512c8e07b1a037d5b04a35afcc651487518ea582e646cf6060682112a85d8baf59e6a9d543eb7beb16c8e3fa7

data/VERSION

@@ -1 +1 @@
-3.3.0
+3.4.0

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -46,4 +46,4 @@ module ActiveJob
     # `:amazon` is the convention used for ActionMailer and ActiveStorage
     AmazonAdapter = AmazonSqsAdapter
   end
-end
+end

data/lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'concurrent'
+
+module ActiveJob
+  module QueueAdapters
+
+    # == Async adapter for Amazon SQS ActiveJob
+    #
+    # This adapter queues jobs asynchronously (ie non-blocking).  Error handler can be configured
+    # with +Aws::Rails::SqsActiveJob.config.async_queue_error_handler+.
+    #
+    # To use this adapter, set up as:
+    #
+    # config.active_job.queue_adapter = :amazon_sqs_async
+    class AmazonSqsAsyncAdapter < AmazonSqsAdapter
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        Concurrent::Promise
+        .execute { super(job, send_message_opts) }
+        .on_error do |e|
+          Rails.logger.error "Failed to queue job #{job}.  Reason: #{e}"
+          error_handler = Aws::Rails::SqsActiveJob.config.async_queue_error_handler
+          error_handler.call(e, job, send_message_opts) if error_handler
+        end
+      end
+    end
+  end
+end

data/lib/aws-sdk-rails.rb

@@ -6,9 +6,11 @@ require_relative 'aws/rails/notifications'
 require_relative 'aws/rails/sqs_active_job/configuration'
 require_relative 'aws/rails/sqs_active_job/executor'
 require_relative 'aws/rails/sqs_active_job/job_runner'
+require_relative 'aws/rails/sqs_active_job/lambda_handler'
 
 require_relative 'action_dispatch/session/dynamodb_store'
 require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+require_relative 'active_job/queue_adapters/amazon_sqs_async_adapter'
 
 require_relative 'generators/aws_record/base'
 

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -14,11 +14,12 @@ module Aws
         yield(config)
       end
 
-      # Holds configuration for AWS SQS ActiveJob
-      # Use the Aws::Rails::SqsActiveJob.config to access.
+      # Configuration for AWS SQS ActiveJob.
+      # Use +Aws::Rails::SqsActiveJob.config+ to access the singleton config instance.
       class Configuration
 
         # Default configuration options
+        # @api private
         DEFAULTS = {
           max_messages:  10,
           visibility_timeout: 120,
@@ -27,38 +28,48 @@ module Aws
           logger: ::Rails.logger
         }
 
+        # @api private
         attr_accessor :queues, :max_messages, :visibility_timeout,
-                      :shutdown_timeout, :client, :logger
+                      :shutdown_timeout, :client, :logger, :async_queue_error_handler
 
+        # Don't use this method directly: Confugration is a singleton class, use
+        # +Aws::Rails::SqsActiveJob.config+ to access the singleton config.
+        #
         # @param [Hash] options
-        # @option options [Hash[Symbol, String]] :queues - A mapping between the
+        # @option options [Hash[Symbol, String]] :queues A mapping between the
         #   active job queue name and the SQS Queue URL. Note: multiple active
         #   job queues can map to the same SQS Queue URL.
         #
-        # @option options  [Integer] :max_messages -
+        # @option options  [Integer] :max_messages
         #    The max number of messages to poll for in a batch.
         #
-        # @option options [Integer] :visibility_timeout -
+        # @option options [Integer] :visibility_timeout
         #   The visibility timeout is the number of seconds
         #   that a message will not be processable by any other consumers.
         #   You should set this value to be longer than your expected job runtime
         #   to prevent other processes from picking up an running job.
         #   See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
         #
-        # @option options [Integer] :shutdown_timeout -
+        # @option options [Integer] :shutdown_timeout
         #   the amount of time to wait
         #   for a clean shutdown.  Jobs that are unable to complete in this time
         #   will not be deleted from the SQS queue and will be retryable after
         #   the visibility timeout.
         #
-        # @option options [ActiveSupport::Logger] :logger - Logger to use
+        # @option options [ActiveSupport::Logger] :logger Logger to use
         #   for the poller.
         #
-        # @option options [String] :config_file -
+        # @option options [String] :config_file
         #   Override file to load configuration from.  If not specified will
         #   attempt to load from config/aws_sqs_active_job.yml.
         #
-        # @option options [SQS::Client] :client - SQS Client to use.  A default
+        # @option options [Callable] :async_queue_error_handler An error handler
+        #   to be called when the async active job adapter experiances an error
+        #   queueing a job.  Only applies when
+        #   +active_job.queue_adapter = :amazon_sqs_async+.  Called with:
+        #   [error, job, job_options]
+        #
+        # @option options [SQS::Client] :client SQS Client to use.  A default
         #   client will be created if none is provided.
         def initialize(options = {})
           options[:config_file] ||= config_file if config_file.exist?
@@ -69,7 +80,7 @@ module Aws
         end
 
         def client
-          @client ||= Aws::SQS::Client.new
+          @client ||= Aws::SQS::Client.new(user_agent_suffix: user_agent)
         end
 
         # Return the queue_url for a given job_queue name
@@ -80,10 +91,12 @@ module Aws
           queues[job_queue.to_sym]
         end
 
+        # @api private
         def to_s
           to_h.to_s
         end
 
+        # @api private
         def to_h
           h = {}
           self.instance_variables.each do |v|
@@ -129,7 +142,11 @@ module Aws
         def config_file_path(options)
           options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
         end
+
+        def user_agent
+          "ft/aws-sdk-rails-activejob/#{Aws::Rails::VERSION}"
+        end
       end
     end
   end
-end
+end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -13,7 +13,6 @@ module Aws
            max_threads:     Concurrent.processor_count,
            auto_terminate:  true,
            idletime:        60, # 1 minute
-           max_queue:       2,
            fallback_policy: :caller_runs # slow down the producer thread
         }.freeze
 
@@ -56,4 +55,4 @@ module Aws
       end
     end
   end
-end
+end

data/lib/aws/rails/sqs_active_job/lambda_handler.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      # A lambda event handler to run jobs from an SQS queue trigger
+      # Trigger the lambda from your SQS queue
+      # Configure the entrypoint to: +config/environment.Aws::Rails::SqsActiveJob.lambda_job_handler+
+      # This will load your Rails environment, and then use this method as the handler.
+      def self.lambda_job_handler(event:, context:)
+        return 'no records to process' unless event['Records']
+
+        event['Records'].each do |record|
+          sqs_msg = to_sqs_msg(record)
+          job = Aws::Rails::SqsActiveJob::JobRunner.new(sqs_msg)
+          puts("Running job: #{job.id}[#{job.class_name}]")
+          job.run
+          sqs_msg.delete
+        end
+        "Processed #{event['Records'].length} jobs."
+      end
+
+      private
+
+      def self.to_sqs_msg(record)
+        msg = Aws::SQS::Types::Message.new(
+          body: record['body'],
+          md5_of_body: record['md5OfBody'],
+          message_attributes: self.to_message_attributes(record),
+          message_id: record['messageId'],
+          receipt_handle: record['receiptHandle']
+        )
+        Aws::SQS::Message.new(
+          queue_url: to_queue_url(record),
+          receipt_handle: msg.receipt_handle,
+          data: msg,
+          client: Aws::Rails::SqsActiveJob.config.client
+        )
+      end
+
+      def self.to_message_attributes(record)
+        record['messageAttributes'].each_with_object({}) do |(key, value), acc|
+          acc[key] = {
+            string_value: value['stringValue'],
+            binary_value: value['binaryValue'],
+            string_list_values: ['stringListValues'],
+            binary_list_values: value['binaryListValues'],
+            data_type: value['dataType']
+          }
+        end
+      end
+
+      def self.to_queue_url(record)
+        source_arn = record['eventSourceARN']
+        raise ArgumentError, "Invalid queue arn: #{source_arn}" unless Aws::ARNParser.arn?(source_arn)
+
+        arn = Aws::ARNParser.parse(source_arn)
+        sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region)
+        "https://sqs.#{arn.region}.#{sfx}/#{arn.account_id}/#{arn.resource}"
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -15,10 +15,11 @@ module Aws
       class Poller
 
         DEFAULT_OPTS = {
-          threads: Concurrent.processor_count,
+          threads: 2*Concurrent.processor_count,
           max_messages: 10,
           visibility_timeout: 60,
           shutdown_timeout: 15,
+          backpressure: 10
         }
 
         def initialize(args = ARGV)
@@ -99,7 +100,7 @@ module Aws
           parser = ::OptionParser.new { |opts|
             opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
             opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
-            opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create.  Defaults to the number of processors available on this system.") { |a| out[:threads] = a }
+            opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.") { |a| out[:threads] = a }
             opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
             opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
             opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
@@ -122,4 +123,4 @@ module Aws
       end
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-rails
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-01 00:00:00.000000000 Z
+date: 2020-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-record
@@ -121,6 +121,7 @@ files:
 - bin/aws_sqs_active_job
 - lib/action_dispatch/session/dynamodb_store.rb
 - lib/active_job/queue_adapters/amazon_sqs_adapter.rb
+- lib/active_job/queue_adapters/amazon_sqs_async_adapter.rb
 - lib/aws-sdk-rails.rb
 - lib/aws/rails/mailer.rb
 - lib/aws/rails/notifications.rb
@@ -128,6 +129,7 @@ files:
 - lib/aws/rails/sqs_active_job/configuration.rb
 - lib/aws/rails/sqs_active_job/executor.rb
 - lib/aws/rails/sqs_active_job/job_runner.rb
+- lib/aws/rails/sqs_active_job/lambda_handler.rb
 - lib/aws/rails/sqs_active_job/poller.rb
 - lib/generators/aws_record/base.rb
 - lib/generators/aws_record/generated_attribute.rb