aws-sdk-rails 2.0.1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9a5ff066628318dc1cc54e08b560bc7dc82370bc
-  data.tar.gz: 4c36681e57448fc61714157c9d3e522ded8d7721
+SHA256:
+  metadata.gz: 2d4664fce310e44e2613d3caf795eead1a767194d48c7b407854a4a3dbe342f4
+  data.tar.gz: c49b2bd9ec55f6b1fc63260c0704bc3e2594d4084742a774a527cbec81dfbad9
 SHA512:
-  metadata.gz: 39b54b648b6650b4e66e82c774dabdcf0fa6992148e87f4998442b86bf8deb2d8dfbe2d516dca7cfcf496bb905d9064c5ef10c56e18fb9597a1c446c55a95735
-  data.tar.gz: 0bd3edda7d062750779b2bd1933555f9dc44a129769f40d2a8a7fc376c493761aee29280cc0addd2804c9ffab1184396c30b82990543f36941ba312bc4025e3e
+  metadata.gz: 8b38a5870854a7be9867aeb95e1ae2f3fd0a332289ae1f8124306811e42aab5d0274c2abdb7914ff33d9ccb162c334e06034376d4c4a702ab4b39c56b2667b7e
+  data.tar.gz: d82eb9df09e04c9426c3191d994568f3565574c7c2f1ed24d70a0ecd8e288179fb9462480f9795409ac857a5e07839096f4b6fd02ab5545d1926e0fc10aa2964

data/VERSION

@@ -0,0 +1 @@
+3.3.0

data/bin/aws_sqs_active_job

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/aws/rails/sqs_active_job/poller'
+
+Aws::Rails::SqsActiveJob::Poller.new.run

data/lib/action_dispatch/session/dynamodb_store.rb

@@ -0,0 +1,32 @@
+require 'aws-sessionstore-dynamodb'
+
+module ActionDispatch
+  module Session
+    # Uses the Dynamo DB Session Store implementation to create a class that
+    # extends ActionDispatch::Session. Rails will create a :dynamodb_store
+    # configuration for session_store from this class name.
+    #
+    # This class will use the Rails secret_key_base unless otherwise provided.
+    #
+    # Configuration can also be provided in YAML files from Rails config, either
+    # in "config/session_store.yml" or "config/session_store/#{Rails.env}.yml".
+    # Configuration files that are environment-specific will take precedence.
+    #
+    # @see https://docs.aws.amazon.com/sdk-for-ruby/aws-sessionstore-dynamodb/api/Aws/SessionStore/DynamoDB/Configuration.html
+    class DynamodbStore < Aws::SessionStore::DynamoDB::RackMiddleware
+      def initialize(app, options = {})
+        options[:config_file] ||= config_file if config_file.exist?
+        options[:secret_key] ||= Rails.application.secret_key_base
+        super
+      end
+
+      private
+
+      def config_file
+        file = Rails.root.join("config/dynamo_db_session_store/#{Rails.env}.yml")
+        file = Rails.root.join('config/dynamo_db_session_store.yml') unless file.exist?
+        file
+      end
+    end
+  end
+end

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+module ActiveJob
+  module QueueAdapters
+
+    class AmazonSqsAdapter
+
+      def enqueue(job)
+        _enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp)
+        delay = (timestamp - Time.now.to_f).floor
+        raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+        _enqueue(job, delay_seconds: delay)
+      end
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        body = job.serialize
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        send_message_opts[:queue_url] = queue_url
+        send_message_opts[:message_body] = Aws::Json.dump(body)
+        send_message_opts[:message_attributes] = message_attributes(job)
+        Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
+      end
+
+      def message_attributes(job)
+        {
+          'aws_sqs_active_job_class' => {
+            string_value: job.class.to_s,
+            data_type: 'String'
+          },
+          'aws_sqs_active_job_version' => {
+            string_value: Aws::Rails::VERSION,
+            data_type: 'String'
+          }
+        }
+      end
+    end
+
+    # create an alias to allow `:amazon` to be used as the adapter name
+    # `:amazon` is the convention used for ActionMailer and ActiveStorage
+    AmazonAdapter = AmazonSqsAdapter
+  end
+end

data/lib/aws-sdk-rails.rb

@@ -1,37 +1,19 @@
-require_relative 'aws/rails/mailer'
-
-module Aws
-  module Rails
+# frozen_string_literal: true
 
-    # @api private
-    class Railtie < ::Rails::Railtie
-      initializer "aws-sdk-rails.initialize", before: :load_config_initializers do |app|
-        # Initialization Actions
-        Aws::Rails.add_action_mailer_delivery_method
-        Aws::Rails.log_to_rails_logger
-      end
-    end
+require_relative 'aws/rails/mailer'
+require_relative 'aws/rails/railtie'
+require_relative 'aws/rails/notifications'
+require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/executor'
+require_relative 'aws/rails/sqs_active_job/job_runner'
 
-    # This is called automatically from the SDK's Railtie, but if you want to
-    # manually specify options for building the Aws::SES::Client object, you
-    # can manually call this method.
-    #
-    # @param [Symbol] name The name of the ActionMailer delivery method to
-    #   register.
-    # @param [Hash] options The options you wish to pass on to the
-    #   Aws::SES::Client initialization method.
-    def self.add_action_mailer_delivery_method(name = :aws_sdk, options = {})
-      ActiveSupport.on_load(:action_mailer) do
-        self.add_delivery_method(name, Aws::Rails::Mailer, options)
-      end
-    end
+require_relative 'action_dispatch/session/dynamodb_store'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
 
-    # Configures the AWS SDK for Ruby's logger to use the Rails logger.
-    def self.log_to_rails_logger
-      Aws.config[:logger] = ::Rails.logger
-      nil
-    end
+require_relative 'generators/aws_record/base'
 
+module Aws
+  module Rails
+    VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
   end
 end
-

data/lib/aws/rails/mailer.rb

@@ -1,23 +1,23 @@
+# frozen_string_literal: true
+
 require 'aws-sdk-ses'
 
 module Aws
   module Rails
-
     # Provides a delivery method for ActionMailer that uses Amazon Simple Email
     # Service.
-    # 
+    #
     # Once you have an SES delivery method you can configure Rails to
     # use this for ActionMailer in your environment configuration
     # (e.g. RAILS_ROOT/config/environments/production.rb)
     #
-    #     config.action_mailer.delivery_method = :aws_sdk
+    #     config.action_mailer.delivery_method = :ses
     #
-    # Uses the AWS SDK for Ruby V2's credential provider chain when creating an
-    # SES client instance.
+    # Uses the AWS SDK for Ruby's credential provider chain when creating an SES
+    # client instance.
     class Mailer
-
       # @param [Hash] options Passes along initialization options to
-      #   [Aws::SES::Client.new](http://docs.aws.amazon.com/sdkforruby/api/Aws/SES/Client.html#initialize-instance_method).
+      #   [Aws::SES::Client.new](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SES/Client.html#initialize-instance_method).
       def initialize(options = {})
         @client = SES::Client.new(options)
       end
@@ -33,15 +33,15 @@ module Aws
           send_opts[:destinations] = message.destinations
         end
 
-        @client.send_raw_email(send_opts)
-
+        @client.send_raw_email(send_opts).tap do |response|
+          message.header[:ses_message_id] = response.message_id
+        end
       end
 
       # ActionMailer expects this method to be present and to return a hash.
       def settings
         {}
       end
-
     end
   end
 end

data/lib/aws/rails/notifications.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-core'
+require 'active_support/notifications'
+
+module Aws
+  module Rails
+
+    # Instruments client operation calls for ActiveSupport::Notifications
+    # Each client operation will produce an event with name:
+    # <operation>.<service>.aws
+    # @api private
+    class Notifications < Seahorse::Client::Plugin
+
+      def add_handlers(handlers, config)
+        # This plugin needs to be first
+        # which means it is called first in the stack, to start recording time,
+        # and returns last
+        handlers.add(Handler, step: :initialize, priority: 99)
+      end
+
+      class Handler < Seahorse::Client::Handler
+
+        def call(context)
+          event_name = "#{context.operation_name}.#{context.config.api.metadata['serviceId']}.aws"
+          ActiveSupport::Notifications.instrument(event_name, context: context) do
+            @handler.call(context)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/railtie.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Aws
+  # Use the Rails namespace.
+  module Rails
+    # @api private
+    class Railtie < ::Rails::Railtie
+      initializer 'aws-sdk-rails.initialize',
+                  before: :load_config_initializers do
+        # Initialization Actions
+        Aws::Rails.use_rails_encrypted_credentials
+        Aws::Rails.add_action_mailer_delivery_method
+        Aws::Rails.log_to_rails_logger
+      end
+
+      rake_tasks do
+        load 'tasks/dynamo_db/session_store.rake'
+        load 'tasks/aws_record/migrate.rake'
+      end
+    end
+
+    # This is called automatically from the SDK's Railtie, but can be manually
+    # called if you want to specify options for building the Aws::SES::Client.
+    #
+    # @param [Symbol] name The name of the ActionMailer delivery method to
+    #   register.
+    # @param [Hash] options The options you wish to pass on to the
+    #   Aws::SES::Client initialization method.
+    def self.add_action_mailer_delivery_method(name = :ses, options = {})
+      ActiveSupport.on_load(:action_mailer) do
+        add_delivery_method(name, Aws::Rails::Mailer, options)
+      end
+    end
+
+    # Configures the AWS SDK for Ruby's logger to use the Rails logger.
+    def self.log_to_rails_logger
+      Aws.config[:logger] = ::Rails.logger
+      nil
+    end
+
+    # Configures the AWS SDK with credentials from Rails encrypted credentials.
+    def self.use_rails_encrypted_credentials
+      # limit the config keys we merge to credentials only
+      aws_credential_keys = %i[access_key_id secret_access_key session_token]
+
+      Aws.config.merge!(
+        ::Rails.application
+          .try(:credentials)
+          .try(:aws)
+          .to_h.slice(*aws_credential_keys)
+      )
+    end
+
+    # Adds ActiveSupport Notifications instrumentation to AWS SDK
+    # client operations.  Each operation will produce an event with a name:
+    # <operation>.<service>.aws.  For example, S3's put_object has an event
+    # name of: put_object.S3.aws
+    def self.instrument_sdk_operations
+      Aws.constants.each do |c|
+        m = Aws.const_get(c)
+        if m.is_a?(Module) && m.const_defined?(:Client) &&
+           m.const_get(:Client).superclass == Seahorse::Client::Base
+          m.const_get(:Client).add_plugin(Aws::Rails::Notifications)
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      # @return [Configuration] the (singleton) Configuration
+      def self.config
+        @config ||= Configuration.new
+      end
+
+      # @yield Configuration
+      def self.configure
+        yield(config)
+      end
+
+      # Holds configuration for AWS SQS ActiveJob
+      # Use the Aws::Rails::SqsActiveJob.config to access.
+      class Configuration
+
+        # Default configuration options
+        DEFAULTS = {
+          max_messages:  10,
+          visibility_timeout: 120,
+          shutdown_timeout: 15,
+          queues: {},
+          logger: ::Rails.logger
+        }
+
+        attr_accessor :queues, :max_messages, :visibility_timeout,
+                      :shutdown_timeout, :client, :logger
+
+        # @param [Hash] options
+        # @option options [Hash[Symbol, String]] :queues - A mapping between the
+        #   active job queue name and the SQS Queue URL. Note: multiple active
+        #   job queues can map to the same SQS Queue URL.
+        #
+        # @option options  [Integer] :max_messages -
+        #    The max number of messages to poll for in a batch.
+        #
+        # @option options [Integer] :visibility_timeout -
+        #   The visibility timeout is the number of seconds
+        #   that a message will not be processable by any other consumers.
+        #   You should set this value to be longer than your expected job runtime
+        #   to prevent other processes from picking up an running job.
+        #   See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
+        #
+        # @option options [Integer] :shutdown_timeout -
+        #   the amount of time to wait
+        #   for a clean shutdown.  Jobs that are unable to complete in this time
+        #   will not be deleted from the SQS queue and will be retryable after
+        #   the visibility timeout.
+        #
+        # @option options [ActiveSupport::Logger] :logger - Logger to use
+        #   for the poller.
+        #
+        # @option options [String] :config_file -
+        #   Override file to load configuration from.  If not specified will
+        #   attempt to load from config/aws_sqs_active_job.yml.
+        #
+        # @option options [SQS::Client] :client - SQS Client to use.  A default
+        #   client will be created if none is provided.
+        def initialize(options = {})
+          options[:config_file] ||= config_file if config_file.exist?
+          options = DEFAULTS
+             .merge(file_options(options))
+             .merge(options)
+          set_attributes(options)
+        end
+
+        def client
+          @client ||= Aws::SQS::Client.new
+        end
+
+        # Return the queue_url for a given job_queue name
+        def queue_url_for(job_queue)
+          job_queue = job_queue.to_sym
+          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
+
+          queues[job_queue.to_sym]
+        end
+
+        def to_s
+          to_h.to_s
+        end
+
+        def to_h
+          h = {}
+          self.instance_variables.each do |v|
+            v_sym = v.to_s.gsub('@', '').to_sym
+            val = self.instance_variable_get(v)
+            h[v_sym] = val
+          end
+          h
+        end
+
+        private
+
+        # Set accessible attributes after merged options.
+        def set_attributes(options)
+          options.keys.each do |opt_name|
+            instance_variable_set("@#{opt_name}", options[opt_name])
+          end
+        end
+
+        def file_options(options = {})
+          file_path = config_file_path(options)
+          if file_path
+            load_from_file(file_path)
+          else
+            {}
+          end
+        end
+
+        def config_file
+          file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
+          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
+          file
+        end
+
+        # Load options from YAML file
+        def load_from_file(file_path)
+          require "erb"
+          opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
+          opts.deep_symbolize_keys
+        end
+
+        # @return [String] Configuration path found in environment or YAML file.
+        def config_file_path(options)
+          options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Aws
+  module Rails
+    module SqsActiveJob
+      # CLI runner for polling for SQS ActiveJobs
+      class Executor
+
+        DEFAULTS = {
+           min_threads:     0,
+           max_threads:     Concurrent.processor_count,
+           auto_terminate:  true,
+           idletime:        60, # 1 minute
+           max_queue:       2,
+           fallback_policy: :caller_runs # slow down the producer thread
+        }.freeze
+
+        def initialize(options = {})
+          @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
+          @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
+        end
+
+        # TODO: Consider catching the exception and sleeping instead of using :caller_runs
+        def execute(message)
+          @executor.post(message) do |message|
+            begin
+              job = JobRunner.new(message)
+              @logger.info("Running job: #{job.id}[#{job.class_name}]")
+              job.run
+              message.delete
+            rescue Aws::Json::ParseError => e
+              @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
+            rescue StandardError => e
+              # message will not be deleted and will be retried
+              job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
+              @logger.info "Error processing job #{job_msg}: #{e}"
+              @logger.debug e.backtrace.join("\n")
+            end
+          end
+        end
+
+        def shutdown(timeout=nil)
+          @executor.shutdown
+          clean_shutdown = @executor.wait_for_termination(timeout)
+          if clean_shutdown
+            @logger.info 'Clean shutdown complete.  All executing jobs finished.'
+          else
+            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have"\
+              " finished cleanly.  Unfinished jobs will not be removed from"\
+              " the queue and can be ru-run once their visibility timeout"\
+              " passes."
+          end
+        end
+      end
+    end
+  end
+end