aws-sdk-rails 4.1.0 → 4.2.0

data/lib/aws/rails/sqs_active_job/job_runner.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module Aws
-  module Rails
-    module SqsActiveJob
-      # @api private
-      class JobRunner
-        attr_reader :id, :class_name
-
-        def initialize(message)
-          @job_data = Aws::Json.load(message.data.body)
-          @class_name = @job_data['job_class'].constantize
-          @id = @job_data['job_id']
-        end
-
-        def run
-          ActiveJob::Base.execute @job_data
-        end
-
-        def exception_executions?
-          @job_data['exception_executions'] &&
-            !@job_data['exception_executions'].empty?
-        end
-      end
-    end
-  end
-end

data/lib/aws/rails/sqs_active_job/lambda_handler.rb

@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-require 'aws-sdk-sqs'
-
-module Aws
-  module Rails
-    # A lambda event handler to run jobs from an SQS queue trigger
-    # Trigger the lambda from your SQS queue
-    # Configure the entrypoint to: +config/environment.Aws::Rails::SqsActiveJob.lambda_job_handler+
-    # This will load your Rails environment, and then use this method as the handler.
-    module SqsActiveJob
-      def self.lambda_job_handler(event:, context:)
-        return 'no records to process' unless event['Records']
-
-        event['Records'].each do |record|
-          sqs_msg = to_sqs_msg(record)
-          job = Aws::Rails::SqsActiveJob::JobRunner.new(sqs_msg)
-          puts("Running job: #{job.id}[#{job.class_name}]")
-          job.run
-          sqs_msg.delete
-        end
-        "Processed #{event['Records'].length} jobs."
-      end
-
-      def self.to_sqs_msg(record)
-        msg = Aws::SQS::Types::Message.new(
-          body: record['body'],
-          md5_of_body: record['md5OfBody'],
-          message_attributes: to_message_attributes(record),
-          message_id: record['messageId'],
-          receipt_handle: record['receiptHandle']
-        )
-        Aws::SQS::Message.new(
-          queue_url: to_queue_url(record),
-          receipt_handle: msg.receipt_handle,
-          data: msg,
-          client: Aws::Rails::SqsActiveJob.config.client
-        )
-      end
-
-      def self.to_message_attributes(record)
-        record['messageAttributes'].transform_values do |value|
-          {
-            string_value: value['stringValue'],
-            binary_value: value['binaryValue'],
-            string_list_values: ['stringListValues'],
-            binary_list_values: value['binaryListValues'],
-            data_type: value['dataType']
-          }
-        end
-      end
-
-      def self.to_queue_url(record)
-        source_arn = record['eventSourceARN']
-        raise ArgumentError, "Invalid queue arn: #{source_arn}" unless Aws::ARNParser.arn?(source_arn)
-
-        arn = Aws::ARNParser.parse(source_arn)
-        sfx = Aws::Partitions::EndpointProvider.dns_suffix_for(arn.region)
-        "https://sqs.#{arn.region}.#{sfx}/#{arn.account_id}/#{arn.resource}"
-      end
-    end
-  end
-end

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -1,160 +0,0 @@
-# frozen_string_literal: true
-
-require 'aws-sdk-sqs'
-require 'optparse'
-require 'concurrent'
-
-module Aws
-  module Rails
-    module SqsActiveJob
-      class Interrupt < StandardError; end
-
-      # CLI runner for polling for SQS ActiveJobs
-      # Use `aws_sqs_active_job --help` for detailed usage
-      class Poller
-        DEFAULT_OPTS = {
-          threads: 2 * Concurrent.processor_count,
-          max_messages: 10,
-          shutdown_timeout: 15,
-          backpressure: 10,
-          retry_standard_errors: true
-        }.freeze
-
-        def initialize(args = ARGV)
-          @options = parse_args(args)
-          # Set_environment must be run before we boot_rails
-          set_environment
-        end
-
-        def set_environment
-          @environment = @options[:environment] || ENV['APP_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
-        end
-
-        def run
-          # exit 0
-          boot_rails
-
-          # cannot load config (from file or initializers) until after
-          # rails has been booted.
-          @options = DEFAULT_OPTS
-                     .merge(Aws::Rails::SqsActiveJob.config.to_h)
-                     .merge(@options.to_h)
-          validate_config
-          # ensure we have a logger configured
-          @logger = @options[:logger] || ActiveSupport::Logger.new($stdout)
-          @logger.info("Starting Poller with options=#{@options}")
-
-          Signal.trap('INT') { raise Interrupt }
-          Signal.trap('TERM') { raise Interrupt }
-          @executor = Executor.new(
-            max_threads: @options[:threads],
-            logger: @logger,
-            max_queue: @options[:backpressure],
-            retry_standard_errors: @options[:retry_standard_errors]
-          )
-
-          poll
-        rescue Interrupt
-          @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.'
-          shutdown
-          exit
-        end
-
-        private
-
-        def shutdown
-          @executor.shutdown(@options[:shutdown_timeout])
-        end
-
-        def poll
-          queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(@options[:queue])
-          @logger.info "Polling on: #{@options[:queue]} => #{queue_url}"
-          client = Aws::Rails::SqsActiveJob.config.client
-          @poller = Aws::SQS::QueuePoller.new(queue_url, client: client)
-          poller_options = {
-            skip_delete: true,
-            max_number_of_messages: @options[:max_messages],
-            visibility_timeout: @options[:visibility_timeout]
-          }
-          # Limit max_number_of_messages for FIFO queues to 1
-          # this ensures jobs with the same message_group_id are processed
-          # in order
-          # Jobs with different message_group_id will be processed in
-          # parallel and may be out of order.
-          poller_options[:max_number_of_messages] = 1 if Aws::Rails::SqsActiveJob.fifo?(queue_url)
-
-          single_message = poller_options[:max_number_of_messages] == 1
-
-          @poller.poll(poller_options) do |msgs|
-            msgs = [msgs] if single_message
-            @logger.info "Processing batch of #{msgs.length} messages"
-            msgs.each do |msg|
-              @executor.execute(Aws::SQS::Message.new(
-                                  queue_url: queue_url,
-                                  receipt_handle: msg.receipt_handle,
-                                  data: msg,
-                                  client: client
-                                ))
-            end
-          end
-        end
-
-        def boot_rails
-          ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
-          require 'rails'
-          require File.expand_path('config/environment.rb')
-        end
-
-        # rubocop:disable Metrics
-        def parse_args(argv)
-          out = {}
-          parser = ::OptionParser.new do |opts|
-            opts.on('-q', '--queue STRING', '[Required] Queue to poll') { |a| out[:queue] = a }
-            opts.on('-e', '--environment STRING',
-                    'Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.') do |a|
-              out[:environment] = a
-            end
-            opts.on('-t', '--threads INTEGER', Integer,
-                    'The maximum number of worker threads to create.  Defaults to 2x the number of processors available on this system.') do |a|
-              out[:threads] = a
-            end
-            opts.on('-b', '--backpressure INTEGER', Integer,
-                    'The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.') do |a|
-              out[:backpressure] = a
-            end
-            opts.on('-m', '--max_messages INTEGER', Integer,
-                    'Max number of messages to receive in a batch from SQS.') do |a|
-              out[:max_messages] = a
-            end
-            opts.on('-v', '--visibility_timeout INTEGER', Integer,
-                    'The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.') do |a|
-              out[:visibility_timeout] = a
-            end
-            opts.on('-s', '--shutdown_timeout INTEGER', Integer,
-                    'The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.') do |a|
-              out[:shutdown_timeout] = a
-            end
-            opts.on('--[no-]retry_standard_errors [FLAG]', TrueClass,
-                    'When set, retry all StandardErrors (leaving failed messages on the SQS Queue). These retries are ON TOP of standard Rails ActiveJob retries set by retry_on in the ActiveJob.') do |a|
-              out[:retry_standard_errors] = a.nil? ? true : a
-            end
-          end
-
-          parser.banner = 'aws_sqs_active_job [options]'
-          parser.on_tail '-h', '--help', 'Show help' do
-            puts parser
-            exit 1
-          end
-
-          parser.parse(argv)
-          out
-        end
-        # rubocop:enable Metrics
-
-        def validate_config
-          raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
-        end
-      end
-    end
-  end
-end

data/lib/aws/rails/sqs_active_job.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../active_job/queue_adapters/sqs_adapter'
-require_relative '../../active_job/queue_adapters/sqs_adapter/params'
-require_relative '../../active_job/queue_adapters/sqs_async_adapter'
-require_relative 'sqs_active_job/configuration'
-require_relative 'sqs_active_job/deduplication'
-require_relative 'sqs_active_job/executor'
-require_relative 'sqs_active_job/job_runner'
-require_relative 'sqs_active_job/lambda_handler'
-
-module Aws
-  module Rails
-    # == AWS SQS ActiveJob.
-    #
-    # SQS-based queuing backend for Active Job.
-    module SqsActiveJob
-      # @return [Configuration] the (singleton) Configuration
-      def self.config
-        @config ||= Configuration.new
-      end
-
-      # @yield Configuration
-      def self.configure
-        yield(config)
-      end
-
-      def self.fifo?(queue_url)
-        queue_url.ends_with? '.fifo'
-      end
-    end
-  end
-end

data/lib/generators/aws_record/base.rb

@@ -1,213 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails/generators'
-require_relative 'generated_attribute'
-require_relative 'secondary_index'
-
-module AwsRecord
-  module Generators
-    class Base < Rails::Generators::NamedBase
-      argument :attributes, type: :array, default: [], banner: 'field[:type][:opts]...',
-                            desc: 'Describes the fields in the model'
-      check_class_collision
-
-      class_option :disable_mutation_tracking, type: :boolean, desc: 'Disables dirty tracking'
-      class_option :timestamps, type: :boolean, desc: 'Adds created, updated timestamps to the model'
-      class_option :table_config, type: :hash, default: {}, banner: 'primary:R-W [SecondaryIndex1:R-W]...',
-                                  desc: 'Declares the r/w units for the model as well as any secondary indexes', required: true
-      class_option :gsi, type: :array, default: [],
-                         banner: 'name:hkey{field_name}[,rkey{field_name},proj_type{ALL|KEYS_ONLY|INCLUDE}]...', desc: 'Allows for the declaration of secondary indexes'
-      class_option :table_name, type: :string, banner: 'model_table_name'
-      class_option :password_digest, type: :boolean, desc: 'Whether to add a password_digest field to the model'
-
-      class_option :required, type: :string, banner: 'field1...',
-                              desc: 'A list of attributes that are required for an instance of the model'
-      class_option :length_validations, type: :hash, default: {}, banner: 'field1:MIN-MAX...',
-                                        desc: 'Validations on the length of attributes in a model'
-
-      attr_accessor :primary_read_units, :primary_write_units, :gsi_rw_units, :gsis, :required_attrs,
-                    :length_validations
-
-      private
-
-      def initialize(args, *options)
-        options[0] << '--skip-table-config' if options[1][:behavior] == :revoke
-        @parse_errors = []
-
-        super
-        ensure_unique_fields
-        ensure_hkey
-        parse_gsis!
-        parse_table_config!
-        parse_validations!
-
-        return if @parse_errors.empty?
-
-        warn 'The following errors were encountered while trying to parse the given attributes'
-        $stderr.puts
-        warn @parse_errors
-        $stderr.puts
-
-        abort('Please fix the errors before proceeding.')
-      end
-
-      def parse_attributes!
-        self.attributes = (attributes || []).map do |attr|
-          GeneratedAttribute.parse(attr)
-        rescue ArgumentError => e
-          @parse_errors << e
-          next
-        end
-        self.attributes = attributes.compact
-
-        if options['password_digest']
-          attributes << GeneratedAttribute.new('password_digest', :string_attr, digest: true)
-        end
-
-        return unless options['timestamps']
-
-        attributes << GeneratedAttribute.parse('created:datetime:default_value{Time.now}')
-        attributes << GeneratedAttribute.parse('updated:datetime:default_value{Time.now}')
-      end
-
-      def ensure_unique_fields
-        used_names = Set.new
-        duplicate_fields = []
-
-        attributes.each do |attr|
-          duplicate_fields << [:attribute, attr.name] if used_names.include? attr.name
-          used_names.add attr.name
-
-          next unless attr.options.key? :database_attribute_name
-
-          raw_db_attr_name = attr.options[:database_attribute_name].delete('"') # db attribute names are wrapped with " to make template generation easier
-
-          duplicate_fields << [:database_attribute_name, raw_db_attr_name] if used_names.include? raw_db_attr_name
-
-          used_names.add raw_db_attr_name
-        end
-
-        return if duplicate_fields.empty?
-
-        duplicate_fields.each do |invalid_attr|
-          @parse_errors << ArgumentError.new("Found duplicated field name: #{invalid_attr[1]}, in attribute#{invalid_attr[0]}")
-        end
-      end
-
-      def ensure_hkey
-        uuid_member = nil
-        hkey_member = nil
-        rkey_member = nil
-
-        attributes.each do |attr|
-          if attr.options.key? :hash_key
-            if hkey_member
-              @parse_errors << ArgumentError.new("Redefinition of hash_key attr: #{attr.name}, original declaration of hash_key on: #{hkey_member.name}")
-              next
-            end
-
-            hkey_member = attr
-          elsif attr.options.key? :range_key
-            if rkey_member
-              @parse_errors << ArgumentError.new("Redefinition of range_key attr: #{attr.name}, original declaration of range_key on: #{hkey_member.name}")
-              next
-            end
-
-            rkey_member = attr
-          end
-
-          uuid_member = attr if attr.name.include? 'uuid'
-        end
-
-        return if hkey_member
-
-        if uuid_member
-          uuid_member.options[:hash_key] = true
-        else
-          attributes.unshift GeneratedAttribute.parse('uuid:hkey')
-        end
-      end
-
-      def mutation_tracking_disabled?
-        options['disable_mutation_tracking']
-      end
-
-      def has_validations?
-        !@required_attrs.empty? || !@length_validations.empty?
-      end
-
-      def parse_table_config!
-        return unless options['table_config']
-
-        @primary_read_units, @primary_write_units = parse_rw_units('primary')
-
-        @gsi_rw_units = @gsis.to_h do |idx|
-          [idx.name, parse_rw_units(idx.name)]
-        end
-
-        options['table_config'].each_key do |config|
-          next if config == 'primary'
-
-          gsi = @gsis.select { |idx| idx.name == config }
-
-          @parse_errors << ArgumentError.new("Could not find a gsi declaration for #{config}") if gsi.empty?
-        end
-      end
-
-      def parse_rw_units(name)
-        if options['table_config'].key? name
-          rw_units = options['table_config'][name]
-          rw_units.gsub(/[,.-]/, ':').split(':').reject(&:empty?)
-        else
-          @parse_errors << ArgumentError.new("Please provide a table_config definition for #{name}")
-        end
-      end
-
-      def parse_gsis!
-        @gsis = (options['gsi'] || []).map do |raw_idx|
-          idx = SecondaryIndex.parse(raw_idx)
-
-          attributes = self.attributes.select { |attr| attr.name == idx.hash_key }
-          if attributes.empty?
-            @parse_errors << ArgumentError.new("Could not find attribute #{idx.hash_key} for gsi #{idx.name} hkey")
-            next
-          end
-
-          if idx.range_key
-            attributes = self.attributes.select { |attr| attr.name == idx.range_key }
-            if attributes.empty?
-              @parse_errors << ArgumentError.new("Could not find attribute #{idx.range_key} for gsi #{idx.name} rkey")
-              next
-            end
-          end
-
-          idx
-        rescue ArgumentError => e
-          @parse_errors << e
-          next
-        end
-
-        @gsis = @gsis.compact
-      end
-
-      def parse_validations!
-        @required_attrs = options['required'] ? options['required'].split(',') : []
-        @required_attrs.each do |val_attr|
-          @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if attributes.none? do |attr|
-                                                                                                       attr.name == val_attr
-                                                                                                     end
-        end
-
-        @length_validations = options['length_validations'].map do |val_attr, bounds|
-          @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if attributes.none? do |attr|
-                                                                                                       attr.name == val_attr
-                                                                                                     end
-
-          bounds = bounds.gsub(/[,.-]/, ':').split(':').reject(&:empty?)
-          [val_attr, "#{bounds[0]}..#{bounds[1]}"]
-        end
-        @length_validations = @length_validations.to_h
-      end
-    end
-  end
-end

data/lib/generators/aws_record/generated_attribute.rb

@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-module AwsRecord
-  module Generators
-    class GeneratedAttribute
-      OPTS = %w[hkey rkey persist_nil db_attr_name ddb_type default_value].freeze
-      INVALID_HKEY_TYPES = %i[map_attr list_attr numeric_set_attr string_set_attr].freeze
-      attr_reader :name, :type
-      attr_accessor :options
-
-      def field_type
-        case @type
-        when :integer_attr then :number_field
-        when :date_attr then :date_select
-        when :datetime_attr then :datetime_select
-        when :boolean_attr then :check_box
-        else :text_field
-        end
-      end
-
-      class << self
-        def parse(field_definition)
-          name, type, opts = field_definition.split(':')
-          type ||= 'string'
-          if OPTS.any? { |opt| type.include? opt }
-            opts = type
-            type = 'string'
-          end
-
-          opts = opts.split(',') if opts
-          type, opts = parse_type_and_options(name, type, opts)
-          validate_opt_combs(name, type, opts)
-
-          new(name, type, opts)
-        end
-
-        private
-
-        def validate_opt_combs(name, type, opts)
-          return unless opts
-
-          is_hkey = opts.key?(:hash_key)
-          is_rkey = opts.key?(:range_key)
-
-          if is_hkey && is_rkey
-            raise ArgumentError,
-                  "Field #{name} cannot be a range key and hash key simultaneously"
-          end
-          return unless is_hkey && INVALID_HKEY_TYPES.include?(type)
-
-          raise ArgumentError,
-                "Field #{name} cannot be a hash key and be of type #{type}"
-        end
-
-        def parse_type_and_options(name, type, opts)
-          opts ||= []
-          [parse_type(name, type), opts.to_h { |opt| parse_option(name, opt) }]
-        end
-
-        def parse_option(name, opt)
-          case opt
-
-          when 'hkey'
-            [:hash_key, true]
-          when 'rkey'
-            [:range_key, true]
-          when 'persist_nil'
-            [:persist_nil, true]
-          when /db_attr_name\{(\w+)\}/
-            [:database_attribute_name, "\"#{::Regexp.last_match(1)}\""]
-          when /ddb_type\{(S|N|B|BOOL|SS|NS|BS|M|L)\}/i
-            [:dynamodb_type, "\"#{::Regexp.last_match(1).upcase}\""]
-          when /default_value\{(.+)\}/
-            [:default_value, ::Regexp.last_match(1)]
-          else
-            raise ArgumentError, "You provided an invalid option for #{name}: #{opt}"
-          end
-        end
-
-        def parse_type(name, type)
-          case type.downcase
-
-          when 'bool', 'boolean'
-            :boolean_attr
-          when 'date'
-            :date_attr
-          when 'datetime'
-            :datetime_attr
-          when 'float'
-            :float_attr
-          when 'int', 'integer'
-            :integer_attr
-          when 'list'
-            :list_attr
-          when 'map'
-            :map_attr
-          when 'num_set', 'numeric_set', 'nset'
-            :numeric_set_attr
-          when 'string_set', 's_set', 'sset'
-            :string_set_attr
-          when 'string'
-            :string_attr
-          else
-            raise ArgumentError, "Invalid type for #{name}: #{type}"
-          end
-        end
-      end
-
-      def initialize(name, type = :string_attr, options = {})
-        @name = name
-        @type = type
-        @options = options
-        @digest = options.delete(:digest)
-      end
-
-      # Methods used by rails scaffolding
-      def password_digest?
-        @digest
-      end
-
-      def polymorphic?
-        false
-      end
-
-      def column_name
-        if @name == 'password_digest'
-          'password'
-        else
-          @name
-        end
-      end
-
-      def human_name
-        name.humanize
-      end
-    end
-  end
-end

data/lib/generators/aws_record/model/USAGE

@@ -1,24 +0,0 @@
-Description:
-    rails generator for aws-record models
-
-    Pass the name of the model (preferably in singular form), and an optional list of attributes
-
-    Attributes are declarations of the fields that you wish to store within a model. You can pass
-    a type and list of options for each attribtue in the form: `name:type:options` if you do not provide
-    a type, it is assumed that the attribute is of type `string_attr`
-
-    Each model should have an hkey, if one is not present a `uuid:hkey` will be created for you.
-
-    Timestamps are not added by default but you can add them using the `--timestamps` flag
-    More information can be found at: https://github.com/awslabs/aws-record-generator/blob/master/README.md
-    
-    You don't have to think up every attribute up front, but it helps to
-    sketch out a few so you can start working with the resource immediately.
-
-Example:
-    rails generate aws_record:model Forum forum_uuid:hkey post_id:rkey post_title post_body tags:sset:default_value{Set.new} created_at:datetime:d_attr_name{PostCreatedAtTime} moderation:boolean:default_value{false}
-
-    This will create:
-        app/models/forum.rb
-        db/table_config/forum_config.rb
-        lib/tasks/table_config_migrate_task.rake # This is created once the first time the generator is run

data/lib/generators/aws_record/model/model_generator.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../base'
-
-module AwsRecord
-  module Generators
-    class ModelGenerator < Base
-      def initialize(args, *options)
-        self.class.source_root File.expand_path('templates', __dir__)
-        super
-      end
-
-      def create_model
-        template 'model.erb', File.join('app/models', class_path, "#{file_name}.rb")
-      end
-
-      def create_table_config
-        return unless options['table_config']
-
-        template 'table_config.erb',
-                 File.join('db/table_config', class_path, "#{file_name}_config.rb")
-      end
-    end
-  end
-end