aws-sdk-rails 3.2.1 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b16c1ca75c59341fdc3696d91648fda6901937f31b5b4c55b5d1ccf3bcc9af47
-  data.tar.gz: 9c9c22d4740ea76b06fa7ee6b8d525450fc982e3c73db0fe81234356dbd71a86
+  metadata.gz: 2d4664fce310e44e2613d3caf795eead1a767194d48c7b407854a4a3dbe342f4
+  data.tar.gz: c49b2bd9ec55f6b1fc63260c0704bc3e2594d4084742a774a527cbec81dfbad9
 SHA512:
-  metadata.gz: 2ef9e3bf5bde8d91b25635e2bed4028a163547f6e8eabb3ece2e08d1482e6edce16386a916240f344dcce27bc8efec3c5c424236fed6f150e4a6912165648914
-  data.tar.gz: 0052b3633ce5fc655f4b6fdd80851049196e1c7c7a7d6cc92e17d52fc6fcc8ae69712bc411d3f0feb369dfd835e23912c67802ff32abe3654cf3b4feee87c02a
+  metadata.gz: 8b38a5870854a7be9867aeb95e1ae2f3fd0a332289ae1f8124306811e42aab5d0274c2abdb7914ff33d9ccb162c334e06034376d4c4a702ab4b39c56b2667b7e
+  data.tar.gz: d82eb9df09e04c9426c3191d994568f3565574c7c2f1ed24d70a0ecd8e288179fb9462480f9795409ac857a5e07839096f4b6fd02ab5545d1926e0fc10aa2964

data/VERSION

@@ -0,0 +1 @@
+3.3.0

data/bin/aws_sqs_active_job

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/aws/rails/sqs_active_job/poller'
+
+Aws::Rails::SqsActiveJob::Poller.new.run

data/lib/active_job/queue_adapters/amazon_sqs_adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+module ActiveJob
+  module QueueAdapters
+
+    class AmazonSqsAdapter
+
+      def enqueue(job)
+        _enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp)
+        delay = (timestamp - Time.now.to_f).floor
+        raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
+        _enqueue(job, delay_seconds: delay)
+      end
+
+      private
+
+      def _enqueue(job, send_message_opts = {})
+        body = job.serialize
+        queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
+        send_message_opts[:queue_url] = queue_url
+        send_message_opts[:message_body] = Aws::Json.dump(body)
+        send_message_opts[:message_attributes] = message_attributes(job)
+        Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
+      end
+
+      def message_attributes(job)
+        {
+          'aws_sqs_active_job_class' => {
+            string_value: job.class.to_s,
+            data_type: 'String'
+          },
+          'aws_sqs_active_job_version' => {
+            string_value: Aws::Rails::VERSION,
+            data_type: 'String'
+          }
+        }
+      end
+    end
+
+    # create an alias to allow `:amazon` to be used as the adapter name
+    # `:amazon` is the convention used for ActionMailer and ActiveStorage
+    AmazonAdapter = AmazonSqsAdapter
+  end
+end

data/lib/aws-sdk-rails.rb

@@ -3,5 +3,17 @@
 require_relative 'aws/rails/mailer'
 require_relative 'aws/rails/railtie'
 require_relative 'aws/rails/notifications'
+require_relative 'aws/rails/sqs_active_job/configuration'
+require_relative 'aws/rails/sqs_active_job/executor'
+require_relative 'aws/rails/sqs_active_job/job_runner'
 
 require_relative 'action_dispatch/session/dynamodb_store'
+require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
+
+require_relative 'generators/aws_record/base'
+
+module Aws
+  module Rails
+    VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
+  end
+end

data/lib/aws/rails/railtie.rb

@@ -15,6 +15,7 @@ module Aws
 
       rake_tasks do
         load 'tasks/dynamo_db/session_store.rake'
+        load 'tasks/aws_record/migrate.rake'
       end
     end
 

data/lib/aws/rails/sqs_active_job/configuration.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      # @return [Configuration] the (singleton) Configuration
+      def self.config
+        @config ||= Configuration.new
+      end
+
+      # @yield Configuration
+      def self.configure
+        yield(config)
+      end
+
+      # Holds configuration for AWS SQS ActiveJob
+      # Use the Aws::Rails::SqsActiveJob.config to access.
+      class Configuration
+
+        # Default configuration options
+        DEFAULTS = {
+          max_messages:  10,
+          visibility_timeout: 120,
+          shutdown_timeout: 15,
+          queues: {},
+          logger: ::Rails.logger
+        }
+
+        attr_accessor :queues, :max_messages, :visibility_timeout,
+                      :shutdown_timeout, :client, :logger
+
+        # @param [Hash] options
+        # @option options [Hash[Symbol, String]] :queues - A mapping between the
+        #   active job queue name and the SQS Queue URL. Note: multiple active
+        #   job queues can map to the same SQS Queue URL.
+        #
+        # @option options  [Integer] :max_messages -
+        #    The max number of messages to poll for in a batch.
+        #
+        # @option options [Integer] :visibility_timeout -
+        #   The visibility timeout is the number of seconds
+        #   that a message will not be processable by any other consumers.
+        #   You should set this value to be longer than your expected job runtime
+        #   to prevent other processes from picking up an running job.
+        #   See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
+        #
+        # @option options [Integer] :shutdown_timeout -
+        #   the amount of time to wait
+        #   for a clean shutdown.  Jobs that are unable to complete in this time
+        #   will not be deleted from the SQS queue and will be retryable after
+        #   the visibility timeout.
+        #
+        # @option options [ActiveSupport::Logger] :logger - Logger to use
+        #   for the poller.
+        #
+        # @option options [String] :config_file -
+        #   Override file to load configuration from.  If not specified will
+        #   attempt to load from config/aws_sqs_active_job.yml.
+        #
+        # @option options [SQS::Client] :client - SQS Client to use.  A default
+        #   client will be created if none is provided.
+        def initialize(options = {})
+          options[:config_file] ||= config_file if config_file.exist?
+          options = DEFAULTS
+             .merge(file_options(options))
+             .merge(options)
+          set_attributes(options)
+        end
+
+        def client
+          @client ||= Aws::SQS::Client.new
+        end
+
+        # Return the queue_url for a given job_queue name
+        def queue_url_for(job_queue)
+          job_queue = job_queue.to_sym
+          raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
+
+          queues[job_queue.to_sym]
+        end
+
+        def to_s
+          to_h.to_s
+        end
+
+        def to_h
+          h = {}
+          self.instance_variables.each do |v|
+            v_sym = v.to_s.gsub('@', '').to_sym
+            val = self.instance_variable_get(v)
+            h[v_sym] = val
+          end
+          h
+        end
+
+        private
+
+        # Set accessible attributes after merged options.
+        def set_attributes(options)
+          options.keys.each do |opt_name|
+            instance_variable_set("@#{opt_name}", options[opt_name])
+          end
+        end
+
+        def file_options(options = {})
+          file_path = config_file_path(options)
+          if file_path
+            load_from_file(file_path)
+          else
+            {}
+          end
+        end
+
+        def config_file
+          file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
+          file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
+          file
+        end
+
+        # Load options from YAML file
+        def load_from_file(file_path)
+          require "erb"
+          opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
+          opts.deep_symbolize_keys
+        end
+
+        # @return [String] Configuration path found in environment or YAML file.
+        def config_file_path(options)
+          options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/executor.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Aws
+  module Rails
+    module SqsActiveJob
+      # CLI runner for polling for SQS ActiveJobs
+      class Executor
+
+        DEFAULTS = {
+           min_threads:     0,
+           max_threads:     Concurrent.processor_count,
+           auto_terminate:  true,
+           idletime:        60, # 1 minute
+           max_queue:       2,
+           fallback_policy: :caller_runs # slow down the producer thread
+        }.freeze
+
+        def initialize(options = {})
+          @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
+          @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
+        end
+
+        # TODO: Consider catching the exception and sleeping instead of using :caller_runs
+        def execute(message)
+          @executor.post(message) do |message|
+            begin
+              job = JobRunner.new(message)
+              @logger.info("Running job: #{job.id}[#{job.class_name}]")
+              job.run
+              message.delete
+            rescue Aws::Json::ParseError => e
+              @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
+            rescue StandardError => e
+              # message will not be deleted and will be retried
+              job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
+              @logger.info "Error processing job #{job_msg}: #{e}"
+              @logger.debug e.backtrace.join("\n")
+            end
+          end
+        end
+
+        def shutdown(timeout=nil)
+          @executor.shutdown
+          clean_shutdown = @executor.wait_for_termination(timeout)
+          if clean_shutdown
+            @logger.info 'Clean shutdown complete.  All executing jobs finished.'
+          else
+            @logger.info "Timeout (#{timeout}) exceeded.  Some jobs may not have"\
+              " finished cleanly.  Unfinished jobs will not be removed from"\
+              " the queue and can be ru-run once their visibility timeout"\
+              " passes."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/job_runner.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      class JobRunner
+        attr_reader :id, :class_name
+
+        def initialize(message)
+          @job_data = Aws::Json.load(message.data.body)
+          @class_name = @job_data['job_class'].constantize
+          @id = @job_data['job_id']
+        end
+
+        def run
+          ActiveJob::Base.execute @job_data
+        end
+      end
+    end
+  end
+end

data/lib/aws/rails/sqs_active_job/poller.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'optparse'
+require 'concurrent'
+
+module Aws
+  module Rails
+    module SqsActiveJob
+
+      class Interrupt < Exception; end
+
+      # CLI runner for polling for SQS ActiveJobs
+      # Use `aws_sqs_active_job --help` for detailed usage
+      class Poller
+
+        DEFAULT_OPTS = {
+          threads: Concurrent.processor_count,
+          max_messages: 10,
+          visibility_timeout: 60,
+          shutdown_timeout: 15,
+        }
+
+        def initialize(args = ARGV)
+          @options = parse_args(args)
+          # Set_environment must be run before we boot_rails
+          set_environment
+        end
+
+        def set_environment
+          @environment = @options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+        end
+
+        def run
+          # exit 0
+          boot_rails
+
+          # cannot load config (from file or initializers) until after
+          # rails has been booted.
+          @options = DEFAULT_OPTS
+                     .merge(Aws::Rails::SqsActiveJob.config.to_h)
+                     .merge(@options.to_h)
+          validate_config
+          # ensure we have a logger configured
+          @logger = @options[:logger] || ActiveSupport::Logger.new(STDOUT)
+          @logger.info("Starting Poller with options=#{@options}")
+
+
+          Signal.trap('INT') { raise Interrupt }
+          Signal.trap('TERM') { raise Interrupt }
+          @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
+
+          poll
+        rescue Interrupt
+          @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.'
+          shutdown
+          exit
+        end
+
+        private
+
+        def shutdown
+          @executor.shutdown(@options[:shutdown_timeout])
+        end
+
+        def poll
+          queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(@options[:queue])
+          @logger.info "Polling on: #{@options[:queue]} => #{queue_url}"
+          client = Aws::Rails::SqsActiveJob.config.client
+          @poller = Aws::SQS::QueuePoller.new(queue_url, client: client)
+          single_message = @options[:max_messages] == 1
+          poller_options = {
+            skip_delete: true,
+            max_number_of_messages: @options[:max_messages],
+            visibility_timeout: @options[:visibility_timeout]
+          }
+          @poller.poll(poller_options) do |msgs|
+            msgs = [msgs] if single_message
+            @logger.info "Processing batch of #{msgs.length} messages"
+            msgs.each do |msg|
+              @executor.execute(Aws::SQS::Message.new(
+                queue_url: queue_url,
+                receipt_handle: msg.receipt_handle,
+                data: msg,
+                client: client
+              ))
+            end
+          end
+        end
+
+        def boot_rails
+          ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
+          require "rails"
+          require File.expand_path("config/environment.rb")
+        end
+
+        def parse_args(argv)
+          out = {}
+          parser = ::OptionParser.new { |opts|
+            opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
+            opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
+            opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create.  Defaults to the number of processors available on this system.") { |a| out[:threads] = a }
+            opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number.  Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
+            opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
+            opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job.  See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
+            opts.on("-s", "--shutdown_timeout INTEGER", Integer, "The amount of time to wait for a clean shutdown.  Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.") { |a| out[:shutdown_timeout] = a }
+          }
+
+          parser.banner = "aws_sqs_active_job [options]"
+          parser.on_tail "-h", "--help", "Show help" do
+            puts parser
+            exit 1
+          end
+
+          parser.parse(argv)
+          out
+        end
+
+        def validate_config
+          raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
+        end
+      end
+    end
+  end
+end

data/lib/generators/aws_record/base.rb

@@ -0,0 +1,217 @@
+require 'rails/generators'
+require_relative 'generated_attribute'
+require_relative 'secondary_index'
+
+module AwsRecord
+    module Generators
+      class Base < Rails::Generators::NamedBase
+        argument :attributes, type: :array, default: [], banner: "field[:type][:opts]...", desc: "Describes the fields in the model"
+        check_class_collision
+
+        class_option :disable_mutation_tracking, type: :boolean, desc: "Disables dirty tracking"
+        class_option :timestamps, type: :boolean, desc: "Adds created, updated timestamps to the model"
+        class_option :table_config, type: :hash, default: {}, banner: "primary:R-W [SecondaryIndex1:R-W]...", desc: "Declares the r/w units for the model as well as any secondary indexes", :required => true
+        class_option :gsi, type: :array, default: [], banner: "name:hkey{field_name}[,rkey{field_name},proj_type{ALL|KEYS_ONLY|INCLUDE}]...", desc: "Allows for the declaration of secondary indexes"
+        class_option :table_name, type: :string, banner: "model_table_name"
+        class_option :password_digest, type: :boolean, desc: "Whether to add a password_digest field to the model"
+
+        class_option :required, type: :string, banner: "field1...", desc: "A list of attributes that are required for an instance of the model"
+        class_option :length_validations, type: :hash, default: {}, banner: "field1:MIN-MAX...", desc: "Validations on the length of attributes in a model"
+
+        attr_accessor :primary_read_units, :primary_write_units, :gsi_rw_units, :gsis, :required_attrs, :length_validations
+
+        private
+
+        def initialize(args, *options)
+          options[0] << "--skip-table-config" if options[1][:behavior] == :revoke
+          @parse_errors = []
+
+          super
+          ensure_unique_fields
+          ensure_hkey
+          parse_gsis!
+          parse_table_config!
+          parse_validations!
+
+          if !@parse_errors.empty?
+            STDERR.puts "The following errors were encountered while trying to parse the given attributes"
+            STDERR.puts
+            STDERR.puts @parse_errors
+            STDERR.puts
+
+            abort("Please fix the errors before proceeding.")
+          end
+        end
+
+        def parse_attributes!
+
+          self.attributes = (attributes || []).map do |attr|
+            begin
+              GeneratedAttribute.parse(attr)
+            rescue ArgumentError => e
+              @parse_errors << e
+              next
+            end
+          end
+          self.attributes = self.attributes.compact
+
+          if options['password_digest']
+            self.attributes << GeneratedAttribute.new("password_digest", :string_attr, :digest => true)
+          end
+
+          if options['timestamps']
+            self.attributes << GeneratedAttribute.parse("created:datetime:default_value{Time.now}")
+            self.attributes << GeneratedAttribute.parse("updated:datetime:default_value{Time.now}")
+          end
+        end
+
+        def ensure_unique_fields
+          used_names = Set.new
+          duplicate_fields = []
+
+          self.attributes.each do |attr|
+
+            if used_names.include? attr.name
+              duplicate_fields << [:attribute, attr.name]
+            end
+            used_names.add attr.name
+
+            if attr.options.key? :database_attribute_name
+              raw_db_attr_name = attr.options[:database_attribute_name].delete('"') # db attribute names are wrapped with " to make template generation easier
+
+              if used_names.include? raw_db_attr_name
+                duplicate_fields << [:database_attribute_name, raw_db_attr_name]
+              end
+
+              used_names.add raw_db_attr_name
+            end
+          end
+
+          if !duplicate_fields.empty?
+            duplicate_fields.each do |invalid_attr|
+              @parse_errors << ArgumentError.new("Found duplicated field name: #{invalid_attr[1]}, in attribute#{invalid_attr[0]}")
+            end
+          end
+        end
+
+        def ensure_hkey
+          uuid_member = nil
+          hkey_member = nil
+          rkey_member = nil
+
+          self.attributes.each do |attr|
+            if attr.options.key? :hash_key
+              if hkey_member
+                @parse_errors << ArgumentError.new("Redefinition of hash_key attr: #{attr.name}, original declaration of hash_key on: #{hkey_member.name}")
+                next
+              end
+
+              hkey_member = attr
+            elsif attr.options.key? :range_key
+              if rkey_member
+                @parse_errors << ArgumentError.new("Redefinition of range_key attr: #{attr.name}, original declaration of range_key on: #{hkey_member.name}")
+                next
+              end
+
+              rkey_member = attr
+            end
+
+            if attr.name.include? "uuid"
+              uuid_member = attr
+            end
+          end
+
+          if !hkey_member
+            if uuid_member
+              uuid_member.options[:hash_key] = true
+            else
+              self.attributes.unshift GeneratedAttribute.parse("uuid:hkey")
+            end
+          end
+        end
+
+        def mutation_tracking_disabled?
+          options['disable_mutation_tracking']
+        end
+
+        def has_validations?
+          !@required_attrs.empty? || !@length_validations.empty?
+        end
+
+        def parse_table_config!
+          return unless options['table_config']
+
+          @primary_read_units, @primary_write_units = parse_rw_units("primary")
+
+          @gsi_rw_units = @gsis.map { |idx|
+            [idx.name, parse_rw_units(idx.name)]
+          }.to_h
+
+          options['table_config'].each do |config, rw_units|
+            if config == "primary"
+              next
+            else
+              gsi = @gsis.select { |idx| idx.name == config}
+
+              if gsi.empty?
+                @parse_errors << ArgumentError.new("Could not find a gsi declaration for #{config}")
+              end
+            end
+          end
+        end
+
+        def parse_rw_units(name)
+          if !options['table_config'].key? name
+            @parse_errors << ArgumentError.new("Please provide a table_config definition for #{name}")
+          else
+            rw_units = options['table_config'][name]
+            return rw_units.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
+          end
+        end
+
+        def parse_gsis!
+          @gsis = (options['gsi'] || []).map do |raw_idx|
+            begin
+              idx = SecondaryIndex.parse(raw_idx)
+
+              attributes = self.attributes.select { |attr| attr.name == idx.hash_key}
+              if attributes.empty?
+                @parse_errors << ArgumentError.new("Could not find attribute #{idx.hash_key} for gsi #{idx.name} hkey")
+                next
+              end
+
+              if idx.range_key
+                attributes = self.attributes.select { |attr| attr.name == idx.range_key}
+                if attributes.empty?
+                  @parse_errors << ArgumentError.new("Could not find attribute #{idx.range_key} for gsi #{idx.name} rkey")
+                  next
+                end
+              end
+
+              idx
+            rescue ArgumentError => e
+              @parse_errors << e
+              next
+            end
+          end
+
+          @gsis = @gsis.compact
+        end
+
+        def parse_validations!
+          @required_attrs = options['required'] ? options['required'].split(',') : []
+          @required_attrs.each do |val_attr|
+            @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
+          end
+
+          @length_validations = options['length_validations'].map do |val_attr, bounds|
+            @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
+
+            bounds = bounds.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
+            [val_attr, "#{bounds[0]}..#{bounds[1]}"]
+          end
+          @length_validations = @length_validations.to_h
+        end
+      end
+    end
+  end

data/lib/generators/aws_record/generated_attribute.rb

@@ -0,0 +1,129 @@
+module AwsRecord
+  module Generators
+    class GeneratedAttribute
+
+      OPTS = %w(hkey rkey persist_nil db_attr_name ddb_type default_value)
+      INVALID_HKEY_TYPES = %i(map_attr list_attr numeric_set_attr string_set_attr)
+      attr_reader :name, :type
+      attr_accessor :options
+
+      def field_type
+        case @type
+          when :integer_attr then :number_field
+          when :date_attr then :date_select
+          when :datetime_attr then :datetime_select
+          when :boolean_attr then :check_box
+          else :text_field
+        end
+      end
+
+      class << self
+
+        def parse(field_definition)
+          name, type, opts = field_definition.split(':')
+          type = "string" if not type
+          type, opts = "string", type if OPTS.any? { |opt| type.include? opt }
+
+          opts = opts.split(',') if opts
+          type, opts = parse_type_and_options(name, type, opts)
+          validate_opt_combs(name, type, opts)
+
+          new(name, type, opts)
+        end
+
+        private
+
+        def validate_opt_combs(name, type, opts)
+          if opts
+              is_hkey = opts.key?(:hash_key)
+              is_rkey = opts.key?(:range_key)
+
+              raise ArgumentError.new("Field #{name} cannot be a range key and hash key simultaneously") if is_hkey && is_rkey
+              raise ArgumentError.new("Field #{name} cannot be a hash key and be of type #{type}") if is_hkey and INVALID_HKEY_TYPES.include? type
+          end
+        end
+
+        def parse_type_and_options(name, type, opts)
+          opts = [] if not opts
+          return parse_type(name, type), opts.map { |opt| parse_option(name, opt) }.to_h
+        end
+
+        def parse_option(name, opt)
+          case opt
+
+          when "hkey"
+            return :hash_key, true
+          when "rkey"
+            return :range_key, true
+          when "persist_nil"
+            return :persist_nil, true
+          when /db_attr_name\{(\w+)\}/
+            return :database_attribute_name, '"' + $1 + '"'
+          when /ddb_type\{(S|N|B|BOOL|SS|NS|BS|M|L)\}/i
+            return :dynamodb_type, '"' + $1.upcase + '"'
+          when /default_value\{(.+)\}/
+            return :default_value, $1
+          else
+            raise ArgumentError.new("You provided an invalid option for #{name}: #{opt}")
+          end
+        end
+
+        def parse_type(name, type)
+          case type.downcase
+
+          when "bool", "boolean"
+            :boolean_attr
+          when "date"
+            :date_attr
+          when "datetime"
+            :datetime_attr
+          when "float"
+            :float_attr
+          when "int", "integer"
+            :integer_attr
+          when "list"
+            :list_attr
+          when "map"
+            :map_attr
+          when "num_set", "numeric_set", "nset"
+            :numeric_set_attr
+          when "string_set", "s_set", "sset"
+            :string_set_attr
+          when "string"
+            :string_attr
+          else
+            raise ArgumentError.new("Invalid type for #{name}: #{type}")
+          end
+        end
+      end
+
+      def initialize(name, type = :string_attr, options = {})
+        @name = name
+        @type = type
+        @options = options
+        @digest = options.delete(:digest)
+      end
+
+      # Methods used by rails scaffolding
+      def password_digest?
+        @digest
+      end
+
+      def polymorphic?
+        false
+      end
+
+      def column_name
+        if @name == "password_digest"
+          "password"
+        else
+          @name
+        end
+      end
+
+      def human_name
+        name.humanize
+      end
+    end
+  end
+end

data/lib/generators/aws_record/model/USAGE

@@ -0,0 +1,24 @@
+Description:
+    rails generator for aws-record models
+
+    Pass the name of the model (preferably in singular form), and an optional list of attributes
+
+    Attributes are declarations of the fields that you wish to store within a model. You can pass
+    a type and list of options for each attribtue in the form: `name:type:options` if you do not provide
+    a type, it is assumed that the attribute is of type `string_attr`
+
+    Each model should have an hkey, if one is not present a `uuid:hkey` will be created for you.
+
+    Timestamps are not added by default but you can add them using the `--timestamps` flag
+    More information can be found at: https://github.com/awslabs/aws-record-generator/blob/master/README.md
+    
+    You don't have to think up every attribute up front, but it helps to
+    sketch out a few so you can start working with the resource immediately.
+
+Example:
+    rails generate aws_record:model Forum forum_uuid:hkey post_id:rkey post_title post_body tags:sset:default_value{Set.new} created_at:datetime:d_attr_name{PostCreatedAtTime} moderation:boolean:default_value{false}
+
+    This will create:
+        app/models/forum.rb
+        db/table_config/forum_config.rb
+        lib/tasks/table_config_migrate_task.rake # This is created once the first time the generator is run

data/lib/generators/aws_record/model/model_generator.rb

@@ -0,0 +1,21 @@
+require_relative '../base'
+
+module AwsRecord
+  module Generators
+    class ModelGenerator < Base
+      def initialize(args, *options)
+        self.class.source_root File.expand_path('../templates', __FILE__)
+        super
+      end
+
+      def create_model
+        template "model.rb", File.join("app/models", class_path, "#{file_name}.rb")
+      end
+
+      def create_table_config
+        template "table_config.rb", File.join("db/table_config", class_path, "#{file_name}_config.rb") if options["table_config"]
+      end
+
+    end
+  end
+end

data/lib/generators/aws_record/model/templates/model.rb

@@ -0,0 +1,48 @@
+require 'aws-record'
+<% if has_validations? -%>
+require 'active_model'
+<% end -%>
+
+<% module_namespacing do -%>
+class <%= class_name %>
+  include Aws::Record
+<% if options.key? :scaffold -%>
+  extend ActiveModel::Naming
+<% end -%>
+<% if has_validations? -%>
+  include ActiveModel::Validations
+<% end -%>
+<% if options.key? :password_digest -%>
+  include ActiveModel::SecurePassword
+<% end -%>
+<% if mutation_tracking_disabled? -%>
+  disable_mutation_tracking
+<% end -%>
+
+<% attributes.each do |attribute| -%>
+  <%= attribute.type %> :<%= attribute.name %><% *opts, last_opt = attribute.options.to_a %><%= ', ' if last_opt %><% opts.each do |opt| %><%= opt[0] %>: <%= opt[1] %>, <% end %><% if last_opt %><%= last_opt[0] %>: <%= last_opt[1] %><% end %>
+<% end -%>
+<% gsis.each do |index| %>
+  global_secondary_index(
+    :<%= index.name %>,
+    hash_key: :<%= index.hash_key -%>,<%- if index.range_key %>
+    range_key: :<%= index.range_key -%>,<%- end %>
+    projection: {
+      projection_type: <%= index.projection_type %>
+    }
+  )
+<% end -%>
+<% if !required_attrs.empty? -%>
+  validates_presence_of <% *req, last_req = required_attrs -%><% req.each do |required_validation|-%>:<%= required_validation %>, <% end -%>:<%= last_req %>
+<% end -%>
+<% length_validations.each do |attribute, validation| -%>
+  validates_length_of :<%= attribute %>, within: <%= validation %>
+<% end -%>
+<% if options['table_name'] -%>
+  set_table_name "<%= options['table_name'] %>"
+<% end -%>
+<% if options.key? :password_digest -%>
+  has_secure_password
+<% end -%>
+end
+<% end -%>

data/lib/generators/aws_record/model/templates/table_config.rb

@@ -0,0 +1,18 @@
+require 'aws-record'
+
+module ModelTableConfig
+  def self.config
+    Aws::Record::TableConfig.define do |t|
+      t.model_class <%= class_name %>
+
+      t.read_capacity_units <%= primary_read_units %>
+      t.write_capacity_units <%= primary_write_units %>
+      <%- gsis.each do |index| %>
+      t.global_secondary_index(:<%= index.name %>) do |i|
+        i.read_capacity_units <%= gsi_rw_units[index.name][0] %>
+        i.write_capacity_units <%= gsi_rw_units[index.name][1] %>
+      end
+      <%- end -%>
+    end
+  end
+end

data/lib/generators/aws_record/secondary_index.rb

@@ -0,0 +1,60 @@
+module AwsRecord
+  module Generators
+    class SecondaryIndex
+
+      PROJ_TYPES = %w(ALL KEYS_ONLY INCLUDE)
+      attr_reader :name, :hash_key, :range_key, :projection_type
+
+      class << self
+        def parse(key_definition)
+          name, index_options = key_definition.split(':')
+          index_options = index_options.split(',') if index_options
+          opts = parse_raw_options(index_options)
+
+          new(name, opts)
+        end
+
+        private
+          def parse_raw_options(raw_opts)
+            raw_opts = [] if not raw_opts
+            raw_opts.map { |opt| get_option_value(opt) }.to_h
+          end
+
+          def get_option_value(raw_option)
+            case raw_option
+
+            when /hkey\{(\w+)\}/
+              return :hash_key, $1
+            when /rkey\{(\w+)\}/
+              return :range_key, $1
+            when /proj_type\{(\w+)\}/
+              return :projection_type, $1
+            else
+              raise ArgumentError.new("Invalid option for secondary index #{raw_option}")
+            end
+          end
+      end
+
+      def initialize(name, opts)
+        raise ArgumentError.new("You must provide a name") if not name
+        raise ArgumentError.new("You must provide a hash key") if not opts[:hash_key]
+
+        if opts.key? :projection_type
+          raise ArgumentError.new("Invalid projection type #{opts[:projection_type]}") if not PROJ_TYPES.include? opts[:projection_type]
+          raise NotImplementedError.new("ALL is the only projection type currently supported") if opts[:projection_type] != "ALL"
+        else
+          opts[:projection_type] = "ALL"
+        end
+
+        if opts[:hash_key] == opts[:range_key]
+          raise ArgumentError.new("#{opts[:hash_key]} cannot be both the rkey and hkey for gsi #{name}")
+        end
+
+        @name = name
+        @hash_key = opts[:hash_key]
+        @range_key = opts[:range_key]
+        @projection_type = '"' + "#{opts[:projection_type]}" + '"'
+      end
+    end
+  end
+end

data/lib/tasks/aws_record/migrate.rake

@@ -0,0 +1,12 @@
+desc 'Run all table configs in table_config folder'
+namespace :aws_record do
+  task migrate: :environment do
+    Dir[File.join('db', 'table_config', '**/*.rb')].each do |filename|
+      puts "running #{filename}"
+      require(File.expand_path(filename))
+
+      table_config = ModelTableConfig.config
+      table_config.migrate! unless table_config.compatible?
+    end
+  end
+end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-rails
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.3.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-13 00:00:00.000000000 Z
+date: 2020-12-01 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-record
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ses
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: aws-sessionstore-dynamodb
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.2.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -70,19 +112,35 @@ description: Integrates the AWS Ruby SDK with Ruby on Rails
 email:
 - mamuller@amazon.com
 - alexwoo@amazon.com
-executables: []
+executables:
+- aws_sqs_active_job
 extensions: []
 extra_rdoc_files: []
 files:
+- VERSION
+- bin/aws_sqs_active_job
 - lib/action_dispatch/session/dynamodb_store.rb
+- lib/active_job/queue_adapters/amazon_sqs_adapter.rb
 - lib/aws-sdk-rails.rb
 - lib/aws/rails/mailer.rb
 - lib/aws/rails/notifications.rb
 - lib/aws/rails/railtie.rb
+- lib/aws/rails/sqs_active_job/configuration.rb
+- lib/aws/rails/sqs_active_job/executor.rb
+- lib/aws/rails/sqs_active_job/job_runner.rb
+- lib/aws/rails/sqs_active_job/poller.rb
+- lib/generators/aws_record/base.rb
+- lib/generators/aws_record/generated_attribute.rb
+- lib/generators/aws_record/model/USAGE
+- lib/generators/aws_record/model/model_generator.rb
+- lib/generators/aws_record/model/templates/model.rb
+- lib/generators/aws_record/model/templates/table_config.rb
+- lib/generators/aws_record/secondary_index.rb
 - lib/generators/dynamo_db/session_store_migration/USAGE
 - lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb
 - lib/generators/dynamo_db/session_store_migration/templates/dynamo_db_session_store.yml
 - lib/generators/dynamo_db/session_store_migration/templates/session_store_migration.rb
+- lib/tasks/aws_record/migrate.rake
 - lib/tasks/dynamo_db/session_store.rake
 homepage: https://github.com/aws/aws-sdk-rails
 licenses: