aws-sdk-rails 3.2.1 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b16c1ca75c59341fdc3696d91648fda6901937f31b5b4c55b5d1ccf3bcc9af47
4
- data.tar.gz: 9c9c22d4740ea76b06fa7ee6b8d525450fc982e3c73db0fe81234356dbd71a86
3
+ metadata.gz: 2d4664fce310e44e2613d3caf795eead1a767194d48c7b407854a4a3dbe342f4
4
+ data.tar.gz: c49b2bd9ec55f6b1fc63260c0704bc3e2594d4084742a774a527cbec81dfbad9
5
5
  SHA512:
6
- metadata.gz: 2ef9e3bf5bde8d91b25635e2bed4028a163547f6e8eabb3ece2e08d1482e6edce16386a916240f344dcce27bc8efec3c5c424236fed6f150e4a6912165648914
7
- data.tar.gz: 0052b3633ce5fc655f4b6fdd80851049196e1c7c7a7d6cc92e17d52fc6fcc8ae69712bc411d3f0feb369dfd835e23912c67802ff32abe3654cf3b4feee87c02a
6
+ metadata.gz: 8b38a5870854a7be9867aeb95e1ae2f3fd0a332289ae1f8124306811e42aab5d0274c2abdb7914ff33d9ccb162c334e06034376d4c4a702ab4b39c56b2667b7e
7
+ data.tar.gz: d82eb9df09e04c9426c3191d994568f3565574c7c2f1ed24d70a0ecd8e288179fb9462480f9795409ac857a5e07839096f4b6fd02ab5545d1926e0fc10aa2964
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 3.3.0
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/aws/rails/sqs_active_job/poller'
4
+
5
+ Aws::Rails::SqsActiveJob::Poller.new.run
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+
5
+ module ActiveJob
6
+ module QueueAdapters
7
+
8
+ class AmazonSqsAdapter
9
+
10
+ def enqueue(job)
11
+ _enqueue(job)
12
+ end
13
+
14
+ def enqueue_at(job, timestamp)
15
+ delay = (timestamp - Time.now.to_f).floor
16
+ raise ArgumentError, 'Unable to queue a job with a delay great than 15 minutes' if delay > 15.minutes
17
+ _enqueue(job, delay_seconds: delay)
18
+ end
19
+
20
+ private
21
+
22
+ def _enqueue(job, send_message_opts = {})
23
+ body = job.serialize
24
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(job.queue_name)
25
+ send_message_opts[:queue_url] = queue_url
26
+ send_message_opts[:message_body] = Aws::Json.dump(body)
27
+ send_message_opts[:message_attributes] = message_attributes(job)
28
+ Aws::Rails::SqsActiveJob.config.client.send_message(send_message_opts)
29
+ end
30
+
31
+ def message_attributes(job)
32
+ {
33
+ 'aws_sqs_active_job_class' => {
34
+ string_value: job.class.to_s,
35
+ data_type: 'String'
36
+ },
37
+ 'aws_sqs_active_job_version' => {
38
+ string_value: Aws::Rails::VERSION,
39
+ data_type: 'String'
40
+ }
41
+ }
42
+ end
43
+ end
44
+
45
+ # create an alias to allow `:amazon` to be used as the adapter name
46
+ # `:amazon` is the convention used for ActionMailer and ActiveStorage
47
+ AmazonAdapter = AmazonSqsAdapter
48
+ end
49
+ end
@@ -3,5 +3,17 @@
3
3
  require_relative 'aws/rails/mailer'
4
4
  require_relative 'aws/rails/railtie'
5
5
  require_relative 'aws/rails/notifications'
6
+ require_relative 'aws/rails/sqs_active_job/configuration'
7
+ require_relative 'aws/rails/sqs_active_job/executor'
8
+ require_relative 'aws/rails/sqs_active_job/job_runner'
6
9
 
7
10
  require_relative 'action_dispatch/session/dynamodb_store'
11
+ require_relative 'active_job/queue_adapters/amazon_sqs_adapter'
12
+
13
+ require_relative 'generators/aws_record/base'
14
+
15
+ module Aws
16
+ module Rails
17
+ VERSION = File.read(File.expand_path('../VERSION', __dir__)).strip
18
+ end
19
+ end
@@ -15,6 +15,7 @@ module Aws
15
15
 
16
16
  rake_tasks do
17
17
  load 'tasks/dynamo_db/session_store.rake'
18
+ load 'tasks/aws_record/migrate.rake'
18
19
  end
19
20
  end
20
21
 
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ # @return [Configuration] the (singleton) Configuration
8
+ def self.config
9
+ @config ||= Configuration.new
10
+ end
11
+
12
+ # @yield Configuration
13
+ def self.configure
14
+ yield(config)
15
+ end
16
+
17
+ # Holds configuration for AWS SQS ActiveJob
18
+ # Use the Aws::Rails::SqsActiveJob.config to access.
19
+ class Configuration
20
+
21
+ # Default configuration options
22
+ DEFAULTS = {
23
+ max_messages: 10,
24
+ visibility_timeout: 120,
25
+ shutdown_timeout: 15,
26
+ queues: {},
27
+ logger: ::Rails.logger
28
+ }
29
+
30
+ attr_accessor :queues, :max_messages, :visibility_timeout,
31
+ :shutdown_timeout, :client, :logger
32
+
33
+ # @param [Hash] options
34
+ # @option options [Hash[Symbol, String]] :queues - A mapping between the
35
+ # active job queue name and the SQS Queue URL. Note: multiple active
36
+ # job queues can map to the same SQS Queue URL.
37
+ #
38
+ # @option options [Integer] :max_messages -
39
+ # The max number of messages to poll for in a batch.
40
+ #
41
+ # @option options [Integer] :visibility_timeout -
42
+ # The visibility timeout is the number of seconds
43
+ # that a message will not be processable by any other consumers.
44
+ # You should set this value to be longer than your expected job runtime
45
+ # to prevent other processes from picking up an running job.
46
+ # See the (SQS Visibility Timeout Documentation)[https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html]
47
+ #
48
+ # @option options [Integer] :shutdown_timeout -
49
+ # the amount of time to wait
50
+ # for a clean shutdown. Jobs that are unable to complete in this time
51
+ # will not be deleted from the SQS queue and will be retryable after
52
+ # the visibility timeout.
53
+ #
54
+ # @option options [ActiveSupport::Logger] :logger - Logger to use
55
+ # for the poller.
56
+ #
57
+ # @option options [String] :config_file -
58
+ # Override file to load configuration from. If not specified will
59
+ # attempt to load from config/aws_sqs_active_job.yml.
60
+ #
61
+ # @option options [SQS::Client] :client - SQS Client to use. A default
62
+ # client will be created if none is provided.
63
+ def initialize(options = {})
64
+ options[:config_file] ||= config_file if config_file.exist?
65
+ options = DEFAULTS
66
+ .merge(file_options(options))
67
+ .merge(options)
68
+ set_attributes(options)
69
+ end
70
+
71
+ def client
72
+ @client ||= Aws::SQS::Client.new
73
+ end
74
+
75
+ # Return the queue_url for a given job_queue name
76
+ def queue_url_for(job_queue)
77
+ job_queue = job_queue.to_sym
78
+ raise ArgumentError, "No queue defined for #{job_queue}" unless queues.key? job_queue
79
+
80
+ queues[job_queue.to_sym]
81
+ end
82
+
83
+ def to_s
84
+ to_h.to_s
85
+ end
86
+
87
+ def to_h
88
+ h = {}
89
+ self.instance_variables.each do |v|
90
+ v_sym = v.to_s.gsub('@', '').to_sym
91
+ val = self.instance_variable_get(v)
92
+ h[v_sym] = val
93
+ end
94
+ h
95
+ end
96
+
97
+ private
98
+
99
+ # Set accessible attributes after merged options.
100
+ def set_attributes(options)
101
+ options.keys.each do |opt_name|
102
+ instance_variable_set("@#{opt_name}", options[opt_name])
103
+ end
104
+ end
105
+
106
+ def file_options(options = {})
107
+ file_path = config_file_path(options)
108
+ if file_path
109
+ load_from_file(file_path)
110
+ else
111
+ {}
112
+ end
113
+ end
114
+
115
+ def config_file
116
+ file = ::Rails.root.join("config/aws_sqs_active_job/#{::Rails.env}.yml")
117
+ file = ::Rails.root.join('config/aws_sqs_active_job.yml') unless file.exist?
118
+ file
119
+ end
120
+
121
+ # Load options from YAML file
122
+ def load_from_file(file_path)
123
+ require "erb"
124
+ opts = YAML.load(ERB.new(File.read(file_path)).result) || {}
125
+ opts.deep_symbolize_keys
126
+ end
127
+
128
+ # @return [String] Configuration path found in environment or YAML file.
129
+ def config_file_path(options)
130
+ options[:config_file] || ENV["AWS_SQS_ACTIVE_JOB_CONFIG_FILE"]
131
+ end
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent'
4
+
5
+ module Aws
6
+ module Rails
7
+ module SqsActiveJob
8
+ # CLI runner for polling for SQS ActiveJobs
9
+ class Executor
10
+
11
+ DEFAULTS = {
12
+ min_threads: 0,
13
+ max_threads: Concurrent.processor_count,
14
+ auto_terminate: true,
15
+ idletime: 60, # 1 minute
16
+ max_queue: 2,
17
+ fallback_policy: :caller_runs # slow down the producer thread
18
+ }.freeze
19
+
20
+ def initialize(options = {})
21
+ @executor = Concurrent::ThreadPoolExecutor.new(DEFAULTS.merge(options))
22
+ @logger = options[:logger] || ActiveSupport::Logger.new(STDOUT)
23
+ end
24
+
25
+ # TODO: Consider catching the exception and sleeping instead of using :caller_runs
26
+ def execute(message)
27
+ @executor.post(message) do |message|
28
+ begin
29
+ job = JobRunner.new(message)
30
+ @logger.info("Running job: #{job.id}[#{job.class_name}]")
31
+ job.run
32
+ message.delete
33
+ rescue Aws::Json::ParseError => e
34
+ @logger.error "Unable to parse message body: #{message.data.body}. Error: #{e}."
35
+ rescue StandardError => e
36
+ # message will not be deleted and will be retried
37
+ job_msg = job ? "#{job.id}[#{job.class_name}]" : 'unknown job'
38
+ @logger.info "Error processing job #{job_msg}: #{e}"
39
+ @logger.debug e.backtrace.join("\n")
40
+ end
41
+ end
42
+ end
43
+
44
+ def shutdown(timeout=nil)
45
+ @executor.shutdown
46
+ clean_shutdown = @executor.wait_for_termination(timeout)
47
+ if clean_shutdown
48
+ @logger.info 'Clean shutdown complete. All executing jobs finished.'
49
+ else
50
+ @logger.info "Timeout (#{timeout}) exceeded. Some jobs may not have"\
51
+ " finished cleanly. Unfinished jobs will not be removed from"\
52
+ " the queue and can be ru-run once their visibility timeout"\
53
+ " passes."
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ class JobRunner
8
+ attr_reader :id, :class_name
9
+
10
+ def initialize(message)
11
+ @job_data = Aws::Json.load(message.data.body)
12
+ @class_name = @job_data['job_class'].constantize
13
+ @id = @job_data['job_id']
14
+ end
15
+
16
+ def run
17
+ ActiveJob::Base.execute @job_data
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+ require 'optparse'
5
+ require 'concurrent'
6
+
7
+ module Aws
8
+ module Rails
9
+ module SqsActiveJob
10
+
11
+ class Interrupt < Exception; end
12
+
13
+ # CLI runner for polling for SQS ActiveJobs
14
+ # Use `aws_sqs_active_job --help` for detailed usage
15
+ class Poller
16
+
17
+ DEFAULT_OPTS = {
18
+ threads: Concurrent.processor_count,
19
+ max_messages: 10,
20
+ visibility_timeout: 60,
21
+ shutdown_timeout: 15,
22
+ }
23
+
24
+ def initialize(args = ARGV)
25
+ @options = parse_args(args)
26
+ # Set_environment must be run before we boot_rails
27
+ set_environment
28
+ end
29
+
30
+ def set_environment
31
+ @environment = @options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
32
+ end
33
+
34
+ def run
35
+ # exit 0
36
+ boot_rails
37
+
38
+ # cannot load config (from file or initializers) until after
39
+ # rails has been booted.
40
+ @options = DEFAULT_OPTS
41
+ .merge(Aws::Rails::SqsActiveJob.config.to_h)
42
+ .merge(@options.to_h)
43
+ validate_config
44
+ # ensure we have a logger configured
45
+ @logger = @options[:logger] || ActiveSupport::Logger.new(STDOUT)
46
+ @logger.info("Starting Poller with options=#{@options}")
47
+
48
+
49
+ Signal.trap('INT') { raise Interrupt }
50
+ Signal.trap('TERM') { raise Interrupt }
51
+ @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
52
+
53
+ poll
54
+ rescue Interrupt
55
+ @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.'
56
+ shutdown
57
+ exit
58
+ end
59
+
60
+ private
61
+
62
+ def shutdown
63
+ @executor.shutdown(@options[:shutdown_timeout])
64
+ end
65
+
66
+ def poll
67
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(@options[:queue])
68
+ @logger.info "Polling on: #{@options[:queue]} => #{queue_url}"
69
+ client = Aws::Rails::SqsActiveJob.config.client
70
+ @poller = Aws::SQS::QueuePoller.new(queue_url, client: client)
71
+ single_message = @options[:max_messages] == 1
72
+ poller_options = {
73
+ skip_delete: true,
74
+ max_number_of_messages: @options[:max_messages],
75
+ visibility_timeout: @options[:visibility_timeout]
76
+ }
77
+ @poller.poll(poller_options) do |msgs|
78
+ msgs = [msgs] if single_message
79
+ @logger.info "Processing batch of #{msgs.length} messages"
80
+ msgs.each do |msg|
81
+ @executor.execute(Aws::SQS::Message.new(
82
+ queue_url: queue_url,
83
+ receipt_handle: msg.receipt_handle,
84
+ data: msg,
85
+ client: client
86
+ ))
87
+ end
88
+ end
89
+ end
90
+
91
+ def boot_rails
92
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
93
+ require "rails"
94
+ require File.expand_path("config/environment.rb")
95
+ end
96
+
97
+ def parse_args(argv)
98
+ out = {}
99
+ parser = ::OptionParser.new { |opts|
100
+ opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
101
+ opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
102
+ opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create. Defaults to the number of processors available on this system.") { |a| out[:threads] = a }
103
+ opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number. Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
104
+ opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
105
+ opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job. See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
106
+ opts.on("-s", "--shutdown_timeout INTEGER", Integer, "The amount of time to wait for a clean shutdown. Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.") { |a| out[:shutdown_timeout] = a }
107
+ }
108
+
109
+ parser.banner = "aws_sqs_active_job [options]"
110
+ parser.on_tail "-h", "--help", "Show help" do
111
+ puts parser
112
+ exit 1
113
+ end
114
+
115
+ parser.parse(argv)
116
+ out
117
+ end
118
+
119
+ def validate_config
120
+ raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,217 @@
1
+ require 'rails/generators'
2
+ require_relative 'generated_attribute'
3
+ require_relative 'secondary_index'
4
+
5
+ module AwsRecord
6
+ module Generators
7
+ class Base < Rails::Generators::NamedBase
8
+ argument :attributes, type: :array, default: [], banner: "field[:type][:opts]...", desc: "Describes the fields in the model"
9
+ check_class_collision
10
+
11
+ class_option :disable_mutation_tracking, type: :boolean, desc: "Disables dirty tracking"
12
+ class_option :timestamps, type: :boolean, desc: "Adds created, updated timestamps to the model"
13
+ class_option :table_config, type: :hash, default: {}, banner: "primary:R-W [SecondaryIndex1:R-W]...", desc: "Declares the r/w units for the model as well as any secondary indexes", :required => true
14
+ class_option :gsi, type: :array, default: [], banner: "name:hkey{field_name}[,rkey{field_name},proj_type{ALL|KEYS_ONLY|INCLUDE}]...", desc: "Allows for the declaration of secondary indexes"
15
+ class_option :table_name, type: :string, banner: "model_table_name"
16
+ class_option :password_digest, type: :boolean, desc: "Whether to add a password_digest field to the model"
17
+
18
+ class_option :required, type: :string, banner: "field1...", desc: "A list of attributes that are required for an instance of the model"
19
+ class_option :length_validations, type: :hash, default: {}, banner: "field1:MIN-MAX...", desc: "Validations on the length of attributes in a model"
20
+
21
+ attr_accessor :primary_read_units, :primary_write_units, :gsi_rw_units, :gsis, :required_attrs, :length_validations
22
+
23
+ private
24
+
25
+ def initialize(args, *options)
26
+ options[0] << "--skip-table-config" if options[1][:behavior] == :revoke
27
+ @parse_errors = []
28
+
29
+ super
30
+ ensure_unique_fields
31
+ ensure_hkey
32
+ parse_gsis!
33
+ parse_table_config!
34
+ parse_validations!
35
+
36
+ if !@parse_errors.empty?
37
+ STDERR.puts "The following errors were encountered while trying to parse the given attributes"
38
+ STDERR.puts
39
+ STDERR.puts @parse_errors
40
+ STDERR.puts
41
+
42
+ abort("Please fix the errors before proceeding.")
43
+ end
44
+ end
45
+
46
+ def parse_attributes!
47
+
48
+ self.attributes = (attributes || []).map do |attr|
49
+ begin
50
+ GeneratedAttribute.parse(attr)
51
+ rescue ArgumentError => e
52
+ @parse_errors << e
53
+ next
54
+ end
55
+ end
56
+ self.attributes = self.attributes.compact
57
+
58
+ if options['password_digest']
59
+ self.attributes << GeneratedAttribute.new("password_digest", :string_attr, :digest => true)
60
+ end
61
+
62
+ if options['timestamps']
63
+ self.attributes << GeneratedAttribute.parse("created:datetime:default_value{Time.now}")
64
+ self.attributes << GeneratedAttribute.parse("updated:datetime:default_value{Time.now}")
65
+ end
66
+ end
67
+
68
+ def ensure_unique_fields
69
+ used_names = Set.new
70
+ duplicate_fields = []
71
+
72
+ self.attributes.each do |attr|
73
+
74
+ if used_names.include? attr.name
75
+ duplicate_fields << [:attribute, attr.name]
76
+ end
77
+ used_names.add attr.name
78
+
79
+ if attr.options.key? :database_attribute_name
80
+ raw_db_attr_name = attr.options[:database_attribute_name].delete('"') # db attribute names are wrapped with " to make template generation easier
81
+
82
+ if used_names.include? raw_db_attr_name
83
+ duplicate_fields << [:database_attribute_name, raw_db_attr_name]
84
+ end
85
+
86
+ used_names.add raw_db_attr_name
87
+ end
88
+ end
89
+
90
+ if !duplicate_fields.empty?
91
+ duplicate_fields.each do |invalid_attr|
92
+ @parse_errors << ArgumentError.new("Found duplicated field name: #{invalid_attr[1]}, in attribute#{invalid_attr[0]}")
93
+ end
94
+ end
95
+ end
96
+
97
+ def ensure_hkey
98
+ uuid_member = nil
99
+ hkey_member = nil
100
+ rkey_member = nil
101
+
102
+ self.attributes.each do |attr|
103
+ if attr.options.key? :hash_key
104
+ if hkey_member
105
+ @parse_errors << ArgumentError.new("Redefinition of hash_key attr: #{attr.name}, original declaration of hash_key on: #{hkey_member.name}")
106
+ next
107
+ end
108
+
109
+ hkey_member = attr
110
+ elsif attr.options.key? :range_key
111
+ if rkey_member
112
+ @parse_errors << ArgumentError.new("Redefinition of range_key attr: #{attr.name}, original declaration of range_key on: #{hkey_member.name}")
113
+ next
114
+ end
115
+
116
+ rkey_member = attr
117
+ end
118
+
119
+ if attr.name.include? "uuid"
120
+ uuid_member = attr
121
+ end
122
+ end
123
+
124
+ if !hkey_member
125
+ if uuid_member
126
+ uuid_member.options[:hash_key] = true
127
+ else
128
+ self.attributes.unshift GeneratedAttribute.parse("uuid:hkey")
129
+ end
130
+ end
131
+ end
132
+
133
+ def mutation_tracking_disabled?
134
+ options['disable_mutation_tracking']
135
+ end
136
+
137
+ def has_validations?
138
+ !@required_attrs.empty? || !@length_validations.empty?
139
+ end
140
+
141
+ def parse_table_config!
142
+ return unless options['table_config']
143
+
144
+ @primary_read_units, @primary_write_units = parse_rw_units("primary")
145
+
146
+ @gsi_rw_units = @gsis.map { |idx|
147
+ [idx.name, parse_rw_units(idx.name)]
148
+ }.to_h
149
+
150
+ options['table_config'].each do |config, rw_units|
151
+ if config == "primary"
152
+ next
153
+ else
154
+ gsi = @gsis.select { |idx| idx.name == config}
155
+
156
+ if gsi.empty?
157
+ @parse_errors << ArgumentError.new("Could not find a gsi declaration for #{config}")
158
+ end
159
+ end
160
+ end
161
+ end
162
+
163
+ def parse_rw_units(name)
164
+ if !options['table_config'].key? name
165
+ @parse_errors << ArgumentError.new("Please provide a table_config definition for #{name}")
166
+ else
167
+ rw_units = options['table_config'][name]
168
+ return rw_units.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
169
+ end
170
+ end
171
+
172
+ def parse_gsis!
173
+ @gsis = (options['gsi'] || []).map do |raw_idx|
174
+ begin
175
+ idx = SecondaryIndex.parse(raw_idx)
176
+
177
+ attributes = self.attributes.select { |attr| attr.name == idx.hash_key}
178
+ if attributes.empty?
179
+ @parse_errors << ArgumentError.new("Could not find attribute #{idx.hash_key} for gsi #{idx.name} hkey")
180
+ next
181
+ end
182
+
183
+ if idx.range_key
184
+ attributes = self.attributes.select { |attr| attr.name == idx.range_key}
185
+ if attributes.empty?
186
+ @parse_errors << ArgumentError.new("Could not find attribute #{idx.range_key} for gsi #{idx.name} rkey")
187
+ next
188
+ end
189
+ end
190
+
191
+ idx
192
+ rescue ArgumentError => e
193
+ @parse_errors << e
194
+ next
195
+ end
196
+ end
197
+
198
+ @gsis = @gsis.compact
199
+ end
200
+
201
+ def parse_validations!
202
+ @required_attrs = options['required'] ? options['required'].split(',') : []
203
+ @required_attrs.each do |val_attr|
204
+ @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
205
+ end
206
+
207
+ @length_validations = options['length_validations'].map do |val_attr, bounds|
208
+ @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
209
+
210
+ bounds = bounds.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
211
+ [val_attr, "#{bounds[0]}..#{bounds[1]}"]
212
+ end
213
+ @length_validations = @length_validations.to_h
214
+ end
215
+ end
216
+ end
217
+ end
@@ -0,0 +1,129 @@
1
+ module AwsRecord
2
+ module Generators
3
+ class GeneratedAttribute
4
+
5
+ OPTS = %w(hkey rkey persist_nil db_attr_name ddb_type default_value)
6
+ INVALID_HKEY_TYPES = %i(map_attr list_attr numeric_set_attr string_set_attr)
7
+ attr_reader :name, :type
8
+ attr_accessor :options
9
+
10
+ def field_type
11
+ case @type
12
+ when :integer_attr then :number_field
13
+ when :date_attr then :date_select
14
+ when :datetime_attr then :datetime_select
15
+ when :boolean_attr then :check_box
16
+ else :text_field
17
+ end
18
+ end
19
+
20
+ class << self
21
+
22
+ def parse(field_definition)
23
+ name, type, opts = field_definition.split(':')
24
+ type = "string" if not type
25
+ type, opts = "string", type if OPTS.any? { |opt| type.include? opt }
26
+
27
+ opts = opts.split(',') if opts
28
+ type, opts = parse_type_and_options(name, type, opts)
29
+ validate_opt_combs(name, type, opts)
30
+
31
+ new(name, type, opts)
32
+ end
33
+
34
+ private
35
+
36
+ def validate_opt_combs(name, type, opts)
37
+ if opts
38
+ is_hkey = opts.key?(:hash_key)
39
+ is_rkey = opts.key?(:range_key)
40
+
41
+ raise ArgumentError.new("Field #{name} cannot be a range key and hash key simultaneously") if is_hkey && is_rkey
42
+ raise ArgumentError.new("Field #{name} cannot be a hash key and be of type #{type}") if is_hkey and INVALID_HKEY_TYPES.include? type
43
+ end
44
+ end
45
+
46
+ def parse_type_and_options(name, type, opts)
47
+ opts = [] if not opts
48
+ return parse_type(name, type), opts.map { |opt| parse_option(name, opt) }.to_h
49
+ end
50
+
51
+ def parse_option(name, opt)
52
+ case opt
53
+
54
+ when "hkey"
55
+ return :hash_key, true
56
+ when "rkey"
57
+ return :range_key, true
58
+ when "persist_nil"
59
+ return :persist_nil, true
60
+ when /db_attr_name\{(\w+)\}/
61
+ return :database_attribute_name, '"' + $1 + '"'
62
+ when /ddb_type\{(S|N|B|BOOL|SS|NS|BS|M|L)\}/i
63
+ return :dynamodb_type, '"' + $1.upcase + '"'
64
+ when /default_value\{(.+)\}/
65
+ return :default_value, $1
66
+ else
67
+ raise ArgumentError.new("You provided an invalid option for #{name}: #{opt}")
68
+ end
69
+ end
70
+
71
+ def parse_type(name, type)
72
+ case type.downcase
73
+
74
+ when "bool", "boolean"
75
+ :boolean_attr
76
+ when "date"
77
+ :date_attr
78
+ when "datetime"
79
+ :datetime_attr
80
+ when "float"
81
+ :float_attr
82
+ when "int", "integer"
83
+ :integer_attr
84
+ when "list"
85
+ :list_attr
86
+ when "map"
87
+ :map_attr
88
+ when "num_set", "numeric_set", "nset"
89
+ :numeric_set_attr
90
+ when "string_set", "s_set", "sset"
91
+ :string_set_attr
92
+ when "string"
93
+ :string_attr
94
+ else
95
+ raise ArgumentError.new("Invalid type for #{name}: #{type}")
96
+ end
97
+ end
98
+ end
99
+
100
+ def initialize(name, type = :string_attr, options = {})
101
+ @name = name
102
+ @type = type
103
+ @options = options
104
+ @digest = options.delete(:digest)
105
+ end
106
+
107
+ # Methods used by rails scaffolding
108
+ def password_digest?
109
+ @digest
110
+ end
111
+
112
+ def polymorphic?
113
+ false
114
+ end
115
+
116
+ def column_name
117
+ if @name == "password_digest"
118
+ "password"
119
+ else
120
+ @name
121
+ end
122
+ end
123
+
124
+ def human_name
125
+ name.humanize
126
+ end
127
+ end
128
+ end
129
+ end
@@ -0,0 +1,24 @@
1
+ Description:
2
+ rails generator for aws-record models
3
+
4
+ Pass the name of the model (preferably in singular form), and an optional list of attributes
5
+
6
+ Attributes are declarations of the fields that you wish to store within a model. You can pass
7
+ a type and list of options for each attribtue in the form: `name:type:options` if you do not provide
8
+ a type, it is assumed that the attribute is of type `string_attr`
9
+
10
+ Each model should have an hkey, if one is not present a `uuid:hkey` will be created for you.
11
+
12
+ Timestamps are not added by default but you can add them using the `--timestamps` flag
13
+ More information can be found at: https://github.com/awslabs/aws-record-generator/blob/master/README.md
14
+
15
+ You don't have to think up every attribute up front, but it helps to
16
+ sketch out a few so you can start working with the resource immediately.
17
+
18
+ Example:
19
+ rails generate aws_record:model Forum forum_uuid:hkey post_id:rkey post_title post_body tags:sset:default_value{Set.new} created_at:datetime:d_attr_name{PostCreatedAtTime} moderation:boolean:default_value{false}
20
+
21
+ This will create:
22
+ app/models/forum.rb
23
+ db/table_config/forum_config.rb
24
+ lib/tasks/table_config_migrate_task.rake # This is created once the first time the generator is run
@@ -0,0 +1,21 @@
1
+ require_relative '../base'
2
+
3
+ module AwsRecord
4
+ module Generators
5
+ class ModelGenerator < Base
6
+ def initialize(args, *options)
7
+ self.class.source_root File.expand_path('../templates', __FILE__)
8
+ super
9
+ end
10
+
11
+ def create_model
12
+ template "model.rb", File.join("app/models", class_path, "#{file_name}.rb")
13
+ end
14
+
15
+ def create_table_config
16
+ template "table_config.rb", File.join("db/table_config", class_path, "#{file_name}_config.rb") if options["table_config"]
17
+ end
18
+
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,48 @@
1
+ require 'aws-record'
2
+ <% if has_validations? -%>
3
+ require 'active_model'
4
+ <% end -%>
5
+
6
+ <% module_namespacing do -%>
7
+ class <%= class_name %>
8
+ include Aws::Record
9
+ <% if options.key? :scaffold -%>
10
+ extend ActiveModel::Naming
11
+ <% end -%>
12
+ <% if has_validations? -%>
13
+ include ActiveModel::Validations
14
+ <% end -%>
15
+ <% if options.key? :password_digest -%>
16
+ include ActiveModel::SecurePassword
17
+ <% end -%>
18
+ <% if mutation_tracking_disabled? -%>
19
+ disable_mutation_tracking
20
+ <% end -%>
21
+
22
+ <% attributes.each do |attribute| -%>
23
+ <%= attribute.type %> :<%= attribute.name %><% *opts, last_opt = attribute.options.to_a %><%= ', ' if last_opt %><% opts.each do |opt| %><%= opt[0] %>: <%= opt[1] %>, <% end %><% if last_opt %><%= last_opt[0] %>: <%= last_opt[1] %><% end %>
24
+ <% end -%>
25
+ <% gsis.each do |index| %>
26
+ global_secondary_index(
27
+ :<%= index.name %>,
28
+ hash_key: :<%= index.hash_key -%>,<%- if index.range_key %>
29
+ range_key: :<%= index.range_key -%>,<%- end %>
30
+ projection: {
31
+ projection_type: <%= index.projection_type %>
32
+ }
33
+ )
34
+ <% end -%>
35
+ <% if !required_attrs.empty? -%>
36
+ validates_presence_of <% *req, last_req = required_attrs -%><% req.each do |required_validation|-%>:<%= required_validation %>, <% end -%>:<%= last_req %>
37
+ <% end -%>
38
+ <% length_validations.each do |attribute, validation| -%>
39
+ validates_length_of :<%= attribute %>, within: <%= validation %>
40
+ <% end -%>
41
+ <% if options['table_name'] -%>
42
+ set_table_name "<%= options['table_name'] %>"
43
+ <% end -%>
44
+ <% if options.key? :password_digest -%>
45
+ has_secure_password
46
+ <% end -%>
47
+ end
48
+ <% end -%>
@@ -0,0 +1,18 @@
1
+ require 'aws-record'
2
+
3
+ module ModelTableConfig
4
+ def self.config
5
+ Aws::Record::TableConfig.define do |t|
6
+ t.model_class <%= class_name %>
7
+
8
+ t.read_capacity_units <%= primary_read_units %>
9
+ t.write_capacity_units <%= primary_write_units %>
10
+ <%- gsis.each do |index| %>
11
+ t.global_secondary_index(:<%= index.name %>) do |i|
12
+ i.read_capacity_units <%= gsi_rw_units[index.name][0] %>
13
+ i.write_capacity_units <%= gsi_rw_units[index.name][1] %>
14
+ end
15
+ <%- end -%>
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,60 @@
1
+ module AwsRecord
2
+ module Generators
3
+ class SecondaryIndex
4
+
5
+ PROJ_TYPES = %w(ALL KEYS_ONLY INCLUDE)
6
+ attr_reader :name, :hash_key, :range_key, :projection_type
7
+
8
+ class << self
9
+ def parse(key_definition)
10
+ name, index_options = key_definition.split(':')
11
+ index_options = index_options.split(',') if index_options
12
+ opts = parse_raw_options(index_options)
13
+
14
+ new(name, opts)
15
+ end
16
+
17
+ private
18
+ def parse_raw_options(raw_opts)
19
+ raw_opts = [] if not raw_opts
20
+ raw_opts.map { |opt| get_option_value(opt) }.to_h
21
+ end
22
+
23
+ def get_option_value(raw_option)
24
+ case raw_option
25
+
26
+ when /hkey\{(\w+)\}/
27
+ return :hash_key, $1
28
+ when /rkey\{(\w+)\}/
29
+ return :range_key, $1
30
+ when /proj_type\{(\w+)\}/
31
+ return :projection_type, $1
32
+ else
33
+ raise ArgumentError.new("Invalid option for secondary index #{raw_option}")
34
+ end
35
+ end
36
+ end
37
+
38
+ def initialize(name, opts)
39
+ raise ArgumentError.new("You must provide a name") if not name
40
+ raise ArgumentError.new("You must provide a hash key") if not opts[:hash_key]
41
+
42
+ if opts.key? :projection_type
43
+ raise ArgumentError.new("Invalid projection type #{opts[:projection_type]}") if not PROJ_TYPES.include? opts[:projection_type]
44
+ raise NotImplementedError.new("ALL is the only projection type currently supported") if opts[:projection_type] != "ALL"
45
+ else
46
+ opts[:projection_type] = "ALL"
47
+ end
48
+
49
+ if opts[:hash_key] == opts[:range_key]
50
+ raise ArgumentError.new("#{opts[:hash_key]} cannot be both the rkey and hkey for gsi #{name}")
51
+ end
52
+
53
+ @name = name
54
+ @hash_key = opts[:hash_key]
55
+ @range_key = opts[:range_key]
56
+ @projection_type = '"' + "#{opts[:projection_type]}" + '"'
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,12 @@
1
+ desc 'Run all table configs in table_config folder'
2
+ namespace :aws_record do
3
+ task migrate: :environment do
4
+ Dir[File.join('db', 'table_config', '**/*.rb')].each do |filename|
5
+ puts "running #{filename}"
6
+ require(File.expand_path(filename))
7
+
8
+ table_config = ModelTableConfig.config
9
+ table_config.migrate! unless table_config.compatible?
10
+ end
11
+ end
12
+ end
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aws-sdk-rails
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.1
4
+ version: 3.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-11-13 00:00:00.000000000 Z
11
+ date: 2020-12-01 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-record
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2'
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: aws-sdk-ses
15
29
  requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,20 @@ dependencies:
24
38
  - - "~>"
25
39
  - !ruby/object:Gem::Version
26
40
  version: '1'
41
+ - !ruby/object:Gem::Dependency
42
+ name: aws-sdk-sqs
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1'
27
55
  - !ruby/object:Gem::Dependency
28
56
  name: aws-sessionstore-dynamodb
29
57
  requirement: !ruby/object:Gem::Requirement
@@ -52,6 +80,20 @@ dependencies:
52
80
  - - ">="
53
81
  - !ruby/object:Gem::Version
54
82
  version: 5.2.0
83
+ - !ruby/object:Gem::Dependency
84
+ name: concurrent-ruby
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1'
55
97
  - !ruby/object:Gem::Dependency
56
98
  name: rails
57
99
  requirement: !ruby/object:Gem::Requirement
@@ -70,19 +112,35 @@ description: Integrates the AWS Ruby SDK with Ruby on Rails
70
112
  email:
71
113
  - mamuller@amazon.com
72
114
  - alexwoo@amazon.com
73
- executables: []
115
+ executables:
116
+ - aws_sqs_active_job
74
117
  extensions: []
75
118
  extra_rdoc_files: []
76
119
  files:
120
+ - VERSION
121
+ - bin/aws_sqs_active_job
77
122
  - lib/action_dispatch/session/dynamodb_store.rb
123
+ - lib/active_job/queue_adapters/amazon_sqs_adapter.rb
78
124
  - lib/aws-sdk-rails.rb
79
125
  - lib/aws/rails/mailer.rb
80
126
  - lib/aws/rails/notifications.rb
81
127
  - lib/aws/rails/railtie.rb
128
+ - lib/aws/rails/sqs_active_job/configuration.rb
129
+ - lib/aws/rails/sqs_active_job/executor.rb
130
+ - lib/aws/rails/sqs_active_job/job_runner.rb
131
+ - lib/aws/rails/sqs_active_job/poller.rb
132
+ - lib/generators/aws_record/base.rb
133
+ - lib/generators/aws_record/generated_attribute.rb
134
+ - lib/generators/aws_record/model/USAGE
135
+ - lib/generators/aws_record/model/model_generator.rb
136
+ - lib/generators/aws_record/model/templates/model.rb
137
+ - lib/generators/aws_record/model/templates/table_config.rb
138
+ - lib/generators/aws_record/secondary_index.rb
82
139
  - lib/generators/dynamo_db/session_store_migration/USAGE
83
140
  - lib/generators/dynamo_db/session_store_migration/session_store_migration_generator.rb
84
141
  - lib/generators/dynamo_db/session_store_migration/templates/dynamo_db_session_store.yml
85
142
  - lib/generators/dynamo_db/session_store_migration/templates/session_store_migration.rb
143
+ - lib/tasks/aws_record/migrate.rake
86
144
  - lib/tasks/dynamo_db/session_store.rake
87
145
  homepage: https://github.com/aws/aws-sdk-rails
88
146
  licenses: