aws-sdk-rails 2.0.1 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module Rails
5
+ module SqsActiveJob
6
+
7
+ class JobRunner
8
+ attr_reader :id, :class_name
9
+
10
+ def initialize(message)
11
+ @job_data = Aws::Json.load(message.data.body)
12
+ @class_name = @job_data['job_class'].constantize
13
+ @id = @job_data['job_id']
14
+ end
15
+
16
+ def run
17
+ ActiveJob::Base.execute @job_data
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-sqs'
4
+ require 'optparse'
5
+ require 'concurrent'
6
+
7
+ module Aws
8
+ module Rails
9
+ module SqsActiveJob
10
+
11
+ class Interrupt < Exception; end
12
+
13
+ # CLI runner for polling for SQS ActiveJobs
14
+ # Use `aws_sqs_active_job --help` for detailed usage
15
+ class Poller
16
+
17
+ DEFAULT_OPTS = {
18
+ threads: Concurrent.processor_count,
19
+ max_messages: 10,
20
+ visibility_timeout: 60,
21
+ shutdown_timeout: 15,
22
+ }
23
+
24
+ def initialize(args = ARGV)
25
+ @options = parse_args(args)
26
+ # Set_environment must be run before we boot_rails
27
+ set_environment
28
+ end
29
+
30
+ def set_environment
31
+ @environment = @options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
32
+ end
33
+
34
+ def run
35
+ # exit 0
36
+ boot_rails
37
+
38
+ # cannot load config (from file or initializers) until after
39
+ # rails has been booted.
40
+ @options = DEFAULT_OPTS
41
+ .merge(Aws::Rails::SqsActiveJob.config.to_h)
42
+ .merge(@options.to_h)
43
+ validate_config
44
+ # ensure we have a logger configured
45
+ @logger = @options[:logger] || ActiveSupport::Logger.new(STDOUT)
46
+ @logger.info("Starting Poller with options=#{@options}")
47
+
48
+
49
+ Signal.trap('INT') { raise Interrupt }
50
+ Signal.trap('TERM') { raise Interrupt }
51
+ @executor = Executor.new(max_threads: @options[:threads], logger: @logger, max_queue: @options[:backpressure])
52
+
53
+ poll
54
+ rescue Interrupt
55
+ @logger.info 'Process Interrupted or killed - attempting to shutdown cleanly.'
56
+ shutdown
57
+ exit
58
+ end
59
+
60
+ private
61
+
62
+ def shutdown
63
+ @executor.shutdown(@options[:shutdown_timeout])
64
+ end
65
+
66
+ def poll
67
+ queue_url = Aws::Rails::SqsActiveJob.config.queue_url_for(@options[:queue])
68
+ @logger.info "Polling on: #{@options[:queue]} => #{queue_url}"
69
+ client = Aws::Rails::SqsActiveJob.config.client
70
+ @poller = Aws::SQS::QueuePoller.new(queue_url, client: client)
71
+ single_message = @options[:max_messages] == 1
72
+ poller_options = {
73
+ skip_delete: true,
74
+ max_number_of_messages: @options[:max_messages],
75
+ visibility_timeout: @options[:visibility_timeout]
76
+ }
77
+ @poller.poll(poller_options) do |msgs|
78
+ msgs = [msgs] if single_message
79
+ @logger.info "Processing batch of #{msgs.length} messages"
80
+ msgs.each do |msg|
81
+ @executor.execute(Aws::SQS::Message.new(
82
+ queue_url: queue_url,
83
+ receipt_handle: msg.receipt_handle,
84
+ data: msg,
85
+ client: client
86
+ ))
87
+ end
88
+ end
89
+ end
90
+
91
+ def boot_rails
92
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = @environment
93
+ require "rails"
94
+ require File.expand_path("config/environment.rb")
95
+ end
96
+
97
+ def parse_args(argv)
98
+ out = {}
99
+ parser = ::OptionParser.new { |opts|
100
+ opts.on("-q", "--queue STRING", "[Required] Queue to poll") { |a| out[:queue] = a }
101
+ opts.on("-e", "--environment STRING", "Rails environment (defaults to development). You can also use the APP_ENV or RAILS_ENV environment variables to specify the environment.") { |a| out[:environment] = a }
102
+ opts.on("-t", "--threads INTEGER", Integer, "The maximum number of worker threads to create. Defaults to the number of processors available on this system.") { |a| out[:threads] = a }
103
+ opts.on("-b", "--backpressure INTEGER", Integer, "The maximum number of messages to have waiting in the Executor queue. This should be a low, but non zero number. Messages in the Executor queue cannot be picked up by other processes and will slow down shutdown.") { |a| out[:backpressure] = a }
104
+ opts.on("-m", "--max_messages INTEGER", Integer, "Max number of messages to receive in a batch from SQS.") { |a| out[:max_messages] = a }
105
+ opts.on("-v", "--visibility_timeout INTEGER", Integer, "The visibility timeout is the number of seconds that a message will not be processable by any other consumers. You should set this value to be longer than your expected job runtime to prevent other processes from picking up an running job. See the SQS Visibility Timeout Documentation at https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html.") { |a| out[:visibility_timeout] = a }
106
+ opts.on("-s", "--shutdown_timeout INTEGER", Integer, "The amount of time to wait for a clean shutdown. Jobs that are unable to complete in this time will not be deleted from the SQS queue and will be retryable after the visibility timeout.") { |a| out[:shutdown_timeout] = a }
107
+ }
108
+
109
+ parser.banner = "aws_sqs_active_job [options]"
110
+ parser.on_tail "-h", "--help", "Show help" do
111
+ puts parser
112
+ exit 1
113
+ end
114
+
115
+ parser.parse(argv)
116
+ out
117
+ end
118
+
119
+ def validate_config
120
+ raise ArgumentError, 'You must specify the name of the queue to process jobs from' unless @options[:queue]
121
+ end
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,217 @@
1
+ require 'rails/generators'
2
+ require_relative 'generated_attribute'
3
+ require_relative 'secondary_index'
4
+
5
+ module AwsRecord
6
+ module Generators
7
+ class Base < Rails::Generators::NamedBase
8
+ argument :attributes, type: :array, default: [], banner: "field[:type][:opts]...", desc: "Describes the fields in the model"
9
+ check_class_collision
10
+
11
+ class_option :disable_mutation_tracking, type: :boolean, desc: "Disables dirty tracking"
12
+ class_option :timestamps, type: :boolean, desc: "Adds created, updated timestamps to the model"
13
+ class_option :table_config, type: :hash, default: {}, banner: "primary:R-W [SecondaryIndex1:R-W]...", desc: "Declares the r/w units for the model as well as any secondary indexes", :required => true
14
+ class_option :gsi, type: :array, default: [], banner: "name:hkey{field_name}[,rkey{field_name},proj_type{ALL|KEYS_ONLY|INCLUDE}]...", desc: "Allows for the declaration of secondary indexes"
15
+ class_option :table_name, type: :string, banner: "model_table_name"
16
+ class_option :password_digest, type: :boolean, desc: "Whether to add a password_digest field to the model"
17
+
18
+ class_option :required, type: :string, banner: "field1...", desc: "A list of attributes that are required for an instance of the model"
19
+ class_option :length_validations, type: :hash, default: {}, banner: "field1:MIN-MAX...", desc: "Validations on the length of attributes in a model"
20
+
21
+ attr_accessor :primary_read_units, :primary_write_units, :gsi_rw_units, :gsis, :required_attrs, :length_validations
22
+
23
+ private
24
+
25
+ def initialize(args, *options)
26
+ options[0] << "--skip-table-config" if options[1][:behavior] == :revoke
27
+ @parse_errors = []
28
+
29
+ super
30
+ ensure_unique_fields
31
+ ensure_hkey
32
+ parse_gsis!
33
+ parse_table_config!
34
+ parse_validations!
35
+
36
+ if !@parse_errors.empty?
37
+ STDERR.puts "The following errors were encountered while trying to parse the given attributes"
38
+ STDERR.puts
39
+ STDERR.puts @parse_errors
40
+ STDERR.puts
41
+
42
+ abort("Please fix the errors before proceeding.")
43
+ end
44
+ end
45
+
46
+ def parse_attributes!
47
+
48
+ self.attributes = (attributes || []).map do |attr|
49
+ begin
50
+ GeneratedAttribute.parse(attr)
51
+ rescue ArgumentError => e
52
+ @parse_errors << e
53
+ next
54
+ end
55
+ end
56
+ self.attributes = self.attributes.compact
57
+
58
+ if options['password_digest']
59
+ self.attributes << GeneratedAttribute.new("password_digest", :string_attr, :digest => true)
60
+ end
61
+
62
+ if options['timestamps']
63
+ self.attributes << GeneratedAttribute.parse("created:datetime:default_value{Time.now}")
64
+ self.attributes << GeneratedAttribute.parse("updated:datetime:default_value{Time.now}")
65
+ end
66
+ end
67
+
68
+ def ensure_unique_fields
69
+ used_names = Set.new
70
+ duplicate_fields = []
71
+
72
+ self.attributes.each do |attr|
73
+
74
+ if used_names.include? attr.name
75
+ duplicate_fields << [:attribute, attr.name]
76
+ end
77
+ used_names.add attr.name
78
+
79
+ if attr.options.key? :database_attribute_name
80
+ raw_db_attr_name = attr.options[:database_attribute_name].delete('"') # db attribute names are wrapped with " to make template generation easier
81
+
82
+ if used_names.include? raw_db_attr_name
83
+ duplicate_fields << [:database_attribute_name, raw_db_attr_name]
84
+ end
85
+
86
+ used_names.add raw_db_attr_name
87
+ end
88
+ end
89
+
90
+ if !duplicate_fields.empty?
91
+ duplicate_fields.each do |invalid_attr|
92
+ @parse_errors << ArgumentError.new("Found duplicated field name: #{invalid_attr[1]}, in attribute#{invalid_attr[0]}")
93
+ end
94
+ end
95
+ end
96
+
97
+ def ensure_hkey
98
+ uuid_member = nil
99
+ hkey_member = nil
100
+ rkey_member = nil
101
+
102
+ self.attributes.each do |attr|
103
+ if attr.options.key? :hash_key
104
+ if hkey_member
105
+ @parse_errors << ArgumentError.new("Redefinition of hash_key attr: #{attr.name}, original declaration of hash_key on: #{hkey_member.name}")
106
+ next
107
+ end
108
+
109
+ hkey_member = attr
110
+ elsif attr.options.key? :range_key
111
+ if rkey_member
112
+ @parse_errors << ArgumentError.new("Redefinition of range_key attr: #{attr.name}, original declaration of range_key on: #{hkey_member.name}")
113
+ next
114
+ end
115
+
116
+ rkey_member = attr
117
+ end
118
+
119
+ if attr.name.include? "uuid"
120
+ uuid_member = attr
121
+ end
122
+ end
123
+
124
+ if !hkey_member
125
+ if uuid_member
126
+ uuid_member.options[:hash_key] = true
127
+ else
128
+ self.attributes.unshift GeneratedAttribute.parse("uuid:hkey")
129
+ end
130
+ end
131
+ end
132
+
133
+ def mutation_tracking_disabled?
134
+ options['disable_mutation_tracking']
135
+ end
136
+
137
+ def has_validations?
138
+ !@required_attrs.empty? || !@length_validations.empty?
139
+ end
140
+
141
+ def parse_table_config!
142
+ return unless options['table_config']
143
+
144
+ @primary_read_units, @primary_write_units = parse_rw_units("primary")
145
+
146
+ @gsi_rw_units = @gsis.map { |idx|
147
+ [idx.name, parse_rw_units(idx.name)]
148
+ }.to_h
149
+
150
+ options['table_config'].each do |config, rw_units|
151
+ if config == "primary"
152
+ next
153
+ else
154
+ gsi = @gsis.select { |idx| idx.name == config}
155
+
156
+ if gsi.empty?
157
+ @parse_errors << ArgumentError.new("Could not find a gsi declaration for #{config}")
158
+ end
159
+ end
160
+ end
161
+ end
162
+
163
+ def parse_rw_units(name)
164
+ if !options['table_config'].key? name
165
+ @parse_errors << ArgumentError.new("Please provide a table_config definition for #{name}")
166
+ else
167
+ rw_units = options['table_config'][name]
168
+ return rw_units.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
169
+ end
170
+ end
171
+
172
+ def parse_gsis!
173
+ @gsis = (options['gsi'] || []).map do |raw_idx|
174
+ begin
175
+ idx = SecondaryIndex.parse(raw_idx)
176
+
177
+ attributes = self.attributes.select { |attr| attr.name == idx.hash_key}
178
+ if attributes.empty?
179
+ @parse_errors << ArgumentError.new("Could not find attribute #{idx.hash_key} for gsi #{idx.name} hkey")
180
+ next
181
+ end
182
+
183
+ if idx.range_key
184
+ attributes = self.attributes.select { |attr| attr.name == idx.range_key}
185
+ if attributes.empty?
186
+ @parse_errors << ArgumentError.new("Could not find attribute #{idx.range_key} for gsi #{idx.name} rkey")
187
+ next
188
+ end
189
+ end
190
+
191
+ idx
192
+ rescue ArgumentError => e
193
+ @parse_errors << e
194
+ next
195
+ end
196
+ end
197
+
198
+ @gsis = @gsis.compact
199
+ end
200
+
201
+ def parse_validations!
202
+ @required_attrs = options['required'] ? options['required'].split(',') : []
203
+ @required_attrs.each do |val_attr|
204
+ @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
205
+ end
206
+
207
+ @length_validations = options['length_validations'].map do |val_attr, bounds|
208
+ @parse_errors << ArgumentError.new("No such field #{val_attr} in required validations") if !self.attributes.any? { |attr| attr.name == val_attr }
209
+
210
+ bounds = bounds.gsub(/[,.-]/, ':').split(':').reject { |s| s.empty? }
211
+ [val_attr, "#{bounds[0]}..#{bounds[1]}"]
212
+ end
213
+ @length_validations = @length_validations.to_h
214
+ end
215
+ end
216
+ end
217
+ end
@@ -0,0 +1,129 @@
1
+ module AwsRecord
2
+ module Generators
3
+ class GeneratedAttribute
4
+
5
+ OPTS = %w(hkey rkey persist_nil db_attr_name ddb_type default_value)
6
+ INVALID_HKEY_TYPES = %i(map_attr list_attr numeric_set_attr string_set_attr)
7
+ attr_reader :name, :type
8
+ attr_accessor :options
9
+
10
+ def field_type
11
+ case @type
12
+ when :integer_attr then :number_field
13
+ when :date_attr then :date_select
14
+ when :datetime_attr then :datetime_select
15
+ when :boolean_attr then :check_box
16
+ else :text_field
17
+ end
18
+ end
19
+
20
+ class << self
21
+
22
+ def parse(field_definition)
23
+ name, type, opts = field_definition.split(':')
24
+ type = "string" if not type
25
+ type, opts = "string", type if OPTS.any? { |opt| type.include? opt }
26
+
27
+ opts = opts.split(',') if opts
28
+ type, opts = parse_type_and_options(name, type, opts)
29
+ validate_opt_combs(name, type, opts)
30
+
31
+ new(name, type, opts)
32
+ end
33
+
34
+ private
35
+
36
+ def validate_opt_combs(name, type, opts)
37
+ if opts
38
+ is_hkey = opts.key?(:hash_key)
39
+ is_rkey = opts.key?(:range_key)
40
+
41
+ raise ArgumentError.new("Field #{name} cannot be a range key and hash key simultaneously") if is_hkey && is_rkey
42
+ raise ArgumentError.new("Field #{name} cannot be a hash key and be of type #{type}") if is_hkey and INVALID_HKEY_TYPES.include? type
43
+ end
44
+ end
45
+
46
+ def parse_type_and_options(name, type, opts)
47
+ opts = [] if not opts
48
+ return parse_type(name, type), opts.map { |opt| parse_option(name, opt) }.to_h
49
+ end
50
+
51
+ def parse_option(name, opt)
52
+ case opt
53
+
54
+ when "hkey"
55
+ return :hash_key, true
56
+ when "rkey"
57
+ return :range_key, true
58
+ when "persist_nil"
59
+ return :persist_nil, true
60
+ when /db_attr_name\{(\w+)\}/
61
+ return :database_attribute_name, '"' + $1 + '"'
62
+ when /ddb_type\{(S|N|B|BOOL|SS|NS|BS|M|L)\}/i
63
+ return :dynamodb_type, '"' + $1.upcase + '"'
64
+ when /default_value\{(.+)\}/
65
+ return :default_value, $1
66
+ else
67
+ raise ArgumentError.new("You provided an invalid option for #{name}: #{opt}")
68
+ end
69
+ end
70
+
71
+ def parse_type(name, type)
72
+ case type.downcase
73
+
74
+ when "bool", "boolean"
75
+ :boolean_attr
76
+ when "date"
77
+ :date_attr
78
+ when "datetime"
79
+ :datetime_attr
80
+ when "float"
81
+ :float_attr
82
+ when "int", "integer"
83
+ :integer_attr
84
+ when "list"
85
+ :list_attr
86
+ when "map"
87
+ :map_attr
88
+ when "num_set", "numeric_set", "nset"
89
+ :numeric_set_attr
90
+ when "string_set", "s_set", "sset"
91
+ :string_set_attr
92
+ when "string"
93
+ :string_attr
94
+ else
95
+ raise ArgumentError.new("Invalid type for #{name}: #{type}")
96
+ end
97
+ end
98
+ end
99
+
100
+ def initialize(name, type = :string_attr, options = {})
101
+ @name = name
102
+ @type = type
103
+ @options = options
104
+ @digest = options.delete(:digest)
105
+ end
106
+
107
+ # Methods used by rails scaffolding
108
+ def password_digest?
109
+ @digest
110
+ end
111
+
112
+ def polymorphic?
113
+ false
114
+ end
115
+
116
+ def column_name
117
+ if @name == "password_digest"
118
+ "password"
119
+ else
120
+ @name
121
+ end
122
+ end
123
+
124
+ def human_name
125
+ name.humanize
126
+ end
127
+ end
128
+ end
129
+ end