openstax_aws 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +6 -0
  6. data/Gemfile.lock +120 -0
  7. data/LICENSE.txt +1 -0
  8. data/README.md +927 -0
  9. data/Rakefile +6 -0
  10. data/TODO.md +1 -0
  11. data/assets/secrets_sequence_diagram.png +0 -0
  12. data/bin/console +14 -0
  13. data/bin/create_development_environment +26 -0
  14. data/bin/get_latest_ubuntu_ami +31 -0
  15. data/bin/setup +8 -0
  16. data/bin/templates/aws_ruby_development.yml +221 -0
  17. data/examples/deployment.rb +90 -0
  18. data/ideas.md +15 -0
  19. data/lib/openstax/aws/auto_scaling_group.rb +28 -0
  20. data/lib/openstax/aws/auto_scaling_instance.rb +96 -0
  21. data/lib/openstax/aws/build_image_command_1.rb +53 -0
  22. data/lib/openstax/aws/change_set.rb +100 -0
  23. data/lib/openstax/aws/deployment_base.rb +372 -0
  24. data/lib/openstax/aws/distribution.rb +56 -0
  25. data/lib/openstax/aws/ec2_instance_data.rb +18 -0
  26. data/lib/openstax/aws/extensions.rb +19 -0
  27. data/lib/openstax/aws/git_helper.rb +18 -0
  28. data/lib/openstax/aws/image.rb +34 -0
  29. data/lib/openstax/aws/msk_cluster.rb +19 -0
  30. data/lib/openstax/aws/packer_1_2_5.rb +63 -0
  31. data/lib/openstax/aws/packer_1_4_1.rb +72 -0
  32. data/lib/openstax/aws/packer_factory.rb +25 -0
  33. data/lib/openstax/aws/rds_instance.rb +25 -0
  34. data/lib/openstax/aws/s3_text_file.rb +50 -0
  35. data/lib/openstax/aws/sam_stack.rb +85 -0
  36. data/lib/openstax/aws/secrets.rb +302 -0
  37. data/lib/openstax/aws/secrets_factory.rb +126 -0
  38. data/lib/openstax/aws/secrets_set.rb +21 -0
  39. data/lib/openstax/aws/secrets_specification.rb +68 -0
  40. data/lib/openstax/aws/stack.rb +465 -0
  41. data/lib/openstax/aws/stack_event.rb +28 -0
  42. data/lib/openstax/aws/stack_factory.rb +153 -0
  43. data/lib/openstax/aws/stack_parameters.rb +19 -0
  44. data/lib/openstax/aws/stack_status.rb +125 -0
  45. data/lib/openstax/aws/system.rb +21 -0
  46. data/lib/openstax/aws/tag.rb +31 -0
  47. data/lib/openstax/aws/template.rb +129 -0
  48. data/lib/openstax/aws/version.rb +5 -0
  49. data/lib/openstax/aws/wait_message.rb +20 -0
  50. data/lib/openstax_aws.rb +154 -0
  51. data/openstax_aws.gemspec +58 -0
  52. metadata +350 -0
@@ -0,0 +1,96 @@
1
+ module OpenStax::Aws
2
+ class AutoScalingInstance
3
+
4
+ attr_reader :raw
5
+
6
+ delegate_missing_to :@raw
7
+
8
+ def initialize(group_name:, id:, region:)
9
+ @raw = Aws::AutoScaling::Instance.new(
10
+ group_name,
11
+ id,
12
+ client: Aws::AutoScaling::Client.new(region: region)
13
+ )
14
+ end
15
+
16
+ def self.me
17
+ instance_id = Ec2InstanceData.instance_id
18
+ region = Ec2InstanceData.region
19
+
20
+ client = Aws::AutoScaling::Client.new(region: region)
21
+ instance_info = client.describe_auto_scaling_instances({instance_ids: [instance_id]})
22
+ .auto_scaling_instances[0]
23
+
24
+ new(
25
+ group_name: instance_info.auto_scaling_group_name,
26
+ id: instance_id,
27
+ region: region
28
+ )
29
+ end
30
+
31
+ def terminate(options = {})
32
+ hook_name = options.delete(:continue_hook_name)
33
+ raw.terminate(options)
34
+ if hook_name
35
+ sleep(terminate_wait_sleep_seconds) until terminating_wait?
36
+ continue_to_termination(hook_name: hook_name)
37
+ end
38
+ end
39
+
40
+ def unless_waiting_for_termination(hook_name:)
41
+ # "Terminating" is a transition state to "Terminating:Wait", but we don't
42
+ # check for it because if we try to continue from "Terminating", AWS freaks
43
+ # out because it needs to continue from the wait state
44
+
45
+ if terminating_wait?
46
+ continue_to_termination(hook_name: hook_name)
47
+ return
48
+ end
49
+
50
+ yield
51
+
52
+ # In case the yield takes a long time and this code isn't called
53
+ # again for a while (e.g. an infrequent cron job), check the terminating
54
+ # state again. If this method is called in a loop, the check here
55
+ # and the next check at the start of this method will not cause duplicate
56
+ # network calls because the lifecycle state is cached for a few seconds.
57
+
58
+ if terminating_wait?
59
+ continue_to_termination(hook_name: hook_name)
60
+ end
61
+ end
62
+
63
+ def recent_lifecycle_state
64
+ if @recent_lifecycle_state_last_refreshed_at.nil? ||
65
+ Time.now - @recent_lifecycle_state_last_refreshed_at > lifecycle_state_refresh_seconds
66
+ reload
67
+ @recent_lifecycle_state_last_refreshed_at = Time.now
68
+ @recent_lifecycle_state = lifecycle_state
69
+ else
70
+ @recent_lifecycle_state
71
+ end
72
+ end
73
+
74
+ def lifecycle_state_refresh_seconds
75
+ 5
76
+ end
77
+
78
+ def terminate_wait_sleep_seconds
79
+ 6
80
+ end
81
+
82
+ def terminating_wait?
83
+ "Terminating:Wait" == recent_lifecycle_state
84
+ end
85
+
86
+ def continue_to_termination(hook_name:)
87
+ raw.client.complete_lifecycle_action({
88
+ lifecycle_hook_name: hook_name,
89
+ auto_scaling_group_name: raw.group_name,
90
+ lifecycle_action_result: "CONTINUE",
91
+ instance_id: raw.id,
92
+ })
93
+ end
94
+
95
+ end
96
+ end
@@ -0,0 +1,53 @@
1
+ module OpenStax
2
+ module Aws
3
+ class BuildImageCommand1
4
+
5
+ # A standardized way to use Packer to build images.
6
+
7
+ def initialize(ami_name_base:, region:,
8
+ verbose: false, debug: false,
9
+ github_org:, repo:, branch: nil, sha: nil,
10
+ packer_absolute_file_path: , playbook_absolute_file_path:,
11
+ dry_run: true)
12
+ if sha.nil?
13
+ branch ||= 'master'
14
+
15
+ sha = OpenStax::Aws::GitHelper.sha_for_branch_name(
16
+ org_slash_repo: "#{github_org}/#{repo}",
17
+ branch: branch
18
+ )
19
+ end
20
+
21
+ ami_name = "#{ami_name_base}@#{sha[0..6]} #{Time.now.utc.strftime("%y%m%d%H%MZ")}"
22
+
23
+ @packer = OpenStax::Aws::PackerFactory.new_packer(
24
+ absolute_file_path: packer_absolute_file_path,
25
+ dry_run: dry_run
26
+ )
27
+
28
+ @packer.only("amazon-ebs")
29
+
30
+ @packer.var("region", region)
31
+ @packer.var("ami_name", ami_name)
32
+ @packer.var("sha", sha)
33
+ @packer.var("playbook_file", playbook_absolute_file_path)
34
+ @packer.var("ami_description", {
35
+ sha: sha,
36
+ github_org: github_org,
37
+ repo: repo
38
+ }.to_json)
39
+
40
+ @packer.verbose! if verbose
41
+ @packer.debug! if debug
42
+ end
43
+
44
+ def run
45
+ @packer.run
46
+ end
47
+
48
+ def to_s
49
+ @packer.to_s
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,100 @@
1
+ module OpenStax::Aws
2
+ class ChangeSet
3
+
4
+ attr_reader :client
5
+
6
+ def initialize(client:)
7
+ @client = client
8
+ end
9
+
10
+ def create(options:)
11
+ create_change_set_output = client.create_change_set(options)
12
+ @id = create_change_set_output.id
13
+
14
+ wait_message = OpenStax::Aws::WaitMessage.new(
15
+ message: "Waiting for change set #{id} to be ready"
16
+ )
17
+
18
+ begin
19
+ client.wait_until(:change_set_create_complete, change_set_name: id) do |w|
20
+ w.delay = OpenStax::Aws.configuration.stack_waiter_delay
21
+ w.before_attempt do |attempts, response|
22
+ wait_message.say_it
23
+ end
24
+ end
25
+ rescue Aws::Waiters::Errors::FailureStateError => ee
26
+ if ee.response&.status_reason =~ /didn't contain changes/
27
+ logger.info("No changes detected, deleting change set")
28
+ delete
29
+ return self
30
+ else
31
+ logger.error(ee.response.status_reason)
32
+ raise
33
+ end
34
+ rescue Aws::Waiters::Errors::WaiterFailed => ee
35
+ logger.error("An error occurred: #{ee.message}")
36
+ raise
37
+ end
38
+
39
+ @description = client.describe_change_set(change_set_name: id)
40
+
41
+ self
42
+ end
43
+
44
+ def created?
45
+ @description.present?
46
+ end
47
+
48
+ def delete
49
+ client.delete_change_set(change_set_name: id)
50
+ end
51
+
52
+ def execute
53
+ client.execute_change_set(change_set_name: id)
54
+ end
55
+
56
+ def id
57
+ @id || raise("ID isn't yet known!")
58
+ end
59
+
60
+ def description
61
+ @description || raise("Description not set; create failed?")
62
+ end
63
+
64
+ def has_change_caused_by?(entity_name)
65
+ description.changes.any? do |change|
66
+ change.resource_change.details.any? do |detail|
67
+ detail.causing_entity == entity_name
68
+ end
69
+ end
70
+ end
71
+
72
+ def parameter_value(parameter_name)
73
+ description.parameters.select do |parameter|
74
+ parameter.parameter_key == parameter_name
75
+ end.first.parameter_value
76
+ end
77
+
78
+ def resource_change_summaries
79
+ summaries = description.changes.flat_map(&:resource_change).map do |change|
80
+ summary = "#{change.action} '#{change.logical_resource_id}' (#{change.resource_type})"
81
+
82
+ case change.action
83
+ when "Modify"
84
+ causes = change.details.map{|detail| [detail.change_source, detail.causing_entity].compact.join(":")}.join(", ")
85
+
86
+ summary = "#{summary}: Replacement=#{change.replacement}; Due to change in #{change.scope}; Causes: #{causes}"
87
+ end
88
+
89
+ summary
90
+ end
91
+ end
92
+
93
+ protected
94
+
95
+ def logger
96
+ OpenStax::Aws.configuration.logger
97
+ end
98
+
99
+ end
100
+ end
@@ -0,0 +1,372 @@
1
+ module OpenStax::Aws
2
+ class DeploymentBase
3
+ class Status
4
+ def initialize(deployment)
5
+ @deployment = deployment
6
+ end
7
+
8
+ def stack_statuses(reload: false)
9
+ @deployment.stacks.each_with_object({}) do |stack, hash|
10
+ hash[stack.name] = stack.status(reload: reload)
11
+ end
12
+ end
13
+
14
+ def failed?(reload: false)
15
+ stack_statuses(reload: reload).values.any?(&:failed?)
16
+ end
17
+
18
+ def succeeded?(reload: false)
19
+ stack_statuses(reload: reload).values.all?(&:succeeded?)
20
+ end
21
+
22
+ def to_h
23
+ {
24
+ stacks: stack_statuses.each_with_object({}) do |(stack_name, stack_status), new_hash|
25
+ new_hash[stack_name] = stack_status.to_h # convert the stack status object to a hash
26
+ end
27
+ }
28
+ end
29
+
30
+ def to_json
31
+ to_h.to_json
32
+ end
33
+ end
34
+
35
+ attr_reader :env_name, :region, :name, :dry_run
36
+
37
+ RESERVED_ENV_NAMES = [
38
+ "external", # used to namespace external secrets in the parameter store
39
+ ]
40
+
41
+ def initialize(env_name: nil, region:, name:, dry_run: true)
42
+ if RESERVED_ENV_NAMES.include?(env_name)
43
+ raise "#{env_name} is a reserved word and cannot be used as an environment name"
44
+ end
45
+
46
+ # Allow a blank env_name but normalize it to `nil`
47
+ @env_name = env_name.blank? ? nil : env_name
48
+
49
+ if @env_name && !@env_name.match(/^[a-zA-Z][a-zA-Z0-9-]*$/)
50
+ raise "The environment name must consist only of letters, numbers, and hyphens, " \
51
+ "and must start with a letter."
52
+ end
53
+
54
+ @region = region
55
+ @name = name
56
+ @dry_run = dry_run
57
+ end
58
+
59
+ def name!
60
+ raise "`name` is blank" if name.blank?
61
+ name
62
+ end
63
+
64
+ def env_name!
65
+ raise "`env_name` is blank" if env_name.blank?
66
+ env_name
67
+ end
68
+
69
+ def tags
70
+ self.class.tags.each_with_object(HashWithIndifferentAccess.new) do |(key, value), hsh|
71
+ hsh[key] = value.is_a?(Proc) ? instance_eval(&value) : value
72
+ end
73
+ end
74
+
75
+ class << self
76
+
77
+ def template_directory(*directory_parts)
78
+ if method_defined?("template_directory")
79
+ raise "Can only set template_directory once per class definition"
80
+ end
81
+
82
+ define_method "template_directory" do
83
+ File.join(*directory_parts)
84
+ end
85
+ end
86
+
87
+ def sam_build_directory(*directory_parts)
88
+ if method_defined?("sam_build_directory")
89
+ raise "Can only set buisam_build_directoryld_directory once per class definition"
90
+ end
91
+
92
+ define_method "sam_build_directory" do
93
+ File.expand_path(File.join(*directory_parts))
94
+ end
95
+ end
96
+
97
+ def secrets(id, &block)
98
+ if id.blank?
99
+ raise "The first argument to `secrets` must be a non-blank ID"
100
+ end
101
+
102
+ if !id.to_s.match(/^[a-zA-Z][a-zA-Z0-9_]*$/)
103
+ raise "The first argument to `secrets` must consist only of letters, numbers, and underscores, " \
104
+ "and must start with a letter."
105
+ end
106
+
107
+ if method_defined?("#{id}_secrets")
108
+ raise "Can only define the `#{id}` secrets once per class definition"
109
+ end
110
+
111
+ if method_defined?("#{id}_stack")
112
+ raise "Cannot define `#{id}` secrets because there is a stack with that ID"
113
+ end
114
+
115
+ define_method("#{id}_secrets") do |for_create_or_update: false|
116
+ secrets_factory = SecretsFactory.new(
117
+ region: region,
118
+ namespace: [env_name, name],
119
+ context: self,
120
+ dry_run: dry_run,
121
+ for_create_or_update: for_create_or_update,
122
+ shared_substitutions_block: @shared_secrets_substitutions_block
123
+ )
124
+
125
+ secrets_factory.namespace(id)
126
+ secrets_factory.instance_exec({}, &block)
127
+ secrets_factory.instance
128
+ end
129
+ end
130
+
131
+ def stack_ids
132
+ @stack_ids ||= []
133
+ end
134
+
135
+ def stack(id, &block)
136
+ if id.blank?
137
+ raise "The first argument to `stack` must be a non-blank ID"
138
+ end
139
+
140
+ if !id.to_s.match(/^[a-zA-Z][a-zA-Z0-9_]*$/)
141
+ raise "The first argument to `stack` must consist only of letters, numbers, and underscores, " \
142
+ "and must start with a letter."
143
+ end
144
+
145
+ if method_defined?("#{id}_secrets")
146
+ raise "Cannot define `#{id}` stack because there are secrets with that ID"
147
+ end
148
+
149
+ stack_ids.push(id)
150
+
151
+ define_method("#{id}_stack") do
152
+ instance_variable_get("@#{id}_stack") || begin
153
+ stack_factory = StackFactory.new(id: id, deployment: self)
154
+ stack_factory.instance_eval(&block) if block_given?
155
+
156
+ # Fill in missing attributes using deployment variables and conventions
157
+
158
+ if stack_factory.name.blank?
159
+ stack_factory.name([env_name,name,id].compact.join("-").gsub("_","-"))
160
+ end
161
+
162
+ if stack_factory.region.blank?
163
+ stack_factory.region(region)
164
+ end
165
+
166
+ if stack_factory.dry_run.nil?
167
+ stack_factory.dry_run(dry_run)
168
+ end
169
+
170
+ if stack_factory.enable_termination_protection.nil?
171
+ stack_factory.enable_termination_protection(is_production?)
172
+ end
173
+
174
+ if stack_factory.absolute_template_path.blank?
175
+ stack_factory.autoset_absolute_template_path(respond_to?(:template_directory) ? template_directory : "")
176
+ end
177
+
178
+ # Populate parameter defaults that match convention names
179
+
180
+ if OpenStax::Aws.configuration.infer_parameter_defaults
181
+ defaults = parameter_defaults_from_template(stack_factory.absolute_template_path)
182
+ defaults.each{|key,value| stack_factory.parameter_defaults[key] ||= value}
183
+ end
184
+
185
+ stack_factory.build.tap do |stack|
186
+ instance_variable_set("@#{id}_stack", stack)
187
+ end
188
+ end
189
+ end
190
+ end
191
+
192
+ def secrets_substitutions(&block)
193
+ define_method("shared_secrets_substitutions_block") do
194
+ block
195
+ end
196
+ end
197
+
198
+ def tag(key, value=nil, &block)
199
+ raise 'The first argument to `tag` must not be blank' if key.blank?
200
+ raise '`tag` must be given a value or a block' if value.nil? && !block_given?
201
+ tags[key] = block_given? ? block : value
202
+ end
203
+
204
+ def tags
205
+ @tags ||= HashWithIndifferentAccess.new
206
+ end
207
+
208
+ def inherited(child_class)
209
+ # Copy any tags defined in the parent to the child so that it can access them
210
+ # while not using class variables that are shared across all classes in the
211
+ # that inherit here
212
+ child_class.instance_variable_set("@tags", tags.dup)
213
+ end
214
+
215
+ def logger
216
+ OpenStax::Aws.configuration.logger
217
+ end
218
+ end
219
+
220
+ def parameter_defaults_from_template(template_or_absolute_template_path)
221
+ template = template_or_absolute_template_path.is_a?(String) ?
222
+ Template.from_absolute_file_path(template_or_absolute_template_path) :
223
+ template_or_absolute_template_path
224
+
225
+ template.parameter_names.each_with_object({}) do |parameter_name, defaults|
226
+ value = parameter_default(parameter_name) ||
227
+ built_in_parameter_default(parameter_name)
228
+
229
+ if !value.nil?
230
+ defaults[parameter_name.underscore.to_sym] = value
231
+ end
232
+ end
233
+ end
234
+
235
+ def stacks
236
+ self.class.stack_ids.map{|id| self.send("#{id}_stack")}
237
+ end
238
+
239
+ def status(reload: false)
240
+ @status = nil if reload
241
+ @status ||= Status.new(self)
242
+ end
243
+
244
+ def deployed_parameters
245
+ stacks.each_with_object({}) do |stack, hash|
246
+ hash[stack.name] = stack.deployed_parameters
247
+ end
248
+ end
249
+
250
+ def built_in_parameter_default(parameter_name)
251
+ case parameter_name
252
+ when "EnvName"
253
+ env_name
254
+ when /(.+)StackName$/
255
+ send("#{$1}Stack".underscore).name rescue nil
256
+ end
257
+ end
258
+
259
+ def shared_secrets_substitutions_block
260
+ nil # can be overridden by the DSL
261
+ end
262
+
263
+ protected
264
+
265
+ def parameter_default(parameter_name)
266
+ nil
267
+ end
268
+
269
+ def is_production?
270
+ env_name == OpenStax::Aws.configuration.production_env_name
271
+ end
272
+
273
+ def subdomain_with_trailing_dot(site_name:)
274
+ parts = []
275
+ parts.push(site_name) if !site_name.blank?
276
+ parts.push(env_name!) unless is_production?
277
+
278
+ subdomain = parts.join("-")
279
+ subdomain.blank? ? "" : subdomain + "."
280
+ end
281
+
282
+ delegate :logger, to: :configuration
283
+
284
+ def configuration
285
+ OpenStax::Aws.configuration
286
+ end
287
+
288
+ def client
289
+ @client ||= ::Aws::CloudFormation::Client.new(region: region)
290
+ end
291
+
292
+ def auto_scaling_client
293
+ @auto_scaling_client ||= ::Aws::AutoScaling::Client.new(region: region)
294
+ end
295
+
296
+ def set_desired_capacity(asg_name:, desired_capacity:)
297
+ auto_scaling_client.set_desired_capacity(auto_scaling_group_name: asg_name, desired_capacity: desired_capacity)
298
+ end
299
+
300
+ def auto_scaling_group(name:)
301
+ ::Aws::AutoScaling::AutoScalingGroup.new(name: name, client: auto_scaling_client)
302
+ end
303
+
304
+ def cloudfront_client
305
+ @cloudfront_client ||= ::Aws::CloudFront::Client.new(region: region)
306
+ end
307
+
308
+ def s3_client
309
+ @s3_client ||= Aws::S3::Client.new(region: region)
310
+ end
311
+
312
+ def s3_key_exists?(bucket:, key:)
313
+ begin
314
+ s3_client.get_object(bucket: bucket, key: key)
315
+ true
316
+ rescue Aws::S3::Errors::NoSuchKey
317
+ false
318
+ end
319
+ end
320
+
321
+ def wait_for_tag_change(resource:, key:, polling_seconds: 20, timeout_seconds: nil)
322
+ keep_polling = true
323
+ started_at = Time.now
324
+ original_value = resource_tag_value(resource: resource, key: key)
325
+
326
+ wait_message = OpenStax::Aws::WaitMessage.new(
327
+ message: "Waiting for the #{key} tag on #{resource.name} to change from #{original_value}"
328
+ )
329
+
330
+ while keep_polling
331
+ wait_message.say_it
332
+
333
+ sleep(polling_seconds)
334
+
335
+ keep_polling = false if resource_tag_value(resource: resource, key: key) != original_value
336
+ keep_polling = false if !timeout_seconds.nil? && Time.now - timeout_seconds > started_at
337
+ end
338
+ end
339
+
340
+ def resource_tag_value(resource:, key:)
341
+ begin
342
+ resource.tag(key).value
343
+ rescue NoMethodError => ee
344
+ nil
345
+ end
346
+ end
347
+
348
+ def get_image_tag(image_id:, key:)
349
+ tag = image(image_id).tags.find{|tag| tag.key == key}
350
+ raise "No tag with key #{key} on AMI #{image_id}" if tag.nil?
351
+ tag.value
352
+ end
353
+
354
+ def image(image_id)
355
+ Aws::EC2::Image.new(image_id, region: region)
356
+ end
357
+
358
+ # Returns the SHA on an AMI
359
+ def image_sha(image_id)
360
+ get_image_tag(image_id: image_id, key: "sha")
361
+ end
362
+
363
+ protected
364
+
365
+ def secrets_namespace(id: 'default')
366
+ raise "Override this method in your deployment class and provide a namespace " \
367
+ "for secrets data in the AWS Parameter Store. The key there will be this namespace " \
368
+ "prefixed by the environment name and suffixed with the secret name."
369
+ end
370
+
371
+ end
372
+ end