openstax_aws 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +19 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG.md +11 -0
  5. data/Gemfile +6 -0
  6. data/Gemfile.lock +120 -0
  7. data/LICENSE.txt +1 -0
  8. data/README.md +927 -0
  9. data/Rakefile +6 -0
  10. data/TODO.md +1 -0
  11. data/assets/secrets_sequence_diagram.png +0 -0
  12. data/bin/console +14 -0
  13. data/bin/create_development_environment +26 -0
  14. data/bin/get_latest_ubuntu_ami +31 -0
  15. data/bin/setup +8 -0
  16. data/bin/templates/aws_ruby_development.yml +221 -0
  17. data/examples/deployment.rb +90 -0
  18. data/ideas.md +15 -0
  19. data/lib/openstax/aws/auto_scaling_group.rb +28 -0
  20. data/lib/openstax/aws/auto_scaling_instance.rb +96 -0
  21. data/lib/openstax/aws/build_image_command_1.rb +53 -0
  22. data/lib/openstax/aws/change_set.rb +100 -0
  23. data/lib/openstax/aws/deployment_base.rb +372 -0
  24. data/lib/openstax/aws/distribution.rb +56 -0
  25. data/lib/openstax/aws/ec2_instance_data.rb +18 -0
  26. data/lib/openstax/aws/extensions.rb +19 -0
  27. data/lib/openstax/aws/git_helper.rb +18 -0
  28. data/lib/openstax/aws/image.rb +34 -0
  29. data/lib/openstax/aws/msk_cluster.rb +19 -0
  30. data/lib/openstax/aws/packer_1_2_5.rb +63 -0
  31. data/lib/openstax/aws/packer_1_4_1.rb +72 -0
  32. data/lib/openstax/aws/packer_factory.rb +25 -0
  33. data/lib/openstax/aws/rds_instance.rb +25 -0
  34. data/lib/openstax/aws/s3_text_file.rb +50 -0
  35. data/lib/openstax/aws/sam_stack.rb +85 -0
  36. data/lib/openstax/aws/secrets.rb +302 -0
  37. data/lib/openstax/aws/secrets_factory.rb +126 -0
  38. data/lib/openstax/aws/secrets_set.rb +21 -0
  39. data/lib/openstax/aws/secrets_specification.rb +68 -0
  40. data/lib/openstax/aws/stack.rb +465 -0
  41. data/lib/openstax/aws/stack_event.rb +28 -0
  42. data/lib/openstax/aws/stack_factory.rb +153 -0
  43. data/lib/openstax/aws/stack_parameters.rb +19 -0
  44. data/lib/openstax/aws/stack_status.rb +125 -0
  45. data/lib/openstax/aws/system.rb +21 -0
  46. data/lib/openstax/aws/tag.rb +31 -0
  47. data/lib/openstax/aws/template.rb +129 -0
  48. data/lib/openstax/aws/version.rb +5 -0
  49. data/lib/openstax/aws/wait_message.rb +20 -0
  50. data/lib/openstax_aws.rb +154 -0
  51. data/openstax_aws.gemspec +58 -0
  52. metadata +350 -0
@@ -0,0 +1,28 @@
1
+
2
+ require_relative 'stack_status'
3
+
4
+ module OpenStax::Aws
5
+ class Stack
6
+ class Event
7
+ def initialize(aws_stack_event)
8
+ @aws_stack_event = aws_stack_event
9
+ end
10
+
11
+ def status_text
12
+ @aws_stack_event.data.resource_status
13
+ end
14
+
15
+ def status_reason
16
+ @aws_stack_event.data.resource_status_reason
17
+ end
18
+
19
+ def failed?
20
+ Status.failure_status_texts.include?(status_text)
21
+ end
22
+
23
+ def user_initiated?
24
+ status_reason == "User Initiated"
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,153 @@
1
+ module OpenStax::Aws
2
+ class StackFactory
3
+ attr_reader :attributes, :id
4
+
5
+ def initialize(id:, deployment:)
6
+ raise "`deployment` cannot be nil" if deployment.nil?
7
+
8
+ @id = id
9
+ @deployment = deployment
10
+ @attributes = {
11
+ parameter_defaults: {},
12
+ tags: {}
13
+ }
14
+ end
15
+
16
+ def self.build(id:, deployment:, &block)
17
+ factory = new(id: id, deployment: deployment)
18
+ factory.instance_eval(&block) if block_given?
19
+ factory.build
20
+ end
21
+
22
+ def method_missing(name, *args, &block)
23
+ if args.empty? && !block_given?
24
+ attributes[name.to_sym]
25
+ else
26
+ attributes[name.to_sym] = args.empty? ?
27
+ @deployment.instance_eval(&block) :
28
+ args[0]
29
+ end
30
+ end
31
+
32
+ def template_directory(*directory_parts)
33
+ if directory_parts.empty? && !block_given?
34
+ attributes[:template_directory]
35
+ else
36
+ directory_parts = yield if directory_parts.empty?
37
+ attributes[:template_directory] = File.join(*directory_parts)
38
+ end
39
+ end
40
+
41
+ def autoset_absolute_template_path(fallback_base_directory)
42
+ base_directory = template_directory || fallback_base_directory
43
+
44
+ if base_directory.blank?
45
+ raise "Tried to autoset the absolute_template_path but didn't have " \
46
+ "access to a base directory (e.g. set with template_directory)"
47
+ end
48
+
49
+ if relative_template_path
50
+ path = File.join(base_directory, relative_template_path)
51
+ else
52
+ path = File.join(base_directory, "#{@id}.yml")
53
+
54
+ if !File.file?(path)
55
+ path = File.join(base_directory, "#{@id}.json")
56
+ if !File.file?(path)
57
+ raise "Couldn't infer an existing template file for stack #{@id}"
58
+ end
59
+ end
60
+ end
61
+
62
+ self.absolute_template_path(path)
63
+ end
64
+
65
+ def parameter_defaults(&block)
66
+ factory = ParameterDefaultsFactory.new(@deployment)
67
+ factory.instance_eval(&block) if block_given?
68
+ attributes[:parameter_defaults].merge!(factory.attributes)
69
+ end
70
+
71
+ def volatile_parameters(&block)
72
+ attributes[:volatile_parameters_block] = block
73
+ end
74
+
75
+ def secrets(&block)
76
+ attributes[:secrets_blocks] ||= []
77
+ attributes[:secrets_blocks].push(block)
78
+ end
79
+
80
+ def cycle_if_different_parameter(name=nil, &block)
81
+ attributes[:cycle_if_different_parameter] = block.present? ? block.call : name
82
+ end
83
+
84
+ def tag(key, value)
85
+ raise 'The first argument to `tag` must not be blank' if key.blank?
86
+ (attributes[:tags] ||= {})[key] = value
87
+ end
88
+
89
+ def build
90
+ autoset_absolute_template_path(nil) if absolute_template_path.blank?
91
+
92
+ initializer_args = {
93
+ id: id,
94
+ name: attributes[:name],
95
+ tags: @deployment.tags.merge(attributes[:tags]),
96
+ region: attributes[:region],
97
+ enable_termination_protection: attributes[:enable_termination_protection],
98
+ absolute_template_path: attributes[:absolute_template_path],
99
+ capabilities: attributes[:capabilities],
100
+ parameter_defaults: attributes[:parameter_defaults],
101
+ volatile_parameters_block: attributes[:volatile_parameters_block],
102
+ secrets_blocks: attributes[:secrets_blocks],
103
+ secrets_context: @deployment,
104
+ secrets_namespace: [@deployment.env_name, @deployment.name],
105
+ shared_secrets_substitutions_block: @deployment.shared_secrets_substitutions_block,
106
+ cycle_if_different_parameter: attributes[:cycle_if_different_parameter],
107
+ dry_run: attributes[:dry_run]
108
+ }
109
+
110
+ template = Template.from_absolute_file_path(absolute_template_path)
111
+ if template.is_sam?
112
+ if !@deployment.respond_to?(:sam_build_directory)
113
+ raise "You must set the SAM build directory with a call to `sam_build_directory`"
114
+ end
115
+
116
+ initializer_args[:build_directory] = @deployment.sam_build_directory
117
+ SamStack.new(**initializer_args)
118
+ else
119
+ Stack.new(**initializer_args)
120
+ end
121
+ end
122
+
123
+ class ParameterDefaultsFactory
124
+ attr_reader :attributes
125
+
126
+ def initialize(context)
127
+ @context = context
128
+ @attributes = {}
129
+ end
130
+
131
+ def method_missing(name, *args, &block)
132
+ attributes[name.to_sym] = args[0] || @context.instance_eval(&block)
133
+ end
134
+ end
135
+
136
+ class VolatileParametersFactory
137
+ attr_reader :attributes
138
+
139
+ def initialize(context)
140
+ raise "context cannot be nil" if context.nil?
141
+ @context = context
142
+ @attributes = {}
143
+ end
144
+
145
+ def method_missing(name, *args, &block)
146
+ raise "Volatile parameter `#{name}` cannot be called with arguments, only a block" if !args.empty?
147
+ raise "Volatile parameter `#{name}` must be called with a block to set the parameter value" if !block_given?
148
+ attributes[name.to_sym] = @context.instance_eval(&block)
149
+ end
150
+ end
151
+
152
+ end
153
+ end
@@ -0,0 +1,19 @@
1
+ module OpenStax::Aws
2
+ class StackParameters
3
+
4
+ def initialize(stack:, params:, recover_previous_values: true)
5
+ @stack = stack
6
+ @raw_params = params
7
+ @recover_previous_values = recover_previous_values
8
+ end
9
+
10
+ def [](key)
11
+ if @recover_previous_values && @raw_params[key] == :use_previous_value
12
+ @stack.deployed_parameters[key]
13
+ else
14
+ @raw_params[key]
15
+ end
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,125 @@
1
+ module OpenStax::Aws
2
+ class Stack
3
+ class Status
4
+ def initialize(stack)
5
+ @stack = stack
6
+ end
7
+
8
+ def status_text
9
+ begin
10
+ @stack.aws_stack.stack_status
11
+ rescue Aws::CloudFormation::Errors::ValidationError => ee
12
+ case ee.message
13
+ when /Stack.*does not exist/
14
+ self.class.does_not_exist_status_text
15
+ else
16
+ raise
17
+ end
18
+ end
19
+ end
20
+
21
+ def failed?
22
+ self.class.failure_status_texts.include?(status_text)
23
+ end
24
+
25
+ def succeeded?
26
+ self.class.success_status_texts.include?(status_text)
27
+ end
28
+
29
+ def updating?
30
+ %w(
31
+ UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
32
+ UPDATE_IN_PROGRESS
33
+ UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
34
+ UPDATE_ROLLBACK_IN_PROGRESS
35
+ ).include?(status_text)
36
+ end
37
+
38
+ def creating?
39
+ "CREATE_IN_PROGRESS" == status_text
40
+ end
41
+
42
+ def deleting?
43
+ "DELETE_IN_PROGRESS" == status_text
44
+ end
45
+
46
+ def exists?
47
+ self.class.does_not_exist_status_text != status_text
48
+ end
49
+
50
+ def self.does_not_exist_status_text
51
+ "DOES_NOT_EXIST"
52
+ end
53
+
54
+ def self.all_status_texts
55
+ %w(
56
+ CREATE_IN_PROGRESS
57
+ CREATE_FAILED
58
+ CREATE_COMPLETE
59
+ ROLLBACK_IN_PROGRESS
60
+ ROLLBACK_FAILED
61
+ ROLLBACK_COMPLETE
62
+ DELETE_IN_PROGRESS
63
+ DELETE_FAILED
64
+ DELETE_COMPLETE
65
+ UPDATE_IN_PROGRESS
66
+ UPDATE_COMPLETE_CLEANUP_IN_PROGRESS
67
+ UPDATE_COMPLETE
68
+ UPDATE_ROLLBACK_IN_PROGRESS
69
+ UPDATE_ROLLBACK_FAILED
70
+ UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS
71
+ UPDATE_ROLLBACK_COMPLETE
72
+ REVIEW_IN_PROGRESS
73
+ IMPORT_IN_PROGRESS
74
+ IMPORT_COMPLETE
75
+ IMPORT_ROLLBACK_IN_PROGRESS
76
+ IMPORT_ROLLBACK_FAILED
77
+ IMPORT_ROLLBACK_COMPLETE
78
+ )
79
+ end
80
+
81
+ def self.active_status_texts
82
+ all_status_texts - %w(CREATE_FAILED DELETE_COMPLETE)
83
+ end
84
+
85
+ def self.failure_status_texts
86
+ %w(
87
+ UPDATE_ROLLBACK_COMPLETE
88
+ ROLLBACK_COMPLETE
89
+ ROLLBACK_IN_PROGRESS
90
+ CREATE_FAILED
91
+ ROLLBACK_FAILED
92
+ DELETE_FAILED
93
+ UPDATE_ROLLBACK_FAILED
94
+ IMPORT_ROLLBACK_FAILED
95
+ )
96
+ end
97
+
98
+ def self.success_status_texts
99
+ %w(
100
+ UPDATE_COMPLETE
101
+ DELETE_COMPLETE
102
+ CREATE_COMPLETE
103
+ )
104
+ end
105
+
106
+ def failed_events_since_last_user_event
107
+ @stack.events.each_with_object([]) do |event, array|
108
+ array.push(event) if event.failed? && event.status_reason #if nil, don't push
109
+ return array if event.user_initiated?
110
+ end
111
+ end
112
+
113
+ def to_h
114
+ {
115
+ status: status_text,
116
+ failed_events_since_last_user_event: failed? ? failed_events_since_last_user_event : []
117
+ }
118
+ end
119
+
120
+ def to_json
121
+ to_h.to_json
122
+ end
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,21 @@
1
+ module OpenStax::Aws
2
+ module System
3
+
4
+ def self.call(command, logger: nil, dry_run:)
5
+ logger&.info("**** DRY RUN ****") if dry_run
6
+ logger&.info("Running: #{command}")
7
+
8
+ if !dry_run
9
+ Open3.popen2e(command) do |stdin, stdout_err, wait_thr|
10
+ while line=stdout_err.gets do
11
+ STDERR.puts(line)
12
+ end
13
+
14
+ exit_status = wait_thr.value.exitstatus
15
+ exit(exit_status) if exit_status != 0
16
+ end
17
+ end
18
+ end
19
+
20
+ end
21
+ end
@@ -0,0 +1,31 @@
1
+ module OpenStax::Aws
2
+ class Tag
3
+
4
+ attr_reader :key, :value
5
+
6
+ AWS_TAG_KEY_REGEX = /\A[\w\-\/.+=:@]{1,128}\z/
7
+ AWS_TAG_VALUE_REGEX = /\A[\w\-\/.+=:@ ]{0,256}\z/
8
+
9
+ def initialize(key, value)
10
+ if key.nil? || !key.match(AWS_TAG_KEY_REGEX)
11
+ raise "The tag key '#{key}' is invalid: must be a non-blank ID matching #{AWS_TAG_KEY_REGEX}"
12
+ end
13
+
14
+ @key = key.to_s
15
+
16
+ if @key.starts_with?("aws:")
17
+ raise "The tag key '#{@key}' is invalid: it cannot start with 'aws:'"
18
+ end
19
+
20
+ if value.nil?
21
+ raise "The tag value for key '#{key}' cannot be nil"
22
+ end
23
+
24
+ if !value.match(AWS_TAG_VALUE_REGEX)
25
+ raise "The tag value '#{value}' must be a tag value matching #{AWS_TAG_VALUE_REGEX}"
26
+ end
27
+
28
+ @value = value.to_s
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,129 @@
1
+ module OpenStax::Aws
2
+ class Template
3
+
4
+ class TemplateInvalid < StandardError
5
+ attr_reader :template_path, :template_body
6
+
7
+ def initialize(msg, template_path, template_body)
8
+ @template_path = template_path
9
+ @template_body = template_body
10
+
11
+ super(msg)
12
+ end
13
+ end
14
+
15
+ attr_reader :absolute_file_path
16
+
17
+ def self.from_absolute_file_path(absolute_file_path)
18
+ new(absolute_file_path: absolute_file_path)
19
+ end
20
+
21
+ def self.from_body(body)
22
+ new(body: body)
23
+ end
24
+
25
+ def basename
26
+ File.basename(absolute_file_path)
27
+ end
28
+
29
+ def body
30
+ @body ||= File.read(absolute_file_path)
31
+ end
32
+
33
+ def hash
34
+ json_hash || yml_hash || raise("Cannot read template #{absolute_file_path}")
35
+ end
36
+
37
+ def valid?
38
+ begin
39
+ validate
40
+ rescue TemplateInvalid
41
+ return false
42
+ end
43
+
44
+ true
45
+ end
46
+
47
+ def parameter_names
48
+ hash["Parameters"].try(:keys) || []
49
+ end
50
+
51
+ def required_capabilities
52
+ has_an_iam_resource = hash["Resources"].any? do |resource_id, resource_body|
53
+ resource_body["Type"].starts_with?("AWS::IAM::")
54
+ end
55
+
56
+ # A bit of a cop out to claim named_iam since we haven't checked
57
+ # to see if any of the IAM resources have custom names, but it will
58
+ # work
59
+ has_an_iam_resource ? [:named_iam] : []
60
+ end
61
+
62
+ def s3_key
63
+ [
64
+ OpenStax::Aws.configuration.cfn_template_bucket_folder,
65
+ s3_folder,
66
+ basename
67
+ ].compact.join("/")
68
+ end
69
+
70
+ def s3_folder
71
+ @unique_s3_folder ||=
72
+ OpenStax::Aws.configuration.fixed_s3_template_folder ||
73
+ Time.now.utc.strftime("%Y%m%d_%H%M%S_#{SecureRandom.hex(4)}")
74
+ end
75
+
76
+ def s3_url
77
+ upload_once
78
+ "https://s3.amazonaws.com/#{OpenStax::Aws.configuration.cfn_template_bucket_name}/#{s3_key}"
79
+ end
80
+
81
+ def upload_once
82
+ return if @uploaded
83
+
84
+ validate
85
+ upload
86
+
87
+ @uploaded = true
88
+ end
89
+
90
+ def is_sam?
91
+ body.match(/Transform: AWS::Serverless/).present?
92
+ end
93
+
94
+ protected
95
+
96
+ def initialize(absolute_file_path: nil, body: nil)
97
+ raise "One of `absolute_file_path` or `body` must be set" if absolute_file_path.blank? && body.nil?
98
+ @absolute_file_path = absolute_file_path
99
+ @body = body.try(:dup)
100
+ end
101
+
102
+ def validate
103
+ begin
104
+ # any region works here
105
+ Aws::CloudFormation::Client.new(region: "us-west-2").validate_template(template_body: body)
106
+ rescue Aws::CloudFormation::Errors::ValidationError => ee
107
+ raise TemplateInvalid.new(basename + ": " + ee.message, absolute_file_path, body)
108
+ end
109
+ end
110
+
111
+ def upload
112
+ client = Aws::S3::Client.new(region: OpenStax::Aws.configuration.cfn_template_bucket_region)
113
+ client.put_object({
114
+ body: body,
115
+ bucket: OpenStax::Aws.configuration.cfn_template_bucket_name,
116
+ key: s3_key
117
+ })
118
+ end
119
+
120
+ def json_hash
121
+ JSON.parse(body) rescue nil
122
+ end
123
+
124
+ def yml_hash
125
+ YAML.load(body) rescue nil
126
+ end
127
+
128
+ end
129
+ end