openstax_aws 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +19 -0
- data/.travis.yml +12 -0
- data/CHANGELOG.md +11 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +120 -0
- data/LICENSE.txt +1 -0
- data/README.md +927 -0
- data/Rakefile +6 -0
- data/TODO.md +1 -0
- data/assets/secrets_sequence_diagram.png +0 -0
- data/bin/console +14 -0
- data/bin/create_development_environment +26 -0
- data/bin/get_latest_ubuntu_ami +31 -0
- data/bin/setup +8 -0
- data/bin/templates/aws_ruby_development.yml +221 -0
- data/examples/deployment.rb +90 -0
- data/ideas.md +15 -0
- data/lib/openstax/aws/auto_scaling_group.rb +28 -0
- data/lib/openstax/aws/auto_scaling_instance.rb +96 -0
- data/lib/openstax/aws/build_image_command_1.rb +53 -0
- data/lib/openstax/aws/change_set.rb +100 -0
- data/lib/openstax/aws/deployment_base.rb +372 -0
- data/lib/openstax/aws/distribution.rb +56 -0
- data/lib/openstax/aws/ec2_instance_data.rb +18 -0
- data/lib/openstax/aws/extensions.rb +19 -0
- data/lib/openstax/aws/git_helper.rb +18 -0
- data/lib/openstax/aws/image.rb +34 -0
- data/lib/openstax/aws/msk_cluster.rb +19 -0
- data/lib/openstax/aws/packer_1_2_5.rb +63 -0
- data/lib/openstax/aws/packer_1_4_1.rb +72 -0
- data/lib/openstax/aws/packer_factory.rb +25 -0
- data/lib/openstax/aws/rds_instance.rb +25 -0
- data/lib/openstax/aws/s3_text_file.rb +50 -0
- data/lib/openstax/aws/sam_stack.rb +85 -0
- data/lib/openstax/aws/secrets.rb +302 -0
- data/lib/openstax/aws/secrets_factory.rb +126 -0
- data/lib/openstax/aws/secrets_set.rb +21 -0
- data/lib/openstax/aws/secrets_specification.rb +68 -0
- data/lib/openstax/aws/stack.rb +465 -0
- data/lib/openstax/aws/stack_event.rb +28 -0
- data/lib/openstax/aws/stack_factory.rb +153 -0
- data/lib/openstax/aws/stack_parameters.rb +19 -0
- data/lib/openstax/aws/stack_status.rb +125 -0
- data/lib/openstax/aws/system.rb +21 -0
- data/lib/openstax/aws/tag.rb +31 -0
- data/lib/openstax/aws/template.rb +129 -0
- data/lib/openstax/aws/version.rb +5 -0
- data/lib/openstax/aws/wait_message.rb +20 -0
- data/lib/openstax_aws.rb +154 -0
- data/openstax_aws.gemspec +58 -0
- metadata +350 -0
@@ -0,0 +1,56 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Distribution
|
3
|
+
|
4
|
+
attr_reader :id, :region, :logger
|
5
|
+
|
6
|
+
def initialize(id:, region:, logger: nil)
|
7
|
+
@id = id
|
8
|
+
@region = region
|
9
|
+
@logger ||= Logger.new(IO::NULL)
|
10
|
+
end
|
11
|
+
|
12
|
+
def invalidate(paths:, wait: false)
|
13
|
+
paths = [paths].flatten.compact
|
14
|
+
|
15
|
+
resp = client.create_invalidation({
|
16
|
+
distribution_id: id,
|
17
|
+
invalidation_batch: {
|
18
|
+
paths: {
|
19
|
+
quantity: paths.length,
|
20
|
+
items: paths,
|
21
|
+
},
|
22
|
+
caller_reference: "#{paths.join(' ')} #{Time.now.utc.strftime("%Y%m%d-%H%M%S")}"
|
23
|
+
},
|
24
|
+
})
|
25
|
+
|
26
|
+
invalidation_id = resp.invalidation.id
|
27
|
+
|
28
|
+
logger.info("Created invalidation #{invalidation_id} for paths #{paths.join(', ')}.")
|
29
|
+
|
30
|
+
wait_message = OpenStax::Aws::WaitMessage.new(
|
31
|
+
message: "Waiting for invalidation #{invalidation_id} to be completed"
|
32
|
+
)
|
33
|
+
|
34
|
+
begin
|
35
|
+
Aws::CloudFront::Waiters::InvalidationCompleted.new(
|
36
|
+
client: client,
|
37
|
+
before_attempt: ->(*) { wait_message.say_it }
|
38
|
+
).wait(
|
39
|
+
distribution_id: id,
|
40
|
+
id: invalidation_id
|
41
|
+
)
|
42
|
+
rescue Aws::Waiters::Errors::WaiterFailed => error
|
43
|
+
logger.error "Waiting failed: #{error.message}"
|
44
|
+
raise
|
45
|
+
end
|
46
|
+
logger.info "Invalidation #{invalidation_id} has been completed!"
|
47
|
+
end
|
48
|
+
|
49
|
+
protected
|
50
|
+
|
51
|
+
def client
|
52
|
+
@cloudfront_client ||= ::Aws::CloudFront::Client.new(region: region)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Ec2InstanceData
|
3
|
+
|
4
|
+
def self.get(key)
|
5
|
+
endpoint = "http://169.254.169.254/latest/#{key}"
|
6
|
+
Net::HTTP.get(URI.parse(endpoint))
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.instance_id
|
10
|
+
get("meta-data/instance-id")
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.region
|
14
|
+
get("meta-data/placement/availability-zone")[0..-2]
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'git'
|
2
|
+
require 'open-uri'
|
3
|
+
|
4
|
+
module OpenStax::Aws
|
5
|
+
module GitHelper
|
6
|
+
|
7
|
+
def self.sha_for_branch_name(org_slash_repo:, branch:)
|
8
|
+
::Git.ls_remote("https://github.com/#{org_slash_repo}")["branches"][branch][:sha]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.file_content_at_sha(org_slash_repo:, sha:, path:)
|
12
|
+
location = "https://raw.githubusercontent.com/#{org_slash_repo}/#{sha}/#{path}"
|
13
|
+
file = open(location)
|
14
|
+
file.read
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class Image
|
3
|
+
|
4
|
+
attr_reader :aws_image
|
5
|
+
|
6
|
+
def initialize(id: nil, region: nil, aws_image: nil)
|
7
|
+
if aws_image
|
8
|
+
@aws_image = aws_image
|
9
|
+
else
|
10
|
+
if id.nil? || region.nil?
|
11
|
+
raise ArgumentError, "`id` and `region` must be provided"
|
12
|
+
end
|
13
|
+
|
14
|
+
@aws_image = Aws::EC2::Image.new(id, region: region)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def get_tag(key:)
|
19
|
+
tag = aws_image.tags.find{|tag| tag.key == key}
|
20
|
+
raise "No tag with key #{key} on AMI #{aws_image.image_id}" if tag.nil?
|
21
|
+
tag.value
|
22
|
+
end
|
23
|
+
|
24
|
+
def sha
|
25
|
+
get_tag(key: "sha")
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.find_by_sha(sha:, region:)
|
29
|
+
Aws::EC2::Client.new(region: region).describe_images({
|
30
|
+
filters: [{name: "tag:sha", values: [sha]}]
|
31
|
+
}).images.map{|aws_image| new(aws_image: aws_image)}
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class MskCluster
|
3
|
+
|
4
|
+
attr_reader :client, :cluster_arn
|
5
|
+
|
6
|
+
delegate_missing_to :@client
|
7
|
+
|
8
|
+
def initialize(cluster_arn:, region:)
|
9
|
+
@cluster_arn = cluster_arn
|
10
|
+
@client = ::Aws::Kafka::Client.new(region: region)
|
11
|
+
end
|
12
|
+
|
13
|
+
def bootstrap_broker_string
|
14
|
+
client.get_bootstrap_brokers(cluster_arn: cluster_arn).bootstrap_broker_string
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'open3'
|
2
|
+
|
3
|
+
module OpenStax::Aws
|
4
|
+
class Packer_1_2_5
|
5
|
+
|
6
|
+
def initialize(absolute_file_path:, dry_run: true)
|
7
|
+
@logger = OpenStax::Aws.configuration.logger
|
8
|
+
@only = []
|
9
|
+
@vars = {}
|
10
|
+
@dry_run = dry_run
|
11
|
+
@verbose = false
|
12
|
+
@debug = false
|
13
|
+
@absolute_file_path = absolute_file_path
|
14
|
+
end
|
15
|
+
|
16
|
+
def only(builders)
|
17
|
+
@only = [builders].flatten
|
18
|
+
end
|
19
|
+
|
20
|
+
def var(key, value)
|
21
|
+
@vars[key.to_s] = value.to_s
|
22
|
+
end
|
23
|
+
|
24
|
+
def verbose!
|
25
|
+
@verbose = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def debug!
|
29
|
+
@debug = true
|
30
|
+
end
|
31
|
+
|
32
|
+
def command
|
33
|
+
cmd = "packer build --only=amazon-ebs"
|
34
|
+
|
35
|
+
cmd = "#{cmd} --only=#{@only.join(',')}" if !@only.empty?
|
36
|
+
|
37
|
+
@vars.each do |key, value|
|
38
|
+
cmd = "#{cmd} --var '#{key}=#{value}'"
|
39
|
+
end
|
40
|
+
|
41
|
+
cmd = "PACKER_LOG=1 #{cmd}" if @verbose
|
42
|
+
cmd = "#{cmd} --debug" if @debug
|
43
|
+
|
44
|
+
cmd = "#{cmd} #{@absolute_file_path}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def run
|
48
|
+
@logger.info("**** DRY RUN ****") if @dry_run
|
49
|
+
@logger.info("Running: #{command}")
|
50
|
+
|
51
|
+
if !@dry_run
|
52
|
+
@logger.info("Printing stderr for desired verbosity")
|
53
|
+
|
54
|
+
Open3.popen2e(command) do |stdin, stdout_err, wait_thr|
|
55
|
+
while line=stdout_err.gets do
|
56
|
+
puts(line)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'open3'
|
2
|
+
|
3
|
+
module OpenStax::Aws
|
4
|
+
class Packer_1_4_1
|
5
|
+
|
6
|
+
def initialize(absolute_file_path:, dry_run: true)
|
7
|
+
@logger = OpenStax::Aws.configuration.logger
|
8
|
+
@only = []
|
9
|
+
@vars = {}
|
10
|
+
@dry_run = dry_run
|
11
|
+
@verbose = false
|
12
|
+
@debug = false
|
13
|
+
@absolute_file_path = absolute_file_path
|
14
|
+
end
|
15
|
+
|
16
|
+
def only(builders)
|
17
|
+
@only = [builders].flatten
|
18
|
+
end
|
19
|
+
|
20
|
+
def var(key, value)
|
21
|
+
@vars[key.to_s] = value.to_s
|
22
|
+
end
|
23
|
+
|
24
|
+
def verbose!
|
25
|
+
@verbose = true
|
26
|
+
end
|
27
|
+
|
28
|
+
def debug!
|
29
|
+
@debug = true
|
30
|
+
end
|
31
|
+
|
32
|
+
def command
|
33
|
+
cmd = "packer build"
|
34
|
+
|
35
|
+
cmd = "#{cmd} --only=#{@only.join(',')}" if !@only.empty?
|
36
|
+
|
37
|
+
@vars.each do |key, value|
|
38
|
+
cmd = "#{cmd} --var '#{key}=#{value}'"
|
39
|
+
end
|
40
|
+
|
41
|
+
cmd = "PACKER_LOG=1 #{cmd}" if @verbose
|
42
|
+
cmd = "#{cmd} --debug" if @debug
|
43
|
+
|
44
|
+
cmd = "#{cmd} #{@absolute_file_path}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def run
|
48
|
+
@logger.info("**** DRY RUN ****") if @dry_run
|
49
|
+
@logger.info("Running: #{command}")
|
50
|
+
|
51
|
+
if !@dry_run
|
52
|
+
@logger.info("Printing stderr for desired verbosity")
|
53
|
+
ami = ""
|
54
|
+
|
55
|
+
Open3.popen2e(command) do |stdin, stdout_err, wait_thr|
|
56
|
+
while line=stdout_err.gets do
|
57
|
+
STDERR.puts(line)
|
58
|
+
matchami = line.match(/AMI: (ami-[0-9\-a-z]*)/i)
|
59
|
+
ami = matchami.captures[0] if matchami
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
puts ami
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def to_s
|
68
|
+
command.to_s
|
69
|
+
end
|
70
|
+
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class PackerFactory
|
3
|
+
|
4
|
+
# There are differences between Packer 1.2 and 1.4. This factory attempts to
|
5
|
+
# provide the right version of packer class.
|
6
|
+
|
7
|
+
def self.new_packer(absolute_file_path:, dry_run: true)
|
8
|
+
packer_version = `packer --version`
|
9
|
+
|
10
|
+
raise "packer is not installed" if packer_version.nil?
|
11
|
+
|
12
|
+
packer_class =
|
13
|
+
case packer_version
|
14
|
+
when /^1.2/
|
15
|
+
Packer_1_2_5
|
16
|
+
else
|
17
|
+
Packer_1_4_1
|
18
|
+
end
|
19
|
+
|
20
|
+
@packer = packer_class.new(absolute_file_path: absolute_file_path,
|
21
|
+
dry_run: dry_run)
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class RdsInstance
|
3
|
+
|
4
|
+
attr_reader :raw
|
5
|
+
|
6
|
+
delegate_missing_to :@raw
|
7
|
+
|
8
|
+
def initialize(db_instance_identifier:, region:)
|
9
|
+
@raw = Aws::RDS::DBInstance.new(
|
10
|
+
db_instance_identifier,
|
11
|
+
client: Aws::RDS::Client.new(region: region)
|
12
|
+
)
|
13
|
+
end
|
14
|
+
|
15
|
+
def set_master_password(password:)
|
16
|
+
raw.modify({
|
17
|
+
apply_immediately: true,
|
18
|
+
master_user_password: password
|
19
|
+
})
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,50 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class S3TextFile
|
3
|
+
|
4
|
+
attr_reader :bucket_name, :bucket_region, :key
|
5
|
+
|
6
|
+
def initialize(bucket_name:, bucket_region:, key:)
|
7
|
+
raise ArgumentError, "bucket_name cannot be nil" if bucket_name.nil?
|
8
|
+
raise ArgumentError, "bucket_region cannot be nil" if bucket_region.nil?
|
9
|
+
raise ArgumentError, "key cannot be nil" if key.nil?
|
10
|
+
|
11
|
+
@bucket_name = bucket_name
|
12
|
+
@bucket_region = bucket_region
|
13
|
+
@key = key
|
14
|
+
end
|
15
|
+
|
16
|
+
def read
|
17
|
+
object.load
|
18
|
+
object.get.body.read
|
19
|
+
end
|
20
|
+
|
21
|
+
def get
|
22
|
+
object.load
|
23
|
+
object.get
|
24
|
+
end
|
25
|
+
|
26
|
+
def write(string_contents:, content_type:'text/plain', cache_control: nil)
|
27
|
+
args = {
|
28
|
+
body: StringIO.new(string_contents)
|
29
|
+
}
|
30
|
+
|
31
|
+
args[:content_type] = content_type if !content_type.nil?
|
32
|
+
args[:cache_control] = cache_control if !cache_control.nil?
|
33
|
+
|
34
|
+
object.put(**args)
|
35
|
+
end
|
36
|
+
|
37
|
+
def delete
|
38
|
+
object.delete
|
39
|
+
end
|
40
|
+
|
41
|
+
def object
|
42
|
+
@object ||= Aws::S3::Object.new(
|
43
|
+
bucket_name,
|
44
|
+
key,
|
45
|
+
client: Aws::S3::Client.new(region: bucket_region)
|
46
|
+
)
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
module OpenStax::Aws
|
2
|
+
class SamStack < Stack
|
3
|
+
|
4
|
+
attr_reader :build_directory
|
5
|
+
|
6
|
+
def initialize(id: nil, name:, tags: {},
|
7
|
+
region:, enable_termination_protection: false,
|
8
|
+
absolute_template_path: nil,
|
9
|
+
capabilities: nil, parameter_defaults: {},
|
10
|
+
volatile_parameters_block: nil,
|
11
|
+
secrets_blocks: [], secrets_context: nil, secrets_namespace: nil,
|
12
|
+
shared_secrets_substitutions_block: nil,
|
13
|
+
cycle_if_different_parameter: nil,
|
14
|
+
build_directory:,
|
15
|
+
dry_run: true)
|
16
|
+
@build_directory = build_directory
|
17
|
+
|
18
|
+
super(id: id,
|
19
|
+
name: name,
|
20
|
+
tags: tags,
|
21
|
+
region: region,
|
22
|
+
enable_termination_protection: enable_termination_protection,
|
23
|
+
absolute_template_path: absolute_template_path,
|
24
|
+
capabilities: capabilities,
|
25
|
+
parameter_defaults: parameter_defaults,
|
26
|
+
volatile_parameters_block: volatile_parameters_block,
|
27
|
+
secrets_blocks: secrets_blocks,
|
28
|
+
secrets_context: secrets_context,
|
29
|
+
secrets_namespace: secrets_namespace,
|
30
|
+
shared_secrets_substitutions_block: shared_secrets_substitutions_block,
|
31
|
+
cycle_if_different_parameter: cycle_if_different_parameter,
|
32
|
+
dry_run: dry_run)
|
33
|
+
end
|
34
|
+
|
35
|
+
def build
|
36
|
+
# SAM doesn't have an API or SDK - we have to make calls to its CLI
|
37
|
+
command = "sam build -t #{absolute_template_path} -b #{build_directory}"
|
38
|
+
System.call(command, logger: logger, dry_run: dry_run)
|
39
|
+
end
|
40
|
+
|
41
|
+
def deploy(bucket_name:, params: {})
|
42
|
+
# SAM doesn't have an API or SDK - we have to make calls to its CLI
|
43
|
+
|
44
|
+
check_for_required_tags
|
45
|
+
|
46
|
+
params = parameter_defaults.merge(params)
|
47
|
+
|
48
|
+
command = "sam deploy" \
|
49
|
+
" --template-file #{build_directory}/template.yaml" \
|
50
|
+
" --capabilities CAPABILITY_IAM" \
|
51
|
+
" --s3-bucket #{bucket_name}" \
|
52
|
+
" --s3-prefix #{name}" \
|
53
|
+
" --stack-name #{name}"
|
54
|
+
|
55
|
+
if params.any?
|
56
|
+
command += " --parameter-overrides #{self.class.format_hash_as_cli_stack_parameters(params)}"
|
57
|
+
end
|
58
|
+
|
59
|
+
if tags.any?
|
60
|
+
command += " --tags #{self.class.format_tags_as_cli_tags(tags)}"
|
61
|
+
end
|
62
|
+
|
63
|
+
System.call(command, logger: logger, dry_run: dry_run)
|
64
|
+
|
65
|
+
if enable_termination_protection
|
66
|
+
client.update_termination_protection({
|
67
|
+
enable_termination_protection: true,
|
68
|
+
stack_name: name
|
69
|
+
})
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.format_hash_as_cli_stack_parameters(params={})
|
74
|
+
params.map{|key, value| "ParameterKey=#{key.to_s.camelcase},ParameterValue=#{value}"}
|
75
|
+
.map{|item| "'" + item + "'"}
|
76
|
+
.join(" ")
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.format_tags_as_cli_tags(tags=[])
|
80
|
+
tags.map{|tag| "'#{tag.key}=#{tag.value}'"}
|
81
|
+
.join(" ")
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
end
|