cloud-mu 3.4.0 → 3.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ansible/roles/mu-nat/tasks/main.yml +3 -0
- data/bin/mu-aws-setup +41 -7
- data/bin/mu-azure-setup +34 -0
- data/bin/mu-configure +214 -119
- data/bin/mu-gcp-setup +37 -2
- data/bin/mu-node-manage +3 -0
- data/bin/mu-refresh-ssl +67 -0
- data/bin/mu-run-tests +14 -4
- data/bin/mu-self-update +30 -10
- data/bin/mu-upload-chef-artifacts +30 -26
- data/cloud-mu.gemspec +8 -6
- data/cookbooks/mu-master/attributes/default.rb +5 -1
- data/cookbooks/mu-master/metadata.rb +2 -2
- data/cookbooks/mu-master/recipes/default.rb +81 -26
- data/cookbooks/mu-master/recipes/init.rb +197 -62
- data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
- data/cookbooks/mu-master/recipes/vault.rb +78 -77
- data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
- data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
- data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
- data/cookbooks/mu-tools/attributes/default.rb +5 -0
- data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
- data/cookbooks/mu-tools/libraries/helper.rb +12 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +6 -0
- data/cookbooks/mu-tools/recipes/aws_api.rb +6 -4
- data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
- data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
- data/cookbooks/mu-tools/recipes/google_api.rb +5 -2
- data/cookbooks/mu-tools/resources/disk.rb +108 -58
- data/extras/Gemfile.lock.bootstrap +394 -0
- data/extras/bucketstubs/error.html +0 -0
- data/extras/bucketstubs/index.html +0 -0
- data/extras/clean-stock-amis +9 -9
- data/extras/git_rpm/build.sh +20 -0
- data/extras/git_rpm/mugit.spec +53 -0
- data/extras/image-generators/VMWare/centos8.yaml +15 -0
- data/extras/openssl_rpm/build.sh +19 -0
- data/extras/openssl_rpm/mussl.spec +46 -0
- data/extras/python_rpm/muthon.spec +14 -4
- data/extras/ruby_rpm/muby.spec +9 -5
- data/extras/sqlite_rpm/build.sh +19 -0
- data/extras/sqlite_rpm/muqlite.spec +47 -0
- data/install/installer +7 -5
- data/modules/mu.rb +12 -5
- data/modules/mu/cloud/machine_images.rb +1 -1
- data/modules/mu/cloud/providers.rb +6 -1
- data/modules/mu/cloud/resource_base.rb +1 -1
- data/modules/mu/cloud/ssh_sessions.rb +4 -0
- data/modules/mu/config.rb +28 -12
- data/modules/mu/config/database.rb +2 -2
- data/modules/mu/config/firewall_rule.rb +1 -1
- data/modules/mu/config/ref.rb +2 -2
- data/modules/mu/config/schema_helpers.rb +12 -3
- data/modules/mu/config/server.rb +10 -4
- data/modules/mu/config/server_pool.rb +2 -2
- data/modules/mu/config/vpc.rb +10 -10
- data/modules/mu/defaults/AWS.yaml +32 -32
- data/modules/mu/deploy.rb +23 -10
- data/modules/mu/groomers/chef.rb +2 -2
- data/modules/mu/master.rb +49 -3
- data/modules/mu/mommacat.rb +8 -5
- data/modules/mu/mommacat/naming.rb +2 -2
- data/modules/mu/mommacat/storage.rb +22 -27
- data/modules/mu/providers/aws.rb +142 -48
- data/modules/mu/providers/aws/alarm.rb +3 -3
- data/modules/mu/providers/aws/bucket.rb +19 -19
- data/modules/mu/providers/aws/cache_cluster.rb +22 -22
- data/modules/mu/providers/aws/cdn.rb +2 -2
- data/modules/mu/providers/aws/collection.rb +14 -14
- data/modules/mu/providers/aws/container_cluster.rb +27 -27
- data/modules/mu/providers/aws/database.rb +40 -39
- data/modules/mu/providers/aws/dnszone.rb +5 -5
- data/modules/mu/providers/aws/endpoint.rb +35 -35
- data/modules/mu/providers/aws/firewall_rule.rb +26 -23
- data/modules/mu/providers/aws/function.rb +28 -28
- data/modules/mu/providers/aws/group.rb +7 -7
- data/modules/mu/providers/aws/habitat.rb +2 -2
- data/modules/mu/providers/aws/job.rb +6 -6
- data/modules/mu/providers/aws/loadbalancer.rb +34 -34
- data/modules/mu/providers/aws/log.rb +14 -14
- data/modules/mu/providers/aws/msg_queue.rb +10 -10
- data/modules/mu/providers/aws/nosqldb.rb +8 -8
- data/modules/mu/providers/aws/notifier.rb +7 -7
- data/modules/mu/providers/aws/role.rb +17 -15
- data/modules/mu/providers/aws/search_domain.rb +10 -10
- data/modules/mu/providers/aws/server.rb +176 -95
- data/modules/mu/providers/aws/server_pool.rb +65 -105
- data/modules/mu/providers/aws/storage_pool.rb +17 -9
- data/modules/mu/providers/aws/user.rb +1 -1
- data/modules/mu/providers/aws/vpc.rb +103 -51
- data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
- data/modules/mu/providers/azure.rb +78 -12
- data/modules/mu/providers/azure/server.rb +18 -3
- data/modules/mu/providers/cloudformation/server.rb +1 -1
- data/modules/mu/providers/google.rb +19 -4
- data/modules/mu/providers/google/folder.rb +6 -2
- data/modules/mu/providers/google/function.rb +65 -30
- data/modules/mu/providers/google/role.rb +1 -1
- data/modules/mu/providers/google/vpc.rb +27 -2
- data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
- data/modules/tests/k8s.yaml +1 -1
- metadata +24 -8
data/modules/mu/groomers/chef.rb
CHANGED
@@ -70,8 +70,8 @@ module MU
|
|
70
70
|
# XXX kludge to get at knife-windows when it's installed from
|
71
71
|
# a git repo and bundler sticks it somewhere in a corner
|
72
72
|
$LOAD_PATH.each { |path|
|
73
|
-
if path.match(/\/gems\/
|
74
|
-
addpath = path.sub(/\/gems\/
|
73
|
+
if path.match(/\/gems\/chef\-\d+\.\d+\.\d+\/lib$/)
|
74
|
+
addpath = path.sub(/\/gems\/chef\-\d+\.\d+\.\d+\/lib$/, "")+"/bundler/gems"
|
75
75
|
Dir.glob(addpath+"/knife-windows-*").each { |version|
|
76
76
|
$LOAD_PATH << version+"/lib"
|
77
77
|
}
|
data/modules/mu/master.rb
CHANGED
@@ -195,9 +195,12 @@ module MU
|
|
195
195
|
temp_dev = "/dev/#{ramdisk}"
|
196
196
|
|
197
197
|
if !File.open("/etc/mtab").read.match(/ #{path} /)
|
198
|
-
realdevice =
|
199
|
-
|
200
|
-
|
198
|
+
realdevice = if MU::Cloud::Google.hosted?
|
199
|
+
"/dev/disk/by-id/google-"+device.gsub(/.*?\/([^\/]+)$/, '\1')
|
200
|
+
elsif MU::Cloud::AWS.hosted?
|
201
|
+
MU::Cloud::AWS.realDevicePath(device.dup)
|
202
|
+
else
|
203
|
+
device.dup
|
201
204
|
end
|
202
205
|
alias_device = cryptfile ? "/dev/mapper/"+path.gsub(/[^0-9a-z_\-]/i, "_") : realdevice
|
203
206
|
|
@@ -216,6 +219,9 @@ module MU
|
|
216
219
|
tag_name: "Name",
|
217
220
|
tag_value: "#{$MU_CFG['hostname']} #{path}"
|
218
221
|
)
|
222
|
+
# the device might be on some arbitrary NVMe slot
|
223
|
+
realdevice = MU::Cloud::AWS.realDevicePath(realdevice)
|
224
|
+
alias_device = cryptfile ? "/dev/mapper/"+path.gsub(/[^0-9a-z_\-]/i, "_") : realdevice
|
219
225
|
elsif MU::Cloud::Google.hosted?
|
220
226
|
dummy_svr = MU::Cloud::Google::Server.new(
|
221
227
|
mu_name: "MU-MASTER",
|
@@ -901,5 +907,45 @@ module MU
|
|
901
907
|
}
|
902
908
|
end
|
903
909
|
|
910
|
+
# Just list our block devices
|
911
|
+
# @return [Array<String>]
|
912
|
+
def self.listBlockDevices
|
913
|
+
if File.executable?("/bin/lsblk")
|
914
|
+
%x{/bin/lsblk -i -p -r -n | egrep ' disk( |$)'}.each_line.map { |l|
|
915
|
+
l.chomp.sub(/ .*/, '')
|
916
|
+
}
|
917
|
+
else
|
918
|
+
# XXX something dumber
|
919
|
+
nil
|
920
|
+
end
|
921
|
+
end
|
922
|
+
|
923
|
+
|
924
|
+
# Retrieve the UUID of a block device, if available
|
925
|
+
# @param dev [String]
|
926
|
+
def self.diskUUID(dev)
|
927
|
+
realdev = if MU::Cloud::Google.hosted?
|
928
|
+
"/dev/disk/by-id/google-"+dev.gsub(/.*?\/([^\/]+)$/, '\1')
|
929
|
+
elsif MU::Cloud::AWS.hosted?
|
930
|
+
MU::Cloud::AWS.realDevicePath(dev)
|
931
|
+
else
|
932
|
+
dev
|
933
|
+
end
|
934
|
+
%x{/sbin/blkid #{realdev} -o export | grep ^UUID=}.chomp
|
935
|
+
end
|
936
|
+
|
937
|
+
# Determine whether we're running in an NVMe-enabled environment
|
938
|
+
def self.nvme?
|
939
|
+
if File.executable?("/bin/lsblk")
|
940
|
+
%x{/bin/lsblk -i -p -r -n}.each_line { |l|
|
941
|
+
return true if l =~ /^\/dev\/nvme\d/
|
942
|
+
}
|
943
|
+
else
|
944
|
+
return true if File.exists?("/dev/nvme0n1")
|
945
|
+
end
|
946
|
+
false
|
947
|
+
end
|
948
|
+
|
949
|
+
|
904
950
|
end
|
905
951
|
end
|
data/modules/mu/mommacat.rb
CHANGED
@@ -173,6 +173,7 @@ module MU
|
|
173
173
|
@public_key = nil
|
174
174
|
@secrets = Hash.new
|
175
175
|
@secrets['instance_secret'] = Hash.new
|
176
|
+
@secrets['windows_admin_password'] = Hash.new
|
176
177
|
@ssh_key_name = ssh_key_name
|
177
178
|
@ssh_private_key = ssh_private_key
|
178
179
|
@ssh_public_key = ssh_public_key
|
@@ -512,6 +513,8 @@ module MU
|
|
512
513
|
@ssh_private_key = File.read("#{ssh_dir}/#{@ssh_key_name}")
|
513
514
|
@ssh_private_key.chomp!
|
514
515
|
|
516
|
+
# XXX the following mess belongs in cloud layers, probably in their initDeploy
|
517
|
+
# methods
|
515
518
|
if numKittens(clouds: ["AWS"], types: ["Server", "ServerPool", "ContainerCluster"]) > 0
|
516
519
|
creds_used = []
|
517
520
|
["servers", "server_pools", "container_clusters"].each { |type|
|
@@ -551,7 +554,7 @@ module MU
|
|
551
554
|
|
552
555
|
begin
|
553
556
|
if !no_write
|
554
|
-
if !MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id, retries:
|
557
|
+
if !MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id, retries: 300)
|
555
558
|
raise MuError, "Failed to get deployment-notifcation lock for #{@deploy_id}"
|
556
559
|
end
|
557
560
|
end
|
@@ -884,13 +887,13 @@ MAIL_HEAD_END
|
|
884
887
|
if resource and resource.config and resource.config['cloud']
|
885
888
|
cloudclass = MU::Cloud.cloudClass(resource.config['cloud'])
|
886
889
|
|
887
|
-
cloudclass.writeDeploySecret(
|
888
|
-
cloudclass.writeDeploySecret(
|
890
|
+
cloudclass.writeDeploySecret(self, cert.to_pem, cert_cn+".crt", credentials: resource.config['credentials'])
|
891
|
+
cloudclass.writeDeploySecret(self, key.to_pem, cert_cn+".key", credentials: resource.config['credentials'])
|
889
892
|
if pfx_cert
|
890
|
-
cloudclass.writeDeploySecret(
|
893
|
+
cloudclass.writeDeploySecret(self, pfx_cert.to_der, cert_cn+".pfx", credentials: resource.config['credentials'])
|
891
894
|
end
|
892
895
|
if winrm_cert
|
893
|
-
cloudclass.writeDeploySecret(
|
896
|
+
cloudclass.writeDeploySecret(self, winrm_cert.to_pem, cert_cn+"-winrm.crt", credentials: resource.config['credentials'])
|
894
897
|
end
|
895
898
|
end
|
896
899
|
|
@@ -164,7 +164,7 @@ module MU
|
|
164
164
|
# @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
|
165
165
|
# @param disallowed_chars [Regexp]: A pattern of characters that are illegal for this resource name, such as +/[^a-zA-Z0-9-]/+
|
166
166
|
# @return [String]: A full name string for this resource
|
167
|
-
def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil)
|
167
|
+
def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil, never_gen_unique: false)
|
168
168
|
if name.nil?
|
169
169
|
raise MuError, "Got no argument to MU::MommaCat.getResourceName"
|
170
170
|
end
|
@@ -219,7 +219,7 @@ module MU
|
|
219
219
|
else
|
220
220
|
# If we have to strip anything, assume we've lost uniqueness and
|
221
221
|
# will have to compensate with #genUniquenessString.
|
222
|
-
need_unique_string = true
|
222
|
+
need_unique_string = true if !never_gen_unique
|
223
223
|
reserved = 4
|
224
224
|
basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
|
225
225
|
basename = basename + "-" + @seed.upcase + "-" + name.upcase
|
@@ -208,25 +208,26 @@ module MU
|
|
208
208
|
if lockid == id
|
209
209
|
thread = Thread.list.select { |t| t.object_id == thread_id }.first
|
210
210
|
if thread.object_id != Thread.current.object_id
|
211
|
-
MU.log "#{thread_id} sitting on #{id}", MU::WARN, thread.backtrace
|
211
|
+
MU.log "#{thread_id} sitting on #{id} (#{thread.thread_variables.map { |v| "#{v.to_s}: #{thread.thread_variable_get(v).to_s}" }.join(", ")})", MU::WARN, thread.backtrace
|
212
212
|
end
|
213
213
|
end
|
214
214
|
}
|
215
215
|
}
|
216
216
|
}
|
217
217
|
}
|
218
|
+
|
218
219
|
begin
|
219
220
|
if nonblock
|
220
221
|
if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
|
221
222
|
if retries > 0
|
222
223
|
success = false
|
223
|
-
MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries) { |cur_retries, _wait|
|
224
|
+
MU.retrier([], loop_if: Proc.new { !success }, loop_msg: "Waiting for lock on #{lockdir}/#{id}.lock...", max: retries, wait: 1, logmsg_interval: 0) { |cur_retries, _wait|
|
224
225
|
success = @locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
|
225
|
-
if !success and cur_retries > 0 and (cur_retries %
|
226
|
-
show_relevant.call
|
226
|
+
if !success and cur_retries > 0 and (cur_retries % 45) == 0
|
227
|
+
show_relevant.call
|
227
228
|
end
|
228
229
|
}
|
229
|
-
show_relevant.call
|
230
|
+
show_relevant.call if !success
|
230
231
|
return success
|
231
232
|
else
|
232
233
|
return false
|
@@ -542,6 +543,22 @@ module MU
|
|
542
543
|
return Dir.exist?(deploy_path)
|
543
544
|
end
|
544
545
|
|
546
|
+
# Write our shared deploy secret out to wherever the cloud provider layers
|
547
|
+
# like to stash it.
|
548
|
+
def writeDeploySecret
|
549
|
+
return if !@deploy_secret
|
550
|
+
credsets = credsUsed
|
551
|
+
return if !credsets
|
552
|
+
if !@original_config['scrub_mu_isms'] and !@no_artifacts
|
553
|
+
cloudsUsed.each { |cloud|
|
554
|
+
credsets.each { |credentials|
|
555
|
+
next if MU::Cloud.cloudClass(cloud).credConfig(credentials).nil? # XXX this is a dumb way to check this, should be able to get credsUsed by cloud
|
556
|
+
MU::Cloud.cloudClass(cloud).writeDeploySecret(self, @deploy_secret, credentials: credentials)
|
557
|
+
}
|
558
|
+
}
|
559
|
+
end
|
560
|
+
end
|
561
|
+
|
545
562
|
private
|
546
563
|
|
547
564
|
def writeFile(filename, contents)
|
@@ -552,30 +569,8 @@ module MU
|
|
552
569
|
|
553
570
|
# Helper for +initialize+
|
554
571
|
def setDeploySecret
|
555
|
-
credsets = {}
|
556
|
-
MU::Cloud.resource_types.values.each { |attrs|
|
557
|
-
if !@original_config[attrs[:cfg_plural]].nil? and @original_config[attrs[:cfg_plural]].size > 0
|
558
|
-
@original_config[attrs[:cfg_plural]].each { |resource|
|
559
|
-
|
560
|
-
credsets[resource['cloud']] ||= []
|
561
|
-
credsets[resource['cloud']] << resource['credentials']
|
562
|
-
@clouds[resource['cloud']] = 0 if !@clouds.has_key?(resource['cloud'])
|
563
|
-
@clouds[resource['cloud']] = @clouds[resource['cloud']] + 1
|
564
|
-
|
565
|
-
}
|
566
|
-
end
|
567
|
-
}
|
568
|
-
|
569
572
|
MU.log "Creating deploy secret for #{MU.deploy_id}"
|
570
573
|
@deploy_secret = Password.random(256)
|
571
|
-
if !@original_config['scrub_mu_isms'] and !@no_artifacts
|
572
|
-
credsets.each_pair { |cloud, creds|
|
573
|
-
creds.uniq!
|
574
|
-
creds.each { |credentials|
|
575
|
-
MU::Cloud.cloudClass(cloud).writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials)
|
576
|
-
}
|
577
|
-
}
|
578
|
-
end
|
579
574
|
end
|
580
575
|
|
581
576
|
def loadObjects(delay_descriptor_load)
|
data/modules/mu/providers/aws.rb
CHANGED
@@ -16,7 +16,6 @@ require "net/http"
|
|
16
16
|
require 'open-uri'
|
17
17
|
require 'timeout'
|
18
18
|
require 'inifile'
|
19
|
-
gem 'aws-sdk-core'
|
20
19
|
autoload :Aws, "aws-sdk-core"
|
21
20
|
|
22
21
|
|
@@ -54,14 +53,19 @@ module MU
|
|
54
53
|
def self.resourceInitHook(cloudobj, _deploy)
|
55
54
|
class << self
|
56
55
|
attr_reader :cloudformation_data
|
56
|
+
attr_reader :region
|
57
57
|
end
|
58
|
+
return if !cloudobj
|
58
59
|
cloudobj.instance_variable_set(:@cloudformation_data, {})
|
60
|
+
|
61
|
+
cloudobj.instance_variable_set(:@region, cloudobj.config['region'])
|
59
62
|
end
|
60
63
|
|
61
64
|
# Load some credentials for using the AWS API
|
62
65
|
# @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those.
|
63
66
|
# @return [Aws::Credentials]
|
64
67
|
def self.loadCredentials(name = nil)
|
68
|
+
gem 'aws-sdk-core'
|
65
69
|
@@creds_loaded ||= {}
|
66
70
|
|
67
71
|
if name.nil?
|
@@ -124,10 +128,14 @@ module MU
|
|
124
128
|
# pull access key and secret from a vault
|
125
129
|
begin
|
126
130
|
vault, item = cred_cfg["credentials"].split(/:/)
|
127
|
-
data =
|
128
|
-
|
131
|
+
data = if !vault or !item
|
132
|
+
raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg
|
133
|
+
else
|
134
|
+
MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
|
135
|
+
end
|
136
|
+
if data and data["access_key"] and data["access_secret"]
|
129
137
|
cred_obj = Aws::Credentials.new(
|
130
|
-
|
138
|
+
data['access_key'], data['access_secret']
|
131
139
|
)
|
132
140
|
if name.nil?
|
133
141
|
# Aws.config = {
|
@@ -137,10 +145,10 @@ module MU
|
|
137
145
|
# }
|
138
146
|
end
|
139
147
|
else
|
140
|
-
|
148
|
+
raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg
|
141
149
|
end
|
142
150
|
rescue MU::Groomer::MuNoSuchSecret
|
143
|
-
|
151
|
+
raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg
|
144
152
|
end
|
145
153
|
end
|
146
154
|
|
@@ -186,6 +194,7 @@ end
|
|
186
194
|
# @param r [String]
|
187
195
|
# @return [String]
|
188
196
|
def self.validate_region(r, credentials: nil)
|
197
|
+
require "aws-sdk-ec2"
|
189
198
|
begin
|
190
199
|
MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
|
191
200
|
rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
|
@@ -204,6 +213,7 @@ end
|
|
204
213
|
# @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style
|
205
214
|
# @return [void]
|
206
215
|
def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
|
216
|
+
require "aws-sdk-ec2"
|
207
217
|
tags = []
|
208
218
|
MU::MommaCat.listStandardTags.each_pair { |name, value|
|
209
219
|
tags << {key: name, value: value} if !value.nil?
|
@@ -261,20 +271,33 @@ end
|
|
261
271
|
@@myVPCObj
|
262
272
|
end
|
263
273
|
|
264
|
-
# If we've configured AWS as a provider, or are simply hosted in AWS,
|
274
|
+
# If we've configured AWS as a provider, or are simply hosted in AWS,
|
265
275
|
# decide what our default region is.
|
266
|
-
def self.myRegion(credentials = nil)
|
267
|
-
|
276
|
+
def self.myRegion(credentials = nil, debug: false)
|
277
|
+
loglevel = debug ? MU::NOTICE : MU::DEBUG
|
278
|
+
if @@myRegion_var
|
279
|
+
MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel
|
280
|
+
return @@myRegion_var
|
281
|
+
end
|
268
282
|
|
283
|
+
MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig
|
284
|
+
MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s
|
285
|
+
MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION']
|
286
|
+
MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws']
|
269
287
|
if credConfig.nil? and !hosted? and !ENV['EC2_REGION']
|
288
|
+
MU.log "AWS.myRegion: nothing of use set, returning", loglevel
|
270
289
|
return nil
|
271
290
|
end
|
272
291
|
|
273
292
|
if $MU_CFG and $MU_CFG['aws']
|
274
293
|
$MU_CFG['aws'].each_pair { |credset, cfg|
|
294
|
+
MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg
|
275
295
|
next if credentials and credset != credentials
|
296
|
+
MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg
|
276
297
|
next if !cfg['region']
|
277
|
-
|
298
|
+
MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset)
|
299
|
+
if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset)
|
300
|
+
MU.log "AWS.myRegion: liking this set", loglevel, details: cfg
|
278
301
|
@@myRegion_var = cfg['region']
|
279
302
|
break if cfg['default'] or credentials
|
280
303
|
end
|
@@ -286,15 +309,21 @@ end
|
|
286
309
|
(Aws.config['access_key'] and Aws.config['access_secret'])
|
287
310
|
)
|
288
311
|
# Make sure this string is valid by way of the API
|
312
|
+
MU.log "AWS.myRegion: using ENV", loglevel, details: ENV
|
289
313
|
@@myRegion_var = ENV['EC2_REGION']
|
290
314
|
end
|
291
315
|
|
292
316
|
if hosted? and !@@myRegion_var
|
293
317
|
# hacky, but useful in a pinch (and if we're hosted in AWS)
|
294
318
|
az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
|
319
|
+
MU.log "AWS.myRegion: using hosted", loglevel, details: az_str
|
295
320
|
@@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str
|
296
321
|
end
|
297
322
|
|
323
|
+
if credConfig and credConfig["region"]
|
324
|
+
@@myRegion_var ||= credConfig["region"]
|
325
|
+
end
|
326
|
+
|
298
327
|
@@myRegion_var
|
299
328
|
end
|
300
329
|
|
@@ -382,8 +411,9 @@ end
|
|
382
411
|
# Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
|
383
412
|
# @param deploy_id [String]: The deploy for which we're writing the secret
|
384
413
|
# @param value [String]: The contents of the secret
|
385
|
-
def self.writeDeploySecret(
|
386
|
-
|
414
|
+
def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
|
415
|
+
require "aws-sdk-s3"
|
416
|
+
name ||= deploy.deploy_id+"-secret"
|
387
417
|
begin
|
388
418
|
MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}"
|
389
419
|
MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object(
|
@@ -401,35 +431,35 @@ end
|
|
401
431
|
def self.cloudtrailBucketPolicy(credentials = nil)
|
402
432
|
cfg = credConfig(credentials)
|
403
433
|
policy_json = '{
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
434
|
+
"Version": "2012-10-17",
|
435
|
+
"Statement": [
|
436
|
+
{
|
437
|
+
"Sid": "AWSCloudTrailAclCheck20131101",
|
438
|
+
"Effect": "Allow",
|
409
439
|
"Principal": {
|
410
440
|
"AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
|
411
441
|
"Service": "cloudtrail.amazonaws.com"
|
412
442
|
},
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
443
|
+
"Action": "s3:GetBucketAcl",
|
444
|
+
"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
|
445
|
+
},
|
446
|
+
{
|
447
|
+
"Sid": "AWSCloudTrailWrite20131101",
|
448
|
+
"Effect": "Allow",
|
419
449
|
"Principal": {
|
420
450
|
"AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root",
|
421
451
|
"Service": "cloudtrail.amazonaws.com"
|
422
452
|
},
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
453
|
+
"Action": "s3:PutObject",
|
454
|
+
"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
|
455
|
+
"Condition": {
|
456
|
+
"StringEquals": {
|
457
|
+
"s3:x-amz-acl": "bucket-owner-full-control"
|
458
|
+
}
|
459
|
+
}
|
460
|
+
}
|
461
|
+
]
|
462
|
+
}'
|
433
463
|
ERB.new(policy_json).result
|
434
464
|
end
|
435
465
|
|
@@ -440,6 +470,53 @@ end
|
|
440
470
|
MU::Cloud::AWS.hosted?
|
441
471
|
end
|
442
472
|
|
473
|
+
# If we're in AWS and NVME-aware, return a mapping of AWS-side device
|
474
|
+
# names to actual NVME devices.
|
475
|
+
# @return [Hash]
|
476
|
+
def self.attachedNVMeDisks
|
477
|
+
if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
|
478
|
+
return {}
|
479
|
+
end
|
480
|
+
map = {}
|
481
|
+
devices = MU::Master.listBlockDevices
|
482
|
+
return {} if !devices
|
483
|
+
devices.each { |d|
|
484
|
+
if d =~ /^\/dev\/nvme/
|
485
|
+
%x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc|
|
486
|
+
if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
|
487
|
+
virt_dev = Regexp.last_match[1]
|
488
|
+
map[virt_dev] = d
|
489
|
+
break
|
490
|
+
end
|
491
|
+
}
|
492
|
+
end
|
493
|
+
}
|
494
|
+
map
|
495
|
+
end
|
496
|
+
|
497
|
+
# Map our own idea of what a block device is called back to whatever AWS
|
498
|
+
# and the operating system decided on amongst themselves. This currently
|
499
|
+
# exists to map generic "xvd[a-z]" style names back to real NVMe devices.
|
500
|
+
# @param dev [String]
|
501
|
+
def self.realDevicePath(dev)
|
502
|
+
return dev if !hosted?
|
503
|
+
value = nil
|
504
|
+
should_retry = Proc.new {
|
505
|
+
!value and MU::Master.nvme?
|
506
|
+
}
|
507
|
+
MU.retrier(loop_if: should_retry, wait: 5, max: 6) {
|
508
|
+
map = attachedNVMeDisks
|
509
|
+
value = if map[dev]
|
510
|
+
map[dev]
|
511
|
+
elsif map[dev.gsub(/.*?\//, '')]
|
512
|
+
map[dev.gsub(/.*?\//, '')]
|
513
|
+
else
|
514
|
+
dev # be nice to actually handle this too
|
515
|
+
end
|
516
|
+
}
|
517
|
+
value
|
518
|
+
end
|
519
|
+
|
443
520
|
# Determine whether we (the Mu master, presumably) are hosted in this
|
444
521
|
# cloud.
|
445
522
|
# @return [Boolean]
|
@@ -457,7 +534,7 @@ end
|
|
457
534
|
|
458
535
|
begin
|
459
536
|
Timeout.timeout(4) do
|
460
|
-
instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
|
537
|
+
instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
|
461
538
|
if !instance_id.nil? and instance_id.size > 0
|
462
539
|
@@is_in_aws = true
|
463
540
|
region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
|
@@ -566,17 +643,18 @@ end
|
|
566
643
|
end
|
567
644
|
|
568
645
|
$MU_CFG['aws'].keys
|
569
|
-
end
|
646
|
+
end
|
570
647
|
|
571
648
|
# Resolve the administrative S3 bucket for a given credential set, or
|
572
649
|
# return a default.
|
573
650
|
# @param credentials [String]
|
574
651
|
# @return [String]
|
575
652
|
def self.adminBucketName(credentials = nil)
|
653
|
+
require "aws-sdk-s3"
|
576
654
|
cfg = credConfig(credentials)
|
577
655
|
return nil if !cfg
|
578
656
|
if !cfg['log_bucket_name']
|
579
|
-
cfg['log_bucket_name'] = $MU_CFG['hostname']
|
657
|
+
cfg['log_bucket_name'] = $MU_CFG['hostname']
|
580
658
|
MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN
|
581
659
|
end
|
582
660
|
resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets
|
@@ -610,7 +688,7 @@ end
|
|
610
688
|
|
611
689
|
# Return the $MU_CFG data associated with a particular profile/name/set of
|
612
690
|
# credentials. If no account name is specified, will return one flagged as
|
613
|
-
# default. Returns nil if AWS is not configured. Throws an exception if
|
691
|
+
# default. Returns nil if AWS is not configured. Throws an exception if
|
614
692
|
# an account name is specified which does not exist.
|
615
693
|
# @param name [String]: The name of the key under 'aws' in mu.yaml to return
|
616
694
|
# @return [Hash,nil]
|
@@ -625,10 +703,13 @@ end
|
|
625
703
|
|
626
704
|
if hosted?
|
627
705
|
begin
|
628
|
-
|
629
|
-
if
|
630
|
-
|
631
|
-
|
706
|
+
iam_blob = getAWSMetaData("iam/info")
|
707
|
+
if iam_blob
|
708
|
+
iam_data = JSON.parse(iam_blob)
|
709
|
+
if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
|
710
|
+
@@my_hosted_cfg = hosted_config
|
711
|
+
return name_only ? "#default" : @@my_hosted_cfg
|
712
|
+
end
|
632
713
|
end
|
633
714
|
rescue JSON::ParserError => e
|
634
715
|
end
|
@@ -692,6 +773,7 @@ end
|
|
692
773
|
# XXX this needs to be "myAccountNumber" or somesuch
|
693
774
|
# XXX and maybe do the IAM thing for arbitrary, non-resident accounts
|
694
775
|
def self.account_number
|
776
|
+
require "aws-sdk-ec2"
|
695
777
|
return nil if credConfig.nil?
|
696
778
|
return @@my_acct_num if @@my_acct_num
|
697
779
|
loadCredentials
|
@@ -749,7 +831,7 @@ end
|
|
749
831
|
@@regions.keys.uniq
|
750
832
|
end
|
751
833
|
|
752
|
-
# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
|
834
|
+
# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
|
753
835
|
# *probably* ok for most purposes? We can't call listAZs on it from out here
|
754
836
|
# apparently, so getting around it is nontrivial
|
755
837
|
# if !@@regions.has_key?("us-gov-west-1")
|
@@ -774,6 +856,7 @@ end
|
|
774
856
|
# @param public_key [String]: The public key
|
775
857
|
# @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key
|
776
858
|
def self.createEc2SSHKey(keyname, public_key, credentials: nil)
|
859
|
+
require "aws-sdk-ec2"
|
777
860
|
# We replicate this key in all regions
|
778
861
|
if !MU::Cloud::CloudFormation.emitCloudFormation
|
779
862
|
MU::Cloud::AWS.listRegions.each { |region|
|
@@ -802,8 +885,16 @@ end
|
|
802
885
|
def self.listInstanceTypes(region = myRegion)
|
803
886
|
return @@instance_types if @@instance_types and @@instance_types[region]
|
804
887
|
return {} if credConfig.nil?
|
888
|
+
if region.nil?
|
889
|
+
region = myRegion(debug: true)
|
890
|
+
end
|
891
|
+
return {} if region.nil?
|
805
892
|
|
806
893
|
human_region = @@regionLookup[region]
|
894
|
+
if human_region.nil?
|
895
|
+
MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR
|
896
|
+
return {}
|
897
|
+
end
|
807
898
|
|
808
899
|
@@instance_types ||= {}
|
809
900
|
@@instance_types[region] ||= {}
|
@@ -857,6 +948,7 @@ end
|
|
857
948
|
# @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
|
858
949
|
# @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
|
859
950
|
def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
|
951
|
+
require "aws-sdk-iam"
|
860
952
|
if (name.nil? or name.empty?) and (id.nil? or id.empty?)
|
861
953
|
raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
|
862
954
|
end
|
@@ -891,7 +983,7 @@ end
|
|
891
983
|
return nil
|
892
984
|
end
|
893
985
|
elsif matches.size > 1
|
894
|
-
raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
|
986
|
+
raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
|
895
987
|
end
|
896
988
|
end
|
897
989
|
|
@@ -953,7 +1045,7 @@ end
|
|
953
1045
|
# Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
|
954
1046
|
# look up and validate the specified certificate. This is intended to be
|
955
1047
|
# invoked from resource implementations' +validateConfig+ methods.
|
956
|
-
# @param certblock [Hash,MU::Config::Ref]:
|
1048
|
+
# @param certblock [Hash,MU::Config::Ref]:
|
957
1049
|
# @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
|
958
1050
|
# @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
|
959
1051
|
# @return [Boolean]
|
@@ -1121,7 +1213,7 @@ end
|
|
1121
1213
|
@@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials)
|
1122
1214
|
@@elasticache_api[credentials][region]
|
1123
1215
|
end
|
1124
|
-
|
1216
|
+
|
1125
1217
|
# Amazon's SNS API
|
1126
1218
|
def self.sns(region: MU.curRegion, credentials: nil)
|
1127
1219
|
region ||= myRegion
|
@@ -1129,7 +1221,7 @@ end
|
|
1129
1221
|
@@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials)
|
1130
1222
|
@@sns_api[credentials][region]
|
1131
1223
|
end
|
1132
|
-
|
1224
|
+
|
1133
1225
|
# Amazon's SQS API
|
1134
1226
|
def self.sqs(region: MU.curRegion, credentials: nil)
|
1135
1227
|
region ||= myRegion
|
@@ -1161,7 +1253,7 @@ end
|
|
1161
1253
|
@@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials)
|
1162
1254
|
@@apig_api[credentials][region]
|
1163
1255
|
end
|
1164
|
-
|
1256
|
+
|
1165
1257
|
# Amazon's Cloudwatch Events API
|
1166
1258
|
def self.cloudwatch_events(region = MU.cureRegion)
|
1167
1259
|
region ||= myRegion
|
@@ -1274,7 +1366,7 @@ end
|
|
1274
1366
|
begin
|
1275
1367
|
response = nil
|
1276
1368
|
Timeout.timeout(1) do
|
1277
|
-
response = open("#{base_url}/#{param}").read
|
1369
|
+
response = URI.open("#{base_url}/#{param}").read
|
1278
1370
|
end
|
1279
1371
|
|
1280
1372
|
response
|
@@ -1299,6 +1391,7 @@ end
|
|
1299
1391
|
tag_value=MU.deploy_id,
|
1300
1392
|
region: MU.curRegion,
|
1301
1393
|
credentials: nil)
|
1394
|
+
require "aws-sdk-ec2"
|
1302
1395
|
attempts = 0
|
1303
1396
|
|
1304
1397
|
return nil if resource.nil?
|
@@ -1339,6 +1432,7 @@ end
|
|
1339
1432
|
# Mu Master, if we're in AWS.
|
1340
1433
|
# @return [void]
|
1341
1434
|
def self.openFirewallForClients
|
1435
|
+
require "aws-sdk-ec2"
|
1342
1436
|
MU::Cloud.resourceClass("AWS", :FirewallRule)
|
1343
1437
|
begin
|
1344
1438
|
if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|