cloud-mu 3.3.0 → 3.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ansible/roles/mu-nat/tasks/main.yml +3 -0
- data/bin/mu-aws-setup +41 -7
- data/bin/mu-azure-setup +36 -2
- data/bin/mu-configure +214 -119
- data/bin/mu-gcp-setup +37 -2
- data/bin/mu-node-manage +3 -0
- data/bin/mu-refresh-ssl +67 -0
- data/bin/mu-run-tests +14 -4
- data/bin/mu-self-update +30 -10
- data/bin/mu-upload-chef-artifacts +30 -26
- data/cloud-mu.gemspec +9 -7
- data/cookbooks/mu-master/attributes/default.rb +5 -1
- data/cookbooks/mu-master/metadata.rb +2 -2
- data/cookbooks/mu-master/recipes/default.rb +81 -26
- data/cookbooks/mu-master/recipes/init.rb +197 -62
- data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
- data/cookbooks/mu-master/recipes/vault.rb +78 -77
- data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
- data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
- data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
- data/cookbooks/mu-tools/attributes/default.rb +12 -0
- data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
- data/cookbooks/mu-tools/libraries/helper.rb +98 -4
- data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +31 -9
- data/cookbooks/mu-tools/recipes/aws_api.rb +8 -2
- data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
- data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
- data/cookbooks/mu-tools/recipes/google_api.rb +7 -0
- data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
- data/cookbooks/mu-tools/resources/disk.rb +113 -42
- data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
- data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
- data/extras/Gemfile.lock.bootstrap +394 -0
- data/extras/bucketstubs/error.html +0 -0
- data/extras/bucketstubs/index.html +0 -0
- data/extras/clean-stock-amis +11 -3
- data/extras/generate-stock-images +6 -3
- data/extras/git_rpm/build.sh +20 -0
- data/extras/git_rpm/mugit.spec +53 -0
- data/extras/image-generators/AWS/centos7.yaml +19 -16
- data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
- data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
- data/extras/image-generators/VMWare/centos8.yaml +15 -0
- data/extras/openssl_rpm/build.sh +19 -0
- data/extras/openssl_rpm/mussl.spec +46 -0
- data/extras/python_rpm/muthon.spec +14 -4
- data/extras/ruby_rpm/muby.spec +9 -5
- data/extras/sqlite_rpm/build.sh +19 -0
- data/extras/sqlite_rpm/muqlite.spec +47 -0
- data/install/installer +7 -5
- data/modules/mommacat.ru +2 -2
- data/modules/mu.rb +12 -5
- data/modules/mu/cloud/machine_images.rb +1 -1
- data/modules/mu/cloud/providers.rb +6 -1
- data/modules/mu/cloud/resource_base.rb +7 -4
- data/modules/mu/cloud/ssh_sessions.rb +5 -1
- data/modules/mu/cloud/wrappers.rb +16 -7
- data/modules/mu/config.rb +28 -12
- data/modules/mu/config/database.rb +2 -2
- data/modules/mu/config/firewall_rule.rb +1 -1
- data/modules/mu/config/ref.rb +3 -3
- data/modules/mu/config/schema_helpers.rb +12 -3
- data/modules/mu/config/server.rb +10 -4
- data/modules/mu/config/server_pool.rb +2 -2
- data/modules/mu/config/vpc.rb +10 -10
- data/modules/mu/defaults/AWS.yaml +96 -96
- data/modules/mu/deploy.rb +27 -14
- data/modules/mu/groomers/chef.rb +2 -2
- data/modules/mu/master.rb +49 -3
- data/modules/mu/mommacat.rb +27 -9
- data/modules/mu/mommacat/naming.rb +2 -2
- data/modules/mu/mommacat/search.rb +16 -5
- data/modules/mu/mommacat/storage.rb +67 -32
- data/modules/mu/providers/aws.rb +185 -71
- data/modules/mu/providers/aws/alarm.rb +3 -3
- data/modules/mu/providers/aws/bucket.rb +19 -19
- data/modules/mu/providers/aws/cache_cluster.rb +22 -22
- data/modules/mu/providers/aws/cdn.rb +2 -2
- data/modules/mu/providers/aws/collection.rb +14 -14
- data/modules/mu/providers/aws/container_cluster.rb +27 -27
- data/modules/mu/providers/aws/database.rb +49 -45
- data/modules/mu/providers/aws/dnszone.rb +5 -5
- data/modules/mu/providers/aws/endpoint.rb +35 -35
- data/modules/mu/providers/aws/firewall_rule.rb +26 -23
- data/modules/mu/providers/aws/function.rb +35 -32
- data/modules/mu/providers/aws/group.rb +7 -7
- data/modules/mu/providers/aws/habitat.rb +2 -2
- data/modules/mu/providers/aws/job.rb +35 -32
- data/modules/mu/providers/aws/loadbalancer.rb +58 -37
- data/modules/mu/providers/aws/log.rb +14 -14
- data/modules/mu/providers/aws/msg_queue.rb +10 -10
- data/modules/mu/providers/aws/nosqldb.rb +8 -8
- data/modules/mu/providers/aws/notifier.rb +7 -7
- data/modules/mu/providers/aws/role.rb +69 -47
- data/modules/mu/providers/aws/search_domain.rb +10 -10
- data/modules/mu/providers/aws/server.rb +198 -110
- data/modules/mu/providers/aws/server_pool.rb +71 -119
- data/modules/mu/providers/aws/storage_pool.rb +17 -9
- data/modules/mu/providers/aws/user.rb +1 -1
- data/modules/mu/providers/aws/vpc.rb +106 -51
- data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
- data/modules/mu/providers/azure.rb +82 -16
- data/modules/mu/providers/azure/server.rb +18 -3
- data/modules/mu/providers/cloudformation/server.rb +1 -1
- data/modules/mu/providers/google.rb +20 -5
- data/modules/mu/providers/google/folder.rb +6 -2
- data/modules/mu/providers/google/function.rb +65 -30
- data/modules/mu/providers/google/role.rb +2 -1
- data/modules/mu/providers/google/vpc.rb +27 -2
- data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
- data/modules/tests/k8s.yaml +1 -1
- metadata +32 -15
data/modules/mu/providers/aws.rb
CHANGED
@@ -16,7 +16,6 @@ require "net/http"
|
|
16
16
|
require 'open-uri'
|
17
17
|
require 'timeout'
|
18
18
|
require 'inifile'
|
19
|
-
gem 'aws-sdk-core'
|
20
19
|
autoload :Aws, "aws-sdk-core"
|
21
20
|
|
22
21
|
|
@@ -54,14 +53,19 @@ module MU
|
|
54
53
|
def self.resourceInitHook(cloudobj, _deploy)
|
55
54
|
class << self
|
56
55
|
attr_reader :cloudformation_data
|
56
|
+
attr_reader :region
|
57
57
|
end
|
58
|
+
return if !cloudobj
|
58
59
|
cloudobj.instance_variable_set(:@cloudformation_data, {})
|
60
|
+
|
61
|
+
cloudobj.instance_variable_set(:@region, cloudobj.config['region'])
|
59
62
|
end
|
60
63
|
|
61
64
|
# Load some credentials for using the AWS API
|
62
65
|
# @param name [String]: The name of the mu.yaml AWS credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those.
|
63
66
|
# @return [Aws::Credentials]
|
64
67
|
def self.loadCredentials(name = nil)
|
68
|
+
gem 'aws-sdk-core'
|
65
69
|
@@creds_loaded ||= {}
|
66
70
|
|
67
71
|
if name.nil?
|
@@ -124,10 +128,14 @@ module MU
|
|
124
128
|
# pull access key and secret from a vault
|
125
129
|
begin
|
126
130
|
vault, item = cred_cfg["credentials"].split(/:/)
|
127
|
-
data =
|
128
|
-
|
131
|
+
data = if !vault or !item
|
132
|
+
raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg
|
133
|
+
else
|
134
|
+
MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h
|
135
|
+
end
|
136
|
+
if data and data["access_key"] and data["access_secret"]
|
129
137
|
cred_obj = Aws::Credentials.new(
|
130
|
-
|
138
|
+
data['access_key'], data['access_secret']
|
131
139
|
)
|
132
140
|
if name.nil?
|
133
141
|
# Aws.config = {
|
@@ -137,10 +145,10 @@ module MU
|
|
137
145
|
# }
|
138
146
|
end
|
139
147
|
else
|
140
|
-
|
148
|
+
raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg
|
141
149
|
end
|
142
150
|
rescue MU::Groomer::MuNoSuchSecret
|
143
|
-
|
151
|
+
raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg
|
144
152
|
end
|
145
153
|
end
|
146
154
|
|
@@ -186,6 +194,7 @@ end
|
|
186
194
|
# @param r [String]
|
187
195
|
# @return [String]
|
188
196
|
def self.validate_region(r, credentials: nil)
|
197
|
+
require "aws-sdk-ec2"
|
189
198
|
begin
|
190
199
|
MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name
|
191
200
|
rescue ::Aws::EC2::Errors::UnauthorizedOperation => e
|
@@ -204,6 +213,7 @@ end
|
|
204
213
|
# @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style
|
205
214
|
# @return [void]
|
206
215
|
def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
|
216
|
+
require "aws-sdk-ec2"
|
207
217
|
tags = []
|
208
218
|
MU::MommaCat.listStandardTags.each_pair { |name, value|
|
209
219
|
tags << {key: name, value: value} if !value.nil?
|
@@ -261,20 +271,33 @@ end
|
|
261
271
|
@@myVPCObj
|
262
272
|
end
|
263
273
|
|
264
|
-
# If we've configured AWS as a provider, or are simply hosted in AWS,
|
274
|
+
# If we've configured AWS as a provider, or are simply hosted in AWS,
|
265
275
|
# decide what our default region is.
|
266
|
-
def self.myRegion(credentials = nil)
|
267
|
-
|
276
|
+
def self.myRegion(credentials = nil, debug: false)
|
277
|
+
loglevel = debug ? MU::NOTICE : MU::DEBUG
|
278
|
+
if @@myRegion_var
|
279
|
+
MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel
|
280
|
+
return @@myRegion_var
|
281
|
+
end
|
268
282
|
|
283
|
+
MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig
|
284
|
+
MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s
|
285
|
+
MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION']
|
286
|
+
MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws']
|
269
287
|
if credConfig.nil? and !hosted? and !ENV['EC2_REGION']
|
288
|
+
MU.log "AWS.myRegion: nothing of use set, returning", loglevel
|
270
289
|
return nil
|
271
290
|
end
|
272
291
|
|
273
292
|
if $MU_CFG and $MU_CFG['aws']
|
274
293
|
$MU_CFG['aws'].each_pair { |credset, cfg|
|
294
|
+
MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg
|
275
295
|
next if credentials and credset != credentials
|
296
|
+
MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg
|
276
297
|
next if !cfg['region']
|
277
|
-
|
298
|
+
MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset)
|
299
|
+
if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset)
|
300
|
+
MU.log "AWS.myRegion: liking this set", loglevel, details: cfg
|
278
301
|
@@myRegion_var = cfg['region']
|
279
302
|
break if cfg['default'] or credentials
|
280
303
|
end
|
@@ -286,15 +309,21 @@ end
|
|
286
309
|
(Aws.config['access_key'] and Aws.config['access_secret'])
|
287
310
|
)
|
288
311
|
# Make sure this string is valid by way of the API
|
312
|
+
MU.log "AWS.myRegion: using ENV", loglevel, details: ENV
|
289
313
|
@@myRegion_var = ENV['EC2_REGION']
|
290
314
|
end
|
291
315
|
|
292
316
|
if hosted? and !@@myRegion_var
|
293
317
|
# hacky, but useful in a pinch (and if we're hosted in AWS)
|
294
318
|
az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone")
|
319
|
+
MU.log "AWS.myRegion: using hosted", loglevel, details: az_str
|
295
320
|
@@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str
|
296
321
|
end
|
297
322
|
|
323
|
+
if credConfig and credConfig["region"]
|
324
|
+
@@myRegion_var ||= credConfig["region"]
|
325
|
+
end
|
326
|
+
|
298
327
|
@@myRegion_var
|
299
328
|
end
|
300
329
|
|
@@ -382,8 +411,9 @@ end
|
|
382
411
|
# Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
|
383
412
|
# @param deploy_id [String]: The deploy for which we're writing the secret
|
384
413
|
# @param value [String]: The contents of the secret
|
385
|
-
def self.writeDeploySecret(
|
386
|
-
|
414
|
+
def self.writeDeploySecret(deploy, value, name = nil, credentials: nil)
|
415
|
+
require "aws-sdk-s3"
|
416
|
+
name ||= deploy.deploy_id+"-secret"
|
387
417
|
begin
|
388
418
|
MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}"
|
389
419
|
MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object(
|
@@ -401,35 +431,35 @@ end
|
|
401
431
|
def self.cloudtrailBucketPolicy(credentials = nil)
|
402
432
|
cfg = credConfig(credentials)
|
403
433
|
policy_json = '{
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
434
|
+
"Version": "2012-10-17",
|
435
|
+
"Statement": [
|
436
|
+
{
|
437
|
+
"Sid": "AWSCloudTrailAclCheck20131101",
|
438
|
+
"Effect": "Allow",
|
409
439
|
"Principal": {
|
410
440
|
"AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root",
|
411
441
|
"Service": "cloudtrail.amazonaws.com"
|
412
442
|
},
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
443
|
+
"Action": "s3:GetBucketAcl",
|
444
|
+
"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'"
|
445
|
+
},
|
446
|
+
{
|
447
|
+
"Sid": "AWSCloudTrailWrite20131101",
|
448
|
+
"Effect": "Allow",
|
419
449
|
"Principal": {
|
420
450
|
"AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root",
|
421
451
|
"Service": "cloudtrail.amazonaws.com"
|
422
452
|
},
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
453
|
+
"Action": "s3:PutObject",
|
454
|
+
"Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*",
|
455
|
+
"Condition": {
|
456
|
+
"StringEquals": {
|
457
|
+
"s3:x-amz-acl": "bucket-owner-full-control"
|
458
|
+
}
|
459
|
+
}
|
460
|
+
}
|
461
|
+
]
|
462
|
+
}'
|
433
463
|
ERB.new(policy_json).result
|
434
464
|
end
|
435
465
|
|
@@ -440,6 +470,53 @@ end
|
|
440
470
|
MU::Cloud::AWS.hosted?
|
441
471
|
end
|
442
472
|
|
473
|
+
# If we're in AWS and NVME-aware, return a mapping of AWS-side device
|
474
|
+
# names to actual NVME devices.
|
475
|
+
# @return [Hash]
|
476
|
+
def self.attachedNVMeDisks
|
477
|
+
if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme")
|
478
|
+
return {}
|
479
|
+
end
|
480
|
+
map = {}
|
481
|
+
devices = MU::Master.listBlockDevices
|
482
|
+
return {} if !devices
|
483
|
+
devices.each { |d|
|
484
|
+
if d =~ /^\/dev\/nvme/
|
485
|
+
%x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc|
|
486
|
+
if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./)
|
487
|
+
virt_dev = Regexp.last_match[1]
|
488
|
+
map[virt_dev] = d
|
489
|
+
break
|
490
|
+
end
|
491
|
+
}
|
492
|
+
end
|
493
|
+
}
|
494
|
+
map
|
495
|
+
end
|
496
|
+
|
497
|
+
# Map our own idea of what a block device is called back to whatever AWS
|
498
|
+
# and the operating system decided on amongst themselves. This currently
|
499
|
+
# exists to map generic "xvd[a-z]" style names back to real NVMe devices.
|
500
|
+
# @param dev [String]
|
501
|
+
def self.realDevicePath(dev)
|
502
|
+
return dev if !hosted?
|
503
|
+
value = nil
|
504
|
+
should_retry = Proc.new {
|
505
|
+
!value and MU::Master.nvme?
|
506
|
+
}
|
507
|
+
MU.retrier(loop_if: should_retry, wait: 5, max: 6) {
|
508
|
+
map = attachedNVMeDisks
|
509
|
+
value = if map[dev]
|
510
|
+
map[dev]
|
511
|
+
elsif map[dev.gsub(/.*?\//, '')]
|
512
|
+
map[dev.gsub(/.*?\//, '')]
|
513
|
+
else
|
514
|
+
dev # be nice to actually handle this too
|
515
|
+
end
|
516
|
+
}
|
517
|
+
value
|
518
|
+
end
|
519
|
+
|
443
520
|
# Determine whether we (the Mu master, presumably) are hosted in this
|
444
521
|
# cloud.
|
445
522
|
# @return [Boolean]
|
@@ -457,7 +534,7 @@ end
|
|
457
534
|
|
458
535
|
begin
|
459
536
|
Timeout.timeout(4) do
|
460
|
-
instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
|
537
|
+
instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read
|
461
538
|
if !instance_id.nil? and instance_id.size > 0
|
462
539
|
@@is_in_aws = true
|
463
540
|
region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "")
|
@@ -550,7 +627,9 @@ end
|
|
550
627
|
def self.credToAcct(name = nil)
|
551
628
|
creds = credConfig(name)
|
552
629
|
|
553
|
-
|
630
|
+
if creds['account_number'] and !creds['account_number'].empty?
|
631
|
+
return creds['account_number']
|
632
|
+
end
|
554
633
|
|
555
634
|
acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
|
556
635
|
acct_num.to_s
|
@@ -564,17 +643,18 @@ end
|
|
564
643
|
end
|
565
644
|
|
566
645
|
$MU_CFG['aws'].keys
|
567
|
-
end
|
646
|
+
end
|
568
647
|
|
569
648
|
# Resolve the administrative S3 bucket for a given credential set, or
|
570
649
|
# return a default.
|
571
650
|
# @param credentials [String]
|
572
651
|
# @return [String]
|
573
652
|
def self.adminBucketName(credentials = nil)
|
653
|
+
require "aws-sdk-s3"
|
574
654
|
cfg = credConfig(credentials)
|
575
655
|
return nil if !cfg
|
576
656
|
if !cfg['log_bucket_name']
|
577
|
-
cfg['log_bucket_name'] = $MU_CFG['hostname']
|
657
|
+
cfg['log_bucket_name'] = $MU_CFG['hostname']
|
578
658
|
MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN
|
579
659
|
end
|
580
660
|
resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets
|
@@ -608,7 +688,7 @@ end
|
|
608
688
|
|
609
689
|
# Return the $MU_CFG data associated with a particular profile/name/set of
|
610
690
|
# credentials. If no account name is specified, will return one flagged as
|
611
|
-
# default. Returns nil if AWS is not configured. Throws an exception if
|
691
|
+
# default. Returns nil if AWS is not configured. Throws an exception if
|
612
692
|
# an account name is specified which does not exist.
|
613
693
|
# @param name [String]: The name of the key under 'aws' in mu.yaml to return
|
614
694
|
# @return [Hash,nil]
|
@@ -623,10 +703,13 @@ end
|
|
623
703
|
|
624
704
|
if hosted?
|
625
705
|
begin
|
626
|
-
|
627
|
-
if
|
628
|
-
|
629
|
-
|
706
|
+
iam_blob = getAWSMetaData("iam/info")
|
707
|
+
if iam_blob
|
708
|
+
iam_data = JSON.parse(iam_blob)
|
709
|
+
if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty?
|
710
|
+
@@my_hosted_cfg = hosted_config
|
711
|
+
return name_only ? "#default" : @@my_hosted_cfg
|
712
|
+
end
|
630
713
|
end
|
631
714
|
rescue JSON::ParserError => e
|
632
715
|
end
|
@@ -672,8 +755,8 @@ end
|
|
672
755
|
next
|
673
756
|
end
|
674
757
|
acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
|
675
|
-
|
676
|
-
|
758
|
+
cfg['account_number'] ||= acct_num.to_s
|
759
|
+
if acct_num.to_s == name.to_s
|
677
760
|
@@acct_to_profile_map[name.to_s] = cfg
|
678
761
|
return name_only ? name.to_s : cfg
|
679
762
|
end
|
@@ -690,6 +773,7 @@ end
|
|
690
773
|
# XXX this needs to be "myAccountNumber" or somesuch
|
691
774
|
# XXX and maybe do the IAM thing for arbitrary, non-resident accounts
|
692
775
|
def self.account_number
|
776
|
+
require "aws-sdk-ec2"
|
693
777
|
return nil if credConfig.nil?
|
694
778
|
return @@my_acct_num if @@my_acct_num
|
695
779
|
loadCredentials
|
@@ -747,7 +831,7 @@ end
|
|
747
831
|
@@regions.keys.uniq
|
748
832
|
end
|
749
833
|
|
750
|
-
# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
|
834
|
+
# XXX GovCloud doesn't show up if you query a commercial endpoint... that's
|
751
835
|
# *probably* ok for most purposes? We can't call listAZs on it from out here
|
752
836
|
# apparently, so getting around it is nontrivial
|
753
837
|
# if !@@regions.has_key?("us-gov-west-1")
|
@@ -772,6 +856,7 @@ end
|
|
772
856
|
# @param public_key [String]: The public key
|
773
857
|
# @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key
|
774
858
|
def self.createEc2SSHKey(keyname, public_key, credentials: nil)
|
859
|
+
require "aws-sdk-ec2"
|
775
860
|
# We replicate this key in all regions
|
776
861
|
if !MU::Cloud::CloudFormation.emitCloudFormation
|
777
862
|
MU::Cloud::AWS.listRegions.each { |region|
|
@@ -800,8 +885,16 @@ end
|
|
800
885
|
def self.listInstanceTypes(region = myRegion)
|
801
886
|
return @@instance_types if @@instance_types and @@instance_types[region]
|
802
887
|
return {} if credConfig.nil?
|
888
|
+
if region.nil?
|
889
|
+
region = myRegion(debug: true)
|
890
|
+
end
|
891
|
+
return {} if region.nil?
|
803
892
|
|
804
893
|
human_region = @@regionLookup[region]
|
894
|
+
if human_region.nil?
|
895
|
+
MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR
|
896
|
+
return {}
|
897
|
+
end
|
805
898
|
|
806
899
|
@@instance_types ||= {}
|
807
900
|
@@instance_types[region] ||= {}
|
@@ -855,6 +948,7 @@ end
|
|
855
948
|
# @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
|
856
949
|
# @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
|
857
950
|
def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
|
951
|
+
require "aws-sdk-iam"
|
858
952
|
if (name.nil? or name.empty?) and (id.nil? or id.empty?)
|
859
953
|
raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
|
860
954
|
end
|
@@ -889,7 +983,7 @@ end
|
|
889
983
|
return nil
|
890
984
|
end
|
891
985
|
elsif matches.size > 1
|
892
|
-
raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
|
986
|
+
raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
|
893
987
|
end
|
894
988
|
end
|
895
989
|
|
@@ -951,7 +1045,7 @@ end
|
|
951
1045
|
# Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
|
952
1046
|
# look up and validate the specified certificate. This is intended to be
|
953
1047
|
# invoked from resource implementations' +validateConfig+ methods.
|
954
|
-
# @param certblock [Hash,MU::Config::Ref]:
|
1048
|
+
# @param certblock [Hash,MU::Config::Ref]:
|
955
1049
|
# @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
|
956
1050
|
# @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
|
957
1051
|
# @return [Boolean]
|
@@ -1119,7 +1213,7 @@ end
|
|
1119
1213
|
@@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials)
|
1120
1214
|
@@elasticache_api[credentials][region]
|
1121
1215
|
end
|
1122
|
-
|
1216
|
+
|
1123
1217
|
# Amazon's SNS API
|
1124
1218
|
def self.sns(region: MU.curRegion, credentials: nil)
|
1125
1219
|
region ||= myRegion
|
@@ -1127,7 +1221,7 @@ end
|
|
1127
1221
|
@@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials)
|
1128
1222
|
@@sns_api[credentials][region]
|
1129
1223
|
end
|
1130
|
-
|
1224
|
+
|
1131
1225
|
# Amazon's SQS API
|
1132
1226
|
def self.sqs(region: MU.curRegion, credentials: nil)
|
1133
1227
|
region ||= myRegion
|
@@ -1159,7 +1253,7 @@ end
|
|
1159
1253
|
@@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials)
|
1160
1254
|
@@apig_api[credentials][region]
|
1161
1255
|
end
|
1162
|
-
|
1256
|
+
|
1163
1257
|
# Amazon's Cloudwatch Events API
|
1164
1258
|
def self.cloudwatch_events(region = MU.cureRegion)
|
1165
1259
|
region ||= myRegion
|
@@ -1272,7 +1366,7 @@ end
|
|
1272
1366
|
begin
|
1273
1367
|
response = nil
|
1274
1368
|
Timeout.timeout(1) do
|
1275
|
-
response = open("#{base_url}/#{param}").read
|
1369
|
+
response = URI.open("#{base_url}/#{param}").read
|
1276
1370
|
end
|
1277
1371
|
|
1278
1372
|
response
|
@@ -1297,6 +1391,7 @@ end
|
|
1297
1391
|
tag_value=MU.deploy_id,
|
1298
1392
|
region: MU.curRegion,
|
1299
1393
|
credentials: nil)
|
1394
|
+
require "aws-sdk-ec2"
|
1300
1395
|
attempts = 0
|
1301
1396
|
|
1302
1397
|
return nil if resource.nil?
|
@@ -1337,6 +1432,7 @@ end
|
|
1337
1432
|
# Mu Master, if we're in AWS.
|
1338
1433
|
# @return [void]
|
1339
1434
|
def self.openFirewallForClients
|
1435
|
+
require "aws-sdk-ec2"
|
1340
1436
|
MU::Cloud.resourceClass("AWS", :FirewallRule)
|
1341
1437
|
begin
|
1342
1438
|
if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|
@@ -1516,6 +1612,7 @@ end
|
|
1516
1612
|
def initialize(region: nil, api: "EC2", credentials: nil)
|
1517
1613
|
@cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
|
1518
1614
|
@credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
|
1615
|
+
@api_name = api
|
1519
1616
|
|
1520
1617
|
if !@cred_obj
|
1521
1618
|
raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
|
@@ -1533,6 +1630,8 @@ end
|
|
1533
1630
|
params[:credentials] = @cred_obj
|
1534
1631
|
|
1535
1632
|
MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
|
1633
|
+
require "aws-sdk-#{api.downcase}"
|
1634
|
+
|
1536
1635
|
@api = Object.const_get("Aws::#{api}::Client").new(params)
|
1537
1636
|
end
|
1538
1637
|
|
@@ -1541,27 +1640,31 @@ end
|
|
1541
1640
|
# rescues for known silly endpoint behavior.
|
1542
1641
|
def method_missing(method_sym, *arguments)
|
1543
1642
|
# make sure error symbols are loaded for our exception handling later
|
1544
|
-
require "aws-sdk-
|
1545
|
-
require "aws-sdk-
|
1546
|
-
require "aws-sdk-
|
1547
|
-
require "aws-sdk-
|
1548
|
-
require "aws-sdk-
|
1549
|
-
require "aws-sdk-
|
1550
|
-
require "aws-sdk-
|
1551
|
-
require "aws-sdk-
|
1552
|
-
require "aws-sdk-
|
1553
|
-
require "aws-sdk-
|
1554
|
-
require "aws-sdk-
|
1555
|
-
require "aws-sdk-
|
1556
|
-
require "aws-sdk-
|
1557
|
-
require "aws-sdk-
|
1558
|
-
require "aws-sdk-
|
1559
|
-
|
1560
|
-
|
1643
|
+
require "aws-sdk-lambda"
|
1644
|
+
require "aws-sdk-rds"
|
1645
|
+
require "aws-sdk-ec2"
|
1646
|
+
require "aws-sdk-route53"
|
1647
|
+
require "aws-sdk-iam"
|
1648
|
+
require "aws-sdk-efs"
|
1649
|
+
require "aws-sdk-pricing"
|
1650
|
+
require "aws-sdk-apigateway"
|
1651
|
+
require "aws-sdk-ecs"
|
1652
|
+
require "aws-sdk-eks"
|
1653
|
+
require "aws-sdk-cloudwatchlogs"
|
1654
|
+
require "aws-sdk-cloudwatchevents"
|
1655
|
+
require "aws-sdk-elasticloadbalancing"
|
1656
|
+
require "aws-sdk-elasticloadbalancingv2"
|
1657
|
+
require "aws-sdk-autoscaling"
|
1658
|
+
|
1659
|
+
known_concats = {
|
1660
|
+
"Pricing" => {
|
1661
|
+
:get_products => :price_list
|
1662
|
+
}
|
1663
|
+
}
|
1561
1664
|
|
1562
1665
|
retries = 0
|
1563
1666
|
begin
|
1564
|
-
MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
|
1667
|
+
MU.log "Calling #{@api_name}.#{method_sym} in #{@region}", MU::DEBUG, details: arguments
|
1565
1668
|
|
1566
1669
|
retval = if !arguments.nil? and arguments.size == 1
|
1567
1670
|
@api.method(method_sym).call(arguments[0])
|
@@ -1590,11 +1693,22 @@ end
|
|
1590
1693
|
|
1591
1694
|
if paginator and new_page and !new_page.empty?
|
1592
1695
|
resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
|
1593
|
-
concat_to =
|
1696
|
+
concat_to = MU.structToHash(resp).keys.reject { |m|
|
1594
1697
|
m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
|
1595
1698
|
}
|
1699
|
+
|
1700
|
+
if concat_to.empty? and known_concats[@api_name] and
|
1701
|
+
known_concats[@api_name][method_sym]
|
1702
|
+
concat_to << known_concats[@api_name][method_sym]
|
1703
|
+
end
|
1704
|
+
|
1705
|
+
if concat_to.empty? and method_sym.to_s.match(/^(?:describe|list)_(.*)/)
|
1706
|
+
my_attr = Regexp.last_match[1].to_sym
|
1707
|
+
concat_to << my_attr if resp.respond_to?(my_attr)
|
1708
|
+
end
|
1709
|
+
|
1596
1710
|
if concat_to.size != 1
|
1597
|
-
|
1711
|
+
raise MuError.new "Tried to figure out where I might append paginated results for a #{@api_name}.#{method_sym}, but failed", details: MU.structToHash(resp).keys
|
1598
1712
|
else
|
1599
1713
|
concat_to = concat_to.first
|
1600
1714
|
new_args = arguments ? arguments.dup : [{}]
|