cloud-mu 3.1.3 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +10 -2
  3. data/bin/mu-adopt +5 -1
  4. data/bin/mu-load-config.rb +2 -3
  5. data/bin/mu-run-tests +112 -27
  6. data/cloud-mu.gemspec +20 -20
  7. data/cookbooks/mu-tools/libraries/helper.rb +2 -1
  8. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  9. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  10. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  11. data/extras/image-generators/Google/centos6.yaml +1 -0
  12. data/extras/image-generators/Google/centos7.yaml +1 -1
  13. data/modules/mommacat.ru +5 -15
  14. data/modules/mu.rb +10 -14
  15. data/modules/mu/adoption.rb +20 -14
  16. data/modules/mu/cleanup.rb +13 -9
  17. data/modules/mu/cloud.rb +26 -26
  18. data/modules/mu/clouds/aws.rb +100 -59
  19. data/modules/mu/clouds/aws/alarm.rb +4 -2
  20. data/modules/mu/clouds/aws/bucket.rb +25 -21
  21. data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
  22. data/modules/mu/clouds/aws/collection.rb +21 -20
  23. data/modules/mu/clouds/aws/container_cluster.rb +47 -26
  24. data/modules/mu/clouds/aws/database.rb +57 -68
  25. data/modules/mu/clouds/aws/dnszone.rb +14 -14
  26. data/modules/mu/clouds/aws/endpoint.rb +20 -16
  27. data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
  28. data/modules/mu/clouds/aws/folder.rb +7 -7
  29. data/modules/mu/clouds/aws/function.rb +15 -12
  30. data/modules/mu/clouds/aws/group.rb +14 -10
  31. data/modules/mu/clouds/aws/habitat.rb +16 -13
  32. data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
  33. data/modules/mu/clouds/aws/log.rb +13 -10
  34. data/modules/mu/clouds/aws/msg_queue.rb +15 -8
  35. data/modules/mu/clouds/aws/nosqldb.rb +18 -11
  36. data/modules/mu/clouds/aws/notifier.rb +11 -6
  37. data/modules/mu/clouds/aws/role.rb +87 -70
  38. data/modules/mu/clouds/aws/search_domain.rb +30 -19
  39. data/modules/mu/clouds/aws/server.rb +102 -72
  40. data/modules/mu/clouds/aws/server_pool.rb +47 -28
  41. data/modules/mu/clouds/aws/storage_pool.rb +5 -6
  42. data/modules/mu/clouds/aws/user.rb +13 -10
  43. data/modules/mu/clouds/aws/vpc.rb +135 -121
  44. data/modules/mu/clouds/azure.rb +16 -9
  45. data/modules/mu/clouds/azure/container_cluster.rb +2 -3
  46. data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
  47. data/modules/mu/clouds/azure/habitat.rb +8 -6
  48. data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
  49. data/modules/mu/clouds/azure/role.rb +8 -10
  50. data/modules/mu/clouds/azure/server.rb +65 -25
  51. data/modules/mu/clouds/azure/user.rb +5 -7
  52. data/modules/mu/clouds/azure/vpc.rb +12 -15
  53. data/modules/mu/clouds/cloudformation.rb +8 -7
  54. data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
  55. data/modules/mu/clouds/google.rb +39 -24
  56. data/modules/mu/clouds/google/bucket.rb +9 -11
  57. data/modules/mu/clouds/google/container_cluster.rb +27 -42
  58. data/modules/mu/clouds/google/database.rb +6 -9
  59. data/modules/mu/clouds/google/firewall_rule.rb +11 -10
  60. data/modules/mu/clouds/google/folder.rb +16 -9
  61. data/modules/mu/clouds/google/function.rb +127 -161
  62. data/modules/mu/clouds/google/group.rb +21 -18
  63. data/modules/mu/clouds/google/habitat.rb +18 -15
  64. data/modules/mu/clouds/google/loadbalancer.rb +14 -16
  65. data/modules/mu/clouds/google/role.rb +48 -31
  66. data/modules/mu/clouds/google/server.rb +105 -105
  67. data/modules/mu/clouds/google/server_pool.rb +12 -31
  68. data/modules/mu/clouds/google/user.rb +67 -13
  69. data/modules/mu/clouds/google/vpc.rb +58 -65
  70. data/modules/mu/config.rb +89 -1738
  71. data/modules/mu/config/bucket.rb +3 -3
  72. data/modules/mu/config/collection.rb +3 -3
  73. data/modules/mu/config/container_cluster.rb +2 -2
  74. data/modules/mu/config/dnszone.rb +5 -5
  75. data/modules/mu/config/doc_helpers.rb +517 -0
  76. data/modules/mu/config/endpoint.rb +3 -3
  77. data/modules/mu/config/firewall_rule.rb +118 -3
  78. data/modules/mu/config/folder.rb +3 -3
  79. data/modules/mu/config/function.rb +2 -2
  80. data/modules/mu/config/group.rb +3 -3
  81. data/modules/mu/config/habitat.rb +3 -3
  82. data/modules/mu/config/loadbalancer.rb +3 -3
  83. data/modules/mu/config/log.rb +3 -3
  84. data/modules/mu/config/msg_queue.rb +3 -3
  85. data/modules/mu/config/nosqldb.rb +3 -3
  86. data/modules/mu/config/notifier.rb +2 -2
  87. data/modules/mu/config/ref.rb +333 -0
  88. data/modules/mu/config/role.rb +3 -3
  89. data/modules/mu/config/schema_helpers.rb +508 -0
  90. data/modules/mu/config/search_domain.rb +3 -3
  91. data/modules/mu/config/server.rb +86 -58
  92. data/modules/mu/config/server_pool.rb +2 -2
  93. data/modules/mu/config/tail.rb +189 -0
  94. data/modules/mu/config/user.rb +3 -3
  95. data/modules/mu/config/vpc.rb +44 -4
  96. data/modules/mu/defaults/Google.yaml +2 -2
  97. data/modules/mu/deploy.rb +13 -10
  98. data/modules/mu/groomer.rb +1 -1
  99. data/modules/mu/groomers/ansible.rb +69 -24
  100. data/modules/mu/groomers/chef.rb +52 -44
  101. data/modules/mu/logger.rb +17 -14
  102. data/modules/mu/master.rb +317 -2
  103. data/modules/mu/master/chef.rb +3 -4
  104. data/modules/mu/master/ldap.rb +3 -3
  105. data/modules/mu/master/ssl.rb +12 -2
  106. data/modules/mu/mommacat.rb +85 -1766
  107. data/modules/mu/mommacat/daemon.rb +394 -0
  108. data/modules/mu/mommacat/naming.rb +366 -0
  109. data/modules/mu/mommacat/storage.rb +689 -0
  110. data/modules/tests/bucket.yml +4 -0
  111. data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
  112. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  113. data/modules/tests/regrooms/bucket.yml +19 -0
  114. metadata +112 -102
@@ -35,7 +35,7 @@ module MU
35
35
  params = genParams
36
36
 
37
37
  MU.log "Creating ElasticSearch domain #{@config['domain_name']}", details: params
38
- resp = MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status
38
+ MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).create_elasticsearch_domain(params).domain_status
39
39
 
40
40
  tagDomain
41
41
 
@@ -57,11 +57,13 @@ module MU
57
57
  waitWhileProcessing # don't return until creation/updating is complete
58
58
  end
59
59
 
60
+ @cloud_desc_cache = nil
60
61
  # Wrapper for cloud_desc method that deals with finding the AWS
61
62
  # domain_name parameter, which isn't what we'd call ourselves if we had
62
63
  # our druthers.
63
- def cloud_desc
64
- if @config['domain_name']
64
+ def cloud_desc(use_cache: true)
65
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
66
+ @cloud_desc_cache = if @config['domain_name']
65
67
  MU::Cloud::AWS.elasticsearch(region: @config['region'], credentials: @config['credentials']).describe_elasticsearch_domain(
66
68
  domain_name: @config['domain_name']
67
69
  ).domain_status
@@ -72,6 +74,7 @@ module MU
72
74
  else
73
75
  raise MuError, "#{@mu_name} can't find its official Elasticsearch domain name!"
74
76
  end
77
+ @cloud_desc_cache
75
78
  end
76
79
 
77
80
  # Canonical Amazon Resource Number for this resource
@@ -117,25 +120,33 @@ module MU
117
120
  # @param region [String]: The cloud provider region
118
121
  # @return [void]
119
122
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
120
- list = MU::Cloud::AWS.elasticsearch(region: region).list_domain_names
123
+ MU.log "AWS::SearchDomain.cleanup: need to support flags['known']", MU::DEBUG, details: flags
124
+
125
+ list = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_domain_names
121
126
  if list and list.domain_names and list.domain_names.size > 0
122
127
  names = list.domain_names.map { |d| d.domain_name }
123
128
  begin
124
129
  # why is this API so obnoxious?
125
130
  sample = names.slice!(0, (names.length >= 5 ? 5 : names.length))
126
- descs = MU::Cloud::AWS.elasticsearch(region: region).describe_elasticsearch_domains(domain_names: sample)
131
+ descs = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).describe_elasticsearch_domains(domain_names: sample)
127
132
 
128
133
  descs.domain_status_list.each { |domain|
129
- tags = MU::Cloud::AWS.elasticsearch(region: region).list_tags(arn: domain.arn)
134
+ tags = MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).list_tags(arn: domain.arn)
135
+ deploy_match = false
136
+ master_match = false
130
137
  tags.tag_list.each { |tag|
131
138
  if tag.key == "MU-ID" and tag.value == MU.deploy_id
132
- MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
133
- if !noop
134
- MU::Cloud::AWS.elasticsearch(region: region).delete_elasticsearch_domain(domain_name: domain.domain_name)
135
- end
136
- break
139
+ deploy_match = true
140
+ elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
141
+ master_match = true
137
142
  end
138
143
  }
144
+ if deploy_match and (master_match or ignoremaster)
145
+ MU.log "Deleting ElasticSearch Domain #{domain.domain_name}"
146
+ if !noop
147
+ MU::Cloud::AWS.elasticsearch(region: region, credentials: credentials).delete_elasticsearch_domain(domain_name: domain.domain_name)
148
+ end
149
+ end
139
150
  }
140
151
  end while names.size > 0
141
152
  end
@@ -143,7 +154,7 @@ module MU
143
154
  unless noop
144
155
  marker = nil
145
156
  begin
146
- resp = MU::Cloud::AWS.iam.list_roles(marker: marker)
157
+ resp = MU::Cloud::AWS.iam(credentials: credentials).list_roles(marker: marker)
147
158
  resp.roles.each{ |role|
148
159
  # XXX Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud::AWS::Server.
149
160
  # MU::Cloud::AWS::Server.removeIAMProfile(role.role_name) if role.role_name.match(/^#{Regexp.quote(MU.deploy_id)}/)
@@ -181,14 +192,14 @@ module MU
181
192
  end
182
193
 
183
194
  # Cloud-specific configuration properties.
184
- # @param config [MU::Config]: The calling MU::Config object
195
+ # @param _config [MU::Config]: The calling MU::Config object
185
196
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
186
- def self.schema(config)
197
+ def self.schema(_config)
187
198
  toplevel_required = ["elasticsearch_version", "instance_type"]
188
199
 
189
200
  versions = begin
190
201
  MU::Cloud::AWS.elasticsearch.list_elasticsearch_versions.elasticsearch_versions
191
- rescue MuError => e
202
+ rescue MuError
192
203
  ["7.1", "6.8", "6.7", "6.5", "6.4", "6.3", "6.2", "6.0", "5.6"]
193
204
  end
194
205
  instance_types = begin
@@ -398,7 +409,7 @@ module MU
398
409
  MU::Cloud::AWS.cognito_ident(region: dom['region']).describe_identity_pool(
399
410
  identity_pool_id: dom['cognito']['identity_pool_id']
400
411
  )
401
- rescue ::Aws::CognitoIdentity::Errors::ValidationException, Aws::CognitoIdentity::Errors::ResourceNotFoundException => e
412
+ rescue ::Aws::CognitoIdentity::Errors::ValidationException, Aws::CognitoIdentity::Errors::ResourceNotFoundException
402
413
  MU.log "Cognito identity pool #{dom['cognito']['identity_pool_id']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
403
414
  ok = false
404
415
  end
@@ -406,7 +417,7 @@ module MU
406
417
  MU::Cloud::AWS.cognito_user(region: dom['region']).describe_user_pool(
407
418
  user_pool_id: dom['cognito']['user_pool_id']
408
419
  )
409
- rescue ::Aws::CognitoIdentityProvider::Errors::InvalidParameterException, Aws::CognitoIdentityProvider::Errors::ResourceNotFoundException => e
420
+ rescue ::Aws::CognitoIdentityProvider::Errors::InvalidParameterException, Aws::CognitoIdentityProvider::Errors::ResourceNotFoundException
410
421
  MU.log "Cognito identity pool #{dom['cognito']['user_pool_id']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
411
422
  ok = false
412
423
  end
@@ -426,7 +437,7 @@ module MU
426
437
  if !found
427
438
  MU.log "IAM role #{dom['cognito']['role_arn']} exists, but not does have the AmazonESCognitoAccess policy attached. SearchDomain '#{dom['name']}' may not have necessary Cognito permissions.", MU::WARN
428
439
  end
429
- rescue Aws::IAM::Errors::NoSuchEntity => e
440
+ rescue Aws::IAM::Errors::NoSuchEntity
430
441
  MU.log "IAM role #{dom['cognito']['role_arn']} malformed or does not exist in SearchDomain '#{dom['name']}'", MU::ERR
431
442
  ok = false
432
443
  end
@@ -626,7 +637,7 @@ module MU
626
637
  # modify an existing group. AWS bug, workaround is to just apply
627
638
  # this in groom phase exclusively.
628
639
  if @config['cognito'] and !ext.nil?
629
- myrole = setIAMPolicies
640
+ setIAMPolicies
630
641
 
631
642
  if ext.nil? or !ext.cognito_options.enabled or
632
643
  ext.cognito_options.user_pool_id != @config['cognito']['user_pool_id'] or
@@ -212,7 +212,7 @@ module MU
212
212
  vol_id = attachment.volume_id
213
213
  vol_dev = attachment.device
214
214
  if vol_parent == instance_id and (vol_dev == device or device.nil?)
215
- MU::MommaCat.createTag(vol_id, tag_name, tag_value, region: region, credentials: credentials)
215
+ MU::Cloud::AWS.createTag(vol_id, tag_name, tag_value, region: region, credentials: credentials)
216
216
  break
217
217
  end
218
218
  }
@@ -241,10 +241,10 @@ module MU
241
241
  MU::MommaCat.unlock(instance.instance_id+"-create")
242
242
  else
243
243
  MU::Cloud::AWS.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
244
- MU::MommaCat.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
244
+ MU::Cloud::AWS.createTag(instance.instance_id, "Name", @mu_name, region: @config['region'], credentials: @config['credentials'])
245
245
  end
246
246
  done = true
247
- rescue Exception => e
247
+ rescue StandardError => e
248
248
  if !instance.nil? and !done
249
249
  MU.log "Aborted before I could finish setting up #{@config['name']}, cleaning it up. Stack trace will print once cleanup is complete.", MU::WARN if !@deploy.nocleanup
250
250
  MU::MommaCat.unlockAll
@@ -266,7 +266,6 @@ module MU
266
266
 
267
267
  # Create an Amazon EC2 instance.
268
268
  def createEc2Instance
269
- name = @config["name"]
270
269
  node = @config['mu_name']
271
270
 
272
271
  instance_descriptor = {
@@ -319,7 +318,6 @@ module MU
319
318
  instance_descriptor[:private_ip_address] = @config['private_ip']
320
319
  end
321
320
 
322
- vpc_id = subnet = nil
323
321
  if !@vpc.nil? and @config.has_key?("vpc")
324
322
  subnet_conf = @config['vpc']
325
323
  subnet_conf = @config['vpc']['subnets'].first if @config['vpc'].has_key?("subnets") and !@config['vpc']['subnets'].empty?
@@ -357,7 +355,6 @@ module MU
357
355
  end
358
356
 
359
357
  configured_storage = Array.new
360
- cfm_volume_map = {}
361
358
  if @config["storage"]
362
359
  @config["storage"].each { |vol|
363
360
  # Drop the "encrypted" flag if a snapshot for this device exists
@@ -368,7 +365,7 @@ module MU
368
365
  vol.delete("encrypted")
369
366
  end
370
367
  end
371
- mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
368
+ mapping, _cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
372
369
  configured_storage << mapping
373
370
  }
374
371
  end
@@ -396,9 +393,9 @@ module MU
396
393
  instance = begin
397
394
  response = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).run_instances(instance_descriptor)
398
395
  if response and response.instances and response.instances.size > 0
399
- instance = response.instances.first
396
+ response.instances.first
400
397
  else
401
- MU.log "halp", MU::ERR, details: response
398
+ MU.log "Got a confusing response from run_instances", MU::ERR, details: response
402
399
  end
403
400
  rescue Aws::EC2::Errors::InvalidRequest => e
404
401
  MU.log e.message, MU::ERR, details: instance_descriptor
@@ -446,7 +443,7 @@ module MU
446
443
  instance_ids: [@cloud_id]
447
444
  )
448
445
  MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).wait_until(:instance_stopped, instance_ids: [@cloud_id]) do |waiter|
449
- waiter.before_attempt do |attempts|
446
+ waiter.before_attempt do
450
447
  MU.log "Waiting for #{@mu_name} to stop for hard reboot"
451
448
  end
452
449
  end
@@ -476,10 +473,9 @@ module MU
476
473
  # Figure out what's needed to SSH into this server.
477
474
  # @return [Array<String>]: nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name, alternate_names
478
475
  def getSSHConfig
479
- node, config, deploydata = describe(cloud_id: @cloud_id)
476
+ describe(cloud_id: @cloud_id)
480
477
  # XXX add some awesome alternate names from metadata and make sure they end
481
478
  # up in MU::MommaCat's ssh config wangling
482
- ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
483
479
  return nil if @config.nil? or @deploy.nil?
484
480
 
485
481
  nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
@@ -524,7 +520,7 @@ module MU
524
520
  if !instance_id.nil?
525
521
  @cloud_id = instance_id
526
522
  end
527
- node, config, deploydata = describe(cloud_id: @cloud_id)
523
+ node, _config, deploydata = describe(cloud_id: @cloud_id)
528
524
  instance = cloud_desc
529
525
  raise MuError, "Couldn't find instance #{@mu_name} (#{@cloud_id})" if !instance
530
526
  @cloud_id = instance.instance_id
@@ -532,17 +528,17 @@ module MU
532
528
  return false if !MU::MommaCat.lock(instance.instance_id+"-groom", true)
533
529
 
534
530
  MU::Cloud::AWS.createStandardTags(instance.instance_id, region: @config['region'], credentials: @config['credentials'])
535
- MU::MommaCat.createTag(instance.instance_id, "Name", node, region: @config['region'], credentials: @config['credentials'])
531
+ MU::Cloud::AWS.createTag(instance.instance_id, "Name", node, region: @config['region'], credentials: @config['credentials'])
536
532
 
537
533
  if @config['optional_tags']
538
534
  MU::MommaCat.listOptionalTags.each { |key, value|
539
- MU::MommaCat.createTag(instance.instance_id, key, value, region: @config['region'], credentials: @config['credentials'])
535
+ MU::Cloud::AWS.createTag(instance.instance_id, key, value, region: @config['region'], credentials: @config['credentials'])
540
536
  }
541
537
  end
542
538
 
543
539
  if !@config['tags'].nil?
544
540
  @config['tags'].each { |tag|
545
- MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
541
+ MU::Cloud::AWS.createTag(instance.instance_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
546
542
  }
547
543
  end
548
544
  MU.log "Tagged #{node} (#{instance.instance_id}) with MU-ID=#{MU.deploy_id}", MU::DEBUG
@@ -673,7 +669,7 @@ module MU
673
669
  if resp.addresses.size > 0 and resp.addresses.first.instance_id == @cloud_id
674
670
  has_elastic_ip = true
675
671
  end
676
- rescue Aws::EC2::Errors::InvalidAddressNotFound => e
672
+ rescue Aws::EC2::Errors::InvalidAddressNotFound
677
673
  # XXX this is ok to ignore, it means the public IP isn't Elastic
678
674
  end
679
675
  end
@@ -682,9 +678,6 @@ module MU
682
678
  ec2config_password = nil
683
679
  sshd_password = nil
684
680
  if windows?
685
- ssh_keydir = "#{Etc.getpwuid(Process.uid).dir}/.ssh"
686
- ssh_key_name = @deploy.ssh_key_name
687
-
688
681
  if @config['use_cloud_provider_windows_password']
689
682
  win_admin_password = getWindowsAdminPassword
690
683
  elsif @config['windows_auth_vault'] && !@config['windows_auth_vault'].empty?
@@ -747,14 +740,14 @@ module MU
747
740
  if !subnet.private? or (!@config['static_ip'].nil? and !@config['static_ip']['assign_ip'].nil?)
748
741
  if !@config['static_ip'].nil?
749
742
  if !@config['static_ip']['ip'].nil?
750
- public_ip = MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: false, ip: @config['static_ip']['ip'])
743
+ MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: false, ip: @config['static_ip']['ip'])
751
744
  elsif !has_elastic_ip
752
- public_ip = MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id)
745
+ MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id)
753
746
  end
754
747
  end
755
748
  end
756
749
 
757
- nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = getSSHConfig
750
+ _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
758
751
  if subnet.private? and !nat_ssh_host and !MU::Cloud::AWS::VPC.haveRouteToInstance?(cloud_desc, region: @config['region'], credentials: @config['credentials'])
759
752
  raise MuError, "#{node} is in a private subnet (#{subnet}), but has no bastion host configured, and I have no other route to it"
760
753
  end
@@ -769,17 +762,17 @@ module MU
769
762
  MU.log "Adding network interface on subnet #{subnet_id} for #{node}"
770
763
  iface = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_network_interface(subnet_id: subnet_id).network_interface
771
764
  MU::Cloud::AWS.createStandardTags(iface.network_interface_id, region: @config['region'], credentials: @config['credentials'])
772
- MU::MommaCat.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'], credentials: @config['credentials'])
765
+ MU::Cloud::AWS.createTag(iface.network_interface_id, "Name", node+"-ETH"+device_index.to_s, region: @config['region'], credentials: @config['credentials'])
773
766
 
774
767
  if @config['optional_tags']
775
768
  MU::MommaCat.listOptionalTags.each { |key, value|
776
- MU::MommaCat.createTag(iface.network_interface_id, key, value, region: @config['region'], credentials: @config['credentials'])
769
+ MU::Cloud::AWS.createTag(iface.network_interface_id, key, value, region: @config['region'], credentials: @config['credentials'])
777
770
  }
778
771
  end
779
772
 
780
773
  if !@config['tags'].nil?
781
774
  @config['tags'].each { |tag|
782
- MU::MommaCat.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
775
+ MU::Cloud::AWS.createTag(iface.network_interface_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
783
776
  }
784
777
  end
785
778
 
@@ -793,9 +786,9 @@ module MU
793
786
  end
794
787
  elsif !@config['static_ip'].nil?
795
788
  if !@config['static_ip']['ip'].nil?
796
- public_ip = MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true, ip: @config['static_ip']['ip'])
789
+ MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true, ip: @config['static_ip']['ip'])
797
790
  elsif !has_elastic_ip
798
- public_ip = MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true)
791
+ MU::Cloud::AWS::Server.associateElasticIp(instance.instance_id, classic: true)
799
792
  end
800
793
  end
801
794
 
@@ -811,8 +804,6 @@ module MU
811
804
  @config["private_ip_address"] = instance.private_ip_address
812
805
  @config["public_ip_address"] = instance.public_ip_address
813
806
 
814
- ext_mappings = MU.structToHash(instance.block_device_mappings)
815
-
816
807
  # Root disk on standard CentOS AMI
817
808
  # tagVolumes(instance.instance_id, "/dev/sda", "Name", "ROOT-"+MU.deploy_id+"-"+@config["name"].upcase)
818
809
  # Root disk on standard Ubuntu AMI
@@ -828,24 +819,24 @@ module MU
828
819
  vol.volumes.each { |volume|
829
820
  volume.attachments.each { |attachment|
830
821
  MU::MommaCat.listStandardTags.each_pair { |key, value|
831
- MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
822
+ MU::Cloud::AWS.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
832
823
 
833
824
  if attachment.device == "/dev/sda" or attachment.device == "/dev/sda1"
834
- MU::MommaCat.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'], credentials: @config['credentials'])
825
+ MU::Cloud::AWS.createTag(attachment.volume_id, "Name", "ROOT-#{MU.deploy_id}-#{@config["name"].upcase}", region: @config['region'], credentials: @config['credentials'])
835
826
  else
836
- MU::MommaCat.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'], credentials: @config['credentials'])
827
+ MU::Cloud::AWS.createTag(attachment.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{attachment.device.upcase}", region: @config['region'], credentials: @config['credentials'])
837
828
  end
838
829
  }
839
830
 
840
831
  if @config['optional_tags']
841
832
  MU::MommaCat.listOptionalTags.each { |key, value|
842
- MU::MommaCat.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
833
+ MU::Cloud::AWS.createTag(attachment.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
843
834
  }
844
835
  end
845
836
 
846
837
  if @config['tags']
847
838
  @config['tags'].each { |tag|
848
- MU::MommaCat.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
839
+ MU::Cloud::AWS.createTag(attachment.volume_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
849
840
  }
850
841
  end
851
842
  }
@@ -874,7 +865,7 @@ module MU
874
865
  if @config['groom'].nil? or @config['groom']
875
866
  if windows?
876
867
  # kick off certificate generation early; WinRM will need it
877
- cert, key = @deploy.nodeSSLCerts(self)
868
+ @deploy.nodeSSLCerts(self)
878
869
  if @config.has_key?("basis")
879
870
  @deploy.nodeSSLCerts(self, true)
880
871
  end
@@ -883,13 +874,13 @@ module MU
883
874
  initialWinRMTasks(session)
884
875
  begin
885
876
  session.close
886
- rescue Exception
877
+ rescue StandardError
887
878
  # this is allowed to fail- we're probably rebooting anyway
888
879
  end
889
880
  else # for an existing Windows node: WinRM, then SSH if it fails
890
881
  begin
891
882
  session = getWinRMSession(1, 60)
892
- rescue Exception # yeah, yeah
883
+ rescue StandardError # yeah, yeah
893
884
  session = getSSHSession(1, 60)
894
885
  # XXX maybe loop at least once if this also fails?
895
886
  end
@@ -984,7 +975,6 @@ module MU
984
975
  def self.find(**args)
985
976
  ip ||= args[:flags]['ip'] if args[:flags] and args[:flags]['ip']
986
977
 
987
- instance = nil
988
978
  if !args[:region].nil?
989
979
  regions = [args[:region]]
990
980
  else
@@ -1049,8 +1039,8 @@ module MU
1049
1039
  end
1050
1040
  }
1051
1041
  rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
1052
- if retries < 5
1053
- retries = retries + 1
1042
+ retries += 1
1043
+ if retries <= 5
1054
1044
  sleep 5
1055
1045
  else
1056
1046
  raise MuError, "#{e.inspect} in region #{r}"
@@ -1114,7 +1104,7 @@ module MU
1114
1104
  # Reverse-map our cloud description into a runnable config hash.
1115
1105
  # We assume that any values we have in +@config+ are placeholders, and
1116
1106
  # calculate our own accordingly based on what's live in the cloud.
1117
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
1107
+ def toKitten(**_args)
1118
1108
  bok = {
1119
1109
  "cloud" => "AWS",
1120
1110
  "credentials" => @config['credentials'],
@@ -1270,9 +1260,6 @@ module MU
1270
1260
  # Return a description of this resource appropriate for deployment
1271
1261
  # metadata. Arguments reflect the return values of the MU::Cloud::[Resource].describe method
1272
1262
  def notify
1273
- node, config, deploydata = describe(cloud_id: @cloud_id, update_cache: true)
1274
- deploydata = {} if deploydata.nil?
1275
-
1276
1263
  if cloud_desc.nil?
1277
1264
  raise MuError, "Failed to load instance metadata for #{@mu_name}/#{@cloud_id}"
1278
1265
  end
@@ -1345,7 +1332,7 @@ module MU
1345
1332
  end
1346
1333
  MU.log "Adding administrative holes for NAT host #{@nat.cloud_desc.private_ip_address} to #{@mu_name}"
1347
1334
  if !@deploy.kittens['firewall_rules'].nil?
1348
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
1335
+ @deploy.kittens['firewall_rules'].values.each { |acl|
1349
1336
  if acl.config["admin"]
1350
1337
  acl.addRule([@nat.cloud_desc.private_ip_address], proto: "tcp")
1351
1338
  acl.addRule([@nat.cloud_desc.private_ip_address], proto: "udp")
@@ -1358,7 +1345,7 @@ module MU
1358
1345
  # Called automatically by {MU::Deploy#createResources}
1359
1346
  def groom
1360
1347
  MU::MommaCat.lock(@cloud_id+"-groom")
1361
- node, config, deploydata = describe(cloud_id: @cloud_id)
1348
+ node, _config, deploydata = describe(cloud_id: @cloud_id)
1362
1349
 
1363
1350
  if node.nil? or node.empty?
1364
1351
  raise MuError, "MU::Cloud::AWS::Server.groom was called without a mu_name"
@@ -1407,7 +1394,7 @@ module MU
1407
1394
  end
1408
1395
  rescue MU::Groomer::RunError => e
1409
1396
  MU.log "Proceeding after failed initial Groomer run, but #{node} may not behave as expected!", MU::WARN, details: e.message
1410
- rescue Exception => e
1397
+ rescue StandardError => e
1411
1398
  MU.log "Caught #{e.inspect} on #{node} in an unexpected place (after @groomer.run on Full Initial Run)", MU::ERR
1412
1399
  end
1413
1400
 
@@ -1462,9 +1449,11 @@ module MU
1462
1449
  "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":instance/"+@cloud_id
1463
1450
  end
1464
1451
 
1452
+ @cloud_desc_cache = nil
1465
1453
  # Return the cloud provider's description for this instance
1466
1454
  # @return [Openstruct]
1467
- def cloud_desc
1455
+ def cloud_desc(use_cache: true)
1456
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
1468
1457
  max_retries = 5
1469
1458
  retries = 0
1470
1459
  if !@cloud_id.nil?
@@ -1473,11 +1462,12 @@ module MU
1473
1462
  if resp and resp.reservations and resp.reservations.first and
1474
1463
  resp.reservations.first.instances and
1475
1464
  resp.reservations.first.instances.first
1476
- return resp.reservations.first.instances.first
1465
+ @cloud_desc_cache = resp.reservations.first.instances.first
1466
+ return @cloud_desc_cache
1477
1467
  end
1478
1468
  rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
1479
1469
  return nil
1480
- rescue NoMethodError => e
1470
+ rescue NoMethodError
1481
1471
  if retries >= max_retries
1482
1472
  raise MuError, "Couldn't get a cloud descriptor for #{@mu_name} (#{@cloud_id})"
1483
1473
  else
@@ -1495,7 +1485,7 @@ module MU
1495
1485
  # bastion hosts that may be in the path, see getSSHConfig if that's what
1496
1486
  # you need.
1497
1487
  def canonicalIP
1498
- mu_name, config, deploydata = describe(cloud_id: @cloud_id)
1488
+ _mu_name, _config, deploydata = describe(cloud_id: @cloud_id)
1499
1489
 
1500
1490
  instance = cloud_desc
1501
1491
 
@@ -1574,7 +1564,7 @@ module MU
1574
1564
  resp = nil
1575
1565
  begin
1576
1566
  resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_image(ami_descriptor)
1577
- rescue Aws::EC2::Errors::InvalidAMINameDuplicate => e
1567
+ rescue Aws::EC2::Errors::InvalidAMINameDuplicate
1578
1568
  MU.log "AMI #{name} already exists, skipping", MU::WARN
1579
1569
  return nil
1580
1570
  end
@@ -1583,7 +1573,7 @@ module MU
1583
1573
 
1584
1574
  ami_ids[region] = ami
1585
1575
  MU::Cloud::AWS.createStandardTags(ami, region: region, credentials: credentials)
1586
- MU::MommaCat.createTag(ami, "Name", name, region: region, credentials: credentials)
1576
+ MU::Cloud::AWS.createTag(ami, "Name", name, region: region, credentials: credentials)
1587
1577
  MU.log "AMI of #{name} in region #{region}: #{ami}"
1588
1578
  if make_public
1589
1579
  MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials)
@@ -1611,10 +1601,10 @@ module MU
1611
1601
  ami_ids[r] = copy.image_id
1612
1602
 
1613
1603
  MU::Cloud::AWS.createStandardTags(copy.image_id, region: r, credentials: credentials)
1614
- MU::MommaCat.createTag(copy.image_id, "Name", name, region: r, credentials: credentials)
1604
+ MU::Cloud::AWS.createTag(copy.image_id, "Name", name, region: r, credentials: credentials)
1615
1605
  if !tags.nil?
1616
1606
  tags.each { |tag|
1617
- MU::MommaCat.createTag(instance.instance_id, tag['key'], tag['value'], region: r, credentials: credentials)
1607
+ MU::Cloud::AWS.createTag(instance.instance_id, tag['key'], tag['value'], region: r, credentials: credentials)
1618
1608
  }
1619
1609
  end
1620
1610
  MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r, credentials: credentials)
@@ -1721,7 +1711,7 @@ module MU
1721
1711
  # return [String]: A password string.
1722
1712
  def getWindowsAdminPassword
1723
1713
  if @cloud_id.nil?
1724
- node, config, deploydata = describe
1714
+ describe
1725
1715
  @cloud_id = cloud_desc.instance_id
1726
1716
  end
1727
1717
  ssh_keydir = "#{Etc.getpwuid(Process.uid).dir}/.ssh"
@@ -1842,7 +1832,6 @@ module MU
1842
1832
  if !resp.nil? and !resp.instances.nil?
1843
1833
  resp.instances.each { |instance|
1844
1834
  az = instance.placement.availability_zone
1845
- d_o_t_changed = true
1846
1835
  mappings = MU.structToHash(instance.block_device_mappings)
1847
1836
  mappings.each { |vol|
1848
1837
  if vol[:ebs]
@@ -1883,9 +1872,9 @@ module MU
1883
1872
 
1884
1873
  if @deploy
1885
1874
  MU::MommaCat.listStandardTags.each_pair { |key, value|
1886
- MU::MommaCat.createTag(creation.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
1875
+ MU::Cloud::AWS.createTag(creation.volume_id, key, value, region: @config['region'], credentials: @config['credentials'])
1887
1876
  }
1888
- MU::MommaCat.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'], credentials: @config['credentials'])
1877
+ MU::Cloud::AWS.createTag(creation.volume_id, "Name", "#{MU.deploy_id}-#{@config["name"].upcase}-#{dev.upcase}", region: @config['region'], credentials: @config['credentials'])
1889
1878
  end
1890
1879
 
1891
1880
  attachment = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).attach_volume(
@@ -2078,7 +2067,6 @@ module MU
2078
2067
  if !ignoremaster
2079
2068
  tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
2080
2069
  end
2081
- instances = Array.new
2082
2070
  unterminated = Array.new
2083
2071
  name_tags = Array.new
2084
2072
 
@@ -2119,7 +2107,7 @@ module MU
2119
2107
  threads << Thread.new(volume) { |myvolume|
2120
2108
  MU.dupGlobals(parent_thread_id)
2121
2109
  Thread.abort_on_exception = true
2122
- MU::Cloud::AWS::Server.delete_volume(myvolume, noop, skipsnapshots, credentials: credentials)
2110
+ delete_volume(myvolume, noop, skipsnapshots, credentials: credentials)
2123
2111
  }
2124
2112
  }
2125
2113
 
@@ -2140,7 +2128,7 @@ module MU
2140
2128
  if id
2141
2129
  begin
2142
2130
  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
2143
- rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
2131
+ rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
2144
2132
  MU.log "Instance #{id} no longer exists", MU::WARN
2145
2133
  end
2146
2134
  if !resp.nil? and !resp.reservations.nil? and !resp.reservations.first.nil?
@@ -2172,7 +2160,7 @@ module MU
2172
2160
 
2173
2161
  begin
2174
2162
  MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_instances(instance_ids: [id])
2175
- rescue Aws::EC2::Errors::InvalidInstanceIDNotFound => e
2163
+ rescue Aws::EC2::Errors::InvalidInstanceIDNotFound
2176
2164
  MU.log "Instance #{id} no longer exists", MU::DEBUG
2177
2165
  end
2178
2166
 
@@ -2336,15 +2324,19 @@ module MU
2336
2324
  end
2337
2325
 
2338
2326
  # Cloud-specific configuration properties.
2339
- # @param config [MU::Config]: The calling MU::Config object
2327
+ # @param _config [MU::Config]: The calling MU::Config object
2340
2328
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
2341
- def self.schema(config)
2329
+ def self.schema(_config)
2342
2330
  toplevel_required = []
2343
2331
  schema = {
2344
2332
  "ami_id" => {
2345
2333
  "type" => "string",
2346
2334
  "description" => "Alias for +image_id+"
2347
2335
  },
2336
+ "windows_admin_username" => {
2337
+ "type" => "string",
2338
+ "default" => "Administrator"
2339
+ },
2348
2340
  "generate_iam_role" => {
2349
2341
  "type" => "boolean",
2350
2342
  "default" => true,
@@ -2387,6 +2379,47 @@ module MU
2387
2379
  }
2388
2380
  }
2389
2381
  }
2382
+ },
2383
+ "ssh_user" => {
2384
+ "type" => "string",
2385
+ "default" => "root",
2386
+ "default_if" => [
2387
+ {
2388
+ "key_is" => "platform",
2389
+ "value_is" => "windows",
2390
+ "set" => "Administrator"
2391
+ },
2392
+ {
2393
+ "key_is" => "platform",
2394
+ "value_is" => "win2k12",
2395
+ "set" => "Administrator"
2396
+ },
2397
+ {
2398
+ "key_is" => "platform",
2399
+ "value_is" => "win2k12r2",
2400
+ "set" => "Administrator"
2401
+ },
2402
+ {
2403
+ "key_is" => "platform",
2404
+ "value_is" => "win2k16",
2405
+ "set" => "Administrator"
2406
+ },
2407
+ {
2408
+ "key_is" => "platform",
2409
+ "value_is" => "rhel7",
2410
+ "set" => "ec2-user"
2411
+ },
2412
+ {
2413
+ "key_is" => "platform",
2414
+ "value_is" => "rhel71",
2415
+ "set" => "ec2-user"
2416
+ },
2417
+ {
2418
+ "key_is" => "platform",
2419
+ "value_is" => "amazon",
2420
+ "set" => "ec2-user"
2421
+ }
2422
+ ]
2390
2423
  }
2391
2424
  }
2392
2425
  [toplevel_required, schema]
@@ -2556,20 +2589,17 @@ module MU
2556
2589
  img = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_images(image_ids: [ami_id]).images.first
2557
2590
  return DateTime.new if img.nil?
2558
2591
  return DateTime.parse(img.creation_date)
2559
- rescue Aws::EC2::Errors::InvalidAMIIDNotFound => e
2592
+ rescue Aws::EC2::Errors::InvalidAMIIDNotFound
2560
2593
  end
2561
2594
 
2562
2595
  return DateTime.new
2563
2596
  end
2564
2597
 
2565
- private
2566
-
2567
2598
  # Destroy a volume.
2568
2599
  # @param volume [OpenStruct]: The cloud provider's description of the volume.
2569
- # @param id [String]: The cloud provider's identifier for the volume, to use if the full description is not available.
2570
2600
  # @param region [String]: The cloud provider region
2571
2601
  # @return [void]
2572
- def self.delete_volume(volume, noop, skipsnapshots, id: nil, region: MU.curRegion, credentials: nil)
2602
+ def self.delete_volume(volume, noop, skipsnapshots, region: MU.curRegion, credentials: nil)
2573
2603
  if !volume.nil?
2574
2604
  resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_volumes(volume_ids: [volume.volume_id])
2575
2605
  volume = resp.data.volumes.first
@@ -2624,7 +2654,7 @@ module MU
2624
2654
  end
2625
2655
  end
2626
2656
  end
2627
-
2657
+ private_class_method :delete_volume
2628
2658
 
2629
2659
  end #class
2630
2660
  end #class