cloud-mu 3.1.6 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. checksums.yaml +4 -4
  2. data/bin/mu-adopt +4 -12
  3. data/bin/mu-azure-tests +57 -0
  4. data/bin/mu-cleanup +2 -4
  5. data/bin/mu-configure +37 -1
  6. data/bin/mu-deploy +3 -3
  7. data/bin/mu-findstray-tests +25 -0
  8. data/bin/mu-gen-docs +2 -4
  9. data/bin/mu-run-tests +23 -10
  10. data/cloud-mu.gemspec +2 -2
  11. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  12. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  13. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  14. data/extras/generate-stock-images +1 -0
  15. data/modules/mu.rb +82 -95
  16. data/modules/mu/adoption.rb +356 -56
  17. data/modules/mu/cleanup.rb +21 -20
  18. data/modules/mu/cloud.rb +79 -1753
  19. data/modules/mu/cloud/database.rb +49 -0
  20. data/modules/mu/cloud/dnszone.rb +46 -0
  21. data/modules/mu/cloud/machine_images.rb +212 -0
  22. data/modules/mu/cloud/providers.rb +81 -0
  23. data/modules/mu/cloud/resource_base.rb +920 -0
  24. data/modules/mu/cloud/server.rb +40 -0
  25. data/modules/mu/cloud/server_pool.rb +1 -0
  26. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  27. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  28. data/modules/mu/cloud/wrappers.rb +165 -0
  29. data/modules/mu/config.rb +122 -80
  30. data/modules/mu/config/alarm.rb +2 -6
  31. data/modules/mu/config/bucket.rb +1 -1
  32. data/modules/mu/config/cache_cluster.rb +1 -1
  33. data/modules/mu/config/collection.rb +1 -1
  34. data/modules/mu/config/container_cluster.rb +2 -2
  35. data/modules/mu/config/database.rb +83 -104
  36. data/modules/mu/config/database.yml +1 -2
  37. data/modules/mu/config/dnszone.rb +1 -1
  38. data/modules/mu/config/doc_helpers.rb +4 -5
  39. data/modules/mu/config/endpoint.rb +1 -1
  40. data/modules/mu/config/firewall_rule.rb +3 -19
  41. data/modules/mu/config/folder.rb +1 -1
  42. data/modules/mu/config/function.rb +1 -1
  43. data/modules/mu/config/group.rb +1 -1
  44. data/modules/mu/config/habitat.rb +1 -1
  45. data/modules/mu/config/loadbalancer.rb +57 -11
  46. data/modules/mu/config/log.rb +1 -1
  47. data/modules/mu/config/msg_queue.rb +1 -1
  48. data/modules/mu/config/nosqldb.rb +1 -1
  49. data/modules/mu/config/notifier.rb +1 -1
  50. data/modules/mu/config/ref.rb +30 -4
  51. data/modules/mu/config/role.rb +1 -1
  52. data/modules/mu/config/schema_helpers.rb +30 -34
  53. data/modules/mu/config/search_domain.rb +1 -1
  54. data/modules/mu/config/server.rb +4 -12
  55. data/modules/mu/config/server_pool.rb +3 -7
  56. data/modules/mu/config/storage_pool.rb +1 -1
  57. data/modules/mu/config/tail.rb +10 -0
  58. data/modules/mu/config/user.rb +1 -1
  59. data/modules/mu/config/vpc.rb +12 -17
  60. data/modules/mu/defaults/AWS.yaml +32 -32
  61. data/modules/mu/defaults/Azure.yaml +1 -0
  62. data/modules/mu/defaults/Google.yaml +1 -0
  63. data/modules/mu/deploy.rb +16 -15
  64. data/modules/mu/groomer.rb +15 -0
  65. data/modules/mu/groomers/chef.rb +3 -0
  66. data/modules/mu/logger.rb +120 -144
  67. data/modules/mu/master.rb +1 -1
  68. data/modules/mu/mommacat.rb +54 -25
  69. data/modules/mu/mommacat/daemon.rb +10 -7
  70. data/modules/mu/mommacat/naming.rb +82 -3
  71. data/modules/mu/mommacat/search.rb +47 -15
  72. data/modules/mu/mommacat/storage.rb +72 -41
  73. data/modules/mu/{clouds → providers}/README.md +1 -1
  74. data/modules/mu/{clouds → providers}/aws.rb +114 -47
  75. data/modules/mu/{clouds → providers}/aws/alarm.rb +1 -1
  76. data/modules/mu/{clouds → providers}/aws/bucket.rb +2 -2
  77. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +10 -46
  78. data/modules/mu/{clouds → providers}/aws/collection.rb +3 -3
  79. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +15 -33
  80. data/modules/mu/providers/aws/database.rb +1744 -0
  81. data/modules/mu/{clouds → providers}/aws/dnszone.rb +2 -5
  82. data/modules/mu/{clouds → providers}/aws/endpoint.rb +2 -11
  83. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +33 -29
  84. data/modules/mu/{clouds → providers}/aws/folder.rb +0 -0
  85. data/modules/mu/{clouds → providers}/aws/function.rb +2 -10
  86. data/modules/mu/{clouds → providers}/aws/group.rb +9 -13
  87. data/modules/mu/{clouds → providers}/aws/habitat.rb +1 -1
  88. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +41 -33
  89. data/modules/mu/{clouds → providers}/aws/log.rb +2 -2
  90. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +2 -8
  91. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +0 -0
  92. data/modules/mu/{clouds → providers}/aws/notifier.rb +0 -0
  93. data/modules/mu/{clouds → providers}/aws/role.rb +7 -7
  94. data/modules/mu/{clouds → providers}/aws/search_domain.rb +8 -13
  95. data/modules/mu/{clouds → providers}/aws/server.rb +55 -90
  96. data/modules/mu/{clouds → providers}/aws/server_pool.rb +10 -33
  97. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +19 -36
  98. data/modules/mu/{clouds → providers}/aws/user.rb +8 -12
  99. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  100. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  101. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
  102. data/modules/mu/{clouds → providers}/aws/vpc.rb +135 -70
  103. data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
  104. data/modules/mu/{clouds → providers}/azure.rb +4 -1
  105. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  106. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  107. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  108. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  109. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  110. data/modules/mu/{clouds → providers}/azure/server.rb +30 -23
  111. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  112. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  113. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  114. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  115. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  116. data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
  117. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  118. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  119. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  120. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  121. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  122. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  123. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  124. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  125. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  126. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  127. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  128. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  129. data/modules/mu/{clouds → providers}/google.rb +14 -6
  130. data/modules/mu/{clouds → providers}/google/bucket.rb +1 -1
  131. data/modules/mu/{clouds → providers}/google/container_cluster.rb +28 -13
  132. data/modules/mu/{clouds → providers}/google/database.rb +1 -8
  133. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +2 -2
  134. data/modules/mu/{clouds → providers}/google/folder.rb +4 -8
  135. data/modules/mu/{clouds → providers}/google/function.rb +3 -3
  136. data/modules/mu/{clouds → providers}/google/group.rb +8 -16
  137. data/modules/mu/{clouds → providers}/google/habitat.rb +3 -7
  138. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +1 -1
  139. data/modules/mu/{clouds → providers}/google/role.rb +42 -34
  140. data/modules/mu/{clouds → providers}/google/server.rb +25 -10
  141. data/modules/mu/{clouds → providers}/google/server_pool.rb +10 -10
  142. data/modules/mu/{clouds → providers}/google/user.rb +31 -21
  143. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  144. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  145. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  146. data/modules/mu/{clouds → providers}/google/vpc.rb +37 -2
  147. data/modules/tests/centos6.yaml +11 -0
  148. data/modules/tests/centos7.yaml +11 -0
  149. data/modules/tests/centos8.yaml +12 -0
  150. data/modules/tests/rds.yaml +108 -0
  151. data/modules/tests/regrooms/rds.yaml +123 -0
  152. data/spec/mu/clouds/azure_spec.rb +2 -2
  153. metadata +108 -89
  154. data/modules/mu/clouds/aws/database.rb +0 -1974
@@ -271,13 +271,13 @@ module MU
271
271
  my_org = MU::Cloud::Google.getOrg(credentials)
272
272
  if my_org
273
273
  scopes["organizations"] = [my_org.name]
274
- folders = MU::Cloud::Google::Folder.find(credentials: credentials)
274
+ folders = MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials)
275
275
  if folders and folders.size > 0
276
276
  scopes["folders"] = folders.keys
277
277
  end
278
278
  end
279
279
 
280
- projects = MU::Cloud::Google::Habitat.find(credentials: credentials)
280
+ projects = MU::Cloud.resourceClass("Google", "Habitat").find(credentials: credentials)
281
281
  if projects and projects.size > 0
282
282
  scopes["projects"] = projects.keys
283
283
  end
@@ -407,12 +407,12 @@ module MU
407
407
  # email field (which is the "real" id most of the time)
408
408
  real_id = nil
409
409
  if entity_type == "group"
410
- found = MU::Cloud::Google::Group.find(cloud_id: entity_id, credentials: credentials)
410
+ found = MU::Cloud.resourceClass("Google", "Group").find(cloud_id: entity_id, credentials: credentials)
411
411
  if found[entity_id]
412
412
  real_id = found[entity_id].id
413
413
  end
414
414
  elsif entity_type == "user"
415
- found = MU::Cloud::Google::User.find(cloud_id: entity_id, credentials: credentials)
415
+ found = MU::Cloud.resourceClass("Google", "User").find(cloud_id: entity_id, credentials: credentials)
416
416
  if found[entity_id]
417
417
  real_id = found[entity_id].id
418
418
  end
@@ -563,7 +563,7 @@ module MU
563
563
  if args[:project]
564
564
  canned = Hash[MU::Cloud::Google.iam(credentials: args[:credentials]).list_roles.roles.map { |r| [r.name, r] }]
565
565
  begin
566
- MU::Cloud::Google::Habitat.bindings(args[:project], credentials: args[:credentials]).each { |binding|
566
+ MU::Cloud.resourceClass("Google", "Habitat").bindings(args[:project], credentials: args[:credentials]).each { |binding|
567
567
  found[binding.role] = canned[binding.role]
568
568
  }
569
569
  rescue ::Google::Apis::ClientError => e
@@ -591,10 +591,15 @@ module MU
591
591
  bindings['by_scope']['projects'][args[:project]]
592
592
  bindings['by_scope']['projects'][args[:project]].keys.each { |r|
593
593
  if r.match(/^roles\//)
594
- role = MU::Cloud::Google.iam(credentials: args[:credentials]).get_role(r)
595
- found[role.name] = role
594
+ begin
595
+ role = MU::Cloud::Google.iam(credentials: args[:credentials]).get_role(r)
596
+ found[role.name] = role
597
+ rescue ::Google::Apis::ClientError => e
598
+ raise e if !e.message.match(/(?:forbidden|notFound): /)
599
+ MU.log "Failed MU::Cloud::Google.iam(credentials: #{args[:credentials]}).get_role(#{r})", MU::WARN, details: e.message
600
+ end
596
601
  elsif !found[r]
597
- MU.log "NEED TO GET #{r}", MU::WARN
602
+ # MU.log "NEED TO GET #{r}", MU::WARN
598
603
  end
599
604
  }
600
605
  end
@@ -688,7 +693,7 @@ module MU
688
693
  ids, _names, _privs = MU::Cloud::Google::Role.privilege_service_to_name(@config['credentials'])
689
694
  cloud_desc.role_privileges.each { |priv|
690
695
  if !ids[priv.service_id]
691
- MU.log "Role privilege defined for a service id with no name I can find, writing with raw id", MU::WARN, details: priv
696
+ MU.log "Role privilege defined for a service id with no name I can find, writing with raw id", MU::DEBUG, details: priv
692
697
  bok["import"] << priv.service_id+"/"+priv.privilege_name
693
698
  else
694
699
  bok["import"] << ids[priv.service_id]+"/"+priv.privilege_name
@@ -742,23 +747,33 @@ module MU
742
747
  entity_types.each_pair { |entity_type, entities|
743
748
  mu_entitytype = (entity_type == "serviceAccount" ? "user" : entity_type)+"s"
744
749
  entities.each { |entity|
750
+ next if entity.nil?
745
751
  foreign = if entity_type == "serviceAccount" and entity.match(/@(.*?)\.iam\.gserviceaccount\.com/)
746
752
  !MU::Cloud::Google.listHabitats(@credentials).include?(Regexp.last_match[1])
747
753
  end
754
+
748
755
  entity_ref = if entity_type == "organizations"
749
756
  { "id" => ((org == my_org.name and @config['credentials']) ? @config['credentials'] : org) }
750
757
  elsif entity_type == "domain"
751
758
  { "id" => entity }
752
759
  else
753
- if foreign
754
- { "id" => entity }
755
- else
756
- MU::Config::Ref.get(
757
- id: entity,
758
- cloud: "Google",
759
- type: mu_entitytype
760
- )
760
+ shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(mu_entitytype)
761
+ if args[:types].include?(shortclass) and
762
+ !(entity_type == "serviceAccount" and
763
+ MU::Cloud::Google::User.cannedServiceAcctName?(entity))
764
+ MU.log "Role #{@cloud_id}: Skipping #{shortclass} binding for #{entity}; we are adopting that type and will set bindings from that resource", MU::DEBUG
765
+ next
761
766
  end
767
+
768
+ MU::Config::Ref.get(
769
+ id: entity,
770
+ cloud: "Google",
771
+ type: mu_entitytype
772
+ )
773
+ end
774
+ if entity_ref.nil?
775
+ MU.log "I somehow ended up with a nil entity reference for #{entity_type} #{entity}", MU::ERR, details: [ bok, bindings ]
776
+ next
762
777
  end
763
778
  refmap ||= {}
764
779
  refmap[entity_ref] ||= {}
@@ -778,6 +793,7 @@ module MU
778
793
  }
779
794
  }
780
795
  }
796
+
781
797
  bok["bindings"] ||= []
782
798
  refmap.each_pair { |entity, scopes|
783
799
  newbinding = { "entity" => entity }
@@ -900,7 +916,7 @@ module MU
900
916
  end
901
917
 
902
918
  role = MU::Cloud::Google.admin_directory(credentials: credentials).get_role(MU::Cloud::Google.customerID(credentials), binding.role_id)
903
- MU.log "Failed to find entity #{binding.assigned_to} referenced in GSuite/Cloud Identity binding to role #{role.role_name}", MU::WARN, details: role
919
+ MU.log "Failed to find entity #{binding.assigned_to} referenced in GSuite/Cloud Identity binding to role #{role.role_name}", MU::DEBUG, details: role
904
920
  }
905
921
 
906
922
  resp = MU::Cloud::Google.resource_manager(credentials: credentials).get_organization_iam_policy(my_org.name)
@@ -908,15 +924,15 @@ module MU
908
924
  insertBinding("organizations", my_org.name, binding)
909
925
  }
910
926
 
911
- MU::Cloud::Google::Folder.find(credentials: credentials).keys.each { |folder|
912
- MU::Cloud::Google::Folder.bindings(folder, credentials: credentials).each { |binding|
927
+ MU::Cloud.resourceClass("Google", "Folder").find(credentials: credentials).keys.each { |folder|
928
+ MU::Cloud.resourceClass("Google", "Folder").bindings(folder, credentials: credentials).each { |binding|
913
929
  insertBinding("folders", folder, binding)
914
930
  }
915
931
  }
916
932
  end
917
- MU::Cloud::Google::Habitat.find(credentials: credentials).keys.each { |project|
933
+ MU::Cloud::Google.listHabitats(credentials).each { |project|
918
934
  begin
919
- MU::Cloud::Google::Habitat.bindings(project, credentials: credentials).each { |binding|
935
+ MU::Cloud.resourceClass("Google", "Habitat").bindings(project, credentials: credentials).each { |binding|
920
936
  insertBinding("projects", project, binding)
921
937
  }
922
938
  rescue ::Google::Apis::ClientError => e
@@ -1099,7 +1115,7 @@ If this value is not specified, and the role name matches the name of an existin
1099
1115
  MU.log "None of the directory service privileges available to credentials #{role['credentials']} map to the ones declared for role #{role['name']}", MU::ERR, details: role['import'].sort
1100
1116
  ok = false
1101
1117
  elsif missing.size > 0
1102
- MU.log "Some directory service privileges declared for role #{role['name']} aren't available to credentials #{role['credentials']}, will skip", MU::WARN, details: missing
1118
+ MU.log "Some directory service privileges declared for role #{role['name']} aren't available to credentials #{role['credentials']}, will skip", MU::DEBUG, details: missing
1103
1119
  end
1104
1120
  end
1105
1121
  end
@@ -1114,11 +1130,7 @@ If this value is not specified, and the role name matches the name of an existin
1114
1130
  if role['role_source'] == "project"
1115
1131
  role['project'] ||= MU::Cloud::Google.defaultProject(role['credentials'])
1116
1132
  if configurator.haveLitterMate?(role['project'], "habitats")
1117
- role['dependencies'] ||= []
1118
- role['dependencies'] << {
1119
- "type" => "habitat",
1120
- "name" => role['project']
1121
- }
1133
+ MU::Config.addDependency(role, role['project'], "habitat")
1122
1134
  end
1123
1135
  end
1124
1136
 
@@ -1126,14 +1138,10 @@ If this value is not specified, and the role name matches the name of an existin
1126
1138
  role['bindings'].each { |binding|
1127
1139
  if binding['entity'] and binding['entity']['name'] and
1128
1140
  configurator.haveLitterMate?(binding['entity']['name'], binding['entity']['type'])
1129
- role['dependencies'] ||= []
1130
- role['dependencies'] << {
1131
- "type" => binding['entity']['type'].sub(/s$/, ''),
1132
- "name" => binding['entity']['name']
1133
- }
1134
-
1141
+ MU::Config.addDependency(role, binding['entity']['name'], binding['entity']['type'])
1135
1142
  end
1136
1143
  }
1144
+ role['bindings'].uniq!
1137
1145
  end
1138
1146
 
1139
1147
  ok
@@ -492,7 +492,7 @@ next if !create
492
492
  return nil if @config.nil? or @deploy.nil?
493
493
 
494
494
  nat_ssh_key = nat_ssh_user = nat_ssh_host = nil
495
- if !@config["vpc"].nil? and !MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
495
+ if !@config["vpc"].nil? and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
496
496
 
497
497
  if !@nat.nil?
498
498
  if @nat.cloud_desc.nil?
@@ -623,7 +623,7 @@ next if !create
623
623
  end
624
624
 
625
625
  _nat_ssh_key, _nat_ssh_user, nat_ssh_host, _canonical_ip, _ssh_user, _ssh_key_name = getSSHConfig
626
- if !nat_ssh_host and !MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
626
+ if !nat_ssh_host and !MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials'])
627
627
  # XXX check if canonical_ip is in the private ranges
628
628
  # raise MuError, "#{node} has no NAT host configured, and I have no other route to it"
629
629
  end
@@ -992,7 +992,7 @@ next if !create
992
992
  # Our deploydata gets corrupted often with server pools, this will cause us to use the wrong IP to identify a node
993
993
  # which will cause us to create certificates, DNS records and other artifacts with incorrect information which will cause our deploy to fail.
994
994
  # The cloud_id is always correct so lets use 'cloud_desc' to get the correct IPs
995
- if MU::Cloud::Google::VPC.haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
995
+ if MU::Cloud.resourceClass("Google", "VPC").haveRouteToInstance?(cloud_desc, credentials: @config['credentials']) or public_ips.size == 0
996
996
  @config['canonical_ip'] = private_ips.first
997
997
  return private_ips.first
998
998
  else
@@ -1001,6 +1001,22 @@ next if !create
1001
1001
  end
1002
1002
  end
1003
1003
 
1004
+ # Return all of the IP addresses, public and private, from all of our
1005
+ # network interfaces.
1006
+ # @return [Array<String>]
1007
+ def listIPs
1008
+ ips = []
1009
+ cloud_desc.network_interfaces.each { |iface|
1010
+ ips << iface.network_ip
1011
+ if iface.access_configs
1012
+ iface.access_configs.each { |acfg|
1013
+ ips << acfg.nat_ip if acfg.nat_ip
1014
+ }
1015
+ end
1016
+ }
1017
+ ips
1018
+ end
1019
+
1004
1020
  # return [String]: A password string.
1005
1021
  def getWindowsAdminPassword(use_cache: true)
1006
1022
  @config['windows_auth_vault'] ||= {
@@ -1276,7 +1292,7 @@ next if !create
1276
1292
  # @return [void]
1277
1293
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
1278
1294
  flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
1279
- return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
1295
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
1280
1296
 
1281
1297
  # XXX make damn sure MU.deploy_id is set
1282
1298
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
@@ -1340,7 +1356,7 @@ next if !create
1340
1356
  def self.schema(config)
1341
1357
  toplevel_required = []
1342
1358
  schema = {
1343
- "roles" => MU::Cloud::Google::User.schema(config)[1]["roles"],
1359
+ "roles" => MU::Cloud.resourceClass("Google", "User").schema(config)[1]["roles"],
1344
1360
  "windows_admin_username" => {
1345
1361
  "type" => "string",
1346
1362
  "default" => "muadmin"
@@ -1451,8 +1467,7 @@ next if !create
1451
1467
  foundmatch = false
1452
1468
  MU::Cloud.availableClouds.each { |cloud|
1453
1469
  next if cloud == "Google"
1454
- cloudbase = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1455
- foreign_types = (cloudbase.listInstanceTypes).values.first
1470
+ foreign_types = (MU::Cloud.cloudClass(cloud).listInstanceTypes).values.first
1456
1471
  if foreign_types.size == 1
1457
1472
  foreign_types = foreign_types.values.first
1458
1473
  end
@@ -1518,12 +1533,12 @@ next if !create
1518
1533
  ok = false
1519
1534
  end
1520
1535
  else
1521
- server = MU::Cloud::Google::User.genericServiceAccount(server, configurator)
1536
+ server = MU::Cloud.resourceClass("Google", "User").genericServiceAccount(server, configurator)
1522
1537
  end
1523
1538
 
1524
1539
  subnets = nil
1525
1540
  if !server['vpc']
1526
- vpcs = MU::Cloud::Google::VPC.find(credentials: server['credentials'])
1541
+ vpcs = MU::Cloud.resourceClass("Google", "VPC").find(credentials: server['credentials'])
1527
1542
  if vpcs["default"]
1528
1543
  server["vpc"] ||= {}
1529
1544
  server["vpc"]["vpc_id"] = vpcs["default"].self_link
@@ -1538,7 +1553,7 @@ next if !create
1538
1553
  if !server['vpc']['subnet_id'] and server['vpc']['subnet_name'].nil?
1539
1554
  if !subnets
1540
1555
  if server["vpc"]["vpc_id"]
1541
- vpcs = MU::Cloud::Google::VPC.find(cloud_id: server["vpc"]["vpc_id"])
1556
+ vpcs = MU::Cloud.resourceClass("Google", "VPC").find(cloud_id: server["vpc"]["vpc_id"])
1542
1557
  subnets = vpcs["default"].subnetworks.sample
1543
1558
  end
1544
1559
  end
@@ -89,8 +89,8 @@ module MU
89
89
  machine_type: size,
90
90
  service_accounts: [@service_acct],
91
91
  labels: labels,
92
- disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
93
- network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
92
+ disks: MU::Cloud.resourceClass("Google", "Server").diskConfig(@config, false, false, credentials: @config['credentials']),
93
+ network_interfaces: MU::Cloud.resourceClass("Google", "Server").interfaceConfig(@config, @vpc),
94
94
  metadata: metadata,
95
95
  tags: MU::Cloud::Google.compute(:Tags).new(items: [MU::Cloud::Google.nameStr(@mu_name)])
96
96
  )
@@ -324,11 +324,11 @@ end
324
324
  def self.schema(config)
325
325
  toplevel_required = []
326
326
  schema = {
327
- "ssh_user" => MU::Cloud::Google::Server.schema(config)[1]["ssh_user"],
328
- "metadata" => MU::Cloud::Google::Server.schema(config)[1]["metadata"],
329
- "service_account" => MU::Cloud::Google::Server.schema(config)[1]["service_account"],
330
- "scopes" => MU::Cloud::Google::Server.schema(config)[1]["scopes"],
331
- "network_tags" => MU::Cloud::Google::Server.schema(config)[1]["network_tags"],
327
+ "ssh_user" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["ssh_user"],
328
+ "metadata" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["metadata"],
329
+ "service_account" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["service_account"],
330
+ "scopes" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["scopes"],
331
+ "network_tags" => MU::Cloud.resourceClass("Google", "Server").schema(config)[1]["network_tags"],
332
332
  "availability_zone" => {
333
333
  "type" => "string",
334
334
  "description" => "Target a specific availability zone for this pool, which will create zonal instance managers and scalers instead of regional ones."
@@ -382,7 +382,7 @@ end
382
382
  if pool['basis']['launch_config']
383
383
  launch = pool["basis"]["launch_config"]
384
384
 
385
- launch['size'] = MU::Cloud::Google::Server.validateInstanceType(launch["size"], pool["region"])
385
+ launch['size'] = MU::Cloud.resourceClass("Google", "Server").validateInstanceType(launch["size"], pool["region"])
386
386
  ok = false if launch['size'].nil?
387
387
 
388
388
  if launch['image_id'].nil?
@@ -397,7 +397,7 @@ end
397
397
 
398
398
  real_image = nil
399
399
  begin
400
- real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
400
+ real_image = MU::Cloud.resourceClass("Google", "Server").fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
401
401
  rescue ::Google::Apis::ClientError => e
402
402
  MU.log e.inspect, MU::WARN
403
403
  end
@@ -433,7 +433,7 @@ end
433
433
  # @return [void]
434
434
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
435
435
  flags["habitat"] ||= MU::Cloud::Google.defaultProject(credentials)
436
- return if !MU::Cloud::Google::Habitat.isLive?(flags["habitat"], credentials)
436
+ return if !MU::Cloud.resourceClass("Google", "Habitat").isLive?(flags["habitat"], credentials)
437
437
  filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
438
438
  if !ignoremaster and MU.mu_public_ip
439
439
  filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
@@ -26,10 +26,12 @@ module MU
26
26
  # If we're being reverse-engineered from a cloud descriptor, use that
27
27
  # to determine what sort of account we are.
28
28
  if args[:from_cloud_desc]
29
+ @cloud_desc_cache = args[:from_cloud_desc]
29
30
  MU::Cloud::Google.admin_directory
30
31
  MU::Cloud::Google.iam
31
32
  if args[:from_cloud_desc].class == ::Google::Apis::AdminDirectoryV1::User
32
33
  @config['type'] = "interactive"
34
+ @cloud_id = args[:from_cloud_desc].primary_email
33
35
  elsif args[:from_cloud_desc].class == ::Google::Apis::IamV1::ServiceAccount
34
36
  @config['type'] = "service"
35
37
  @config['name'] = args[:from_cloud_desc].display_name
@@ -48,6 +50,10 @@ module MU
48
50
  @config['name']
49
51
  end
50
52
 
53
+ if @config['type'] == "interactive" and @config['email']
54
+ @cloud_id ||= @config['email']
55
+ end
56
+
51
57
  end
52
58
 
53
59
  # Called automatically by {MU::Deploy#createResources}
@@ -58,7 +64,7 @@ module MU
58
64
  account_id: acct_id,
59
65
  service_account: MU::Cloud::Google.iam(:ServiceAccount).new(
60
66
  display_name: @mu_name,
61
- description: @config['scrub_mu_isms'] ? nil : @deploy.deploy_id
67
+ description: @config['scrub_mu_isms'] ? @config['description'] : @deploy.deploy_id
62
68
  )
63
69
  )
64
70
  if @config['use_if_exists']
@@ -90,7 +96,7 @@ module MU
90
96
  end
91
97
  elsif @config['external']
92
98
  @cloud_id = @config['email']
93
- MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
99
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
94
100
  else
95
101
  if !@config['email']
96
102
  domains = MU::Cloud::Google.admin_directory(credentials: @credentials).list_domains(@customer)
@@ -122,10 +128,10 @@ module MU
122
128
  # Called automatically by {MU::Deploy#createResources}
123
129
  def groom
124
130
  if @config['external']
125
- MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
131
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
126
132
  elsif @config['type'] == "interactive"
127
133
  need_update = false
128
- MU::Cloud::Google::Role.bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
134
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("user", @cloud_id, @config['roles'], credentials: @config['credentials'])
129
135
 
130
136
  if @config['force_password_change'] and !cloud_desc.change_password_at_next_login
131
137
  MU.log "Forcing #{@mu_name} to change their password at next login", MU::NOTICE
@@ -170,7 +176,7 @@ module MU
170
176
  end
171
177
 
172
178
  else
173
- MU::Cloud::Google::Role.bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
179
+ MU::Cloud.resourceClass("Google", "Role").bindFromConfig("serviceAccount", @cloud_id.gsub(/.*?\/([^\/]+)$/, '\1'), @config['roles'], credentials: @config['credentials'])
174
180
  if @config['create_api_key']
175
181
  resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
176
182
  cloud_desc.name
@@ -195,6 +201,7 @@ module MU
195
201
  if @config['type'] == "interactive" or !@config['type']
196
202
  @config['type'] ||= "interactive"
197
203
  if !@config['external']
204
+ @cloud_id ||= @config['email']
198
205
  @cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
199
206
  else
200
207
  return nil
@@ -226,7 +233,7 @@ module MU
226
233
  else
227
234
  {}
228
235
  end
229
- description.delete(:etag)
236
+ description.delete(:etag) if description
230
237
  description
231
238
  end
232
239
 
@@ -275,7 +282,7 @@ module MU
275
282
  next if user_email.nil?
276
283
  next if !user_email.match(/^[^\/]+@[^\/]+$/)
277
284
 
278
- MU::Cloud::Google::Role.removeBindings("user", user_email, credentials: credentials, noop: noop)
285
+ MU::Cloud.resourceClass("Google", "Role").removeBindings("user", user_email, credentials: credentials, noop: noop)
279
286
  }
280
287
 
281
288
  end
@@ -356,8 +363,10 @@ module MU
356
363
  else
357
364
  if cred_cfg['masquerade_as']
358
365
  resp = MU::Cloud::Google.admin_directory(credentials: args[:credentials]).list_users(customer: MU::Cloud::Google.customerID(args[:credentials]), show_deleted: false)
366
+ # XXX this ain't exactly performant, do some caching or something
359
367
  if resp and resp.users
360
368
  resp.users.each { |u|
369
+ next if args[:cloud_id] and !args[:cloud_id] != u.primary_email
361
370
  found[u.primary_email] = u
362
371
  }
363
372
  end
@@ -374,6 +383,7 @@ module MU
374
383
  name.match(/\b\d+\-compute@developer\.gserviceaccount\.com$/) or
375
384
  name.match(/\bproject-\d+@storage-transfer-service\.iam\.gserviceaccount\.com$/) or
376
385
  name.match(/\b\d+@cloudbuild\.gserviceaccount\.com$/) or
386
+ name.match(/\b\d+@cloudservices\.gserviceaccount\.com$/) or
377
387
  name.match(/\bservice-\d+@containerregistry\.iam\.gserviceaccount\.com$/) or
378
388
  name.match(/\bservice-\d+@container-analysis\.iam\.gserviceaccount\.com$/) or
379
389
  name.match(/\bservice-\d+@compute-system\.iam\.gserviceaccount\.com$/) or
@@ -416,7 +426,7 @@ module MU
416
426
  return nil
417
427
  end
418
428
 
419
- user_roles = MU::Cloud::Google::Role.getAllBindings(@config['credentials'])["by_entity"]
429
+ user_roles = MU::Cloud.resourceClass("Google", "Role").getAllBindings(@config['credentials'])["by_entity"]
420
430
 
421
431
  if cloud_desc.nil?
422
432
  MU.log "FAILED TO FIND CLOUD DESCRIPTOR FOR #{self}", MU::ERR, details: @config
@@ -429,6 +439,10 @@ module MU
429
439
 
430
440
  if bok['type'] == "service"
431
441
  bok['name'].gsub!(/@.*/, '')
442
+ if cloud_desc.description and !cloud_desc.description.empty? and
443
+ !cloud_desc.description.match(/^[A-Z0-9_-]+-[A-Z0-9_-]+-\d{10}-[A-Z]{2}$/)
444
+ bok['description'] = cloud_desc.description
445
+ end
432
446
  bok['project'] = @project_id
433
447
  keys = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(@cloud_id)
434
448
 
@@ -439,13 +453,13 @@ module MU
439
453
  if user_roles["serviceAccount"] and
440
454
  user_roles["serviceAccount"][bok['cloud_id']] and
441
455
  user_roles["serviceAccount"][bok['cloud_id']].size > 0
442
- bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
456
+ bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["serviceAccount"][bok['cloud_id']])
443
457
  end
444
458
  else
445
459
  if user_roles["user"] and
446
460
  user_roles["user"][bok['cloud_id']] and
447
461
  user_roles["user"][bok['cloud_id']].size > 0
448
- bok['roles'] = MU::Cloud::Google::Role.entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
462
+ bok['roles'] = MU::Cloud.resourceClass("Google", "Role").entityBindingsToSchema(user_roles["user"][bok['cloud_id']], credentials: @config['credentials'])
449
463
  end
450
464
  bok['given_name'] = cloud_desc.name.given_name if cloud_desc.name.given_name and !cloud_desc.name.given_name.empty?
451
465
  bok['family_name'] = cloud_desc.name.family_name if cloud_desc.name.family_name and !cloud_desc.name.family_name.empty?
@@ -501,6 +515,10 @@ If we are binding (rather than creating) a user and no roles are specified, we w
501
515
  "type" => "string",
502
516
  "description" => "Alias for +family_name+"
503
517
  },
518
+ "description" => {
519
+ "type" => "string",
520
+ "description" => "Comment field for service accounts, which we normally use to store the originating deploy's deploy id, since GCP service accounts do not have labels. This field is only honored if +scrub_mu_isms+ is set."
521
+ },
504
522
  "email" => {
505
523
  "type" => "string",
506
524
  "description" => "Canonical email address for a +directory+ user. If not specified, will be set to +name@domain+."
@@ -528,7 +546,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
528
546
  "roles" => {
529
547
  "type" => "array",
530
548
  "description" => "One or more Google IAM roles to associate with this user.",
531
- "items" => MU::Cloud::Google::Role.ref_schema
549
+ "items" => MU::Cloud.resourceClass("Google", "Role").ref_schema
532
550
  }
533
551
  }
534
552
  [toplevel_required, schema]
@@ -614,15 +632,11 @@ If we are binding (rather than creating) a user and no roles are specified, we w
614
632
  ok = false
615
633
  end
616
634
 
617
- user['dependencies'] ||= []
618
635
  if user['roles']
619
636
  user['roles'].each { |r|
620
637
  if r['role'] and r['role']['name'] and
621
638
  (!r['role']['deploy_id'] and !r['role']['id'])
622
- user['dependencies'] << {
623
- "type" => "role",
624
- "name" => r['role']['name']
625
- }
639
+ MU::Config.addDependency(user, r['role']['name'], "role")
626
640
  end
627
641
 
628
642
  if !r["projects"] and !r["organizations"] and !r["folders"]
@@ -661,7 +675,6 @@ If we are binding (rather than creating) a user and no roles are specified, we w
661
675
  user['roles'] = parent['roles'].dup
662
676
  end
663
677
  configurator.insertKitten(user, "users", true)
664
- parent['dependencies'] ||= []
665
678
  parent['service_account'] = MU::Config::Ref.get(
666
679
  type: "users",
667
680
  cloud: "Google",
@@ -669,10 +682,7 @@ If we are binding (rather than creating) a user and no roles are specified, we w
669
682
  project: user["project"],
670
683
  credentials: user["credentials"]
671
684
  )
672
- parent['dependencies'] << {
673
- "type" => "user",
674
- "name" => user["name"]
675
- }
685
+ MU::Config.addDependency(parent, user['name'], "user")
676
686
 
677
687
  parent
678
688
  end