cloud-mu 3.1.4 → 3.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (203) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +5 -1
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +16 -12
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -1
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +37 -12
  23. data/cloud-mu.gemspec +5 -3
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  27. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  28. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  29. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  30. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  31. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  32. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  33. data/extras/clean-stock-amis +25 -19
  34. data/extras/generate-stock-images +1 -0
  35. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  36. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  37. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  38. data/modules/mommacat.ru +1 -1
  39. data/modules/mu.rb +158 -107
  40. data/modules/mu/adoption.rb +386 -59
  41. data/modules/mu/cleanup.rb +214 -303
  42. data/modules/mu/cloud.rb +128 -1632
  43. data/modules/mu/cloud/database.rb +49 -0
  44. data/modules/mu/cloud/dnszone.rb +44 -0
  45. data/modules/mu/cloud/machine_images.rb +212 -0
  46. data/modules/mu/cloud/providers.rb +81 -0
  47. data/modules/mu/cloud/resource_base.rb +926 -0
  48. data/modules/mu/cloud/server.rb +40 -0
  49. data/modules/mu/cloud/server_pool.rb +1 -0
  50. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  51. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  52. data/modules/mu/cloud/wrappers.rb +169 -0
  53. data/modules/mu/config.rb +135 -82
  54. data/modules/mu/config/alarm.rb +2 -6
  55. data/modules/mu/config/bucket.rb +32 -3
  56. data/modules/mu/config/cache_cluster.rb +2 -2
  57. data/modules/mu/config/cdn.rb +100 -0
  58. data/modules/mu/config/collection.rb +1 -1
  59. data/modules/mu/config/container_cluster.rb +7 -2
  60. data/modules/mu/config/database.rb +84 -105
  61. data/modules/mu/config/database.yml +1 -2
  62. data/modules/mu/config/dnszone.rb +5 -4
  63. data/modules/mu/config/doc_helpers.rb +5 -6
  64. data/modules/mu/config/endpoint.rb +2 -1
  65. data/modules/mu/config/firewall_rule.rb +3 -19
  66. data/modules/mu/config/folder.rb +1 -1
  67. data/modules/mu/config/function.rb +17 -8
  68. data/modules/mu/config/group.rb +1 -1
  69. data/modules/mu/config/habitat.rb +1 -1
  70. data/modules/mu/config/job.rb +89 -0
  71. data/modules/mu/config/loadbalancer.rb +57 -11
  72. data/modules/mu/config/log.rb +1 -1
  73. data/modules/mu/config/msg_queue.rb +1 -1
  74. data/modules/mu/config/nosqldb.rb +1 -1
  75. data/modules/mu/config/notifier.rb +8 -19
  76. data/modules/mu/config/ref.rb +92 -14
  77. data/modules/mu/config/role.rb +1 -1
  78. data/modules/mu/config/schema_helpers.rb +38 -37
  79. data/modules/mu/config/search_domain.rb +1 -1
  80. data/modules/mu/config/server.rb +12 -13
  81. data/modules/mu/config/server.yml +1 -0
  82. data/modules/mu/config/server_pool.rb +3 -7
  83. data/modules/mu/config/storage_pool.rb +1 -1
  84. data/modules/mu/config/tail.rb +11 -0
  85. data/modules/mu/config/user.rb +1 -1
  86. data/modules/mu/config/vpc.rb +27 -23
  87. data/modules/mu/config/vpc.yml +0 -1
  88. data/modules/mu/defaults/AWS.yaml +91 -68
  89. data/modules/mu/defaults/Azure.yaml +1 -0
  90. data/modules/mu/defaults/Google.yaml +1 -0
  91. data/modules/mu/deploy.rb +33 -19
  92. data/modules/mu/groomer.rb +16 -1
  93. data/modules/mu/groomers/ansible.rb +123 -21
  94. data/modules/mu/groomers/chef.rb +64 -11
  95. data/modules/mu/logger.rb +120 -144
  96. data/modules/mu/master.rb +97 -4
  97. data/modules/mu/master/ssl.rb +0 -1
  98. data/modules/mu/mommacat.rb +154 -867
  99. data/modules/mu/mommacat/daemon.rb +23 -14
  100. data/modules/mu/mommacat/naming.rb +110 -3
  101. data/modules/mu/mommacat/search.rb +495 -0
  102. data/modules/mu/mommacat/storage.rb +225 -192
  103. data/modules/mu/{clouds → providers}/README.md +1 -1
  104. data/modules/mu/{clouds → providers}/aws.rb +281 -64
  105. data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
  106. data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
  107. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
  108. data/modules/mu/providers/aws/cdn.rb +782 -0
  109. data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
  110. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
  111. data/modules/mu/providers/aws/database.rb +1744 -0
  112. data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
  113. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  114. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
  115. data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
  116. data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
  117. data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
  118. data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
  119. data/modules/mu/providers/aws/job.rb +466 -0
  120. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
  121. data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
  122. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
  123. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
  124. data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
  125. data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
  126. data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
  127. data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
  128. data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
  129. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
  130. data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
  131. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  132. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  133. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  134. data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
  135. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  136. data/modules/mu/{clouds → providers}/azure.rb +13 -0
  137. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
  138. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
  139. data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
  140. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
  141. data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
  142. data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
  143. data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
  144. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  145. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  146. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  147. data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
  148. data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
  149. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  150. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  151. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  152. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  153. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  156. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  158. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  159. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
  160. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  161. data/modules/mu/{clouds → providers}/google.rb +29 -6
  162. data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
  163. data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
  164. data/modules/mu/{clouds → providers}/google/database.rb +5 -12
  165. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
  166. data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
  167. data/modules/mu/{clouds → providers}/google/function.rb +14 -8
  168. data/modules/mu/{clouds → providers}/google/group.rb +9 -17
  169. data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
  170. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
  171. data/modules/mu/{clouds → providers}/google/role.rb +50 -31
  172. data/modules/mu/{clouds → providers}/google/server.rb +142 -55
  173. data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
  174. data/modules/mu/{clouds → providers}/google/user.rb +34 -24
  175. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  176. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  177. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  178. data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
  179. data/modules/tests/aws-jobs-functions.yaml +46 -0
  180. data/modules/tests/centos6.yaml +15 -0
  181. data/modules/tests/centos7.yaml +15 -0
  182. data/modules/tests/centos8.yaml +12 -0
  183. data/modules/tests/ecs.yaml +23 -0
  184. data/modules/tests/eks.yaml +1 -1
  185. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  186. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/microservice_app.yaml +288 -0
  189. data/modules/tests/rds.yaml +108 -0
  190. data/modules/tests/regrooms/rds.yaml +123 -0
  191. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  192. data/modules/tests/super_complex_bok.yml +2 -2
  193. data/modules/tests/super_simple_bok.yml +3 -5
  194. data/modules/tests/win2k12.yaml +25 -0
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +169 -93
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1974
  202. data/modules/mu/clouds/aws/endpoint.rb +0 -596
  203. data/modules/tests/needwork/win2k12.yaml +0 -13
@@ -107,7 +107,7 @@ module MU
107
107
  def groom
108
108
  if @config['roles']
109
109
  @config['roles'].each { |role|
110
- MU::Cloud::Azure::Role.assignTo(cloud_desc.principal_id, role_name: role, credentials: @config['credentials'])
110
+ MU::Cloud.resourceClass("Azure", "Role").assignTo(cloud_desc.principal_id, role_name: role, credentials: @config['credentials'])
111
111
  }
112
112
  end
113
113
  end
@@ -335,8 +335,10 @@ module MU
335
335
  return nil if cloud_desc.name == "default" # parent project builds these
336
336
  bok = {
337
337
  "cloud" => "Azure",
338
+ "name" => cloud_desc.name,
338
339
  "project" => @config['project'],
339
- "credentials" => @config['credentials']
340
+ "credentials" => @config['credentials'],
341
+ "cloud_id" => @cloud_id.to_s
340
342
  }
341
343
 
342
344
  bok
@@ -424,11 +426,7 @@ module MU
424
426
  }
425
427
  ]
426
428
  }
427
- vpc["dependencies"] ||= []
428
- vpc["dependencies"] << {
429
- "type" => "firewall_rule",
430
- "name" => vpc['name']+"-defaultfw"
431
- }
429
+ MU::Config.addDependency(vpc, vpc['name']+"-defaultfw", "firewall_rule")
432
430
 
433
431
  if !configurator.insertKitten(default_acl, "firewall_rules", true)
434
432
  ok = false
@@ -28,6 +28,16 @@ module MU
28
28
 
29
29
  @@cloudformation_mode = false
30
30
 
31
+ # Is this a "real" cloud provider, or a stub like CloudFormation?
32
+ def self.virtual?
33
+ true
34
+ end
35
+
36
+ # List all AWS projects available to our credentials
37
+ def self.listHabitats(credentials = nil, use_cache: true)
38
+ MU::Cloud::AWS.listHabitats(credentials)
39
+ end
40
+
31
41
  # Return what we think of as a cloud object's habitat. In AWS, this means
32
42
  # the +account_number+ in which it's resident. If this is not applicable,
33
43
  # such as for a {Habitat} or {Folder}, returns nil.
@@ -129,7 +129,7 @@ module MU
129
129
  # @param config [MU::Config]: The calling MU::Config object
130
130
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
131
131
  def self.schema(config)
132
- MU::Cloud::AWS::Alarm.schema(config)
132
+ MU::Cloud.resourceClass("AWS", "Alarm").schema(config)
133
133
  end
134
134
 
135
135
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -137,14 +137,14 @@ module MU
137
137
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
138
138
  # @return [Boolean]: True if validation succeeded, False otherwise
139
139
  def self.validateConfig(server, configurator)
140
- MU::Cloud::AWS::Alarm.validateConfig(server, configurator)
140
+ MU::Cloud.resourceClass("AWS", "Alarm").validateConfig(server, configurator)
141
141
  end
142
142
 
143
143
  # Does this resource type exist as a global (cloud-wide) artifact, or
144
144
  # is it localized to a region/zone?
145
145
  # @return [Boolean]
146
146
  def self.isGlobal?
147
- MU::Cloud::AWS::Alarm.isGlobal?
147
+ MU::Cloud.resourceClass("AWS", "Alarm").isGlobal?
148
148
  end
149
149
 
150
150
 
@@ -150,7 +150,7 @@ module MU
150
150
  # @param config [MU::Config]: The calling MU::Config object
151
151
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
152
152
  def self.schema(config)
153
- MU::Cloud::AWS::CacheCluster.schema(config)
153
+ MU::Cloud.resourceClass("AWS", "CacheCluster").schema(config)
154
154
  end
155
155
 
156
156
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -158,14 +158,14 @@ module MU
158
158
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
159
159
  # @return [Boolean]: True if validation succeeded, False otherwise
160
160
  def self.validateConfig(server, configurator)
161
- MU::Cloud::AWS::CacheCluster.validateConfig(server, configurator)
161
+ MU::Cloud.resourceClass("AWS", "CacheCluster").validateConfig(server, configurator)
162
162
  end
163
163
 
164
164
  # Does this resource type exist as a global (cloud-wide) artifact, or
165
165
  # is it localized to a region/zone?
166
166
  # @return [Boolean]
167
167
  def self.isGlobal?
168
- MU::Cloud::AWS::CacheCluster.isGlobal?
168
+ MU::Cloud.resourceClass("AWS", "CacheCluster").isGlobal?
169
169
  end
170
170
 
171
171
  end
@@ -100,7 +100,7 @@ module MU
100
100
  # @param config [MU::Config]: The calling MU::Config object
101
101
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
102
102
  def self.schema(config)
103
- MU::Cloud::AWS::Collection.schema(config)
103
+ MU::Cloud.resourceClass("AWS", "Collection").schema(config)
104
104
  end
105
105
 
106
106
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -108,14 +108,14 @@ module MU
108
108
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
109
109
  # @return [Boolean]: True if validation succeeded, False otherwise
110
110
  def self.validateConfig(server, configurator)
111
- MU::Cloud::AWS::Collection.validateConfig(server, configurator)
111
+ MU::Cloud.resourceClass("AWS", "Collection").validateConfig(server, configurator)
112
112
  end
113
113
 
114
114
  # Does this resource type exist as a global (cloud-wide) artifact, or
115
115
  # is it localized to a region/zone?
116
116
  # @return [Boolean]
117
117
  def self.isGlobal?
118
- MU::Cloud::AWS::Collection.isGlobal?
118
+ MU::Cloud.resourceClass("AWS", "Collection").isGlobal?
119
119
  end
120
120
 
121
121
  end
@@ -64,8 +64,8 @@ module MU
64
64
  basename = @config["name"].to_s
65
65
  basename = basename+@deploy.timestamp+MU.seed.downcase if !@config['scrub_mu_isms']
66
66
  basename.gsub!(/[^a-z0-9]/i, "")
67
- @config["db_name"] = MU::Cloud::AWS::Database.getName(basename, type: "dbname", config: @config)
68
- @config['master_user'] = MU::Cloud::AWS::Database.getName(basename, type: "dbuser", config: @config)
67
+ @config["db_name"] = MU::Cloud.resourceClass("AWS", "Database").getName(basename, type: "dbname", config: @config)
68
+ @config['master_user'] = MU::Cloud.resourceClass("AWS", "Database").getName(basename, type: "dbuser", config: @config)
69
69
 
70
70
  if @config["create_cluster"]
71
71
  @cfm_name, @cfm_template = MU::Cloud::CloudFormation.cloudFormationBase("dbcluster", self, tags: @config['tags'], scrub_mu_isms: @config['scrub_mu_isms']) if @cfm_template.nil?
@@ -214,18 +214,7 @@ module MU
214
214
  elsif @config['db_name']
215
215
  MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_name], "DBName", @config['db_name'])
216
216
  end
217
- if @config['password'].nil?
218
- if @config['auth_vault'] && !@config['auth_vault'].empty?
219
- @config['password'] = @groomclass.getSecret(
220
- vault: @config['auth_vault']['vault'],
221
- item: @config['auth_vault']['item'],
222
- field: @config['auth_vault']['password_field']
223
- )
224
- else
225
- # Should we use random instead?
226
- @config['password'] = Password.pronounceable(10..12)
227
- end
228
- end
217
+ getPassword
229
218
  MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_name], "MasterUserPassword", @config['password'])
230
219
  end
231
220
  end
@@ -260,7 +249,7 @@ module MU
260
249
  # @param config [MU::Config]: The calling MU::Config object
261
250
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
262
251
  def self.schema(config)
263
- MU::Cloud::AWS::Database.schema(config)
252
+ MU::Cloud.resourceClass("AWS", "Database").schema(config)
264
253
  end
265
254
 
266
255
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -268,14 +257,14 @@ module MU
268
257
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
269
258
  # @return [Boolean]: True if validation succeeded, False otherwise
270
259
  def self.validateConfig(server, configurator)
271
- MU::Cloud::AWS::Database.validateConfig(server, configurator)
260
+ MU::Cloud.resourceClass("AWS", "Database").validateConfig(server, configurator)
272
261
  end
273
262
 
274
263
  # Does this resource type exist as a global (cloud-wide) artifact, or
275
264
  # is it localized to a region/zone?
276
265
  # @return [Boolean]
277
266
  def self.isGlobal?
278
- MU::Cloud::AWS::Database.isGlobal?
267
+ MU::Cloud.resourceClass("AWS", "Database").isGlobal?
279
268
  end
280
269
 
281
270
 
@@ -257,7 +257,7 @@ module MU
257
257
  # @param config [MU::Config]: The calling MU::Config object
258
258
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
259
259
  def self.schema(config)
260
- MU::Cloud::AWS::DNSZone.schema(config)
260
+ MU::Cloud.resourceClass("AWS", "DNSZone").schema(config)
261
261
  end
262
262
 
263
263
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -265,14 +265,14 @@ module MU
265
265
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
266
266
  # @return [Boolean]: True if validation succeeded, False otherwise
267
267
  def self.validateConfig(server, configurator)
268
- MU::Cloud::AWS::DNSZone.validateConfig(server, configurator)
268
+ MU::Cloud.resourceClass("AWS", "DNSZone").validateConfig(server, configurator)
269
269
  end
270
270
 
271
271
  # Does this resource type exist as a global (cloud-wide) artifact, or
272
272
  # is it localized to a region/zone?
273
273
  # @return [Boolean]
274
274
  def self.isGlobal?
275
- MU::Cloud::AWS::DNSZone.isGlobal?
275
+ MU::Cloud.resourceClass("AWS", "DNSZone").isGlobal?
276
276
  end
277
277
 
278
278
  end
@@ -137,7 +137,7 @@ module MU
137
137
  # @return [Boolean]: True if validation succeeded, False otherwise
138
138
  def self.validateConfig(acl, config)
139
139
  # Just use the AWS implemention
140
- MU::Cloud::AWS::FirewallRule.validateConfig(acl, config)
140
+ MU::Cloud.resourceClass("AWS", "FirewallRule").validateConfig(acl, config)
141
141
  end
142
142
 
143
143
  private
@@ -291,14 +291,14 @@ module MU
291
291
  # @param config [MU::Config]: The calling MU::Config object
292
292
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
293
293
  def self.schema(config)
294
- MU::Cloud::AWS::FirewallRule.schema(config)
294
+ MU::Cloud.resourceClass("AWS", "FirewallRule").schema(config)
295
295
  end
296
296
 
297
297
  # Does this resource type exist as a global (cloud-wide) artifact, or
298
298
  # is it localized to a region/zone?
299
299
  # @return [Boolean]
300
300
  def self.isGlobal?
301
- MU::Cloud::AWS::FirewallRule.isGlobal?
301
+ MU::Cloud.resourceClass("AWS", "FirewallRule").isGlobal?
302
302
  end
303
303
 
304
304
  end #class
@@ -176,7 +176,7 @@ module MU
176
176
  # @param config [MU::Config]: The calling MU::Config object
177
177
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
178
178
  def self.schema(config)
179
- MU::Cloud::AWS::LoadBalancer.schema(config)
179
+ MU::Cloud.resourceClass("AWS", "LoadBalancer").schema(config)
180
180
  end
181
181
 
182
182
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -184,14 +184,14 @@ module MU
184
184
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
185
185
  # @return [Boolean]: True if validation succeeded, False otherwise
186
186
  def self.validateConfig(server, configurator)
187
- MU::Cloud::AWS::LoadBalancer.validateConfig(server, configurator)
187
+ MU::Cloud.resourceClass("AWS", "LoadBalancer").validateConfig(server, configurator)
188
188
  end
189
189
 
190
190
  # Does this resource type exist as a global (cloud-wide) artifact, or
191
191
  # is it localized to a region/zone?
192
192
  # @return [Boolean]
193
193
  def self.isGlobal?
194
- MU::Cloud::AWS::LoadBalancer.isGlobal?
194
+ MU::Cloud.resourceClass("AWS", "LoadBalancer").isGlobal?
195
195
  end
196
196
 
197
197
  end
@@ -153,7 +153,7 @@ module MU
153
153
  # @param config [MU::Config]: The calling MU::Config object
154
154
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
155
155
  def self.schema(config)
156
- MU::Cloud::AWS::Log.schema(config)
156
+ MU::Cloud.resourceClass("AWS", "Log").schema(config)
157
157
  end
158
158
 
159
159
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -161,14 +161,14 @@ module MU
161
161
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
162
162
  # @return [Boolean]: True if validation succeeded, False otherwise
163
163
  def self.validateConfig(server, configurator)
164
- MU::Cloud::AWS::Log.validateConfig(server, configurator)
164
+ MU::Cloud.resourceClass("AWS", "Log").validateConfig(server, configurator)
165
165
  end
166
166
 
167
167
  # Does this resource type exist as a global (cloud-wide) artifact, or
168
168
  # is it localized to a region/zone?
169
169
  # @return [Boolean]
170
170
  def self.isGlobal?
171
- MU::Cloud::AWS::Log.isGlobal?
171
+ MU::Cloud.resourceClass("AWS", "Log").isGlobal?
172
172
  end
173
173
 
174
174
  end
@@ -55,8 +55,8 @@ module MU
55
55
  scrub_mu_isms: @config['scrub_mu_isms']
56
56
  )
57
57
 
58
- @disk_devices = MU::Cloud::AWS::Server.disk_devices
59
- @ephemeral_mappings = MU::Cloud::AWS::Server.ephemeral_mappings
58
+ @disk_devices = MU::Cloud.resourceClass("AWS", "Server").disk_devices
59
+ @ephemeral_mappings = MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings
60
60
 
61
61
  if !mu_name.nil?
62
62
  @mu_name = mu_name
@@ -190,7 +190,7 @@ module MU
190
190
  cfm_volume_map = {}
191
191
  if @config["storage"]
192
192
  @config["storage"].each { |vol|
193
- mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
193
+ mapping, cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
194
194
  configured_storage << mapping
195
195
  # vol_name, vol_template = MU::Cloud::CloudFormation.cloudFormationBase("volume", name: "volume"+@cfm_name+mapping[:device_name])
196
196
  # MU::Cloud::CloudFormation.setCloudFormationProp(vol_template[vol_name], "Size", mapping[:ebs][:volume_size].to_s)
@@ -353,7 +353,7 @@ module MU
353
353
  # @param config [MU::Config]: The calling MU::Config object
354
354
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
355
355
  def self.schema(config)
356
- MU::Cloud::AWS::Server.schema(config)
356
+ MU::Cloud.resourceClass("AWS", "Server").schema(config)
357
357
  end
358
358
 
359
359
  # Confirm that the given instance size is valid for the given region.
@@ -362,7 +362,7 @@ module MU
362
362
  # @param region [String]: Region to check against
363
363
  # @return [String,nil]
364
364
  def self.validateInstanceType(size, region)
365
- MU::Cloud::AWS::Server.validateInstanceType(size, region)
365
+ MU::Cloud.resourceClass("AWS", "Server").validateInstanceType(size, region)
366
366
  end
367
367
 
368
368
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -370,14 +370,14 @@ module MU
370
370
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
371
371
  # @return [Boolean]: True if validation succeeded, False otherwise
372
372
  def self.validateConfig(server, configurator)
373
- MU::Cloud::AWS::Server.validateConfig(server, configurator)
373
+ MU::Cloud.resourceClass("AWS", "Server").validateConfig(server, configurator)
374
374
  end
375
375
 
376
376
  # Does this resource type exist as a global (cloud-wide) artifact, or
377
377
  # is it localized to a region/zone?
378
378
  # @return [Boolean]
379
379
  def self.isGlobal?
380
- MU::Cloud::AWS::Server.isGlobal?
380
+ MU::Cloud.resourceClass("AWS", "Server").isGlobal?
381
381
  end
382
382
 
383
383
  end #class
@@ -129,13 +129,13 @@ module MU
129
129
 
130
130
  if launch_desc["storage"]
131
131
  launch_desc["storage"].each { |vol|
132
- mapping, cfm_mapping = MU::Cloud::AWS::Server.convertBlockDeviceMapping(vol)
132
+ mapping, cfm_mapping = MU::Cloud.resourceClass("AWS", "Server").convertBlockDeviceMapping(vol)
133
133
  if cfm_mapping.size > 0
134
134
  MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", cfm_mapping)
135
135
  end
136
136
  }
137
137
  end
138
- MU::Cloud::AWS::Server.ephemeral_mappings.each { |mapping|
138
+ MU::Cloud.resourceClass("AWS", "Server.ephemeral_mappings").each { |mapping|
139
139
  MU::Cloud::CloudFormation.setCloudFormationProp(@cfm_template[@cfm_launch_name], "BlockDeviceMappings", { "DeviceName" => mapping[:device_name], "VirtualName" => mapping[:virtual_name] })
140
140
  }
141
141
 
@@ -263,7 +263,7 @@ module MU
263
263
  # @param config [MU::Config]: The calling MU::Config object
264
264
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
265
265
  def self.schema(config)
266
- MU::Cloud::AWS::ServerPool.schema(config)
266
+ MU::Cloud.resourceClass("AWS", "ServerPool").schema(config)
267
267
  end
268
268
 
269
269
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -271,14 +271,14 @@ module MU
271
271
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
272
272
  # @return [Boolean]: True if validation succeeded, False otherwise
273
273
  def self.validateConfig(server, configurator)
274
- MU::Cloud::AWS::ServerPool.validateConfig(server, configurator)
274
+ MU::Cloud.resourceClass("AWS", "ServerPool").validateConfig(server, configurator)
275
275
  end
276
276
 
277
277
  # Does this resource type exist as a global (cloud-wide) artifact, or
278
278
  # is it localized to a region/zone?
279
279
  # @return [Boolean]
280
280
  def self.isGlobal?
281
- MU::Cloud::AWS::ServerPool.isGlobal?
281
+ MU::Cloud.resourceClass("AWS", "ServerPool").isGlobal?
282
282
  end
283
283
 
284
284
  end
@@ -301,7 +301,7 @@ module MU
301
301
  # @param config [MU::Config]: The calling MU::Config object
302
302
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
303
303
  def self.schema(config)
304
- MU::Cloud::AWS::VPC.schema(config)
304
+ MU::Cloud.resourceClass("AWS", "VPC").schema(config)
305
305
  end
306
306
 
307
307
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::servers}, bare and unvalidated.
@@ -309,14 +309,14 @@ module MU
309
309
  # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
310
310
  # @return [Boolean]: True if validation succeeded, False otherwise
311
311
  def self.validateConfig(server, configurator)
312
- MU::Cloud::AWS::VPC.validateConfig(server, configurator)
312
+ MU::Cloud.resourceClass("AWS", "VPC").validateConfig(server, configurator)
313
313
  end
314
314
 
315
315
  # Does this resource type exist as a global (cloud-wide) artifact, or
316
316
  # is it localized to a region/zone?
317
317
  # @return [Boolean]
318
318
  def self.isGlobal?
319
- MU::Cloud::AWS::VPC.isGlobal?
319
+ MU::Cloud.resourceClass("AWS", "VPC").isGlobal?
320
320
  end
321
321
 
322
322
  end #class
@@ -52,6 +52,11 @@ module MU
52
52
  [:url]
53
53
  end
54
54
 
55
+ # Is this a "real" cloud provider, or a stub like CloudFormation?
56
+ def self.virtual?
57
+ false
58
+ end
59
+
55
60
  # Most of our resource implementation +find+ methods have to mangle their
56
61
  # args to make sure they've extracted a project or location argument from
57
62
  # other available information. This does it for them.
@@ -231,7 +236,7 @@ module MU
231
236
  # @param sibling_only [Boolean]
232
237
  # @return [MU::Config::Habitat,nil]
233
238
  def self.projectLookup(name, deploy = MU.mommacat, raise_on_fail: true, sibling_only: false)
234
- project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy
239
+ project_obj = deploy.findLitterMate(type: "habitats", name: name) if deploy if !caller.grep(/`findLitterMate'/) # XXX the dumbest
235
240
 
236
241
  if !project_obj and !sibling_only
237
242
  resp = MU::MommaCat.findStray(
@@ -337,6 +342,7 @@ module MU
337
342
  # etc)
338
343
  # @param deploy_id [MU::MommaCat]
339
344
  def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
345
+ removeDeploySecretsAndRoles(deploy_id, noop: noop, credentials: credentials)
340
346
  end
341
347
 
342
348
  # Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
@@ -548,7 +554,7 @@ MU.log e.message, MU::WARN, details: e.inspect
548
554
  begin
549
555
  listRegions(credentials: credentials)
550
556
  listInstanceTypes(credentials: credentials)
551
- listProjects(credentials)
557
+ listHabitats(credentials)
552
558
  rescue ::Google::Apis::ClientError
553
559
  MU.log "Found machine credentials #{@@svc_account_name}, but these don't appear to have sufficient permissions or scopes", MU::WARN, details: scopes
554
560
  @@authorizers.delete(credentials)
@@ -700,13 +706,26 @@ MU.log e.message, MU::WARN, details: e.inspect
700
706
  nil
701
707
  end
702
708
 
709
+ @allprojects = []
710
+
703
711
  # List all Google Cloud Platform projects available to our credentials
704
- def self.listProjects(credentials = nil)
712
+ def self.listHabitats(credentials = nil, use_cache: true)
705
713
  cfg = credConfig(credentials)
706
- return [] if !cfg or !cfg['project']
714
+ return [] if !cfg
715
+ if cfg['restrict_to_habitats'] and cfg['restrict_to_habitats'].is_a?(Array)
716
+ cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
717
+ return cfg['restrict_to_habitats'].uniq
718
+ end
719
+ if @allprojects and !@allprojects.empty? and use_cache
720
+ return @allprojects
721
+ end
707
722
  result = MU::Cloud::Google.resource_manager(credentials: credentials).list_projects
708
723
  result.projects.reject! { |p| p.lifecycle_state == "DELETE_REQUESTED" }
709
- result.projects.map { |p| p.project_id }
724
+ @allprojects = result.projects.map { |p| p.project_id }
725
+ if cfg['ignore_habitats'] and cfg['ignore_habitats'].is_a?(Array)
726
+ @allprojects.reject! { |p| cfg['ignore_habitats'].include?(p) }
727
+ end
728
+ @allprojects
710
729
  end
711
730
 
712
731
  @@regions = {}
@@ -1005,14 +1024,17 @@ MU.log e.message, MU::WARN, details: e.inspect
1005
1024
  # @return [Array<OpenStruct>],nil]
1006
1025
  def self.getOrg(credentials = nil, with_id: nil)
1007
1026
  creds = MU::Cloud::Google.credConfig(credentials)
1027
+ return nil if !creds
1008
1028
  credname = if creds and creds['name']
1009
1029
  creds['name']
1010
1030
  else
1011
1031
  "default"
1012
1032
  end
1013
1033
 
1034
+ with_id ||= creds['org'] if creds['org']
1014
1035
  return @@orgmap[credname] if @@orgmap.has_key?(credname)
1015
1036
  resp = MU::Cloud::Google.resource_manager(credentials: credname).search_organizations
1037
+
1016
1038
  if resp and resp.organizations
1017
1039
  # XXX no idea if it's possible to be a member of multiple orgs
1018
1040
  if !with_id
@@ -1020,7 +1042,8 @@ MU.log e.message, MU::WARN, details: e.inspect
1020
1042
  return resp.organizations.first
1021
1043
  else
1022
1044
  resp.organizations.each { |org|
1023
- if org.name == with_id
1045
+ if org.name == with_id or org.display_name == with_id or
1046
+ org.name == "organizations/#{with_id}"
1024
1047
  @@orgmap[credname] = org
1025
1048
  return org
1026
1049
  end