cloud-mu 3.4.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. checksums.yaml +4 -4
  2. data/ansible/roles/mu-nat/tasks/main.yml +3 -0
  3. data/bin/mu-aws-setup +41 -7
  4. data/bin/mu-azure-setup +34 -0
  5. data/bin/mu-configure +214 -119
  6. data/bin/mu-gcp-setup +37 -2
  7. data/bin/mu-node-manage +3 -0
  8. data/bin/mu-refresh-ssl +67 -0
  9. data/bin/mu-run-tests +14 -4
  10. data/bin/mu-self-update +30 -10
  11. data/bin/mu-upload-chef-artifacts +30 -26
  12. data/cloud-mu.gemspec +8 -6
  13. data/cookbooks/mu-master/attributes/default.rb +5 -1
  14. data/cookbooks/mu-master/metadata.rb +2 -2
  15. data/cookbooks/mu-master/recipes/default.rb +81 -26
  16. data/cookbooks/mu-master/recipes/init.rb +197 -62
  17. data/cookbooks/mu-master/recipes/update_nagios_only.rb +1 -1
  18. data/cookbooks/mu-master/recipes/vault.rb +78 -77
  19. data/cookbooks/mu-master/templates/default/mods/rewrite.conf.erb +1 -0
  20. data/cookbooks/mu-master/templates/default/nagios.conf.erb +103 -0
  21. data/cookbooks/mu-master/templates/default/web_app.conf.erb +14 -30
  22. data/cookbooks/mu-tools/attributes/default.rb +5 -0
  23. data/cookbooks/mu-tools/files/centos-6/CentOS-Base.repo +47 -0
  24. data/cookbooks/mu-tools/libraries/helper.rb +12 -2
  25. data/cookbooks/mu-tools/libraries/monkey.rb +1 -1
  26. data/cookbooks/mu-tools/recipes/apply_security.rb +6 -0
  27. data/cookbooks/mu-tools/recipes/aws_api.rb +6 -4
  28. data/cookbooks/mu-tools/recipes/base_repositories.rb +1 -1
  29. data/cookbooks/mu-tools/recipes/gcloud.rb +2 -9
  30. data/cookbooks/mu-tools/recipes/google_api.rb +5 -2
  31. data/cookbooks/mu-tools/resources/disk.rb +108 -58
  32. data/extras/Gemfile.lock.bootstrap +394 -0
  33. data/extras/bucketstubs/error.html +0 -0
  34. data/extras/bucketstubs/index.html +0 -0
  35. data/extras/clean-stock-amis +9 -9
  36. data/extras/git_rpm/build.sh +20 -0
  37. data/extras/git_rpm/mugit.spec +53 -0
  38. data/extras/image-generators/VMWare/centos8.yaml +15 -0
  39. data/extras/openssl_rpm/build.sh +19 -0
  40. data/extras/openssl_rpm/mussl.spec +46 -0
  41. data/extras/python_rpm/muthon.spec +14 -4
  42. data/extras/ruby_rpm/muby.spec +9 -5
  43. data/extras/sqlite_rpm/build.sh +19 -0
  44. data/extras/sqlite_rpm/muqlite.spec +47 -0
  45. data/install/installer +7 -5
  46. data/modules/mu.rb +12 -5
  47. data/modules/mu/cloud/machine_images.rb +1 -1
  48. data/modules/mu/cloud/providers.rb +6 -1
  49. data/modules/mu/cloud/resource_base.rb +1 -1
  50. data/modules/mu/cloud/ssh_sessions.rb +4 -0
  51. data/modules/mu/config.rb +28 -12
  52. data/modules/mu/config/database.rb +2 -2
  53. data/modules/mu/config/firewall_rule.rb +1 -1
  54. data/modules/mu/config/ref.rb +2 -2
  55. data/modules/mu/config/schema_helpers.rb +12 -3
  56. data/modules/mu/config/server.rb +10 -4
  57. data/modules/mu/config/server_pool.rb +2 -2
  58. data/modules/mu/config/vpc.rb +10 -10
  59. data/modules/mu/defaults/AWS.yaml +32 -32
  60. data/modules/mu/deploy.rb +23 -10
  61. data/modules/mu/groomers/chef.rb +2 -2
  62. data/modules/mu/master.rb +49 -3
  63. data/modules/mu/mommacat.rb +8 -5
  64. data/modules/mu/mommacat/naming.rb +2 -2
  65. data/modules/mu/mommacat/storage.rb +22 -27
  66. data/modules/mu/providers/aws.rb +142 -48
  67. data/modules/mu/providers/aws/alarm.rb +3 -3
  68. data/modules/mu/providers/aws/bucket.rb +19 -19
  69. data/modules/mu/providers/aws/cache_cluster.rb +22 -22
  70. data/modules/mu/providers/aws/cdn.rb +2 -2
  71. data/modules/mu/providers/aws/collection.rb +14 -14
  72. data/modules/mu/providers/aws/container_cluster.rb +27 -27
  73. data/modules/mu/providers/aws/database.rb +40 -39
  74. data/modules/mu/providers/aws/dnszone.rb +5 -5
  75. data/modules/mu/providers/aws/endpoint.rb +35 -35
  76. data/modules/mu/providers/aws/firewall_rule.rb +26 -23
  77. data/modules/mu/providers/aws/function.rb +28 -28
  78. data/modules/mu/providers/aws/group.rb +7 -7
  79. data/modules/mu/providers/aws/habitat.rb +2 -2
  80. data/modules/mu/providers/aws/job.rb +6 -6
  81. data/modules/mu/providers/aws/loadbalancer.rb +34 -34
  82. data/modules/mu/providers/aws/log.rb +14 -14
  83. data/modules/mu/providers/aws/msg_queue.rb +10 -10
  84. data/modules/mu/providers/aws/nosqldb.rb +8 -8
  85. data/modules/mu/providers/aws/notifier.rb +7 -7
  86. data/modules/mu/providers/aws/role.rb +17 -15
  87. data/modules/mu/providers/aws/search_domain.rb +10 -10
  88. data/modules/mu/providers/aws/server.rb +176 -95
  89. data/modules/mu/providers/aws/server_pool.rb +65 -105
  90. data/modules/mu/providers/aws/storage_pool.rb +17 -9
  91. data/modules/mu/providers/aws/user.rb +1 -1
  92. data/modules/mu/providers/aws/vpc.rb +103 -51
  93. data/modules/mu/providers/aws/vpc_subnet.rb +43 -39
  94. data/modules/mu/providers/azure.rb +78 -12
  95. data/modules/mu/providers/azure/server.rb +18 -3
  96. data/modules/mu/providers/cloudformation/server.rb +1 -1
  97. data/modules/mu/providers/google.rb +19 -4
  98. data/modules/mu/providers/google/folder.rb +6 -2
  99. data/modules/mu/providers/google/function.rb +65 -30
  100. data/modules/mu/providers/google/role.rb +1 -1
  101. data/modules/mu/providers/google/vpc.rb +27 -2
  102. data/modules/tests/aws-servers-with-handrolled-iam.yaml +37 -0
  103. data/modules/tests/k8s.yaml +1 -1
  104. metadata +24 -8
@@ -27,7 +27,7 @@ module MU
27
27
 
28
28
  # Called automatically by {MU::Deploy#createResources}
29
29
  def create
30
- MU.setVar("curRegion", @config['region']) if !@config['region'].nil?
30
+ MU.setVar("curRegion", @region) if !@region.nil?
31
31
 
32
32
  createUpdateLaunchConfig
33
33
 
@@ -37,7 +37,7 @@ module MU
37
37
 
38
38
  zones_to_try = @config["zones"]
39
39
  begin
40
- asg = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_auto_scaling_group(asg_options)
40
+ asg = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).create_auto_scaling_group(asg_options)
41
41
  rescue Aws::AutoScaling::Errors::ValidationError => e
42
42
  if zones_to_try != nil and zones_to_try.size > 0
43
43
  MU.log "#{e.message}, retrying with individual AZs", MU::WARN
@@ -52,7 +52,7 @@ module MU
52
52
  if zones_to_try != nil and zones_to_try.size < @config["zones"].size
53
53
  zones_to_try.each { |zone|
54
54
  begin
55
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
55
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).update_auto_scaling_group(
56
56
  auto_scaling_group_name: @mu_name,
57
57
  availability_zones: [zone]
58
58
  )
@@ -70,11 +70,11 @@ module MU
70
70
  attempts = 0
71
71
  begin
72
72
  sleep 5
73
- desc = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
73
+ desc = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
74
74
  MU.log "Looking for #{desc.min_size} instances in #{@mu_name}, found #{desc.instances.size}", MU::DEBUG
75
75
  attempts = attempts + 1
76
76
  if attempts > 25 and desc.instances.size == 0
77
- MU.log "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}", MU::ERR, details: MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_scaling_activities(auto_scaling_group_name: @mu_name).activities
77
+ MU.log "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}", MU::ERR, details: MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_scaling_activities(auto_scaling_group_name: @mu_name).activities
78
78
  raise MuError, "No instances spun up after #{5*attempts} seconds, something's wrong with Autoscale group #{@mu_name}"
79
79
  end
80
80
  end while desc.instances.size < desc.min_size
@@ -131,7 +131,7 @@ module MU
131
131
  t.join
132
132
  }
133
133
  MU.log "Setting min_size to #{@config['min_size']} and max_size to #{@config['max_size']}"
134
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
134
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).update_auto_scaling_group(
135
135
  auto_scaling_group_name: @mu_name,
136
136
  min_size: @config['min_size'],
137
137
  max_size: @config['max_size']
@@ -151,7 +151,7 @@ module MU
151
151
  def setScaleInProtection(need_instances = @config['min_size'])
152
152
  live_instances = []
153
153
  begin
154
- desc = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
154
+ desc = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_auto_scaling_groups(auto_scaling_group_names: [@mu_name]).auto_scaling_groups.first
155
155
 
156
156
  live_instances = desc.instances.map { |i| i.instance_id }
157
157
  already_set = 0
@@ -163,7 +163,7 @@ module MU
163
163
  elsif already_set > need_instances
164
164
  unset_me = live_instances.sample(already_set - need_instances)
165
165
  MU.log "Disabling scale-in protection for #{unset_me.size.to_s} instances in #{@mu_name}", MU::NOTICE, details: unset_me
166
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).set_instance_protection(
166
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).set_instance_protection(
167
167
  auto_scaling_group_name: @mu_name,
168
168
  instance_ids: unset_me,
169
169
  protected_from_scale_in: false
@@ -172,7 +172,7 @@ module MU
172
172
  live_instances = live_instances.sample(need_instances)
173
173
  MU.log "Enabling scale-in protection for #{@config['scale_in_protection']} instances in #{@mu_name}", details: live_instances
174
174
  begin
175
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).set_instance_protection(
175
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).set_instance_protection(
176
176
  auto_scaling_group_name: @mu_name,
177
177
  instance_ids: live_instances,
178
178
  protected_from_scale_in: true
@@ -194,10 +194,9 @@ module MU
194
194
  def listNodes
195
195
  nodes = []
196
196
  me = MU::Cloud::AWS::ServerPool.find(cloud_id: cloud_id).values.first
197
- pp me
198
197
  if me and me.instances
199
198
  me.instances.each { |instance|
200
- found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @config["region"], dummy_ok: true)
199
+ found = MU::MommaCat.findStray("AWS", "server", cloud_id: instance.instance_id, region: @region, dummy_ok: true)
201
200
  nodes.concat(found)
202
201
  }
203
202
  end
@@ -211,7 +210,7 @@ module MU
211
210
  arn = if @config['notifications']['topic'].match(/^arn:/)
212
211
  @config['notifications']['topic']
213
212
  else
214
- "arn:#{MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws"}:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@config['notifications']['topic']}"
213
+ "arn:#{MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws"}:sns:#{@region}:#{MU::Cloud::AWS.credToAcct(@credentials)}:#{@config['notifications']['topic']}"
215
214
  end
216
215
  eventmap = {
217
216
  "launch" => "autoscaling:EC2_INSTANCE_LAUNCH",
@@ -220,7 +219,7 @@ module MU
220
219
  "failed_terminate" => "autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
221
220
  }
222
221
  MU.log "Sending simple notifications (#{@config['notifications']['events'].join(", ")}) to #{arn}"
223
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_notification_configuration(
222
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).put_notification_configuration(
224
223
  auto_scaling_group_name: @mu_name,
225
224
  topic_arn: arn,
226
225
  notification_types: @config['notifications']['events'].map { |e|
@@ -230,7 +229,7 @@ module MU
230
229
  end
231
230
 
232
231
  if @config['schedule']
233
- ext_actions = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_scheduled_actions(
232
+ ext_actions = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_scheduled_actions(
234
233
  auto_scaling_group_name: @mu_name
235
234
  ).scheduled_update_group_actions
236
235
 
@@ -251,7 +250,7 @@ module MU
251
250
  if s['action_name'] == ext.scheduled_action_name
252
251
  if !MU.hashCmp(MU.structToHash(ext), sched_config, missing_is_default: true)
253
252
  MU.log "Removing scheduled action #{s['action_name']} from AutoScale group #{@mu_name}"
254
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_scheduled_action(
253
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).delete_scheduled_action(
255
254
  auto_scaling_group_name: @mu_name,
256
255
  scheduled_action_name: s['action_name']
257
256
  )
@@ -263,7 +262,7 @@ module MU
263
262
  }
264
263
  if !action_already_correct
265
264
  MU.log "Adding scheduled action to AutoScale group #{@mu_name}", MU::NOTICE, details: sched_config
266
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scheduled_update_group_action(
265
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).put_scheduled_update_group_action(
267
266
  sched_config
268
267
  )
269
268
  end
@@ -291,30 +290,29 @@ module MU
291
290
  if need_tag_update
292
291
  MU.log "Updating ServerPool #{@mu_name} with new tags", MU::NOTICE, details: tag_conf[:tags]
293
292
 
294
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_or_update_tags(tag_conf)
293
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).create_or_update_tags(tag_conf)
295
294
  current.instances.each { |instance|
296
295
  tag_conf[:tags].each { |t|
297
- MU::Cloud::AWS.createTag(instance.instance_id, t[:key], t[:value], region: @config['region'], credentials: @config['credentials'])
296
+ MU::Cloud::AWS.createTag(instance.instance_id, t[:key], t[:value], region: @region, credentials: @credentials)
298
297
  }
299
298
  }
300
299
  end
301
300
 
302
301
  # XXX actually compare for changes instead of just blindly updating
303
- #pp current
304
- #pp asg_options
302
+
305
303
  asg_options.delete(:tags)
306
304
  asg_options[:min_size] = @config["min_size"]
307
305
  asg_options[:max_size] = @config["max_size"]
308
306
  asg_options[:new_instances_protected_from_scale_in] = (@config['scale_in_protection'] == "all")
309
307
  if asg_options[:target_group_arns]
310
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).attach_load_balancer_target_groups(
308
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).attach_load_balancer_target_groups(
311
309
  auto_scaling_group_name: @mu_name,
312
310
  target_group_arns: asg_options[:target_group_arns]
313
311
  )
314
312
  asg_options.delete(:target_group_arns)
315
313
  end
316
314
 
317
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(asg_options)
315
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).update_auto_scaling_group(asg_options)
318
316
 
319
317
  if @config['scale_in_protection']
320
318
  if @config['scale_in_protection'] == "all"
@@ -328,7 +326,7 @@ module MU
328
326
  setScaleInProtection(0)
329
327
  end
330
328
 
331
- ext_pols = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_policies(
329
+ ext_pols = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_policies(
332
330
  auto_scaling_group_name: @mu_name
333
331
  ).scaling_policies
334
332
  if @config["scaling_policies"] and @config["scaling_policies"].size > 0
@@ -340,7 +338,7 @@ module MU
340
338
  ext_pols.each { |ext|
341
339
  if !legit_policies.include?(ext.policy_name)
342
340
  MU.log "Scaling policy #{ext.policy_name} is not named in scaling_policies, removing from #{@mu_name}", MU::NOTICE, details: ext
343
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_policy(
341
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).delete_policy(
344
342
  auto_scaling_group_name: @mu_name,
345
343
  policy_name: ext.policy_name
346
344
  )
@@ -401,7 +399,7 @@ module MU
401
399
  ext_pols.each { |ext|
402
400
  if ext.policy_name == policy_name
403
401
  if !MU.hashCmp(MU.structToHash(ext), policy_params, missing_is_default: true)
404
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_policy(
402
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).delete_policy(
405
403
  auto_scaling_group_name: @mu_name,
406
404
  policy_name: policy_name
407
405
  )
@@ -413,7 +411,7 @@ module MU
413
411
  }
414
412
  if !policy_already_correct
415
413
  MU.log "Putting scaling policy #{policy_name} for #{@mu_name}", MU::NOTICE, details: policy_params
416
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_scaling_policy(policy_params)
414
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).put_scaling_policy(policy_params)
417
415
  end
418
416
 
419
417
  }
@@ -427,7 +425,7 @@ module MU
427
425
  def cloud_desc(use_cache: true)
428
426
  return @cloud_desc_cache if @cloud_desc_cache and use_cache
429
427
  return nil if !@cloud_id
430
- @cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_auto_scaling_groups(
428
+ @cloud_desc_cache = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_auto_scaling_groups(
431
429
  auto_scaling_group_names: [@mu_name]
432
430
  ).auto_scaling_groups.first
433
431
  @cloud_desc_cache
@@ -491,9 +489,9 @@ module MU
491
489
  def toKitten(**_args)
492
490
  bok = {
493
491
  "cloud" => "AWS",
494
- "credentials" => @config['credentials'],
492
+ "credentials" => @credentials,
495
493
  "cloud_id" => @cloud_id,
496
- "region" => @config['region']
494
+ "region" => @region
497
495
  }
498
496
 
499
497
  if !cloud_desc
@@ -518,7 +516,7 @@ module MU
518
516
  bok['max_size'] = cloud_desc.max_size
519
517
 
520
518
  if cloud_desc.launch_configuration_name
521
- launch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @credentials).describe_launch_configurations(
519
+ launch = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_launch_configurations(
522
520
  launch_configuration_names: [cloud_desc.launch_configuration_name]
523
521
  ).launch_configurations.first
524
522
  bok['basis'] = {
@@ -534,7 +532,7 @@ module MU
534
532
  !cloud_desc.vpc_zone_identifier.empty?
535
533
  nets = cloud_desc.vpc_zone_identifier.split(/,/)
536
534
  begin
537
- resp = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
535
+ resp = MU::Cloud::AWS.ec2(region: @region, credentials: @credentials).describe_subnets(subnet_ids: nets).subnets.first
538
536
  bok['vpc'] = MU::Config::Ref.get(
539
537
  id: resp.vpc_id,
540
538
  cloud: "AWS",
@@ -910,41 +908,13 @@ module MU
910
908
  MU.log "Cannot mix iam_policies with generate_iam_role set to false", MU::ERR
911
909
  ok = false
912
910
  end
913
- else
914
- s3_objs = ['arn:'+(MU::Cloud::AWS.isGovCloud?(pool['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(pool['credentials'])+'/Mu_CA.pem']
915
-
916
- role = {
917
- "name" => pool["name"],
918
- "cloud" => "AWS",
919
- "strip_path" => pool["role_strip_path"],
920
- "can_assume" => [
921
- {
922
- "entity_id" => "ec2.amazonaws.com",
923
- "entity_type" => "service"
924
- }
925
- ],
926
- "policies" => [
927
- {
928
- "name" => "MuSecrets",
929
- "permissions" => ["s3:GetObject"],
930
- "targets" => s3_objs.map { |f| { "identifier" => f } }
931
- }
932
- ]
933
- }
934
- if launch['iam_policies']
935
- role['iam_policies'] = launch['iam_policies'].dup
936
- end
937
- if pool['canned_iam_policies']
938
- role['import'] = pool['canned_iam_policies'].dup
939
- end
940
- if pool['iam_role']
941
- # XXX maybe break this down into policies and add those?
942
- end
943
-
944
- role['credentials'] = pool['credentials'] if pool['credentials']
945
- configurator.insertKitten(role, "roles")
946
- MU::Config.addDependency(pool, pool['name'], "role")
947
911
  end
912
+
913
+ ["generate_iam_role", "iam_role", "canned_iam_policies", "iam_policies"].each { |key|
914
+ pool[key] = launch[key] if !launch[key].nil?
915
+ }
916
+ MU::Cloud.resourceClass("AWS", "Server").generateStandardRole(pool, configurator)
917
+
948
918
  launch["ami_id"] ||= launch["image_id"]
949
919
  if launch["server"].nil? and launch["instance_id"].nil? and launch["ami_id"].nil?
950
920
  img_id = MU::Cloud.getStockImage("AWS", platform: pool['platform'], region: pool['region'])
@@ -957,7 +927,7 @@ module MU
957
927
  end
958
928
  end
959
929
  if launch["server"] != nil
960
- MU::Config.addDependency(pool, launch["server"], "server", phase: "groom")
930
+ MU::Config.addDependency(pool, launch["server"], "server", their_phase: "groom")
961
931
  # XXX I dunno, maybe toss an error if this isn't done already
962
932
  # servers.each { |server|
963
933
  # if server["name"] == launch["server"]
@@ -1157,20 +1127,20 @@ module MU
1157
1127
  @config['basis']['launch_config']["ami_id"] = MU::Cloud.resourceClass("AWS", "Server").createImage(
1158
1128
  name: @mu_name,
1159
1129
  instance_id: @config['basis']['launch_config']["instance_id"],
1160
- credentials: @config['credentials'],
1161
- region: @config['region']
1162
- )[@config['region']]
1130
+ credentials: @credentials,
1131
+ region: @region
1132
+ )[@region]
1163
1133
  end
1164
- MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"], credentials: @config['credentials'])
1134
+ MU::Cloud.resourceClass("AWS", "Server").waitForAMI(@config['basis']['launch_config']["ami_id"].to_s, credentials: @credentials)
1165
1135
 
1166
- oldlaunch = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_launch_configurations(
1136
+ oldlaunch = MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).describe_launch_configurations(
1167
1137
  launch_configuration_names: [@mu_name]
1168
1138
  ).launch_configurations.first
1169
1139
 
1170
1140
  userdata = MU::Cloud.fetchUserdata(
1171
1141
  platform: @config["platform"],
1172
1142
  cloud: "AWS",
1173
- credentials: @config['credentials'],
1143
+ credentials: @credentials,
1174
1144
  template_variables: {
1175
1145
  "deployKey" => Base64.urlsafe_encode64(@deploy.public_key),
1176
1146
  "deploySSHKey" => @deploy.ssh_public_key,
@@ -1223,16 +1193,6 @@ module MU
1223
1193
 
1224
1194
  storage.concat(MU::Cloud.resourceClass("AWS", "Server").ephemeral_mappings)
1225
1195
 
1226
- if @config['basis']['launch_config']['generate_iam_role']
1227
- role = @deploy.findLitterMate(name: @config['name'], type: "roles")
1228
- if role
1229
- s3_objs = ["#{@deploy.deploy_id}-secret", "#{role.mu_name}.pfx", "#{role.mu_name}.crt", "#{role.mu_name}.key", "#{role.mu_name}-winrm.crt", "#{role.mu_name}-winrm.key"].map { |file|
1230
- 'arn:'+(MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(@credentials)+'/'+file
1231
- }
1232
- role.cloudobj.injectPolicyTargets("MuSecrets", s3_objs)
1233
- end
1234
- end
1235
-
1236
1196
  if !oldlaunch.nil?
1237
1197
  olduserdata = Base64.decode64(oldlaunch.user_data)
1238
1198
  if userdata == olduserdata and
@@ -1249,7 +1209,7 @@ module MU
1249
1209
  # Put our Autoscale group onto a temporary launch config
1250
1210
  begin
1251
1211
 
1252
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_launch_configuration(
1212
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).create_launch_configuration(
1253
1213
  launch_configuration_name: @mu_name+"-TMP",
1254
1214
  user_data: Base64.encode64(olduserdata),
1255
1215
  image_id: oldlaunch.image_id,
@@ -1272,12 +1232,12 @@ module MU
1272
1232
  end
1273
1233
 
1274
1234
 
1275
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
1235
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).update_auto_scaling_group(
1276
1236
  auto_scaling_group_name: @mu_name,
1277
1237
  launch_configuration_name: @mu_name+"-TMP"
1278
1238
  )
1279
1239
  # ...now back to an identical one with the "real" name
1280
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_launch_configuration(
1240
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).delete_launch_configuration(
1281
1241
  launch_configuration_name: @mu_name
1282
1242
  )
1283
1243
  end
@@ -1310,30 +1270,30 @@ module MU
1310
1270
  end
1311
1271
  }
1312
1272
  rolename = nil
1273
+
1313
1274
  ['generate_iam_role', 'iam_policies', 'canned_iam_policies', 'iam_role'].each { |field|
1314
- @config['basis']['launch_config'][field] ||= @config[field]
1275
+ if !@config['basis']['launch_config'].nil?
1276
+ @config[field] = @config['basis']['launch_config'][field]
1277
+ else
1278
+ @config['basis']['launch_config'][field] = @config[field]
1279
+ end
1315
1280
  }
1316
1281
 
1317
- if @config['basis']['launch_config']['generate_iam_role']
1318
- role = @deploy.findLitterMate(name: @config['name'], type: "roles")
1319
-
1320
- @config['iam_role'] = role.mu_name
1321
-
1322
- launch_options[:iam_instance_profile] = role.cloudobj.createInstanceProfile
1323
- elsif @config['basis']['launch_config']['iam_role'].nil?
1324
- raise MuError, "#{@mu_name} has generate_iam_role set to false, but no iam_role assigned."
1325
- else
1326
- launch_options[:iam_instance_profile] = @config['basis']['launch_config']['iam_role']
1327
- end
1328
-
1329
- @config['iam_role'] = rolename ? rolename : launch_options[:iam_instance_profile]
1282
+ @config['iam_role'] = @config['basis']['launch_config']['iam_role'] = launch_options[:iam_instance_profile] = MU::Cloud.resourceClass("AWS", "Server").getIAMProfile(
1283
+ @config['name'],
1284
+ @deploy,
1285
+ generated: @config['basis']['launch_config']['generate_iam_role'],
1286
+ role_name: @config['basis']['launch_config']['iam_role'],
1287
+ region: @region,
1288
+ credentials: @credentials
1289
+ ).values.first
1330
1290
 
1331
1291
  lc_attempts = 0
1332
1292
  begin
1333
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).create_launch_configuration(launch_options)
1293
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).create_launch_configuration(launch_options)
1334
1294
  rescue Aws::AutoScaling::Errors::ValidationError => e
1335
1295
  if lc_attempts > 3
1336
- MU.log "Got error while creating #{@mu_name} Launch Config#{@config['credentials'] ? " with credentials #{@config['credentials']}" : ""}: #{e.message}, retrying in 10s", MU::WARN, details: launch_options.reject { |k,_v | k == :user_data }
1296
+ MU.log "Got error while creating #{@mu_name} Launch Config#{@credentials ? " with credentials #{@credentials}" : ""}: #{e.message}, retrying in 10s", MU::WARN, details: launch_options.reject { |k,_v | k == :user_data }
1337
1297
  end
1338
1298
  sleep 5
1339
1299
  lc_attempts += 1
@@ -1342,11 +1302,11 @@ module MU
1342
1302
 
1343
1303
  if !oldlaunch.nil?
1344
1304
  # Tell the ASG to use the new one, and nuke the old one
1345
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).update_auto_scaling_group(
1305
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).update_auto_scaling_group(
1346
1306
  auto_scaling_group_name: @mu_name,
1347
1307
  launch_configuration_name: @mu_name
1348
1308
  )
1349
- MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).delete_launch_configuration(
1309
+ MU::Cloud::AWS.autoscale(region: @region, credentials: @credentials).delete_launch_configuration(
1350
1310
  launch_configuration_name: @mu_name+"-TMP"
1351
1311
  )
1352
1312
  MU.log "Launch Configuration #{@mu_name} replaced"
@@ -1486,8 +1446,8 @@ module MU
1486
1446
  # Do the dance of specifying individual zones if we haven't asked to
1487
1447
  # use particular VPC subnets.
1488
1448
  if @config['zones'].nil? and asg_options[:vpc_zone_identifier].nil?
1489
- @config["zones"] = MU::Cloud::AWS.listAZs(region: @config['region'])
1490
- MU.log "Using zones from #{@config['region']}", MU::DEBUG, details: @config['zones']
1449
+ @config["zones"] = MU::Cloud::AWS.listAZs(region: @region)
1450
+ MU.log "Using zones from #{@region}", MU::DEBUG, details: @config['zones']
1491
1451
  end
1492
1452
  asg_options[:availability_zones] = @config["zones"] if @config["zones"] != nil
1493
1453
  asg_options
@@ -29,7 +29,7 @@ module MU
29
29
  # @return [String]: The cloud provider's identifier for this storage pool.
30
30
  def create
31
31
  MU.log "Creating storage pool #{@mu_name}"
32
- resp = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).create_file_system(
32
+ resp = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).create_file_system(
33
33
  creation_token: @mu_name,
34
34
  performance_mode: @config['storage_type']
35
35
  )
@@ -37,7 +37,7 @@ module MU
37
37
  attempts = 0
38
38
  loop do
39
39
  MU.log "Waiting for #{@mu_name}: #{resp.file_system_id} to become available" if attempts % 5 == 0
40
- storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
40
+ storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
41
41
  creation_token: @mu_name
42
42
  ).file_systems.first
43
43
  break if storage_pool.life_cycle_state == "available"
@@ -47,7 +47,7 @@ module MU
47
47
  raise MuError, "timed out waiting for #{resp.mount_target_id }" if attempts >= 20
48
48
  end
49
49
 
50
- addStandardTags(cloud_id: resp.file_system_id, region: @config['region'], credentials: @config['credentials'])
50
+ addStandardTags(cloud_id: resp.file_system_id, region: @region, credentials: @credentials)
51
51
  @cloud_id = resp.file_system_id
52
52
 
53
53
  if @config['mount_points'] && !@config['mount_points'].empty?
@@ -82,8 +82,8 @@ module MU
82
82
  ip_address: target['ip_address'],
83
83
  subnet_id: target['vpc']['subnet_id'],
84
84
  security_groups: sgs,
85
- credentials: @config['credentials'],
86
- region: @config['region']
85
+ credentials: @credentials,
86
+ region: @region
87
87
  )
88
88
  target['cloud_id'] = mount_target.mount_target_id
89
89
  }
@@ -100,7 +100,7 @@ module MU
100
100
  # Canonical Amazon Resource Number for this resource
101
101
  # @return [String]
102
102
  def arn
103
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":file-system/"+@cloud_id
103
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@region) ? "aws-us-gov" : "aws")+":elasticfilesystem:"+@region+":"+MU::Cloud::AWS.credToAcct(@credentials)+":file-system/"+@cloud_id
104
104
  end
105
105
 
106
106
  # Locate an existing storage pool and return an array containing matching AWS resource descriptors for those that match.
@@ -254,14 +254,14 @@ module MU
254
254
 
255
255
  # Register a description of this storage pool with this deployment's metadata.
256
256
  def notify
257
- storage_pool = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_file_systems(
257
+ storage_pool = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_file_systems(
258
258
  creation_token: @mu_name
259
259
  ).file_systems.first
260
260
 
261
261
  targets = {}
262
262
 
263
263
  if @config['mount_points'] && !@config['mount_points'].empty?
264
- mount_targets = MU::Cloud::AWS.efs(region: @config['region'], credentials: @config['credentials']).describe_mount_targets(
264
+ mount_targets = MU::Cloud::AWS.efs(region: @region, credentials: @credentials).describe_mount_targets(
265
265
  file_system_id: storage_pool.file_system_id
266
266
  ).mount_targets
267
267
 
@@ -274,6 +274,10 @@ module MU
274
274
  subnet_obj = mp_vpc.subnets.select { |s|
275
275
  s.name == mp["vpc"]["subnet_name"] or s.cloud_id == mp["vpc"]["subnet_id"]
276
276
  }.first
277
+ if !subnet_obj
278
+ MU.log "Failed to find live subnet matching configured mount_point", MU::WARN, details: mp["vpc"]
279
+ next
280
+ end
277
281
  mount_target = nil
278
282
  mount_targets.each { |t|
279
283
  subnet_cidr_obj = NetAddr::IPv4Net.parse(subnet_obj.ip_block)
@@ -283,6 +287,10 @@ module MU
283
287
  break
284
288
  end
285
289
  }
290
+ if !mount_target
291
+ MU.log "Failed to find live mount_target corresponding to configured mount_point", MU::WARN, details: mp
292
+ next
293
+ end
286
294
 
287
295
  targets[mp["name"]] = {
288
296
  "owner_id" => mount_target.owner_id,
@@ -294,7 +302,7 @@ module MU
294
302
  "availability_zone" => subnet.availability_zone,
295
303
  "state" => mount_target.life_cycle_state,
296
304
  "ip_address" => mount_target.ip_address,
297
- "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@config['region']}.amazonaws.com",
305
+ "endpoint" => "#{subnet.availability_zone}.#{mount_target.file_system_id}.efs.#{@region}.amazonaws.com",
298
306
  "network_interface_id" => mount_target.network_interface_id
299
307
  }
300
308
  }