cloud-mu 3.1.3 → 3.1.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +10 -2
  3. data/bin/mu-adopt +5 -1
  4. data/bin/mu-load-config.rb +2 -3
  5. data/bin/mu-run-tests +112 -27
  6. data/cloud-mu.gemspec +20 -20
  7. data/cookbooks/mu-tools/libraries/helper.rb +2 -1
  8. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  9. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  10. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  11. data/extras/image-generators/Google/centos6.yaml +1 -0
  12. data/extras/image-generators/Google/centos7.yaml +1 -1
  13. data/modules/mommacat.ru +5 -15
  14. data/modules/mu.rb +10 -14
  15. data/modules/mu/adoption.rb +20 -14
  16. data/modules/mu/cleanup.rb +13 -9
  17. data/modules/mu/cloud.rb +26 -26
  18. data/modules/mu/clouds/aws.rb +100 -59
  19. data/modules/mu/clouds/aws/alarm.rb +4 -2
  20. data/modules/mu/clouds/aws/bucket.rb +25 -21
  21. data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
  22. data/modules/mu/clouds/aws/collection.rb +21 -20
  23. data/modules/mu/clouds/aws/container_cluster.rb +47 -26
  24. data/modules/mu/clouds/aws/database.rb +57 -68
  25. data/modules/mu/clouds/aws/dnszone.rb +14 -14
  26. data/modules/mu/clouds/aws/endpoint.rb +20 -16
  27. data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
  28. data/modules/mu/clouds/aws/folder.rb +7 -7
  29. data/modules/mu/clouds/aws/function.rb +15 -12
  30. data/modules/mu/clouds/aws/group.rb +14 -10
  31. data/modules/mu/clouds/aws/habitat.rb +16 -13
  32. data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
  33. data/modules/mu/clouds/aws/log.rb +13 -10
  34. data/modules/mu/clouds/aws/msg_queue.rb +15 -8
  35. data/modules/mu/clouds/aws/nosqldb.rb +18 -11
  36. data/modules/mu/clouds/aws/notifier.rb +11 -6
  37. data/modules/mu/clouds/aws/role.rb +87 -70
  38. data/modules/mu/clouds/aws/search_domain.rb +30 -19
  39. data/modules/mu/clouds/aws/server.rb +102 -72
  40. data/modules/mu/clouds/aws/server_pool.rb +47 -28
  41. data/modules/mu/clouds/aws/storage_pool.rb +5 -6
  42. data/modules/mu/clouds/aws/user.rb +13 -10
  43. data/modules/mu/clouds/aws/vpc.rb +135 -121
  44. data/modules/mu/clouds/azure.rb +16 -9
  45. data/modules/mu/clouds/azure/container_cluster.rb +2 -3
  46. data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
  47. data/modules/mu/clouds/azure/habitat.rb +8 -6
  48. data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
  49. data/modules/mu/clouds/azure/role.rb +8 -10
  50. data/modules/mu/clouds/azure/server.rb +65 -25
  51. data/modules/mu/clouds/azure/user.rb +5 -7
  52. data/modules/mu/clouds/azure/vpc.rb +12 -15
  53. data/modules/mu/clouds/cloudformation.rb +8 -7
  54. data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
  55. data/modules/mu/clouds/google.rb +39 -24
  56. data/modules/mu/clouds/google/bucket.rb +9 -11
  57. data/modules/mu/clouds/google/container_cluster.rb +27 -42
  58. data/modules/mu/clouds/google/database.rb +6 -9
  59. data/modules/mu/clouds/google/firewall_rule.rb +11 -10
  60. data/modules/mu/clouds/google/folder.rb +16 -9
  61. data/modules/mu/clouds/google/function.rb +127 -161
  62. data/modules/mu/clouds/google/group.rb +21 -18
  63. data/modules/mu/clouds/google/habitat.rb +18 -15
  64. data/modules/mu/clouds/google/loadbalancer.rb +14 -16
  65. data/modules/mu/clouds/google/role.rb +48 -31
  66. data/modules/mu/clouds/google/server.rb +105 -105
  67. data/modules/mu/clouds/google/server_pool.rb +12 -31
  68. data/modules/mu/clouds/google/user.rb +67 -13
  69. data/modules/mu/clouds/google/vpc.rb +58 -65
  70. data/modules/mu/config.rb +89 -1738
  71. data/modules/mu/config/bucket.rb +3 -3
  72. data/modules/mu/config/collection.rb +3 -3
  73. data/modules/mu/config/container_cluster.rb +2 -2
  74. data/modules/mu/config/dnszone.rb +5 -5
  75. data/modules/mu/config/doc_helpers.rb +517 -0
  76. data/modules/mu/config/endpoint.rb +3 -3
  77. data/modules/mu/config/firewall_rule.rb +118 -3
  78. data/modules/mu/config/folder.rb +3 -3
  79. data/modules/mu/config/function.rb +2 -2
  80. data/modules/mu/config/group.rb +3 -3
  81. data/modules/mu/config/habitat.rb +3 -3
  82. data/modules/mu/config/loadbalancer.rb +3 -3
  83. data/modules/mu/config/log.rb +3 -3
  84. data/modules/mu/config/msg_queue.rb +3 -3
  85. data/modules/mu/config/nosqldb.rb +3 -3
  86. data/modules/mu/config/notifier.rb +2 -2
  87. data/modules/mu/config/ref.rb +333 -0
  88. data/modules/mu/config/role.rb +3 -3
  89. data/modules/mu/config/schema_helpers.rb +508 -0
  90. data/modules/mu/config/search_domain.rb +3 -3
  91. data/modules/mu/config/server.rb +86 -58
  92. data/modules/mu/config/server_pool.rb +2 -2
  93. data/modules/mu/config/tail.rb +189 -0
  94. data/modules/mu/config/user.rb +3 -3
  95. data/modules/mu/config/vpc.rb +44 -4
  96. data/modules/mu/defaults/Google.yaml +2 -2
  97. data/modules/mu/deploy.rb +13 -10
  98. data/modules/mu/groomer.rb +1 -1
  99. data/modules/mu/groomers/ansible.rb +69 -24
  100. data/modules/mu/groomers/chef.rb +52 -44
  101. data/modules/mu/logger.rb +17 -14
  102. data/modules/mu/master.rb +317 -2
  103. data/modules/mu/master/chef.rb +3 -4
  104. data/modules/mu/master/ldap.rb +3 -3
  105. data/modules/mu/master/ssl.rb +12 -2
  106. data/modules/mu/mommacat.rb +85 -1766
  107. data/modules/mu/mommacat/daemon.rb +394 -0
  108. data/modules/mu/mommacat/naming.rb +366 -0
  109. data/modules/mu/mommacat/storage.rb +689 -0
  110. data/modules/tests/bucket.yml +4 -0
  111. data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
  112. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  113. data/modules/tests/regrooms/bucket.yml +19 -0
  114. metadata +112 -102
@@ -187,21 +187,35 @@ module MU
187
187
  end
188
188
  end
189
189
 
190
+ @cloud_desc_cache = nil
190
191
  # Retrieve the cloud descriptor for this resource.
191
192
  # @return [Google::Apis::Core::Hashable]
192
- def cloud_desc
193
+ def cloud_desc(use_cache: true)
194
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
193
195
  if @config['type'] == "interactive" or !@config['type']
194
196
  @config['type'] ||= "interactive"
195
197
  if !@config['external']
196
- return MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
198
+ @cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
197
199
  else
198
200
  return nil
199
201
  end
200
202
  else
201
203
  @config['type'] ||= "service"
202
- MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
204
+ # this often fails even when it succeeded earlier, so try to be
205
+ # resilient on GCP's behalf
206
+ retries = 0
207
+ begin
208
+ @cloud_desc_cache = MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
209
+ rescue ::Google::Apis::ClientError => e
210
+ if e.message.match(/notFound:/) and retries < 10
211
+ sleep 3
212
+ retries += 1
213
+ retry
214
+ end
215
+ end
203
216
  end
204
217
 
218
+ @cloud_desc_cache
205
219
  end
206
220
 
207
221
  # Return the metadata for this user configuration
@@ -232,12 +246,17 @@ module MU
232
246
  # Remove all users associated with the currently loaded deployment.
233
247
  # @param noop [Boolean]: If true, will only print what would be done
234
248
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
235
- # @param region [String]: The cloud provider region
236
249
  # @return [void]
237
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
238
- my_domains = MU::Cloud::Google.getDomains(credentials)
250
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
251
+ MU::Cloud::Google.getDomains(credentials)
239
252
  my_org = MU::Cloud::Google.getOrg(credentials)
240
253
 
254
+ filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
255
+ if !ignoremaster and MU.mu_public_ip
256
+ filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
257
+ end
258
+ MU.log "Placeholder: Google User artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
259
+
241
260
  # We don't have a good way of tagging directory users, so we rely
242
261
  # on the known parameter, which is pulled from deployment metadata
243
262
  if flags['known'] and my_org
@@ -319,7 +338,7 @@ module MU
319
338
  MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
320
339
  "projects/"+args[:project]
321
340
  )
322
- rescue ::Google::Apis::ClientError => e
341
+ rescue ::Google::Apis::ClientError
323
342
  MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
324
343
  end
325
344
 
@@ -382,7 +401,7 @@ module MU
382
401
  # Reverse-map our cloud description into a runnable config hash.
383
402
  # We assume that any values we have in +@config+ are placeholders, and
384
403
  # calculate our own accordingly based on what's live in the cloud.
385
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
404
+ def toKitten(**_args)
386
405
  if MU::Cloud::Google::User.cannedServiceAcctName?(@cloud_id)
387
406
  return nil
388
407
  end
@@ -441,9 +460,9 @@ module MU
441
460
  end
442
461
 
443
462
  # Cloud-specific configuration properties.
444
- # @param config [MU::Config]: The calling MU::Config object
463
+ # @param _config [MU::Config]: The calling MU::Config object
445
464
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
446
- def self.schema(config)
465
+ def self.schema(_config)
447
466
  toplevel_required = []
448
467
  schema = {
449
468
  "name" => {
@@ -517,9 +536,9 @@ If we are binding (rather than creating) a user and no roles are specified, we w
517
536
 
518
537
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
519
538
  # @param user [Hash]: The resource to process and validate
520
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
539
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
521
540
  # @return [Boolean]: True if validation succeeded, False otherwise
522
- def self.validateConfig(user, configurator)
541
+ def self.validateConfig(user, _configurator)
523
542
  ok = true
524
543
 
525
544
  my_domains = MU::Cloud::Google.getDomains(user['credentials'])
@@ -621,7 +640,42 @@ If we are binding (rather than creating) a user and no roles are specified, we w
621
640
  ok
622
641
  end
623
642
 
624
- private
643
+ # Create and inject a service account on behalf of the parent resource.
644
+ # Return the modified parent configuration hash with references to the
645
+ # new addition.
646
+ # @param parent [Hash]
647
+ # @param configurator [MU::Config]
648
+ # @return [Hash]
649
+ def self.genericServiceAccount(parent, configurator)
650
+ user = {
651
+ "name" => parent['name'],
652
+ "cloud" => "Google",
653
+ "project" => parent["project"],
654
+ "credentials" => parent["credentials"],
655
+ "type" => "service"
656
+ }
657
+ if user["name"].length < 6
658
+ user["name"] += Password.pronounceable(6)
659
+ end
660
+ if parent['roles']
661
+ user['roles'] = parent['roles'].dup
662
+ end
663
+ configurator.insertKitten(user, "users", true)
664
+ parent['dependencies'] ||= []
665
+ parent['service_account'] = MU::Config::Ref.get(
666
+ type: "users",
667
+ cloud: "Google",
668
+ name: user["name"],
669
+ project: user["project"],
670
+ credentials: user["credentials"]
671
+ )
672
+ parent['dependencies'] << {
673
+ "type" => "user",
674
+ "name" => user["name"]
675
+ }
676
+
677
+ parent
678
+ end
625
679
 
626
680
  end
627
681
  end
@@ -36,12 +36,10 @@ module MU
36
36
 
37
37
  # Called automatically by {MU::Deploy#createResources}
38
38
  def create
39
-
40
39
  networkobj = MU::Cloud::Google.compute(:Network).new(
41
40
  name: MU::Cloud::Google.nameStr(@mu_name),
42
41
  description: @deploy.deploy_id,
43
42
  auto_create_subnetworks: false
44
- # i_pv4_range: @config['ip_block']
45
43
  )
46
44
  MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@project_id}", details: networkobj
47
45
 
@@ -58,7 +56,7 @@ module MU
58
56
  subnet_name = @config['name']+subnet['name']
59
57
 
60
58
  subnet_mu_name = @config['scrub_mu_isms'] ? @cloud_id+subnet_name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet_name, max_length: 61))
61
- MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id}", details: subnet
59
+ MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id} region #{subnet['availability_zone']}", details: subnet
62
60
  subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
63
61
  name: subnet_mu_name,
64
62
  description: @deploy.deploy_id,
@@ -72,9 +70,17 @@ module MU
72
70
  subnetdesc = nil
73
71
  begin
74
72
  subnetdesc = MU::Cloud::Google.compute(credentials: @config['credentials']).get_subnetwork(@project_id, subnet['availability_zone'], subnet_mu_name)
73
+ if !subnetdesc.nil?
74
+ subnet_cfg = {}
75
+ subnet_cfg["ip_block"] = subnet['ip_block']
76
+ subnet_cfg["name"] = subnet_name
77
+ subnet_cfg['mu_name'] = subnet_mu_name
78
+ subnet_cfg["cloud_id"] = subnetdesc.self_link.gsub(/.*?\/([^\/]+)$/, '\1')
79
+ subnet_cfg['az'] = subnet['availability_zone']
80
+ @subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet_cfg, precache_description: false)
81
+ end
75
82
  sleep 1
76
83
  end while subnetdesc.nil?
77
-
78
84
  }
79
85
  }
80
86
  subnetthreads.each do |t|
@@ -82,7 +88,6 @@ module MU
82
88
  end
83
89
  end
84
90
 
85
- route_table_ids = []
86
91
  if !@config['route_tables'].nil?
87
92
  @config['route_tables'].each { |rtb|
88
93
  rtb['routes'].each { |route|
@@ -120,8 +125,8 @@ module MU
120
125
 
121
126
  # Describe this VPC from the cloud platform's perspective
122
127
  # @return [Google::Apis::Core::Hashable]
123
- def cloud_desc
124
- if @cloud_desc_cache
128
+ def cloud_desc(use_cache: true)
129
+ if @cloud_desc_cache and use_cache
125
130
  return @cloud_desc_cache
126
131
  end
127
132
 
@@ -230,8 +235,8 @@ end
230
235
  # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
231
236
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
232
237
  def self.find(**args)
233
- args[:project] ||= args[:habitat]
234
- args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
238
+ args = MU::Cloud::Google.findLocationArgs(args)
239
+
235
240
  resp = {}
236
241
  if args[:cloud_id] and args[:project]
237
242
  begin
@@ -240,7 +245,7 @@ end
240
245
  args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1')
241
246
  )
242
247
  resp[args[:cloud_id]] = vpc if !vpc.nil?
243
- rescue ::Google::Apis::ClientError => e
248
+ rescue ::Google::Apis::ClientError
244
249
  MU.log "VPC #{args[:cloud_id]} in project #{args[:project]} does not exist, or I do not have permission to view it", MU::WARN
245
250
  end
246
251
  else # XXX other criteria
@@ -477,9 +482,8 @@ end
477
482
  # directly at child nodes in peered VPCs, the public internet, and the
478
483
  # like.
479
484
  # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
480
- # @param region [String]: The cloud provider region of the target subnet.
481
485
  # @return [Boolean]
482
- def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)
486
+ def self.haveRouteToInstance?(target_instance, credentials: nil)
483
487
  project ||= MU::Cloud::Google.defaultProject(credentials)
484
488
  return false if MU.myCloud != "Google"
485
489
  # XXX see if we reside in the same Network and overlap subnets
@@ -536,11 +540,15 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
536
540
  # Remove all VPC resources associated with the currently loaded deployment.
537
541
  # @param noop [Boolean]: If true, will only print what would be done
538
542
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
539
- # @param region [String]: The cloud provider region
540
543
  # @return [void]
541
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
544
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
542
545
  flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
543
546
  return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
547
+ filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
548
+ if !ignoremaster and MU.mu_public_ip
549
+ filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
550
+ end
551
+ MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
544
552
 
545
553
  purge_subnets(noop, project: flags['project'], credentials: credentials)
546
554
  ["route", "network"].each { |type|
@@ -562,7 +570,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
562
570
  if e.message.match(/Failed to delete network (.+)/)
563
571
  network_name = Regexp.last_match[1]
564
572
  fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
565
- fwrules.reject! { |name, desc|
573
+ fwrules.reject! { |_name, desc|
566
574
  !desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
567
575
  }
568
576
  fwrules.keys.each { |name|
@@ -586,7 +594,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
586
594
  # We assume that any values we have in +@config+ are placeholders, and
587
595
  # calculate our own accordingly based on what's live in the cloud.
588
596
  # XXX add flag to return the diff between @config and live cloud
589
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
597
+ def toKitten(**_args)
590
598
  return nil if cloud_desc.name == "default" # parent project builds these
591
599
  bok = {
592
600
  "cloud" => "Google",
@@ -595,8 +603,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
595
603
  }
596
604
  MU::Cloud::Google.listRegions.size
597
605
 
598
- diff = {}
599
- schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
606
+ _schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
600
607
  return [nil, nil] if !valid
601
608
  # pp schema
602
609
  # MU.log "++++++++++++++++++++++++++++++++"
@@ -609,6 +616,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
609
616
  bok['subnets'] = []
610
617
  regions_seen = []
611
618
  names_seen = []
619
+ @subnets.reject! { |x| x.cloud_desc.nil? }
612
620
  @subnets.map { |x| x.cloud_desc }.each { |s|
613
621
  subnet_name = s.name.dup
614
622
  names_seen << s.name.dup
@@ -630,7 +638,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
630
638
  end
631
639
  end
632
640
 
633
- peer_names = []
634
641
  if cloud_desc.peerings and cloud_desc.peerings.size > 0
635
642
  bok['peers'] = []
636
643
  cloud_desc.peerings.each { |peer|
@@ -688,9 +695,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
688
695
  end
689
696
 
690
697
  # Cloud-specific configuration properties.
691
- # @param config [MU::Config]: The calling MU::Config object
698
+ # @param _config [MU::Config]: The calling MU::Config object
692
699
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
693
- def self.schema(config = nil)
700
+ def self.schema(_config = nil)
694
701
  toplevel_required = []
695
702
  schema = {
696
703
  "regions" => {
@@ -736,7 +743,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
736
743
 
737
744
  # see if one of this thing's siblings declared a subnet_pref we can
738
745
  # use to guess which one we should marry ourselves to
739
- configurator.kittens.each_pair { |type, siblings|
746
+ configurator.kittens.values.each { |siblings|
740
747
  siblings.each { |sibling|
741
748
  next if !sibling['dependencies']
742
749
  sibling['dependencies'].each { |dep|
@@ -900,7 +907,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
900
907
  "destination_network"=>"0.0.0.0/0"
901
908
  }
902
909
  end
903
- nat_count = 0
910
+
904
911
  # You know what, let's just guarantee that we'll have a route from
905
912
  # this master, always
906
913
  # XXX this confuses machines that don't have public IPs
@@ -945,7 +952,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
945
952
 
946
953
  private
947
954
 
948
- def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true, credentials: nil)
955
+ def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
949
956
  private_acl = {
950
957
  "name" => vpc_name+"-rt",
951
958
  "cloud" => "Google",
@@ -973,6 +980,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
973
980
  # end
974
981
  configurator.insertKitten(private_acl, "firewall_rules", true)
975
982
  end
983
+ private_class_method :genStandardSubnetACLs
976
984
 
977
985
  # Helper method for manufacturing routes. Expect to be called from
978
986
  # {MU::Cloud::Google::VPC#create} or {MU::Cloud::Google::VPC#groom}.
@@ -1039,7 +1047,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1039
1047
  rescue ::Google::Apis::ClientError, MU::MuError => e
1040
1048
  if e.message.match(/notFound/)
1041
1049
  MU.log "Creating route #{routename} in project #{@project_id}", details: routeobj
1042
- resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj)
1050
+ MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj)
1043
1051
  else
1044
1052
  # TODO can't update GCP routes, would have to delete and re-create
1045
1053
  end
@@ -1047,44 +1055,12 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1047
1055
  end
1048
1056
  end
1049
1057
 
1050
-
1051
- # Remove all network gateways associated with the currently loaded deployment.
1052
- # @param noop [Boolean]: If true, will only print what would be done
1053
- # @param region [String]: The cloud provider region
1054
- # @return [void]
1055
- def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
1056
- end
1057
-
1058
- # Remove all NAT gateways associated with the VPC of the currently loaded deployment.
1059
- # @param noop [Boolean]: If true, will only print what would be done
1060
- # @param vpc_id [String]: The cloud provider's unique VPC identifier
1061
- # @param region [String]: The cloud provider region
1062
- # @return [void]
1063
- def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion)
1064
- end
1065
-
1066
- # Remove all VPC endpoints associated with the VPC of the currently loaded deployment.
1067
- # @param noop [Boolean]: If true, will only print what would be done
1068
- # @param vpc_id [String]: The cloud provider's unique VPC identifier
1069
- # @param region [String]: The cloud provider region
1070
- # @return [void]
1071
- def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion)
1072
- end
1073
-
1074
- # Remove all network interfaces associated with the currently loaded deployment.
1075
- # @param noop [Boolean]: If true, will only print what would be done
1076
- # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
1077
- # @param region [String]: The cloud provider region
1078
- # @return [void]
1079
- def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
1080
- end
1081
-
1082
1058
  # Remove all subnets associated with the currently loaded deployment.
1083
1059
  # @param noop [Boolean]: If true, will only print what would be done
1084
- # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
1060
+ # @param _tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
1085
1061
  # @param regions [Array<String>]: The cloud provider regions to check
1086
1062
  # @return [void]
1087
- def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
1063
+ def self.purge_subnets(noop = false, _tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
1088
1064
  project ||= MU::Cloud::Google.defaultProject(credentials)
1089
1065
  parent_thread_id = Thread.current.object_id
1090
1066
  regionthreads = []
@@ -1098,7 +1074,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1098
1074
  r,
1099
1075
  noop
1100
1076
  )
1101
- rescue MU::Cloud::MuDefunctHabitat => e
1077
+ rescue MU::Cloud::MuDefunctHabitat
1102
1078
  Thread.exit
1103
1079
  end
1104
1080
  }
@@ -1107,8 +1083,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1107
1083
  t.join
1108
1084
  end
1109
1085
  end
1110
-
1111
- protected
1086
+ private_class_method :purge_subnets
1112
1087
 
1113
1088
  # Subnets are almost a first-class resource. So let's kinda sorta treat
1114
1089
  # them like one. This should only be invoked on objects that already
@@ -1116,7 +1091,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1116
1091
  class Subnet < MU::Cloud::Google::VPC
1117
1092
 
1118
1093
  attr_reader :cloud_id
1119
- attr_reader :url
1120
1094
  attr_reader :ip_block
1121
1095
  attr_reader :mu_name
1122
1096
  attr_reader :name
@@ -1149,10 +1123,29 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
1149
1123
  MU.structToHash(cloud_desc)
1150
1124
  end
1151
1125
 
1126
+ # Return the +self_link+ to this subnet
1127
+ def url
1128
+ cloud_desc if !@url
1129
+ @url
1130
+ end
1131
+
1132
+ @cloud_desc_cache = nil
1152
1133
  # Describe this VPC Subnet from the cloud platform's perspective
1153
1134
  # @return [Google::Apis::Core::Hashable]
1154
- def cloud_desc
1155
- @cloud_desc_cache ||= MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @config['az'], @config['cloud_id'])
1135
+ def cloud_desc(use_cache: true)
1136
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
1137
+
1138
+ begin
1139
+ @cloud_desc_cache = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @az, @cloud_id)
1140
+ rescue ::Google::Apis::ClientError => e
1141
+ if e.message.match(/notFound: /)
1142
+ MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
1143
+ return nil
1144
+ else
1145
+ raise e
1146
+ end
1147
+ end
1148
+ @url ||= @cloud_desc_cache.self_link
1156
1149
  @cloud_desc_cache
1157
1150
  end
1158
1151
 
@@ -18,6 +18,10 @@ require 'erb'
18
18
  require 'pp'
19
19
  require 'json-schema'
20
20
  require 'net/http'
21
+ require 'mu/config/schema_helpers'
22
+ require 'mu/config/tail'
23
+ require 'mu/config/ref'
24
+ require 'mu/config/doc_helpers'
21
25
  autoload :GraphViz, 'graphviz'
22
26
  autoload :ChronicDuration, 'chronic_duration'
23
27
 
@@ -35,35 +39,6 @@ module MU
35
39
  class DeployParamError < MuError
36
40
  end
37
41
 
38
- # The default cloud provider for new resources. Must exist in MU.supportedClouds
39
- # return [String]
40
- def self.defaultCloud
41
- configured = {}
42
- MU::Cloud.supportedClouds.each { |cloud|
43
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
44
- if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
45
- configured[cloud] = $MU_CFG[cloud.downcase].size
46
- configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker
47
- end
48
- }
49
- if configured.size > 0
50
- return configured.keys.sort { |a, b|
51
- configured[b] <=> configured[a]
52
- }.first
53
- else
54
- MU::Cloud.supportedClouds.each { |cloud|
55
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
56
- return cloud if cloudclass.hosted?
57
- }
58
- return MU::Cloud.supportedClouds.first
59
- end
60
- end
61
-
62
- # The default grooming agent for new resources. Must exist in MU.supportedGroomers.
63
- def self.defaultGroomer
64
- MU.localOnly ? "Ansible" : "Chef"
65
- end
66
-
67
42
  attr_accessor :nat_routes
68
43
  attr_reader :skipinitialupdates
69
44
 
@@ -75,120 +50,6 @@ module MU
75
50
  @@config_path
76
51
  end
77
52
 
78
- # Accessor for our Basket of Kittens schema definition
79
- def self.schema
80
- @@schema
81
- end
82
-
83
- # Deep merge a configuration hash so we can meld different cloud providers'
84
- # schemas together, while preserving documentation differences
85
- def self.schemaMerge(orig, new, cloud)
86
- if new.is_a?(Hash)
87
- new.each_pair { |k, v|
88
- if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty?
89
- new[k] = "+"+cloud.upcase+"+: "+v
90
- end
91
- if orig and orig.has_key?(k)
92
- elsif orig
93
- orig[k] = new[k]
94
- else
95
- orig = new
96
- end
97
- schemaMerge(orig[k], new[k], cloud)
98
- }
99
- elsif orig.is_a?(Array) and new
100
- orig.concat(new)
101
- orig.uniq!
102
- elsif new.is_a?(String)
103
- orig ||= ""
104
- orig += "\n" if !orig.empty?
105
- orig += "+#{cloud.upcase}+: "+new
106
- else
107
- # XXX I think this is a NOOP?
108
- end
109
- end
110
-
111
- # Accessor for our Basket of Kittens schema definition, with various
112
- # cloud-specific details merged so we can generate documentation for them.
113
- def self.docSchema
114
- docschema = Marshal.load(Marshal.dump(@@schema))
115
- only_children = {}
116
- MU::Cloud.resource_types.each_pair { |classname, attrs|
117
- MU::Cloud.supportedClouds.each { |cloud|
118
- begin
119
- require "mu/clouds/#{cloud.downcase}/#{attrs[:cfg_name]}"
120
- rescue LoadError => e
121
- next
122
- end
123
- res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname)
124
- required, res_schema = res_class.schema(self)
125
- docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= ""
126
- docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+res_class.quality
127
- res_schema.each { |key, cfg|
128
- if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
129
- only_children[attrs[:cfg_plural]] ||= {}
130
- only_children[attrs[:cfg_plural]][key] ||= {}
131
- only_children[attrs[:cfg_plural]][key][cloud] = cfg
132
- end
133
- }
134
- }
135
- }
136
-
137
- # recursively chase down description fields in arrays and objects of our
138
- # schema and prepend stuff to them for documentation
139
- def self.prepend_descriptions(prefix, cfg)
140
- cfg["prefix"] = prefix
141
- if cfg["type"] == "array" and cfg["items"]
142
- cfg["items"] = prepend_descriptions(prefix, cfg["items"])
143
- elsif cfg["type"] == "object" and cfg["properties"]
144
- cfg["properties"].each_pair { |key, subcfg|
145
- cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key])
146
- }
147
- end
148
- cfg
149
- end
150
-
151
- MU::Cloud.resource_types.each_pair { |classname, attrs|
152
- MU::Cloud.supportedClouds.each { |cloud|
153
- res_class = nil
154
- begin
155
- res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname)
156
- rescue MU::Cloud::MuCloudResourceNotImplemented
157
- next
158
- end
159
- required, res_schema = res_class.schema(self)
160
- next if required.size == 0 and res_schema.size == 0
161
- res_schema.each { |key, cfg|
162
- cfg["description"] ||= ""
163
- if !cfg["description"].empty?
164
- cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"]
165
- end
166
- if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
167
- schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud)
168
- docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= ""
169
- docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"]
170
- MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
171
- else
172
- if only_children[attrs[:cfg_plural]][key]
173
- prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY"
174
- cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section
175
- cfg = prepend_descriptions(prefix, cfg)
176
- end
177
-
178
- docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg
179
- end
180
- docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {}
181
- docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg
182
- }
183
-
184
- docschema['required'].concat(required)
185
- docschema['required'].uniq!
186
- }
187
- }
188
-
189
- docschema
190
- end
191
-
192
53
  attr_reader :config
193
54
 
194
55
  @@parameters = {}
@@ -243,495 +104,16 @@ module MU
243
104
  MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true)
244
105
  end
245
106
 
246
- # A wrapper class for resources to refer to other resources, whether they
247
- # be a sibling object in the current deploy, an object in another deploy,
248
- # or a plain cloud id from outside of Mu.
249
- class Ref
250
- attr_reader :name
251
- attr_reader :type
252
- attr_reader :cloud
253
- attr_reader :deploy_id
254
- attr_reader :region
255
- attr_reader :credentials
256
- attr_reader :habitat
257
- attr_reader :mommacat
258
- attr_reader :tag_key
259
- attr_reader :tag_value
260
- attr_reader :obj
261
-
262
- @@refs = []
263
- @@ref_semaphore = Mutex.new
264
-
265
- # Little bit of a factory pattern... given a hash of options for a {MU::Config::Ref} objects, first see if we have an existing one that matches our more immutable attributes (+cloud+, +id+, etc). If we do, return that. If we do not, create one, add that to our inventory, and return that instead.
266
- # @param cfg [Hash]:
267
- # @return [MU::Config::Ref]
268
- def self.get(cfg)
269
- return cfg if cfg.is_a?(MU::Config::Ref)
270
- checkfields = cfg.keys.map { |k| k.to_sym }
271
- required = [:id, :type]
272
-
273
- @@ref_semaphore.synchronize {
274
- match = nil
275
- @@refs.each { |ref|
276
- saw_mismatch = false
277
- saw_match = false
278
- needed_values = []
279
- checkfields.each { |field|
280
- next if !cfg[field]
281
- ext_value = ref.instance_variable_get("@#{field.to_s}".to_sym)
282
- if !ext_value
283
- needed_values << field
284
- next
285
- end
286
- if cfg[field] != ext_value
287
- saw_mismatch = true
288
- elsif required.include?(field) and cfg[field] == ext_value
289
- saw_match = true
290
- end
291
- }
292
- if saw_match and !saw_mismatch
293
- # populate empty fields we got from this request
294
- if needed_values.size > 0
295
- newref = ref.dup
296
- needed_values.each { |field|
297
- newref.instance_variable_set("@#{field.to_s}".to_sym, cfg[field])
298
- if !newref.respond_to?(field)
299
- newref.singleton_class.instance_eval { attr_reader field.to_sym }
300
- end
301
- }
302
- @@refs << newref
303
- return newref
304
- else
305
- return ref
306
- end
307
- end
308
- }
309
-
310
- }
311
-
312
- # if we get here, there was no match
313
- newref = MU::Config::Ref.new(cfg)
314
- @@ref_semaphore.synchronize {
315
- @@refs << newref
316
- return newref
317
- }
318
- end
319
-
320
- # A way of dynamically defining +attr_reader+ without leaking memory
321
- def self.define_reader(name)
322
- define_method(name) {
323
- instance_variable_get("@#{name.to_s}")
324
- }
325
- end
326
-
327
- # @param cfg [Hash]: A Basket of Kittens configuration hash containing
328
- # lookup information for a cloud object
329
- def initialize(cfg)
330
- cfg.keys.each { |field|
331
- next if field == "tag"
332
- if !cfg[field].nil?
333
- self.instance_variable_set("@#{field}".to_sym, cfg[field])
334
- elsif !cfg[field.to_sym].nil?
335
- self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym])
336
- end
337
- MU::Config::Ref.define_reader(field)
338
- }
339
- if cfg['tag'] and cfg['tag']['key'] and
340
- !cfg['tag']['key'].empty? and cfg['tag']['value']
341
- @tag_key = cfg['tag']['key']
342
- @tag_value = cfg['tag']['value']
343
- end
344
-
345
- if @deploy_id and !@mommacat
346
- @mommacat = MU::MommaCat.getLitter(@deploy_id, set_context_to_me: false)
347
- elsif @mommacat and !@deploy_id
348
- @deploy_id = @mommacat.deploy_id
349
- end
350
-
351
- kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
352
- end
353
-
354
- # Comparison operator
355
- def <=>(other)
356
- return 1 if other.nil?
357
- self.to_s <=> other.to_s
358
- end
359
-
360
- # Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
361
- # @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
362
- # @return [Hash]
363
- def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
364
- parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
365
- desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
366
- schema = {
367
- "type" => "object",
368
- "#MU_REFERENCE" => true,
369
- "minProperties" => 1,
370
- "description" => desc,
371
- "properties" => {
372
- "id" => {
373
- "type" => "string",
374
- "description" => "Cloud identifier of a resource we want to reference, typically used when leveraging resources not managed by MU"
375
- },
376
- "name" => {
377
- "type" => "string",
378
- "description" => "The short (internal Mu) name of a resource we're attempting to reference. Typically used when referring to a sibling resource elsewhere in the same deploy, or in another known Mu deploy in conjunction with +deploy_id+."
379
- },
380
- "type" => {
381
- "type" => "string",
382
- "description" => "The resource type we're attempting to reference.",
383
- "enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }
384
- },
385
- "deploy_id" => {
386
- "type" => "string",
387
- "description" => "Our target resource should be found in this Mu deploy."
388
- },
389
- "credentials" => MU::Config.credentials_primitive,
390
- "region" => MU::Config.region_primitive,
391
- "cloud" => MU::Config.cloud_primitive,
392
- "tag" => {
393
- "type" => "object",
394
- "description" => "If the target resource supports tagging and our resource implementations +find+ method supports it, we can attempt to locate it by tag.",
395
- "properties" => {
396
- "key" => {
397
- "type" => "string",
398
- "description" => "The tag or label key to search against"
399
- },
400
- "value" => {
401
- "type" => "string",
402
- "description" => "The tag or label value to match"
403
- }
404
- }
405
- }
406
- }
407
- }
408
- if !["folders", "habitats"].include?(type)
409
- schema["properties"]["habitat"] = MU::Config::Habitat.reference
410
- end
411
-
412
- if omit_fields
413
- omit_fields.each { |f|
414
- schema["properties"].delete(f)
415
- }
416
- end
417
-
418
- if !type.nil?
419
- schema["required"] = ["type"]
420
- schema["properties"]["type"]["default"] = type
421
- schema["properties"]["type"]["enum"] = [type]
422
- end
423
-
424
- aliases.each { |a|
425
- a.each_pair { |k, v|
426
- if schema["properties"][v]
427
- schema["properties"][k] = schema["properties"][v].dup
428
- schema["properties"][k]["description"] = "Alias for <tt>#{v}</tt>"
429
- else
430
- MU.log "Reference schema alias #{k} wants to alias #{v}, but no such attribute exists", MU::WARN, details: caller[4]
431
- end
432
- }
433
- }
434
-
435
- schema
436
- end
437
-
438
- # Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
439
- # of the sort that would have been used to declare this reference in the
440
- # first place.
441
- def to_h
442
- me = { }
443
-
444
- self.instance_variables.each { |var|
445
- next if [:@obj, :@mommacat, :@tag_key, :@tag_value].include?(var)
446
- val = self.instance_variable_get(var)
447
- next if val.nil?
448
- val = val.to_h if val.is_a?(MU::Config::Ref)
449
- me[var.to_s.sub(/^@/, '')] = val
450
- }
451
- if @tag_key and !@tag_key.empty?
452
- me['tag'] = {
453
- 'key' => @tag_key,
454
- 'value' => @tag_value
455
- }
456
- end
457
- me
458
- end
459
-
460
- # Getter for the #{id} instance variable that attempts to populate it if
461
- # it's not set.
462
- # @return [String,nil]
463
- def id
464
- return @id if @id
465
- kitten # if it's not defined, attempt to define it
466
- @id
467
- end
468
-
469
- # Alias for {id}
470
- # @return [String,nil]
471
- def cloud_id
472
- id
473
- end
474
-
475
- # Return a {MU::Cloud} object for this reference. This is only meant to be
476
- # called in a live deploy, which is to say that if called during initial
477
- # configuration parsing, results may be incorrect.
478
- # @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
479
- def kitten(mommacat = @mommacat, shallow: false)
480
- return nil if !@cloud or !@type
481
-
482
- if @obj
483
- @deploy_id ||= @obj.deploy_id
484
- @id ||= @obj.cloud_id
485
- @name ||= @obj.config['name']
486
- return @obj
487
- end
488
-
489
- if mommacat
490
- @obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
491
- if @obj # initialize missing attributes, if we can
492
- @id ||= @obj.cloud_id
493
- @mommacat ||= mommacat
494
- @obj.intoDeploy(@mommacat) # make real sure these are set
495
- @deploy_id ||= mommacat.deploy_id
496
- if !@name
497
- if @obj.config and @obj.config['name']
498
- @name = @obj.config['name']
499
- elsif @obj.mu_name
500
- if @type == "folders"
501
- MU.log "would assign name '#{@obj.mu_name}' in ref to this folder if I were feeling aggressive", MU::WARN, details: self.to_h
502
- end
503
- # @name = @obj.mu_name
504
- end
505
- end
506
- return @obj
507
- else
508
- # MU.log "Failed to find a live '#{@type.to_s}' object named #{@name}#{@id ? " (#{@id})" : "" }#{ @habitat ? " in habitat #{@habitat}" : "" }", MU::WARN, details: self
509
- end
510
- end
511
-
512
- if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
513
-
514
- begin
515
- hab_arg = if @habitat.nil?
516
- [nil]
517
- elsif @habitat.is_a?(MU::Config::Ref)
518
- [@habitat.id]
519
- elsif @habitat.is_a?(Hash)
520
- [@habitat["id"]]
521
- else
522
- [@habitat.to_s]
523
- end
524
-
525
- found = MU::MommaCat.findStray(
526
- @cloud,
527
- @type,
528
- name: @name,
529
- cloud_id: @id,
530
- deploy_id: @deploy_id,
531
- region: @region,
532
- habitats: hab_arg,
533
- credentials: @credentials,
534
- dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
535
- )
536
- @obj ||= found.first if found
537
- rescue ThreadError => e
538
- # Sometimes MommaCat calls us in a potential deadlock situation;
539
- # don't be the cause of a fatal error if so, we don't need this
540
- # object that badly.
541
- raise e if !e.message.match(/recursive locking/)
542
- rescue SystemExit => e
543
- # XXX this is temporary, to cope with some debug stuff that's in findStray
544
- # for the nonce
545
- return
546
- end
547
- end
548
-
549
- if @obj
550
- @deploy_id ||= @obj.deploy_id
551
- @id ||= @obj.cloud_id
552
- @name ||= @obj.config['name']
553
- end
554
-
555
- @obj
556
- end
557
-
558
- end
559
-
560
- # A wrapper for config leaves that came from ERB parameters instead of raw
561
- # YAML or JSON. Will behave like a string for things that expect that
562
- # sort of thing. Code that needs to know that this leaf was the result of
563
- # a parameter will be able to tell by the object class being something
564
- # other than a plain string, array, or hash.
565
- class Tail
566
- @value = nil
567
- @name = nil
568
- @prettyname = nil
569
- @description = nil
570
- @prefix = ""
571
- @suffix = ""
572
- @is_list_element = false
573
- @pseudo = false
574
- @runtimecode = nil
575
- @valid_values = []
576
- @index = 0
577
- attr_reader :description
578
- attr_reader :pseudo
579
- attr_reader :index
580
- attr_reader :value
581
- attr_reader :runtimecode
582
- attr_reader :valid_values
583
- attr_reader :is_list_element
584
-
585
- def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0)
586
- @name = name
587
- @bindings = {}
588
- @value = value
589
- @valid_values = valid_values
590
- @pseudo = pseudo
591
- @index = index
592
- @runtimecode = runtimecode
593
- @cloudtype = cloudtype
594
- @is_list_element = is_list_element
595
- @description ||=
596
- if !description.nil?
597
- description
598
- else
599
- ""
600
- end
601
- @prettyname ||=
602
- if !prettyname.nil?
603
- prettyname
604
- else
605
- @name.capitalize.gsub(/[^a-z0-9]/i, "")
606
- end
607
- @prefix = prefix if !prefix.nil?
608
- @suffix = suffix if !suffix.nil?
609
- end
610
-
611
- # Return the parameter name of this Tail
612
- def getName
613
- @name
614
- end
615
- # Return the platform-specific cloud type of this Tail
616
- def getCloudType
617
- @cloudtype
618
- end
619
- # Return the human-friendly name of this Tail
620
- def getPrettyName
621
- @prettyname
622
- end
623
- # Walk like a String
624
- def to_s
625
- @prefix.to_s+@value.to_s+@suffix.to_s
626
- end
627
- # Quack like a String
628
- def to_str
629
- to_s
630
- end
631
- # Upcase like a String
632
- def upcase
633
- to_s.upcase
634
- end
635
- # Downcase like a String
636
- def downcase
637
- to_s.downcase
638
- end
639
- # Check for emptiness like a String
640
- def empty?
641
- to_s.empty?
642
- end
643
- # Match like a String
644
- def match(*args)
645
- to_s.match(*args)
646
- end
647
- # Check for equality like a String
648
- def ==(o)
649
- (o.class == self.class or o.class == "String") && o.to_s == to_s
650
- end
651
- # Concatenate like a string
652
- def +(o)
653
- return to_s if o.nil?
654
- to_s + o.to_s
655
- end
656
- # Perform global substitutions like a String
657
- def gsub(*args)
658
- to_s.gsub(*args)
659
- end
660
- end
661
-
662
- # Wrapper method for creating a {MU::Config::Tail} object as a reference to
663
- # a parameter that's valid in the loaded configuration.
664
- # @param param [<String>]: The name of the parameter to which this should be tied.
665
- # @param value [<String>]: The value of the parameter to return when asked
666
- # @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like
667
- # @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id
668
- # @param valid_values [Array<String>]: A list of acceptable String values for the given parameter.
669
- # @param description [<String>]: A long-form description of what the parameter does.
670
- # @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer.
671
- # @param prefix [<String>]: A static String that should be prefixed to the stored value when queried
672
- # @param suffix [<String>]: A static String that should be appended to the stored value when queried
673
- # @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input.
674
- # @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation
675
- def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
676
- if value.nil?
677
- if @@parameters.nil? or !@@parameters.has_key?(param)
678
- MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
679
- return nil
680
- # raise DeployParamError
681
- else
682
- value = @@parameters[param]
683
- end
684
- end
685
- if !prettyname.nil?
686
- prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions
687
- end
688
- if value.is_a?(MU::Config::Tail)
689
- MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN
690
- tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
691
- elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array))
692
- tail = []
693
- count = 0
694
- value.split(/\s*,\s*/).each { |subval|
695
- if @@tails.has_key?(param) and !@@tails[param][count].nil?
696
- subval = @@tails[param][count].values.first.to_s if subval.nil?
697
- list_of = @@tails[param][count].values.first.getName if list_of.nil?
698
- prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil?
699
- description = @@tails[param][count].values.first.description if description.nil?
700
- valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty?
701
- cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String"
702
- end
703
- prettyname = param.capitalize if prettyname.nil?
704
- tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) }
705
- count = count + 1
706
- }
707
- else
708
- if @@tails.has_key?(param)
709
- pseudo = @@tails[param].pseudo
710
- value = @@tails[param].to_s if value.nil?
711
- prettyname = @@tails[param].getPrettyName if prettyname.nil?
712
- description = @@tails[param].description if description.nil?
713
- valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty?
714
- cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String"
715
- end
716
- tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
717
- end
718
-
719
- if valid_values and valid_values.size > 0 and value
720
- if !valid_values.include?(value)
721
- raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'"
722
- end
723
- end
724
- @@tails[param] = tail
725
-
726
- tail
727
- end
728
-
729
107
  # Load up our YAML or JSON and parse it through ERB, optionally substituting
730
108
  # externally-supplied parameters.
731
109
  def resolveConfig(path: @@config_path, param_pass: false, cloud: nil)
732
110
  config = nil
733
111
  @param_pass = param_pass
734
112
 
113
+ if cloud
114
+ MU.log "Exposing cloud variable to ERB with value of #{cloud}", MU::DEBUG
115
+ end
116
+
735
117
  # Catch calls to missing variables in Basket of Kittens files when being
736
118
  # parsed by ERB, and replace with placeholders for parameters. This
737
119
  # method_missing is only defined innside {MU::Config.resolveConfig}
@@ -862,17 +244,15 @@ return
862
244
  # @param cloud [String]: Sets a parameter named 'cloud', and insert it as the default cloud platform if not already declared
863
245
  # @return [Hash]: The complete validated configuration for a deployment.
864
246
  def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil)
865
- $myPublicIp = MU::Cloud::AWS.getAWSMetaData("public-ipv4")
866
- $myRoot = MU.myRoot
247
+ $myPublicIp ||= MU.mu_public_ip
248
+ $myRoot ||= MU.myRoot
867
249
  $myRoot.freeze
868
250
 
869
- $myAZ = MU.myAZ.freeze
251
+ $myAZ ||= MU.myAZ.freeze
870
252
  $myAZ.freeze
871
- $myRegion = MU.curRegion.freeze
253
+ $myRegion ||= MU.curRegion.freeze
872
254
  $myRegion.freeze
873
255
 
874
- $myAppName = nil
875
-
876
256
  @kittens = {}
877
257
  @kittencfg_semaphore = Mutex.new
878
258
  @@config_path = path
@@ -920,7 +300,7 @@ return
920
300
  # you can't specify parameters in an included file, because ERB is what's
921
301
  # doing the including, and parameters need to already be resolved so that
922
302
  # ERB can use them.
923
- param_cfg, raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud)
303
+ param_cfg, _raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud)
924
304
  if param_cfg.has_key?("parameters")
925
305
  param_cfg["parameters"].each { |param|
926
306
  if param.has_key?("default") and param["default"].nil?
@@ -976,7 +356,7 @@ return
976
356
  $parameters = @@parameters.dup
977
357
  $parameters.freeze
978
358
 
979
- tmp_cfg, raw_erb = resolveConfig(path: @@config_path, cloud: cloud)
359
+ tmp_cfg, _raw_erb = resolveConfig(path: @@config_path, cloud: cloud)
980
360
 
981
361
  # Convert parameter entries that constitute whole config keys into
982
362
  # {MU::Config::Tail} objects.
@@ -1022,8 +402,6 @@ return
1022
402
  exit 1
1023
403
  end
1024
404
 
1025
- types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
1026
-
1027
405
  MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type|
1028
406
  if @config[type]
1029
407
  @config[type].each { |k|
@@ -1041,164 +419,6 @@ return
1041
419
  @config.freeze
1042
420
  end
1043
421
 
1044
- # Output the dependencies of this BoK stack as a directed acyclic graph.
1045
- # Very useful for debugging.
1046
- def visualizeDependencies
1047
- # GraphViz won't like MU::Config::Tail, pare down to plain Strings
1048
- config = MU::Config.stripConfig(@config)
1049
- begin
1050
- g = GraphViz.new(:G, :type => :digraph)
1051
- # Generate a GraphViz node for each resource in this stack
1052
- nodes = {}
1053
- MU::Cloud.resource_types.each_pair { |classname, attrs|
1054
- nodes[attrs[:cfg_name]] = {}
1055
- if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
1056
- config[attrs[:cfg_plural]].each { |resource|
1057
- nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}")
1058
- }
1059
- end
1060
- }
1061
- # Now add edges corresponding to the dependencies they list
1062
- MU::Cloud.resource_types.each_pair { |classname, attrs|
1063
- if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
1064
- config[attrs[:cfg_plural]].each { |resource|
1065
- if resource.has_key?("dependencies")
1066
- me = nodes[attrs[:cfg_name]][resource['name']]
1067
- resource["dependencies"].each { |dep|
1068
- parent = nodes[dep['type']][dep['name']]
1069
- g.add_edges(me, parent)
1070
- }
1071
- end
1072
- }
1073
- end
1074
- }
1075
- # Spew some output?
1076
- MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE
1077
- g.output(:jpg => "/tmp/#{config['appname']}.jpg")
1078
- rescue Exception => e
1079
- MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace
1080
- end
1081
- end
1082
-
1083
- # Generate a documentation-friendly dummy Ruby class for our mu.yaml main
1084
- # config.
1085
- def self.emitConfigAsRuby
1086
- example = %Q{---
1087
- public_address: 1.2.3.4
1088
- mu_admin_email: egtlabs@eglobaltech.com
1089
- mu_admin_name: Joe Schmoe
1090
- mommacat_port: 2260
1091
- banner: My Example Mu Master
1092
- mu_repository: git://github.com/cloudamatic/mu.git
1093
- repos:
1094
- - https://github.com/cloudamatic/mu_demo_platform
1095
- allow_invade_foreign_vpcs: true
1096
- ansible_dir:
1097
- aws:
1098
- egtdev:
1099
- region: us-east-1
1100
- log_bucket_name: egt-mu-log-bucket
1101
- default: true
1102
- name: egtdev
1103
- personal:
1104
- region: us-east-2
1105
- log_bucket_name: my-mu-log-bucket
1106
- name: personal
1107
- google:
1108
- egtlabs:
1109
- project: egt-labs-admin
1110
- credentials_file: /opt/mu/etc/google.json
1111
- region: us-east4
1112
- log_bucket_name: hexabucket-761234
1113
- default: true
1114
- }
1115
- mu_yaml_schema = eval(%Q{
1116
- $NOOP = true
1117
- load "#{MU.myRoot}/bin/mu-configure"
1118
- $CONFIGURABLES
1119
- })
1120
- return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash)
1121
- muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb"
1122
- MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}"
1123
- muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
1124
- muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}."
1125
- muyaml_rb.puts "#"
1126
- muyaml_rb.puts "# Example:"
1127
- muyaml_rb.puts "#"
1128
- muyaml_rb.puts "# <pre>"
1129
- example.split(/\n/).each { |line|
1130
- muyaml_rb.puts "# "+line+" " # markdooooown
1131
- }
1132
- muyaml_rb.puts "# </pre>"
1133
- muyaml_rb.puts "module MuYAML"
1134
- muyaml_rb.puts "\t# The configuration file format for Mu's main config file."
1135
- self.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema })
1136
- muyaml_rb.puts "end"
1137
- muyaml_rb.close
1138
- end
1139
-
1140
- # Take the schema we've defined and create a dummy Ruby class tree out of
1141
- # it, basically so we can leverage Yard to document it.
1142
- def self.emitSchemaAsRuby
1143
- kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb"
1144
- MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}"
1145
- kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
1146
- kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
1147
- kitten_rb.puts "#"
1148
- kitten_rb.puts "#"
1149
- kitten_rb.puts "#"
1150
- kitten_rb.puts "module MU"
1151
- kitten_rb.puts "class Config"
1152
- kitten_rb.puts "\t# The configuration file format for Mu application stacks."
1153
- self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema)
1154
- kitten_rb.puts "end"
1155
- kitten_rb.puts "end"
1156
- kitten_rb.close
1157
-
1158
- end
1159
-
1160
- # Take an IP block and split it into a more-or-less arbitrary number of
1161
- # subnets.
1162
- # @param ip_block [String]: CIDR of the network to subdivide
1163
- # @param subnets_desired [Integer]: Number of subnets we want back
1164
- # @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider)
1165
- # @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred.
1166
- def divideNetwork(ip_block, subnets_desired, max_mask = 28)
1167
- cidr = NetAddr::IPv4Net.parse(ip_block.to_s)
1168
-
1169
- # Ugly but reliable method of landing on the right subnet size
1170
- subnet_bits = cidr.netmask.prefix_len
1171
- begin
1172
- subnet_bits += 1
1173
- if subnet_bits > max_mask
1174
- MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR
1175
- raise MuError, "Subnets smaller than /#{max_mask} not permitted"
1176
- end
1177
- end while cidr.subnet_count(subnet_bits) < subnets_desired
1178
-
1179
- if cidr.subnet_count(subnet_bits) > subnets_desired
1180
- MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE
1181
- end
1182
-
1183
- begin
1184
- subnets = []
1185
- (0..subnets_desired).each { |x|
1186
- subnets << cidr.nth_subnet(subnet_bits, x).to_s
1187
- }
1188
- rescue RuntimeError => e
1189
- if e.message.match(/exceeds subnets available for allocation/)
1190
- MU.log e.message, MU::ERR
1191
- MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR
1192
- return nil
1193
- else
1194
- raise e
1195
- end
1196
- end
1197
-
1198
- subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] }
1199
- subnets
1200
- end
1201
-
1202
422
  # See if a given resource is configured in the current stack
1203
423
  # @param name [String]: The name of the resource being checked
1204
424
  # @param type [String]: The type of resource being checked
@@ -1206,7 +426,7 @@ $CONFIGURABLES
1206
426
  def haveLitterMate?(name, type, has_multiple: false)
1207
427
  @kittencfg_semaphore.synchronize {
1208
428
  matches = []
1209
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
429
+ _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
1210
430
  if @kittens[cfg_plural]
1211
431
  @kittens[cfg_plural].each { |kitten|
1212
432
  if kitten['name'].to_s == name.to_s or
@@ -1233,7 +453,7 @@ $CONFIGURABLES
1233
453
  # @param type [String]: The type of resource being removed
1234
454
  def removeKitten(name, type)
1235
455
  @kittencfg_semaphore.synchronize {
1236
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
456
+ _shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
1237
457
  deletia = nil
1238
458
  if @kittens[cfg_plural]
1239
459
  @kittens[cfg_plural].each { |kitten|
@@ -1247,42 +467,6 @@ $CONFIGURABLES
1247
467
  }
1248
468
  end
1249
469
 
1250
- # FirewallRules can reference other FirewallRules, which means we need to do
1251
- # an extra pass to make sure we get all intra-stack dependencies correct.
1252
- # @param acl [Hash]: The configuration hash for the FirewallRule to check
1253
- # @return [Hash]
1254
- def resolveIntraStackFirewallRefs(acl, delay_validation = false)
1255
- acl["rules"].each { |acl_include|
1256
- if acl_include['sgs']
1257
- acl_include['sgs'].each { |sg_ref|
1258
- if haveLitterMate?(sg_ref, "firewall_rules")
1259
- acl["dependencies"] ||= []
1260
- found = false
1261
- acl["dependencies"].each { |dep|
1262
- if dep["type"] == "firewall_rule" and dep["name"] == sg_ref
1263
- dep["no_create_wait"] = true
1264
- found = true
1265
- end
1266
- }
1267
- if !found
1268
- acl["dependencies"] << {
1269
- "type" => "firewall_rule",
1270
- "name" => sg_ref,
1271
- "no_create_wait" => true
1272
- }
1273
- end
1274
- siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
1275
- if !siblingfw["#MU_VALIDATED"]
1276
- # XXX raise failure somehow
1277
- insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
1278
- end
1279
- end
1280
- }
1281
- end
1282
- }
1283
- acl
1284
- end
1285
-
1286
470
  # Insert a resource into the current stack
1287
471
  # @param descriptor [Hash]: The configuration description, as from a Basket of Kittens
1288
472
  # @param type [String]: The type of resource being added
@@ -1290,7 +474,7 @@ $CONFIGURABLES
1290
474
  # @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a +name+ field that's already in use
1291
475
  def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false)
1292
476
  append = false
1293
- start = Time.now
477
+ # start = Time.now
1294
478
  shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
1295
479
  MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0]
1296
480
 
@@ -1617,7 +801,7 @@ $CONFIGURABLES
1617
801
  plain_cfg.delete("parent_block") if cfg_plural == "vpcs"
1618
802
  begin
1619
803
  JSON::Validator.validate!(myschema, plain_cfg)
1620
- rescue JSON::Schema::ValidationError => e
804
+ rescue JSON::Schema::ValidationError
1621
805
  pp plain_cfg
1622
806
  # Use fully_validate to get the complete error list, save some time
1623
807
  errors = JSON::Validator.fully_validate(myschema, plain_cfg)
@@ -1665,165 +849,63 @@ $CONFIGURABLES
1665
849
  ok
1666
850
  end
1667
851
 
1668
- @@allregions = []
1669
- MU::Cloud.availableClouds.each { |cloud|
1670
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1671
- regions = cloudclass.listRegions()
1672
- @@allregions.concat(regions) if regions
1673
- }
852
+ # For our resources which specify intra-stack dependencies, make sure those
853
+ # dependencies are actually declared.
854
+ # TODO check for loops
855
+ def self.check_dependencies(config)
856
+ ok = true
1674
857
 
1675
- # Configuration chunk for choosing a provider region
1676
- # @return [Hash]
1677
- def self.region_primitive
1678
- if !@@allregions or @@allregions.empty?
1679
- @@allregions = []
1680
- MU::Cloud.availableClouds.each { |cloud|
1681
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1682
- return @allregions if !cloudclass.listRegions()
1683
- @@allregions.concat(cloudclass.listRegions())
1684
- }
1685
- end
1686
- {
1687
- "type" => "string",
1688
- "enum" => @@allregions
858
+ config.each_pair { |type, values|
859
+ if values.instance_of?(Array)
860
+ values.each { |resource|
861
+ if resource.kind_of?(Hash) and !resource["dependencies"].nil?
862
+ append = []
863
+ delete = []
864
+ resource["dependencies"].each { |dependency|
865
+ _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(dependency["type"])
866
+ found = false
867
+ names_seen = []
868
+ if !config[cfg_plural].nil?
869
+ config[cfg_plural].each { |service|
870
+ names_seen << service["name"].to_s
871
+ found = true if service["name"].to_s == dependency["name"].to_s
872
+ if service["virtual_name"]
873
+ names_seen << service["virtual_name"].to_s
874
+ if service["virtual_name"].to_s == dependency["name"].to_s
875
+ found = true
876
+ append_me = dependency.dup
877
+ append_me['name'] = service['name']
878
+ append << append_me
879
+ delete << dependency
880
+ end
881
+ end
882
+ }
883
+ end
884
+ if !found
885
+ MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen
886
+ ok = false
887
+ end
888
+ }
889
+ if append.size > 0
890
+ append.uniq!
891
+ resource["dependencies"].concat(append)
892
+ end
893
+ if delete.size > 0
894
+ delete.each { |delete_me|
895
+ resource["dependencies"].delete(delete_me)
896
+ }
897
+ end
898
+ end
899
+ }
900
+ end
1689
901
  }
902
+ return ok
1690
903
  end
1691
904
 
1692
- # Configuration chunk for choosing a set of cloud credentials
1693
- # @return [Hash]
1694
- def self.credentials_primitive
1695
- {
1696
- "type" => "string",
1697
- "description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If "
1698
- }
1699
- end
1700
-
1701
- # Configuration chunk for creating resource tags as an array of key/value
1702
- # pairs.
1703
- # @return [Hash]
1704
- def self.optional_tags_primitive
1705
- {
1706
- "type" => "boolean",
1707
- "description" => "Tag the resource with our optional tags (+MU-HANDLE+, +MU-MASTER-NAME+, +MU-OWNER+).",
1708
- "default" => true
1709
- }
1710
- end
1711
-
1712
- # Configuration chunk for creating resource tags as an array of key/value
1713
- # pairs.
1714
- # @return [Hash]
1715
- def self.tags_primitive
1716
- {
1717
- "type" => "array",
1718
- "minItems" => 1,
1719
- "items" => {
1720
- "description" => "Tags to apply to this resource. Will apply at the cloud provider level and in node groomers, where applicable.",
1721
- "type" => "object",
1722
- "title" => "tags",
1723
- "required" => ["key", "value"],
1724
- "additionalProperties" => false,
1725
- "properties" => {
1726
- "key" => {
1727
- "type" => "string",
1728
- },
1729
- "value" => {
1730
- "type" => "string",
1731
- }
1732
- }
1733
- }
1734
- }
1735
- end
1736
-
1737
- # Configuration chunk for choosing a cloud provider
1738
- # @return [Hash]
1739
- def self.cloud_primitive
1740
- {
1741
- "type" => "string",
1742
- # "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better
1743
- "enum" => MU::Cloud.supportedClouds
1744
- }
1745
- end
1746
-
1747
- # Generate configuration for the general-pursose ADMIN firewall rulesets
1748
- # (security groups in AWS). Note that these are unique to regions and
1749
- # individual VPCs (as well as Classic, which is just a degenerate case of
1750
- # a VPC for our purposes.
1751
- # @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC).
1752
- # @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource.
1753
- # @param cloud [String]: The parent resource's cloud plugin identifier
1754
- # @param region [String]: Cloud provider region, if applicable.
1755
- # @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
1756
- def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
1757
- if !cloud or (cloud == "AWS" and !region)
1758
- raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
1759
- end
1760
- hosts = Array.new
1761
- hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
1762
- hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
1763
- hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
1764
- hosts << "#{admin_ip}/32" if admin_ip
1765
- hosts.uniq!
1766
-
1767
- rules = []
1768
- if cloud == "Google"
1769
- rules = [
1770
- { "ingress" => true, "proto" => "all", "hosts" => hosts },
1771
- { "egress" => true, "proto" => "all", "hosts" => hosts }
1772
- ]
1773
- else
1774
- rules = [
1775
- { "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts },
1776
- { "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
1777
- { "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
1778
- ]
1779
- end
1780
-
1781
- resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("FirewallRule")
1782
-
1783
- if rules_only
1784
- return rules
1785
- end
1786
-
1787
- name = "admin"
1788
- name += credentials.to_s if credentials
1789
- realvpc = nil
1790
- if vpc
1791
- realvpc = {}
1792
- ['vpc_name', 'vpc_id'].each { |p|
1793
- if vpc[p]
1794
- vpc[p.sub(/^vpc_/, '')] = vpc[p]
1795
- vpc.delete(p)
1796
- end
1797
- }
1798
- ['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
1799
- realvpc[field] = vpc[field] if !vpc[field].nil?
1800
- }
1801
- if !realvpc['id'].nil? and !realvpc['id'].empty?
1802
- # Stupid kludge for Google cloud_ids which are sometimes URLs and
1803
- # sometimes not. Requirements are inconsistent from scenario to
1804
- # scenario.
1805
- name = name + "-" + realvpc['id'].gsub(/.*\//, "")
1806
- realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
1807
- elsif !realvpc['name'].nil?
1808
- name = name + "-" + realvpc['name']
1809
- end
1810
- end
1811
-
1812
-
1813
- acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
1814
- if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
1815
- acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
1816
- end
1817
- acl.delete("vpc") if !acl["vpc"]
1818
- if !resclass.isGlobal? and !region.nil? and !region.empty?
1819
- acl["region"] = region
1820
- end
1821
- @admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
1822
- return {"type" => "firewall_rule", "name" => name}
1823
- end
1824
-
1825
- private
1826
-
905
+ # Ugly text-manipulation to recursively resolve some placeholder strings
906
+ # we put in for ERB include() directives.
907
+ # @param lines [String]
908
+ # @return [String]
1827
909
  def self.resolveYAMLAnchors(lines)
1828
910
  new_text = ""
1829
911
  lines.each_line { |line|
@@ -1843,7 +925,6 @@ $CONFIGURABLES
1843
925
  return new_text
1844
926
  end
1845
927
 
1846
-
1847
928
  # Given a path to a config file, try to guess whether it's YAML or JSON.
1848
929
  # @param path [String]: The path to the file to check.
1849
930
  def self.guessFormat(path)
@@ -1852,10 +933,10 @@ $CONFIGURABLES
1852
933
  stripped = raw.gsub(/<%.*?%>,?/, "").gsub(/,[\n\s]*([\]\}])/, '\1')
1853
934
  begin
1854
935
  JSON.parse(stripped)
1855
- rescue JSON::ParserError => e
936
+ rescue JSON::ParserError
1856
937
  begin
1857
938
  YAML.load(raw.gsub(/<%.*?%>/, ""))
1858
- rescue Psych::SyntaxError => e
939
+ rescue Psych::SyntaxError
1859
940
  # Ok, well neither of those worked, let's assume that filenames are
1860
941
  # meaningful.
1861
942
  if path.match(/\.(yaml|yml)$/i)
@@ -1935,7 +1016,7 @@ $CONFIGURABLES
1935
1016
  end
1936
1017
  begin
1937
1018
  erb = ERB.new(File.read(file), nil, "<>")
1938
- rescue Errno::ENOENT => e
1019
+ rescue Errno::ENOENT
1939
1020
  retries = retries + 1
1940
1021
  if retries == 1
1941
1022
  file = File.dirname(MU::Config.config_path)+"/"+orig_filename
@@ -1960,12 +1041,12 @@ $CONFIGURABLES
1960
1041
  parsed_cfg = nil
1961
1042
  begin
1962
1043
  parsed_cfg = JSON.parse(erb.result(binding))
1963
- parsed_as = :json
1044
+ # parsed_as = :json
1964
1045
  rescue JSON::ParserError => e
1965
1046
  MU.log e.inspect, MU::DEBUG
1966
1047
  begin
1967
1048
  parsed_cfg = YAML.load(MU::Config.resolveYAMLAnchors(erb.result(binding)))
1968
- parsed_as = :yaml
1049
+ # parsed_as = :yaml
1969
1050
  rescue Psych::SyntaxError => e
1970
1051
  MU.log e.inspect, MU::DEBUG
1971
1052
  MU.log "#{file} parsed neither as JSON nor as YAML, including as raw text", MU::WARN if @param_pass
@@ -1980,16 +1061,11 @@ $CONFIGURABLES
1980
1061
  $yaml_refs[file] = ""+YAML.dump(parsed_cfg).sub(/^---\n/, "")
1981
1062
  return "# MU::Config.include PLACEHOLDER #{file} REDLOHECALP"
1982
1063
  end
1983
- rescue SyntaxError => e
1064
+ rescue SyntaxError
1984
1065
  raise ValidationError, "ERB in #{file} threw a syntax error"
1985
1066
  end
1986
1067
  end
1987
1068
 
1988
- # (see #include)
1989
- def include(file)
1990
- MU::Config.include(file, get_binding(@@tails.keys.sort), param_pass = @param_pass)
1991
- end
1992
-
1993
1069
  @@bindings = {}
1994
1070
  # Keep a cache of bindings we've created as sandbox contexts for ERB
1995
1071
  # processing, so we don't keep reloading the entire Mu library inside new
@@ -1998,6 +1074,13 @@ $CONFIGURABLES
1998
1074
  @@bindings
1999
1075
  end
2000
1076
 
1077
+ private
1078
+
1079
+ # (see #include)
1080
+ def include(file)
1081
+ MU::Config.include(file, get_binding(@@tails.keys.sort), @param_pass)
1082
+ end
1083
+
2001
1084
  # Namespace magic to pass to ERB's result method.
2002
1085
  def get_binding(keyset)
2003
1086
  environment = $environment
@@ -2012,242 +1095,6 @@ $CONFIGURABLES
2012
1095
  MU::Config.global_bindings[keyset]
2013
1096
  end
2014
1097
 
2015
- def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil)
2016
- return if schema_chunk.nil?
2017
-
2018
- if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)
2019
-
2020
- if schema_chunk["properties"]["creation_style"].nil? or
2021
- schema_chunk["properties"]["creation_style"] != "existing"
2022
- schema_chunk["properties"].each_pair { |key, subschema|
2023
- shortclass = if conf_chunk[key]
2024
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(key)
2025
- shortclass
2026
- else
2027
- nil
2028
- end
2029
-
2030
- new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup
2031
-
2032
- conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) if !new_val.nil?
2033
- }
2034
- end
2035
- elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
2036
- conf_chunk.map! { |item|
2037
- # If we're working on a resource type, go get implementation-specific
2038
- # schema information so that we set those defaults correctly.
2039
- realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
2040
-
2041
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
2042
- toplevel_required, cloudschema = cloudclass.schema(self)
2043
-
2044
- newschema = schema_chunk["items"].dup
2045
- newschema["properties"].merge!(cloudschema)
2046
- newschema
2047
- else
2048
- schema_chunk["items"].dup
2049
- end
2050
-
2051
- applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup
2052
- }
2053
- else
2054
- if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil?
2055
- schema_chunk["default_if"].each { |cond|
2056
- if siblings[cond["key_is"]] == cond["value_is"]
2057
- return Marshal.load(Marshal.dump(cond["set"]))
2058
- end
2059
- }
2060
- end
2061
- if conf_chunk.nil? and schema_chunk["default"] != nil
2062
- return Marshal.load(Marshal.dump(schema_chunk["default"]))
2063
- end
2064
- end
2065
-
2066
- return conf_chunk
2067
- end
2068
-
2069
- # For our resources which specify intra-stack dependencies, make sure those
2070
- # dependencies are actually declared.
2071
- # TODO check for loops
2072
- def self.check_dependencies(config)
2073
- ok = true
2074
-
2075
- config.each_pair { |type, values|
2076
- if values.instance_of?(Array)
2077
- values.each { |resource|
2078
- if resource.kind_of?(Hash) and !resource["dependencies"].nil?
2079
- append = []
2080
- delete = []
2081
- resource["dependencies"].each { |dependency|
2082
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(dependency["type"])
2083
- found = false
2084
- names_seen = []
2085
- if !config[cfg_plural].nil?
2086
- config[cfg_plural].each { |service|
2087
- names_seen << service["name"].to_s
2088
- found = true if service["name"].to_s == dependency["name"].to_s
2089
- if service["virtual_name"]
2090
- names_seen << service["virtual_name"].to_s
2091
- if service["virtual_name"].to_s == dependency["name"].to_s
2092
- found = true
2093
- append_me = dependency.dup
2094
- append_me['name'] = service['name']
2095
- append << append_me
2096
- delete << dependency
2097
- end
2098
- end
2099
- }
2100
- end
2101
- if !found
2102
- MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen
2103
- ok = false
2104
- end
2105
- }
2106
- if append.size > 0
2107
- append.uniq!
2108
- resource["dependencies"].concat(append)
2109
- end
2110
- if delete.size > 0
2111
- delete.each { |delete_me|
2112
- resource["dependencies"].delete(delete_me)
2113
- }
2114
- end
2115
- end
2116
- }
2117
- end
2118
- }
2119
- return ok
2120
- end
2121
-
2122
-
2123
- # Verify that a server or server_pool has a valid AD config referencing
2124
- # valid Vaults for credentials.
2125
- def self.check_vault_refs(server)
2126
- ok = true
2127
- server['vault_access'] = [] if server['vault_access'].nil?
2128
- server['groomer'] ||= self.defaultGroomer
2129
- groomclass = MU::Groomer.loadGroomer(server['groomer'])
2130
-
2131
- begin
2132
- if !server['active_directory'].nil?
2133
- ["domain_admin_vault", "domain_join_vault"].each { |vault_class|
2134
- server['vault_access'] << {
2135
- "vault" => server['active_directory'][vault_class]['vault'],
2136
- "item" => server['active_directory'][vault_class]['item']
2137
- }
2138
- item = groomclass.getSecret(
2139
- vault: server['active_directory'][vault_class]['vault'],
2140
- item: server['active_directory'][vault_class]['item'],
2141
- )
2142
- ["username_field", "password_field"].each { |field|
2143
- if !item.has_key?(server['active_directory'][vault_class][field])
2144
- ok = false
2145
- MU.log "I don't see a value named #{field} in Chef Vault #{server['active_directory'][vault_class]['vault']}:#{server['active_directory'][vault_class]['item']}", MU::ERR
2146
- end
2147
- }
2148
- }
2149
- end
2150
-
2151
- if !server['windows_auth_vault'].nil?
2152
- server['use_cloud_provider_windows_password'] = false
2153
-
2154
- server['vault_access'] << {
2155
- "vault" => server['windows_auth_vault']['vault'],
2156
- "item" => server['windows_auth_vault']['item']
2157
- }
2158
- item = groomclass.getSecret(
2159
- vault: server['windows_auth_vault']['vault'],
2160
- item: server['windows_auth_vault']['item']
2161
- )
2162
- ["password_field", "ec2config_password_field", "sshd_password_field"].each { |field|
2163
- if !item.has_key?(server['windows_auth_vault'][field])
2164
- MU.log "No value named #{field} in Chef Vault #{server['windows_auth_vault']['vault']}:#{server['windows_auth_vault']['item']}, will use a generated password.", MU::NOTICE
2165
- server['windows_auth_vault'].delete(field)
2166
- end
2167
- }
2168
- end
2169
- # Check all of the non-special ones while we're at it
2170
- server['vault_access'].each { |v|
2171
- next if v['vault'] == "splunk" and v['item'] == "admin_user"
2172
- item = groomclass.getSecret(vault: v['vault'], item: v['item'])
2173
- }
2174
- rescue MuError
2175
- MU.log "Can't load a Chef Vault I was configured to use. Does it exist?", MU::ERR
2176
- ok = false
2177
- end
2178
- return ok
2179
- end
2180
-
2181
-
2182
- # Given a bare hash describing a resource, insert default values which can
2183
- # be inherited from its parent or from the root of the BoK.
2184
- # @param kitten [Hash]: A resource descriptor
2185
- # @param type [String]: The type of resource this is ("servers" etc)
2186
- def applyInheritedDefaults(kitten, type)
2187
- return if !kitten.is_a?(Hash)
2188
- kitten['cloud'] ||= @config['cloud']
2189
- kitten['cloud'] ||= MU::Config.defaultCloud
2190
-
2191
- if !MU::Cloud.supportedClouds.include?(kitten['cloud'])
2192
- return
2193
- end
2194
-
2195
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
2196
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
2197
- resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
2198
-
2199
- schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
2200
- if !resclass.isGlobal?
2201
- kitten['region'] ||= @config['region']
2202
- kitten['region'] ||= cloudclass.myRegion(kitten['credentials'])
2203
- schema_fields << "region"
2204
- end
2205
-
2206
- kitten['credentials'] ||= @config['credentials']
2207
- kitten['credentials'] ||= cloudclass.credConfig(name_only: true)
2208
-
2209
- kitten['us_only'] ||= @config['us_only']
2210
- kitten['us_only'] ||= false
2211
-
2212
- kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
2213
- kitten['scrub_mu_isms'] ||= false
2214
-
2215
- if kitten['cloud'] == "Google"
2216
- # TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions)
2217
- if resclass.canLiveIn.include?(:Habitat)
2218
- kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
2219
- schema_fields << "project"
2220
- end
2221
- if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and
2222
- !resclass.isGlobal? and
2223
- ![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS'])
2224
- if MU::Cloud::Google.myRegion((kitten['credentials'])).nil?
2225
- raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set"
2226
- end
2227
- kitten['region'] ||= MU::Cloud::Google.myRegion
2228
- end
2229
- elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region']
2230
- if MU::Cloud::AWS.myRegion.nil?
2231
- raise ValidationError, "AWS resource declared without a region, but no default AWS region found"
2232
- end
2233
- kitten['region'] ||= MU::Cloud::AWS.myRegion
2234
- end
2235
-
2236
-
2237
- kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct']
2238
-
2239
- kitten["dependencies"] ||= []
2240
-
2241
- # Make sure the schema knows about these "new" fields, so that validation
2242
- # doesn't trip over them.
2243
- schema_fields.each { |field|
2244
- if @@schema["properties"][field]
2245
- MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field]
2246
- @@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field]
2247
- end
2248
- }
2249
- end
2250
-
2251
1098
  def validate(config = @config)
2252
1099
  ok = true
2253
1100
 
@@ -2277,9 +1124,11 @@ $CONFIGURABLES
2277
1124
  }
2278
1125
  }
2279
1126
 
1127
+ newrules = []
2280
1128
  @kittens["firewall_rules"].each { |acl|
2281
- acl = resolveIntraStackFirewallRefs(acl)
1129
+ newrules << resolveIntraStackFirewallRefs(acl)
2282
1130
  }
1131
+ @kittens["firewall_rules"] = newrules
2283
1132
 
2284
1133
  # VPCs do complex things in their cloud-layer validation that other
2285
1134
  # resources tend to need, like subnet allocation, so hit them early.
@@ -2321,7 +1170,7 @@ $CONFIGURABLES
2321
1170
  ruleset = haveLitterMate?("database"+db['name'], "firewall_rules")
2322
1171
  if ruleset
2323
1172
  ["server_pools", "servers"].each { |type|
2324
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
1173
+ _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
2325
1174
  @kittens[cfg_plural].each { |server|
2326
1175
  server["dependencies"].each { |dep|
2327
1176
  if dep["type"] == "database" and dep["name"] == db["name"]
@@ -2378,510 +1227,13 @@ $CONFIGURABLES
2378
1227
  # end
2379
1228
  end
2380
1229
 
2381
- # Emit our mu.yaml schema in a format that YARD can comprehend and turn into
2382
- # documentation.
2383
- def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
2384
- return if schema.nil?
2385
- if schema["subtree"]
2386
- printme = Array.new
2387
- # order sub-elements by whether they're required, so we can use YARD's
2388
- # grouping tags on them
2389
- have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] }
2390
- prop_list = schema["subtree"].keys.sort { |a, b|
2391
- if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"]
2392
- -1
2393
- elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"]
2394
- 1
2395
- else
2396
- a <=> b
2397
- end
2398
- }
2399
-
2400
- req = false
2401
- printme << "# @!group Optional parameters" if !have_required
2402
- prop_list.each { |name|
2403
- prop = schema["subtree"][name]
2404
- if prop["required"]
2405
- printme << "# @!group Required parameters" if !req
2406
- req = true
2407
- else
2408
- if req
2409
- printme << "# @!endgroup"
2410
- printme << "# @!group Optional parameters"
2411
- end
2412
- req = false
2413
- end
2414
-
2415
- printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req)
2416
- }
2417
- printme << "# @!endgroup"
2418
-
2419
- desc = (schema['desc'] || schema['title'])
2420
-
2421
- tabs = 1
2422
- class_hierarchy.each { |classname|
2423
- if classname == class_hierarchy.last and desc
2424
- muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n"
2425
- end
2426
- muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
2427
- tabs = tabs + 1
2428
- }
2429
- printme.each { |lines|
2430
- if !lines.nil? and lines.is_a?(String)
2431
- lines.lines.each { |line|
2432
- muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
2433
- }
2434
- end
2435
- }
2436
-
2437
- class_hierarchy.each { |classname|
2438
- tabs = tabs - 1
2439
- muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
2440
- }
2441
-
2442
- # And now that we've dealt with our children, pass our own rendered
2443
- # commentary back up to our caller.
2444
- name = class_hierarchy.last
2445
- if in_array
2446
- type = "Array<#{class_hierarchy.join("::")}>"
2447
- else
2448
- type = class_hierarchy.join("::")
2449
- end
2450
-
2451
- docstring = "\n"
2452
- docstring = docstring + "# **REQUIRED**\n" if required
2453
- # docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
2454
- docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc
2455
- docstring = docstring + "#\n"
2456
- docstring = docstring + "# @return [#{type}]\n"
2457
- docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
2458
- docstring = docstring + "attr_accessor :#{name}"
2459
- return docstring
2460
-
2461
- else
2462
- in_array = schema["array"]
2463
- name = class_hierarchy.last
2464
- type = if schema['boolean']
2465
- "Boolean"
2466
- else
2467
- "String"
2468
- end
2469
- if in_array
2470
- type = "Array<#{type}>"
2471
- end
2472
- docstring = "\n"
2473
-
2474
- prefixes = []
2475
- prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil?
2476
- # prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
2477
- prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
2478
- if !schema['pattern'].nil?
2479
- # XXX unquoted regex chars confuse the hell out of YARD. How do we
2480
- # quote {}[] etc in YARD-speak?
2481
- prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**"
2482
- end
2483
-
2484
- desc = (schema['desc'] || schema['title'])
2485
- if prefixes.size > 0
2486
- docstring += prefixes.join(",\n")
2487
- if desc and desc.size > 1
2488
- docstring += " - "
2489
- end
2490
- docstring += "\n"
2491
- end
2492
-
2493
- docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil?
2494
- docstring = docstring + "#\n"
2495
- docstring = docstring + "# @return [#{type}]\n"
2496
- docstring = docstring + "attr_accessor :#{name}"
2497
-
2498
- return docstring
2499
- end
2500
-
2501
- end
2502
-
2503
- # Emit our Basket of Kittens schema in a format that YARD can comprehend
2504
- # and turn into documentation.
2505
- def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
2506
- return if schema.nil?
2507
-
2508
- if schema["type"] == "object"
2509
- printme = []
2510
-
2511
- if !schema["properties"].nil?
2512
- # order sub-elements by whether they're required, so we can use YARD's
2513
- # grouping tags on them
2514
- if !schema["required"].nil? and schema["required"].size > 0
2515
- prop_list = schema["properties"].keys.sort_by { |name|
2516
- schema["required"].include?(name) ? 0 : 1
2517
- }
2518
- else
2519
- prop_list = schema["properties"].keys
2520
- end
2521
- req = false
2522
- printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0
2523
- prop_list.each { |name|
2524
- prop = schema["properties"][name]
2525
-
2526
- if class_hierarchy.size == 1
2527
-
2528
- _shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name)
2529
- if cfg_name
2530
- example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
2531
- if File.exist?(example_path)
2532
- example = "#\n# Examples:\n#\n"
2533
- # XXX these variables are all parameters from the BoKs in
2534
- # modules/tests. A really clever implementation would read
2535
- # and parse them to get default values, perhaps, instead of
2536
- # hard-coding them here.
2537
- instance_type = "t2.medium"
2538
- db_size = "db.t2.medium"
2539
- vpc_name = "some_vpc"
2540
- logs_name = "some_loggroup"
2541
- queues_name = "some_queue"
2542
- server_pools_name = "some_server_pool"
2543
- ["simple", "complex"].each { |complexity|
2544
- erb = ERB.new(File.read(example_path), nil, "<>")
2545
- example += "# !!!yaml\n"
2546
- example += "# ---\n"
2547
- example += "# appname: #{complexity}\n"
2548
- example += "# #{cfg_plural}:\n"
2549
- firstline = true
2550
- erb.result(binding).split(/\n/).each { |l|
2551
- l.chomp!
2552
- l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
2553
- next if l.empty? or l.match(/^\s+$/)
2554
- if firstline
2555
- l = "- "+l
2556
- firstline = false
2557
- else
2558
- l = " "+l
2559
- end
2560
- example += "# "+l+" "+"\n"
2561
- }
2562
- example += "# &nbsp;\n#\n" if complexity == "simple"
2563
- }
2564
- schema["properties"][name]["items"]["description"] ||= ""
2565
- if !schema["properties"][name]["items"]["description"].empty?
2566
- schema["properties"][name]["items"]["description"] += "\n"
2567
- end
2568
- schema["properties"][name]["items"]["description"] += example
2569
- end
2570
- end
2571
- end
2572
-
2573
- if !schema["required"].nil? and schema["required"].include?(name)
2574
- printme << "# @!group Required parameters" if !req
2575
- req = true
2576
- else
2577
- if req
2578
- printme << "# @!endgroup"
2579
- printme << "# @!group Optional parameters"
2580
- end
2581
- req = false
2582
- end
2583
-
2584
- printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
2585
- }
2586
- printme << "# @!endgroup"
2587
- end
2588
-
2589
- tabs = 1
2590
- class_hierarchy.each { |classname|
2591
- if classname == class_hierarchy.last and !schema['description'].nil?
2592
- kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
2593
- end
2594
- kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
2595
- tabs = tabs + 1
2596
- }
2597
- printme.each { |lines|
2598
- if !lines.nil? and lines.is_a?(String)
2599
- lines.lines.each { |line|
2600
- kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
2601
- }
2602
- end
2603
- }
2604
-
2605
- class_hierarchy.each { |classname|
2606
- tabs = tabs - 1
2607
- kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
2608
- }
2609
-
2610
- # And now that we've dealt with our children, pass our own rendered
2611
- # commentary back up to our caller.
2612
- name = class_hierarchy.last
2613
- if in_array
2614
- type = "Array<#{class_hierarchy.join("::")}>"
2615
- else
2616
- type = class_hierarchy.join("::")
2617
- end
2618
-
2619
- docstring = "\n"
2620
- docstring = docstring + "# **REQUIRED**\n" if required
2621
- docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
2622
- docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
2623
- docstring = docstring + "#\n"
2624
- docstring = docstring + "# @return [#{type}]\n"
2625
- docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
2626
- docstring = docstring + "attr_accessor :#{name}"
2627
- return docstring
2628
-
2629
- elsif schema["type"] == "array"
2630
- return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix)
2631
- else
2632
- name = class_hierarchy.last
2633
- if schema['type'].nil?
2634
- MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema
2635
- return nil
2636
- end
2637
- if in_array
2638
- type = "Array<#{schema['type'].capitalize}>"
2639
- else
2640
- type = schema['type'].capitalize
2641
- end
2642
- docstring = "\n"
2643
-
2644
- prefixes = []
2645
- prefixes << "# **REQUIRED**" if required and schema['default'].nil?
2646
- prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
2647
- prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
2648
- if !schema['enum'].nil? and !schema["enum"].empty?
2649
- prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**"
2650
- elsif !schema['pattern'].nil?
2651
- # XXX unquoted regex chars confuse the hell out of YARD. How do we
2652
- # quote {}[] etc in YARD-speak?
2653
- prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**"
2654
- end
2655
-
2656
- if prefixes.size > 0
2657
- docstring += prefixes.join(",\n")
2658
- if schema['description'] and schema['description'].size > 1
2659
- docstring += " - "
2660
- end
2661
- docstring += "\n"
2662
- end
2663
-
2664
- docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
2665
- docstring = docstring + "#\n"
2666
- docstring = docstring + "# @return [#{type}]\n"
2667
- docstring = docstring + "attr_accessor :#{name}"
2668
-
2669
- return docstring
2670
- end
2671
-
2672
- end
2673
-
2674
- def self.dependencies_primitive
2675
- {
2676
- "type" => "array",
2677
- "items" => {
2678
- "type" => "object",
2679
- "description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
2680
- "required" => ["name", "type"],
2681
- "additionalProperties" => false,
2682
- "properties" => {
2683
- "name" => {"type" => "string"},
2684
- "type" => {
2685
- "type" => "string",
2686
- "enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
2687
- },
2688
- "phase" => {
2689
- "type" => "string",
2690
- "description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
2691
- "enum" => ["create", "groom"]
2692
- },
2693
- "no_create_wait" => {
2694
- "type" => "boolean",
2695
- "default" => false,
2696
- "description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
2697
- }
2698
- }
2699
- }
2700
- }
2701
- end
2702
-
2703
- CIDR_PATTERN = "^\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}$"
2704
- CIDR_DESCRIPTION = "CIDR-formatted IP block, e.g. 1.2.3.4/32"
2705
- CIDR_PRIMITIVE = {
2706
- "type" => "string",
2707
- "pattern" => CIDR_PATTERN,
2708
- "description" => CIDR_DESCRIPTION
2709
- }
2710
-
2711
- # Have a default value available for config schema elements that take an
2712
- # email address.
2713
- # @return [String]
2714
- def self.notification_email
2715
- if MU.chef_user == "mu"
2716
- ENV['MU_ADMIN_EMAIL']
2717
- else
2718
- MU.userEmail
2719
- end
2720
- end
2721
-
2722
- # Load and validate the schema for an individual resource class, optionally
2723
- # merging cloud-specific schema components.
2724
- # @param type [String]: The resource type to load
2725
- # @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge
2726
- # @return [Hash]
2727
- def self.loadResourceSchema(type, cloud: nil)
2728
- valid = true
2729
- shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
2730
- schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)
2731
-
2732
- [:schema, :validate].each { |method|
2733
- if !schemaclass.respond_to?(method)
2734
- MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
2735
- return [nil, false] if method == :schema
2736
- valid = false
2737
- end
2738
- }
2739
-
2740
- schema = schemaclass.schema.dup
2741
-
2742
- schema["properties"]["virtual_name"] = {
2743
- "description" => "Internal use.",
2744
- "type" => "string"
2745
- }
2746
- schema["properties"]["dependencies"] = MU::Config.dependencies_primitive
2747
- schema["properties"]["cloud"] = MU::Config.cloud_primitive
2748
- schema["properties"]["credentials"] = MU::Config.credentials_primitive
2749
- schema["title"] = type.to_s
2750
-
2751
- if cloud
2752
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
2753
-
2754
- if cloudclass.respond_to?(:schema)
2755
- reqd, cloudschema = cloudclass.schema
2756
- cloudschema.each { |key, cfg|
2757
- if schema["properties"][key]
2758
- schemaMerge(schema["properties"][key], cfg, cloud)
2759
- else
2760
- schema["properties"][key] = cfg.dup
2761
- end
2762
- }
2763
- else
2764
- MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
2765
- valid = false
2766
- end
2767
-
2768
- end
2769
-
2770
- return [schema, valid]
2771
- end
2772
-
2773
- @@schema = {
2774
- "$schema" => "http://json-schema.org/draft-04/schema#",
2775
- "title" => "MU Application",
2776
- "type" => "object",
2777
- "description" => "A MU application stack, consisting of at least one resource.",
2778
- "required" => ["admins", "appname"],
2779
- "properties" => {
2780
- "appname" => {
2781
- "type" => "string",
2782
- "description" => "A name for your application stack. Should be short, but easy to differentiate from other applications.",
2783
- },
2784
- "scrub_mu_isms" => {
2785
- "type" => "boolean",
2786
- "description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template. Setting this flag here will override declarations in individual resources."
2787
- },
2788
- "project" => {
2789
- "type" => "string",
2790
- "description" => "**GOOGLE ONLY**: The project into which to deploy resources"
2791
- },
2792
- "billing_acct" => {
2793
- "type" => "string",
2794
- "description" => "**GOOGLE ONLY**: Billing account ID to associate with a newly-created Google Project. If not specified, will attempt to locate a billing account associated with the default project for our credentials.",
2795
- },
2796
- "region" => MU::Config.region_primitive,
2797
- "credentials" => MU::Config.credentials_primitive,
2798
- "us_only" => {
2799
- "type" => "boolean",
2800
- "description" => "For resources which span regions, restrict to regions inside the United States",
2801
- "default" => false
2802
- },
2803
- "conditions" => {
2804
- "type" => "array",
2805
- "items" => {
2806
- "type" => "object",
2807
- "required" => ["name", "cloudcode"],
2808
- "description" => "CloudFormation-specific. Define Conditions as in http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/conditions-section-structure.html. Arguments must use the cloudCode() macro.",
2809
- "properties" => {
2810
- "name" => { "required" => true, "type" => "string" },
2811
- "cloudcode" => { "required" => true, "type" => "string" },
2812
- }
2813
- }
2814
- },
2815
- "parameters" => {
2816
- "type" => "array",
2817
- "items" => {
2818
- "type" => "object",
2819
- "title" => "parameter",
2820
- "description" => "Parameters to be substituted elsewhere in this Basket of Kittens as ERB variables (<%= varname %>)",
2821
- "additionalProperties" => false,
2822
- "properties" => {
2823
- "name" => { "required" => true, "type" => "string" },
2824
- "default" => { "type" => "string" },
2825
- "list_of" => {
2826
- "type" => "string",
2827
- "description" => "Treat the value as a comma-separated list of values with this key name, equivalent to CloudFormation's various List<> types. For example, set to 'subnet_id' to pass values as an array of subnet identifiers as the 'subnets' argument of a VPC stanza."
2828
- },
2829
- "prettyname" => {
2830
- "type" => "string",
2831
- "description" => "An alternative name to use when generating parameter fields in, for example, CloudFormation templates"
2832
- },
2833
- "description" => {"type" => "string"},
2834
- "cloudtype" => {
2835
- "type" => "string",
2836
- "description" => "A platform-specific string describing the type of validation to use for this parameter. E.g. when generating a CloudFormation template, set to AWS::EC2::Image::Id to validate input as an AMI identifier."
2837
- },
2838
- "required" => {
2839
- "type" => "boolean",
2840
- "default" => true
2841
- },
2842
- "valid_values" => {
2843
- "type" => "array",
2844
- "description" => "List of valid values for this parameter. Can only be a list of static strings, for now.",
2845
- "items" => {
2846
- "type" => "string"
2847
- }
2848
- }
2849
- }
2850
- }
2851
- },
2852
- # TODO availability zones (or an array thereof)
2853
-
2854
- "admins" => {
2855
- "type" => "array",
2856
- "items" => {
2857
- "type" => "object",
2858
- "title" => "admin",
2859
- "description" => "Administrative contacts for this application stack. Will be automatically set to invoking Mu user, if not specified.",
2860
- "required" => ["name", "email"],
2861
- "additionalProperties" => false,
2862
- "properties" => {
2863
- "name" => {"type" => "string"},
2864
- "email" => {"type" => "string"},
2865
- "public_key" => {
2866
- "type" => "string",
2867
- "description" => "An OpenSSH-style public key string. This will be installed on all instances created in this deployment."
2868
- }
2869
- }
2870
- },
2871
- "minItems" => 1,
2872
- "uniqueItems" => true
2873
- }
2874
- },
2875
- "additionalProperties" => false
2876
- }
2877
-
2878
1230
  failed = []
2879
1231
 
2880
1232
  # Load all of the config stub files at the Ruby level
2881
1233
  MU::Cloud.resource_types.each_pair { |type, cfg|
2882
1234
  begin
2883
1235
  require "mu/config/#{cfg[:cfg_name]}"
2884
- rescue LoadError => e
1236
+ rescue LoadError
2885
1237
  # raise MuError, "MU::Config implemention of #{type} missing from modules/mu/config/#{cfg[:cfg_name]}.rb"
2886
1238
  MU.log "MU::Config::#{type} stub class is missing", MU::ERR
2887
1239
  failed << type
@@ -2889,7 +1241,6 @@ $CONFIGURABLES
2889
1241
  end
2890
1242
  }
2891
1243
 
2892
-
2893
1244
  MU::Cloud.resource_types.each_pair { |type, cfg|
2894
1245
  begin
2895
1246
  schema, valid = loadResourceSchema(type)