cloud-mu 3.1.3 → 3.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +10 -2
- data/bin/mu-adopt +5 -1
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-run-tests +112 -27
- data/cloud-mu.gemspec +20 -20
- data/cookbooks/mu-tools/libraries/helper.rb +2 -1
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +5 -15
- data/modules/mu.rb +10 -14
- data/modules/mu/adoption.rb +20 -14
- data/modules/mu/cleanup.rb +13 -9
- data/modules/mu/cloud.rb +26 -26
- data/modules/mu/clouds/aws.rb +100 -59
- data/modules/mu/clouds/aws/alarm.rb +4 -2
- data/modules/mu/clouds/aws/bucket.rb +25 -21
- data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
- data/modules/mu/clouds/aws/collection.rb +21 -20
- data/modules/mu/clouds/aws/container_cluster.rb +47 -26
- data/modules/mu/clouds/aws/database.rb +57 -68
- data/modules/mu/clouds/aws/dnszone.rb +14 -14
- data/modules/mu/clouds/aws/endpoint.rb +20 -16
- data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
- data/modules/mu/clouds/aws/folder.rb +7 -7
- data/modules/mu/clouds/aws/function.rb +15 -12
- data/modules/mu/clouds/aws/group.rb +14 -10
- data/modules/mu/clouds/aws/habitat.rb +16 -13
- data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
- data/modules/mu/clouds/aws/log.rb +13 -10
- data/modules/mu/clouds/aws/msg_queue.rb +15 -8
- data/modules/mu/clouds/aws/nosqldb.rb +18 -11
- data/modules/mu/clouds/aws/notifier.rb +11 -6
- data/modules/mu/clouds/aws/role.rb +87 -70
- data/modules/mu/clouds/aws/search_domain.rb +30 -19
- data/modules/mu/clouds/aws/server.rb +102 -72
- data/modules/mu/clouds/aws/server_pool.rb +47 -28
- data/modules/mu/clouds/aws/storage_pool.rb +5 -6
- data/modules/mu/clouds/aws/user.rb +13 -10
- data/modules/mu/clouds/aws/vpc.rb +135 -121
- data/modules/mu/clouds/azure.rb +16 -9
- data/modules/mu/clouds/azure/container_cluster.rb +2 -3
- data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
- data/modules/mu/clouds/azure/habitat.rb +8 -6
- data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
- data/modules/mu/clouds/azure/role.rb +8 -10
- data/modules/mu/clouds/azure/server.rb +65 -25
- data/modules/mu/clouds/azure/user.rb +5 -7
- data/modules/mu/clouds/azure/vpc.rb +12 -15
- data/modules/mu/clouds/cloudformation.rb +8 -7
- data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
- data/modules/mu/clouds/google.rb +39 -24
- data/modules/mu/clouds/google/bucket.rb +9 -11
- data/modules/mu/clouds/google/container_cluster.rb +27 -42
- data/modules/mu/clouds/google/database.rb +6 -9
- data/modules/mu/clouds/google/firewall_rule.rb +11 -10
- data/modules/mu/clouds/google/folder.rb +16 -9
- data/modules/mu/clouds/google/function.rb +127 -161
- data/modules/mu/clouds/google/group.rb +21 -18
- data/modules/mu/clouds/google/habitat.rb +18 -15
- data/modules/mu/clouds/google/loadbalancer.rb +14 -16
- data/modules/mu/clouds/google/role.rb +48 -31
- data/modules/mu/clouds/google/server.rb +105 -105
- data/modules/mu/clouds/google/server_pool.rb +12 -31
- data/modules/mu/clouds/google/user.rb +67 -13
- data/modules/mu/clouds/google/vpc.rb +58 -65
- data/modules/mu/config.rb +89 -1738
- data/modules/mu/config/bucket.rb +3 -3
- data/modules/mu/config/collection.rb +3 -3
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/dnszone.rb +5 -5
- data/modules/mu/config/doc_helpers.rb +517 -0
- data/modules/mu/config/endpoint.rb +3 -3
- data/modules/mu/config/firewall_rule.rb +118 -3
- data/modules/mu/config/folder.rb +3 -3
- data/modules/mu/config/function.rb +2 -2
- data/modules/mu/config/group.rb +3 -3
- data/modules/mu/config/habitat.rb +3 -3
- data/modules/mu/config/loadbalancer.rb +3 -3
- data/modules/mu/config/log.rb +3 -3
- data/modules/mu/config/msg_queue.rb +3 -3
- data/modules/mu/config/nosqldb.rb +3 -3
- data/modules/mu/config/notifier.rb +2 -2
- data/modules/mu/config/ref.rb +333 -0
- data/modules/mu/config/role.rb +3 -3
- data/modules/mu/config/schema_helpers.rb +508 -0
- data/modules/mu/config/search_domain.rb +3 -3
- data/modules/mu/config/server.rb +86 -58
- data/modules/mu/config/server_pool.rb +2 -2
- data/modules/mu/config/tail.rb +189 -0
- data/modules/mu/config/user.rb +3 -3
- data/modules/mu/config/vpc.rb +44 -4
- data/modules/mu/defaults/Google.yaml +2 -2
- data/modules/mu/deploy.rb +13 -10
- data/modules/mu/groomer.rb +1 -1
- data/modules/mu/groomers/ansible.rb +69 -24
- data/modules/mu/groomers/chef.rb +52 -44
- data/modules/mu/logger.rb +17 -14
- data/modules/mu/master.rb +317 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -2
- data/modules/mu/mommacat.rb +85 -1766
- data/modules/mu/mommacat/daemon.rb +394 -0
- data/modules/mu/mommacat/naming.rb +366 -0
- data/modules/mu/mommacat/storage.rb +689 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- metadata +112 -102
@@ -187,21 +187,35 @@ module MU
|
|
187
187
|
end
|
188
188
|
end
|
189
189
|
|
190
|
+
@cloud_desc_cache = nil
|
190
191
|
# Retrieve the cloud descriptor for this resource.
|
191
192
|
# @return [Google::Apis::Core::Hashable]
|
192
|
-
def cloud_desc
|
193
|
+
def cloud_desc(use_cache: true)
|
194
|
+
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
193
195
|
if @config['type'] == "interactive" or !@config['type']
|
194
196
|
@config['type'] ||= "interactive"
|
195
197
|
if !@config['external']
|
196
|
-
|
198
|
+
@cloud_desc_cache = MU::Cloud::Google.admin_directory(credentials: @config['credentials']).get_user(@cloud_id)
|
197
199
|
else
|
198
200
|
return nil
|
199
201
|
end
|
200
202
|
else
|
201
203
|
@config['type'] ||= "service"
|
202
|
-
|
204
|
+
# this often fails even when it succeeded earlier, so try to be
|
205
|
+
# resilient on GCP's behalf
|
206
|
+
retries = 0
|
207
|
+
begin
|
208
|
+
@cloud_desc_cache = MU::Cloud::Google.iam(credentials: @config['credentials']).get_project_service_account(@cloud_id)
|
209
|
+
rescue ::Google::Apis::ClientError => e
|
210
|
+
if e.message.match(/notFound:/) and retries < 10
|
211
|
+
sleep 3
|
212
|
+
retries += 1
|
213
|
+
retry
|
214
|
+
end
|
215
|
+
end
|
203
216
|
end
|
204
217
|
|
218
|
+
@cloud_desc_cache
|
205
219
|
end
|
206
220
|
|
207
221
|
# Return the metadata for this user configuration
|
@@ -232,12 +246,17 @@ module MU
|
|
232
246
|
# Remove all users associated with the currently loaded deployment.
|
233
247
|
# @param noop [Boolean]: If true, will only print what would be done
|
234
248
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
235
|
-
# @param region [String]: The cloud provider region
|
236
249
|
# @return [void]
|
237
|
-
def self.cleanup(noop: false, ignoremaster: false,
|
238
|
-
|
250
|
+
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
251
|
+
MU::Cloud::Google.getDomains(credentials)
|
239
252
|
my_org = MU::Cloud::Google.getOrg(credentials)
|
240
253
|
|
254
|
+
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
255
|
+
if !ignoremaster and MU.mu_public_ip
|
256
|
+
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
257
|
+
end
|
258
|
+
MU.log "Placeholder: Google User artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
|
259
|
+
|
241
260
|
# We don't have a good way of tagging directory users, so we rely
|
242
261
|
# on the known parameter, which is pulled from deployment metadata
|
243
262
|
if flags['known'] and my_org
|
@@ -319,7 +338,7 @@ module MU
|
|
319
338
|
MU::Cloud::Google.iam(credentials: args[:credentials]).list_project_service_accounts(
|
320
339
|
"projects/"+args[:project]
|
321
340
|
)
|
322
|
-
rescue ::Google::Apis::ClientError
|
341
|
+
rescue ::Google::Apis::ClientError
|
323
342
|
MU.log "Do not have permissions to retrieve service accounts for project #{args[:project]}", MU::WARN
|
324
343
|
end
|
325
344
|
|
@@ -382,7 +401,7 @@ module MU
|
|
382
401
|
# Reverse-map our cloud description into a runnable config hash.
|
383
402
|
# We assume that any values we have in +@config+ are placeholders, and
|
384
403
|
# calculate our own accordingly based on what's live in the cloud.
|
385
|
-
def toKitten(
|
404
|
+
def toKitten(**_args)
|
386
405
|
if MU::Cloud::Google::User.cannedServiceAcctName?(@cloud_id)
|
387
406
|
return nil
|
388
407
|
end
|
@@ -441,9 +460,9 @@ module MU
|
|
441
460
|
end
|
442
461
|
|
443
462
|
# Cloud-specific configuration properties.
|
444
|
-
# @param
|
463
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
445
464
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
446
|
-
def self.schema(
|
465
|
+
def self.schema(_config)
|
447
466
|
toplevel_required = []
|
448
467
|
schema = {
|
449
468
|
"name" => {
|
@@ -517,9 +536,9 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
517
536
|
|
518
537
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::users}, bare and unvalidated.
|
519
538
|
# @param user [Hash]: The resource to process and validate
|
520
|
-
# @param
|
539
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
521
540
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
522
|
-
def self.validateConfig(user,
|
541
|
+
def self.validateConfig(user, _configurator)
|
523
542
|
ok = true
|
524
543
|
|
525
544
|
my_domains = MU::Cloud::Google.getDomains(user['credentials'])
|
@@ -621,7 +640,42 @@ If we are binding (rather than creating) a user and no roles are specified, we w
|
|
621
640
|
ok
|
622
641
|
end
|
623
642
|
|
624
|
-
|
643
|
+
# Create and inject a service account on behalf of the parent resource.
|
644
|
+
# Return the modified parent configuration hash with references to the
|
645
|
+
# new addition.
|
646
|
+
# @param parent [Hash]
|
647
|
+
# @param configurator [MU::Config]
|
648
|
+
# @return [Hash]
|
649
|
+
def self.genericServiceAccount(parent, configurator)
|
650
|
+
user = {
|
651
|
+
"name" => parent['name'],
|
652
|
+
"cloud" => "Google",
|
653
|
+
"project" => parent["project"],
|
654
|
+
"credentials" => parent["credentials"],
|
655
|
+
"type" => "service"
|
656
|
+
}
|
657
|
+
if user["name"].length < 6
|
658
|
+
user["name"] += Password.pronounceable(6)
|
659
|
+
end
|
660
|
+
if parent['roles']
|
661
|
+
user['roles'] = parent['roles'].dup
|
662
|
+
end
|
663
|
+
configurator.insertKitten(user, "users", true)
|
664
|
+
parent['dependencies'] ||= []
|
665
|
+
parent['service_account'] = MU::Config::Ref.get(
|
666
|
+
type: "users",
|
667
|
+
cloud: "Google",
|
668
|
+
name: user["name"],
|
669
|
+
project: user["project"],
|
670
|
+
credentials: user["credentials"]
|
671
|
+
)
|
672
|
+
parent['dependencies'] << {
|
673
|
+
"type" => "user",
|
674
|
+
"name" => user["name"]
|
675
|
+
}
|
676
|
+
|
677
|
+
parent
|
678
|
+
end
|
625
679
|
|
626
680
|
end
|
627
681
|
end
|
@@ -36,12 +36,10 @@ module MU
|
|
36
36
|
|
37
37
|
# Called automatically by {MU::Deploy#createResources}
|
38
38
|
def create
|
39
|
-
|
40
39
|
networkobj = MU::Cloud::Google.compute(:Network).new(
|
41
40
|
name: MU::Cloud::Google.nameStr(@mu_name),
|
42
41
|
description: @deploy.deploy_id,
|
43
42
|
auto_create_subnetworks: false
|
44
|
-
# i_pv4_range: @config['ip_block']
|
45
43
|
)
|
46
44
|
MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@project_id}", details: networkobj
|
47
45
|
|
@@ -58,7 +56,7 @@ module MU
|
|
58
56
|
subnet_name = @config['name']+subnet['name']
|
59
57
|
|
60
58
|
subnet_mu_name = @config['scrub_mu_isms'] ? @cloud_id+subnet_name.downcase : MU::Cloud::Google.nameStr(@deploy.getResourceName(subnet_name, max_length: 61))
|
61
|
-
MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id}", details: subnet
|
59
|
+
MU.log "Creating subnetwork #{subnet_mu_name} (#{subnet['ip_block']}) in project #{@project_id} region #{subnet['availability_zone']}", details: subnet
|
62
60
|
subnetobj = MU::Cloud::Google.compute(:Subnetwork).new(
|
63
61
|
name: subnet_mu_name,
|
64
62
|
description: @deploy.deploy_id,
|
@@ -72,9 +70,17 @@ module MU
|
|
72
70
|
subnetdesc = nil
|
73
71
|
begin
|
74
72
|
subnetdesc = MU::Cloud::Google.compute(credentials: @config['credentials']).get_subnetwork(@project_id, subnet['availability_zone'], subnet_mu_name)
|
73
|
+
if !subnetdesc.nil?
|
74
|
+
subnet_cfg = {}
|
75
|
+
subnet_cfg["ip_block"] = subnet['ip_block']
|
76
|
+
subnet_cfg["name"] = subnet_name
|
77
|
+
subnet_cfg['mu_name'] = subnet_mu_name
|
78
|
+
subnet_cfg["cloud_id"] = subnetdesc.self_link.gsub(/.*?\/([^\/]+)$/, '\1')
|
79
|
+
subnet_cfg['az'] = subnet['availability_zone']
|
80
|
+
@subnets << MU::Cloud::Google::VPC::Subnet.new(self, subnet_cfg, precache_description: false)
|
81
|
+
end
|
75
82
|
sleep 1
|
76
83
|
end while subnetdesc.nil?
|
77
|
-
|
78
84
|
}
|
79
85
|
}
|
80
86
|
subnetthreads.each do |t|
|
@@ -82,7 +88,6 @@ module MU
|
|
82
88
|
end
|
83
89
|
end
|
84
90
|
|
85
|
-
route_table_ids = []
|
86
91
|
if !@config['route_tables'].nil?
|
87
92
|
@config['route_tables'].each { |rtb|
|
88
93
|
rtb['routes'].each { |route|
|
@@ -120,8 +125,8 @@ module MU
|
|
120
125
|
|
121
126
|
# Describe this VPC from the cloud platform's perspective
|
122
127
|
# @return [Google::Apis::Core::Hashable]
|
123
|
-
def cloud_desc
|
124
|
-
if @cloud_desc_cache
|
128
|
+
def cloud_desc(use_cache: true)
|
129
|
+
if @cloud_desc_cache and use_cache
|
125
130
|
return @cloud_desc_cache
|
126
131
|
end
|
127
132
|
|
@@ -230,8 +235,8 @@ end
|
|
230
235
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
231
236
|
# @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
|
232
237
|
def self.find(**args)
|
233
|
-
args
|
234
|
-
|
238
|
+
args = MU::Cloud::Google.findLocationArgs(args)
|
239
|
+
|
235
240
|
resp = {}
|
236
241
|
if args[:cloud_id] and args[:project]
|
237
242
|
begin
|
@@ -240,7 +245,7 @@ end
|
|
240
245
|
args[:cloud_id].to_s.sub(/^.*?\/([^\/]+)$/, '\1')
|
241
246
|
)
|
242
247
|
resp[args[:cloud_id]] = vpc if !vpc.nil?
|
243
|
-
rescue ::Google::Apis::ClientError
|
248
|
+
rescue ::Google::Apis::ClientError
|
244
249
|
MU.log "VPC #{args[:cloud_id]} in project #{args[:project]} does not exist, or I do not have permission to view it", MU::WARN
|
245
250
|
end
|
246
251
|
else # XXX other criteria
|
@@ -477,9 +482,8 @@ end
|
|
477
482
|
# directly at child nodes in peered VPCs, the public internet, and the
|
478
483
|
# like.
|
479
484
|
# @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
|
480
|
-
# @param region [String]: The cloud provider region of the target subnet.
|
481
485
|
# @return [Boolean]
|
482
|
-
def self.haveRouteToInstance?(target_instance,
|
486
|
+
def self.haveRouteToInstance?(target_instance, credentials: nil)
|
483
487
|
project ||= MU::Cloud::Google.defaultProject(credentials)
|
484
488
|
return false if MU.myCloud != "Google"
|
485
489
|
# XXX see if we reside in the same Network and overlap subnets
|
@@ -536,11 +540,15 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
536
540
|
# Remove all VPC resources associated with the currently loaded deployment.
|
537
541
|
# @param noop [Boolean]: If true, will only print what would be done
|
538
542
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
539
|
-
# @param region [String]: The cloud provider region
|
540
543
|
# @return [void]
|
541
|
-
def self.cleanup(noop: false, ignoremaster: false,
|
544
|
+
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
542
545
|
flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
|
543
546
|
return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
|
547
|
+
filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
|
548
|
+
if !ignoremaster and MU.mu_public_ip
|
549
|
+
filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
|
550
|
+
end
|
551
|
+
MU.log "Placeholder: Google VPC artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
|
544
552
|
|
545
553
|
purge_subnets(noop, project: flags['project'], credentials: credentials)
|
546
554
|
["route", "network"].each { |type|
|
@@ -562,7 +570,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
562
570
|
if e.message.match(/Failed to delete network (.+)/)
|
563
571
|
network_name = Regexp.last_match[1]
|
564
572
|
fwrules = MU::Cloud::Google::FirewallRule.find(project: flags['project'], credentials: credentials)
|
565
|
-
fwrules.reject! { |
|
573
|
+
fwrules.reject! { |_name, desc|
|
566
574
|
!desc.network.match(/.*?\/#{Regexp.quote(network_name)}$/)
|
567
575
|
}
|
568
576
|
fwrules.keys.each { |name|
|
@@ -586,7 +594,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
586
594
|
# We assume that any values we have in +@config+ are placeholders, and
|
587
595
|
# calculate our own accordingly based on what's live in the cloud.
|
588
596
|
# XXX add flag to return the diff between @config and live cloud
|
589
|
-
def toKitten(
|
597
|
+
def toKitten(**_args)
|
590
598
|
return nil if cloud_desc.name == "default" # parent project builds these
|
591
599
|
bok = {
|
592
600
|
"cloud" => "Google",
|
@@ -595,8 +603,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
595
603
|
}
|
596
604
|
MU::Cloud::Google.listRegions.size
|
597
605
|
|
598
|
-
|
599
|
-
schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
|
606
|
+
_schema, valid = MU::Config.loadResourceSchema("VPC", cloud: "Google")
|
600
607
|
return [nil, nil] if !valid
|
601
608
|
# pp schema
|
602
609
|
# MU.log "++++++++++++++++++++++++++++++++"
|
@@ -609,6 +616,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
609
616
|
bok['subnets'] = []
|
610
617
|
regions_seen = []
|
611
618
|
names_seen = []
|
619
|
+
@subnets.reject! { |x| x.cloud_desc.nil? }
|
612
620
|
@subnets.map { |x| x.cloud_desc }.each { |s|
|
613
621
|
subnet_name = s.name.dup
|
614
622
|
names_seen << s.name.dup
|
@@ -630,7 +638,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
630
638
|
end
|
631
639
|
end
|
632
640
|
|
633
|
-
peer_names = []
|
634
641
|
if cloud_desc.peerings and cloud_desc.peerings.size > 0
|
635
642
|
bok['peers'] = []
|
636
643
|
cloud_desc.peerings.each { |peer|
|
@@ -688,9 +695,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
688
695
|
end
|
689
696
|
|
690
697
|
# Cloud-specific configuration properties.
|
691
|
-
# @param
|
698
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
692
699
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
693
|
-
def self.schema(
|
700
|
+
def self.schema(_config = nil)
|
694
701
|
toplevel_required = []
|
695
702
|
schema = {
|
696
703
|
"regions" => {
|
@@ -736,7 +743,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
736
743
|
|
737
744
|
# see if one of this thing's siblings declared a subnet_pref we can
|
738
745
|
# use to guess which one we should marry ourselves to
|
739
|
-
configurator.kittens.
|
746
|
+
configurator.kittens.values.each { |siblings|
|
740
747
|
siblings.each { |sibling|
|
741
748
|
next if !sibling['dependencies']
|
742
749
|
sibling['dependencies'].each { |dep|
|
@@ -900,7 +907,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
900
907
|
"destination_network"=>"0.0.0.0/0"
|
901
908
|
}
|
902
909
|
end
|
903
|
-
|
910
|
+
|
904
911
|
# You know what, let's just guarantee that we'll have a route from
|
905
912
|
# this master, always
|
906
913
|
# XXX this confuses machines that don't have public IPs
|
@@ -945,7 +952,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
945
952
|
|
946
953
|
private
|
947
954
|
|
948
|
-
def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project,
|
955
|
+
def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, _publicroute = true, credentials: nil)
|
949
956
|
private_acl = {
|
950
957
|
"name" => vpc_name+"-rt",
|
951
958
|
"cloud" => "Google",
|
@@ -973,6 +980,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
973
980
|
# end
|
974
981
|
configurator.insertKitten(private_acl, "firewall_rules", true)
|
975
982
|
end
|
983
|
+
private_class_method :genStandardSubnetACLs
|
976
984
|
|
977
985
|
# Helper method for manufacturing routes. Expect to be called from
|
978
986
|
# {MU::Cloud::Google::VPC#create} or {MU::Cloud::Google::VPC#groom}.
|
@@ -1039,7 +1047,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1039
1047
|
rescue ::Google::Apis::ClientError, MU::MuError => e
|
1040
1048
|
if e.message.match(/notFound/)
|
1041
1049
|
MU.log "Creating route #{routename} in project #{@project_id}", details: routeobj
|
1042
|
-
|
1050
|
+
MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@project_id, routeobj)
|
1043
1051
|
else
|
1044
1052
|
# TODO can't update GCP routes, would have to delete and re-create
|
1045
1053
|
end
|
@@ -1047,44 +1055,12 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1047
1055
|
end
|
1048
1056
|
end
|
1049
1057
|
|
1050
|
-
|
1051
|
-
# Remove all network gateways associated with the currently loaded deployment.
|
1052
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
1053
|
-
# @param region [String]: The cloud provider region
|
1054
|
-
# @return [void]
|
1055
|
-
def self.purge_gateways(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
|
1056
|
-
end
|
1057
|
-
|
1058
|
-
# Remove all NAT gateways associated with the VPC of the currently loaded deployment.
|
1059
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
1060
|
-
# @param vpc_id [String]: The cloud provider's unique VPC identifier
|
1061
|
-
# @param region [String]: The cloud provider region
|
1062
|
-
# @return [void]
|
1063
|
-
def self.purge_nat_gateways(noop = false, vpc_id: nil, region: MU.curRegion)
|
1064
|
-
end
|
1065
|
-
|
1066
|
-
# Remove all VPC endpoints associated with the VPC of the currently loaded deployment.
|
1067
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
1068
|
-
# @param vpc_id [String]: The cloud provider's unique VPC identifier
|
1069
|
-
# @param region [String]: The cloud provider region
|
1070
|
-
# @return [void]
|
1071
|
-
def self.purge_endpoints(noop = false, vpc_id: nil, region: MU.curRegion)
|
1072
|
-
end
|
1073
|
-
|
1074
|
-
# Remove all network interfaces associated with the currently loaded deployment.
|
1075
|
-
# @param noop [Boolean]: If true, will only print what would be done
|
1076
|
-
# @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
|
1077
|
-
# @param region [String]: The cloud provider region
|
1078
|
-
# @return [void]
|
1079
|
-
def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
|
1080
|
-
end
|
1081
|
-
|
1082
1058
|
# Remove all subnets associated with the currently loaded deployment.
|
1083
1059
|
# @param noop [Boolean]: If true, will only print what would be done
|
1084
|
-
# @param
|
1060
|
+
# @param _tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
|
1085
1061
|
# @param regions [Array<String>]: The cloud provider regions to check
|
1086
1062
|
# @return [void]
|
1087
|
-
def self.purge_subnets(noop = false,
|
1063
|
+
def self.purge_subnets(noop = false, _tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
|
1088
1064
|
project ||= MU::Cloud::Google.defaultProject(credentials)
|
1089
1065
|
parent_thread_id = Thread.current.object_id
|
1090
1066
|
regionthreads = []
|
@@ -1098,7 +1074,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1098
1074
|
r,
|
1099
1075
|
noop
|
1100
1076
|
)
|
1101
|
-
rescue MU::Cloud::MuDefunctHabitat
|
1077
|
+
rescue MU::Cloud::MuDefunctHabitat
|
1102
1078
|
Thread.exit
|
1103
1079
|
end
|
1104
1080
|
}
|
@@ -1107,8 +1083,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1107
1083
|
t.join
|
1108
1084
|
end
|
1109
1085
|
end
|
1110
|
-
|
1111
|
-
protected
|
1086
|
+
private_class_method :purge_subnets
|
1112
1087
|
|
1113
1088
|
# Subnets are almost a first-class resource. So let's kinda sorta treat
|
1114
1089
|
# them like one. This should only be invoked on objects that already
|
@@ -1116,7 +1091,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1116
1091
|
class Subnet < MU::Cloud::Google::VPC
|
1117
1092
|
|
1118
1093
|
attr_reader :cloud_id
|
1119
|
-
attr_reader :url
|
1120
1094
|
attr_reader :ip_block
|
1121
1095
|
attr_reader :mu_name
|
1122
1096
|
attr_reader :name
|
@@ -1149,10 +1123,29 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
|
|
1149
1123
|
MU.structToHash(cloud_desc)
|
1150
1124
|
end
|
1151
1125
|
|
1126
|
+
# Return the +self_link+ to this subnet
|
1127
|
+
def url
|
1128
|
+
cloud_desc if !@url
|
1129
|
+
@url
|
1130
|
+
end
|
1131
|
+
|
1132
|
+
@cloud_desc_cache = nil
|
1152
1133
|
# Describe this VPC Subnet from the cloud platform's perspective
|
1153
1134
|
# @return [Google::Apis::Core::Hashable]
|
1154
|
-
def cloud_desc
|
1155
|
-
@cloud_desc_cache
|
1135
|
+
def cloud_desc(use_cache: true)
|
1136
|
+
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
1137
|
+
|
1138
|
+
begin
|
1139
|
+
@cloud_desc_cache = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).get_subnetwork(@parent.habitat_id, @az, @cloud_id)
|
1140
|
+
rescue ::Google::Apis::ClientError => e
|
1141
|
+
if e.message.match(/notFound: /)
|
1142
|
+
MU.log "Failed to fetch cloud description for Google subnet #{@cloud_id}", MU::WARN, details: { "project" => @parent.habitat_id, "region" => @az, "name" => @cloud_id }
|
1143
|
+
return nil
|
1144
|
+
else
|
1145
|
+
raise e
|
1146
|
+
end
|
1147
|
+
end
|
1148
|
+
@url ||= @cloud_desc_cache.self_link
|
1156
1149
|
@cloud_desc_cache
|
1157
1150
|
end
|
1158
1151
|
|
data/modules/mu/config.rb
CHANGED
@@ -18,6 +18,10 @@ require 'erb'
|
|
18
18
|
require 'pp'
|
19
19
|
require 'json-schema'
|
20
20
|
require 'net/http'
|
21
|
+
require 'mu/config/schema_helpers'
|
22
|
+
require 'mu/config/tail'
|
23
|
+
require 'mu/config/ref'
|
24
|
+
require 'mu/config/doc_helpers'
|
21
25
|
autoload :GraphViz, 'graphviz'
|
22
26
|
autoload :ChronicDuration, 'chronic_duration'
|
23
27
|
|
@@ -35,35 +39,6 @@ module MU
|
|
35
39
|
class DeployParamError < MuError
|
36
40
|
end
|
37
41
|
|
38
|
-
# The default cloud provider for new resources. Must exist in MU.supportedClouds
|
39
|
-
# return [String]
|
40
|
-
def self.defaultCloud
|
41
|
-
configured = {}
|
42
|
-
MU::Cloud.supportedClouds.each { |cloud|
|
43
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
|
44
|
-
if $MU_CFG[cloud.downcase] and !$MU_CFG[cloud.downcase].empty?
|
45
|
-
configured[cloud] = $MU_CFG[cloud.downcase].size
|
46
|
-
configured[cloud] += 0.5 if cloudclass.hosted? # tiebreaker
|
47
|
-
end
|
48
|
-
}
|
49
|
-
if configured.size > 0
|
50
|
-
return configured.keys.sort { |a, b|
|
51
|
-
configured[b] <=> configured[a]
|
52
|
-
}.first
|
53
|
-
else
|
54
|
-
MU::Cloud.supportedClouds.each { |cloud|
|
55
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
|
56
|
-
return cloud if cloudclass.hosted?
|
57
|
-
}
|
58
|
-
return MU::Cloud.supportedClouds.first
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
# The default grooming agent for new resources. Must exist in MU.supportedGroomers.
|
63
|
-
def self.defaultGroomer
|
64
|
-
MU.localOnly ? "Ansible" : "Chef"
|
65
|
-
end
|
66
|
-
|
67
42
|
attr_accessor :nat_routes
|
68
43
|
attr_reader :skipinitialupdates
|
69
44
|
|
@@ -75,120 +50,6 @@ module MU
|
|
75
50
|
@@config_path
|
76
51
|
end
|
77
52
|
|
78
|
-
# Accessor for our Basket of Kittens schema definition
|
79
|
-
def self.schema
|
80
|
-
@@schema
|
81
|
-
end
|
82
|
-
|
83
|
-
# Deep merge a configuration hash so we can meld different cloud providers'
|
84
|
-
# schemas together, while preserving documentation differences
|
85
|
-
def self.schemaMerge(orig, new, cloud)
|
86
|
-
if new.is_a?(Hash)
|
87
|
-
new.each_pair { |k, v|
|
88
|
-
if cloud and k == "description" and v.is_a?(String) and !v.match(/\b#{Regexp.quote(cloud.upcase)}\b/) and !v.empty?
|
89
|
-
new[k] = "+"+cloud.upcase+"+: "+v
|
90
|
-
end
|
91
|
-
if orig and orig.has_key?(k)
|
92
|
-
elsif orig
|
93
|
-
orig[k] = new[k]
|
94
|
-
else
|
95
|
-
orig = new
|
96
|
-
end
|
97
|
-
schemaMerge(orig[k], new[k], cloud)
|
98
|
-
}
|
99
|
-
elsif orig.is_a?(Array) and new
|
100
|
-
orig.concat(new)
|
101
|
-
orig.uniq!
|
102
|
-
elsif new.is_a?(String)
|
103
|
-
orig ||= ""
|
104
|
-
orig += "\n" if !orig.empty?
|
105
|
-
orig += "+#{cloud.upcase}+: "+new
|
106
|
-
else
|
107
|
-
# XXX I think this is a NOOP?
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
# Accessor for our Basket of Kittens schema definition, with various
|
112
|
-
# cloud-specific details merged so we can generate documentation for them.
|
113
|
-
def self.docSchema
|
114
|
-
docschema = Marshal.load(Marshal.dump(@@schema))
|
115
|
-
only_children = {}
|
116
|
-
MU::Cloud.resource_types.each_pair { |classname, attrs|
|
117
|
-
MU::Cloud.supportedClouds.each { |cloud|
|
118
|
-
begin
|
119
|
-
require "mu/clouds/#{cloud.downcase}/#{attrs[:cfg_name]}"
|
120
|
-
rescue LoadError => e
|
121
|
-
next
|
122
|
-
end
|
123
|
-
res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname)
|
124
|
-
required, res_schema = res_class.schema(self)
|
125
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["description"] ||= ""
|
126
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["description"] += "\n#\n# `#{cloud}`: "+res_class.quality
|
127
|
-
res_schema.each { |key, cfg|
|
128
|
-
if !docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
|
129
|
-
only_children[attrs[:cfg_plural]] ||= {}
|
130
|
-
only_children[attrs[:cfg_plural]][key] ||= {}
|
131
|
-
only_children[attrs[:cfg_plural]][key][cloud] = cfg
|
132
|
-
end
|
133
|
-
}
|
134
|
-
}
|
135
|
-
}
|
136
|
-
|
137
|
-
# recursively chase down description fields in arrays and objects of our
|
138
|
-
# schema and prepend stuff to them for documentation
|
139
|
-
def self.prepend_descriptions(prefix, cfg)
|
140
|
-
cfg["prefix"] = prefix
|
141
|
-
if cfg["type"] == "array" and cfg["items"]
|
142
|
-
cfg["items"] = prepend_descriptions(prefix, cfg["items"])
|
143
|
-
elsif cfg["type"] == "object" and cfg["properties"]
|
144
|
-
cfg["properties"].each_pair { |key, subcfg|
|
145
|
-
cfg["properties"][key] = prepend_descriptions(prefix, cfg["properties"][key])
|
146
|
-
}
|
147
|
-
end
|
148
|
-
cfg
|
149
|
-
end
|
150
|
-
|
151
|
-
MU::Cloud.resource_types.each_pair { |classname, attrs|
|
152
|
-
MU::Cloud.supportedClouds.each { |cloud|
|
153
|
-
res_class = nil
|
154
|
-
begin
|
155
|
-
res_class = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(classname)
|
156
|
-
rescue MU::Cloud::MuCloudResourceNotImplemented
|
157
|
-
next
|
158
|
-
end
|
159
|
-
required, res_schema = res_class.schema(self)
|
160
|
-
next if required.size == 0 and res_schema.size == 0
|
161
|
-
res_schema.each { |key, cfg|
|
162
|
-
cfg["description"] ||= ""
|
163
|
-
if !cfg["description"].empty?
|
164
|
-
cfg["description"] = "\n# +"+cloud.upcase+"+: "+cfg["description"]
|
165
|
-
end
|
166
|
-
if docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
|
167
|
-
schemaMerge(docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key], cfg, cloud)
|
168
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] ||= ""
|
169
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["description"] += "\n"+(cfg["description"].match(/^#/) ? "" : "# ")+cfg["description"]
|
170
|
-
MU.log "Munging #{cloud}-specific #{classname.to_s} schema into BasketofKittens => #{attrs[:cfg_plural]} => #{key}", MU::DEBUG, details: docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]
|
171
|
-
else
|
172
|
-
if only_children[attrs[:cfg_plural]][key]
|
173
|
-
prefix = only_children[attrs[:cfg_plural]][key].keys.map{ |x| x.upcase }.join(" & ")+" ONLY"
|
174
|
-
cfg["description"].gsub!(/^\n#/, '') # so we don't leave the description blank in the "optional parameters" section
|
175
|
-
cfg = prepend_descriptions(prefix, cfg)
|
176
|
-
end
|
177
|
-
|
178
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key] = cfg
|
179
|
-
end
|
180
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"] = {}
|
181
|
-
docschema["properties"][attrs[:cfg_plural]]["items"]["properties"][key]["clouds"][cloud] = cfg
|
182
|
-
}
|
183
|
-
|
184
|
-
docschema['required'].concat(required)
|
185
|
-
docschema['required'].uniq!
|
186
|
-
}
|
187
|
-
}
|
188
|
-
|
189
|
-
docschema
|
190
|
-
end
|
191
|
-
|
192
53
|
attr_reader :config
|
193
54
|
|
194
55
|
@@parameters = {}
|
@@ -243,495 +104,16 @@ module MU
|
|
243
104
|
MU::Config.manxify(Marshal.load(Marshal.dump(MU.structToHash(config.dup))), remove_runtime_keys: true)
|
244
105
|
end
|
245
106
|
|
246
|
-
# A wrapper class for resources to refer to other resources, whether they
|
247
|
-
# be a sibling object in the current deploy, an object in another deploy,
|
248
|
-
# or a plain cloud id from outside of Mu.
|
249
|
-
class Ref
|
250
|
-
attr_reader :name
|
251
|
-
attr_reader :type
|
252
|
-
attr_reader :cloud
|
253
|
-
attr_reader :deploy_id
|
254
|
-
attr_reader :region
|
255
|
-
attr_reader :credentials
|
256
|
-
attr_reader :habitat
|
257
|
-
attr_reader :mommacat
|
258
|
-
attr_reader :tag_key
|
259
|
-
attr_reader :tag_value
|
260
|
-
attr_reader :obj
|
261
|
-
|
262
|
-
@@refs = []
|
263
|
-
@@ref_semaphore = Mutex.new
|
264
|
-
|
265
|
-
# Little bit of a factory pattern... given a hash of options for a {MU::Config::Ref} objects, first see if we have an existing one that matches our more immutable attributes (+cloud+, +id+, etc). If we do, return that. If we do not, create one, add that to our inventory, and return that instead.
|
266
|
-
# @param cfg [Hash]:
|
267
|
-
# @return [MU::Config::Ref]
|
268
|
-
def self.get(cfg)
|
269
|
-
return cfg if cfg.is_a?(MU::Config::Ref)
|
270
|
-
checkfields = cfg.keys.map { |k| k.to_sym }
|
271
|
-
required = [:id, :type]
|
272
|
-
|
273
|
-
@@ref_semaphore.synchronize {
|
274
|
-
match = nil
|
275
|
-
@@refs.each { |ref|
|
276
|
-
saw_mismatch = false
|
277
|
-
saw_match = false
|
278
|
-
needed_values = []
|
279
|
-
checkfields.each { |field|
|
280
|
-
next if !cfg[field]
|
281
|
-
ext_value = ref.instance_variable_get("@#{field.to_s}".to_sym)
|
282
|
-
if !ext_value
|
283
|
-
needed_values << field
|
284
|
-
next
|
285
|
-
end
|
286
|
-
if cfg[field] != ext_value
|
287
|
-
saw_mismatch = true
|
288
|
-
elsif required.include?(field) and cfg[field] == ext_value
|
289
|
-
saw_match = true
|
290
|
-
end
|
291
|
-
}
|
292
|
-
if saw_match and !saw_mismatch
|
293
|
-
# populate empty fields we got from this request
|
294
|
-
if needed_values.size > 0
|
295
|
-
newref = ref.dup
|
296
|
-
needed_values.each { |field|
|
297
|
-
newref.instance_variable_set("@#{field.to_s}".to_sym, cfg[field])
|
298
|
-
if !newref.respond_to?(field)
|
299
|
-
newref.singleton_class.instance_eval { attr_reader field.to_sym }
|
300
|
-
end
|
301
|
-
}
|
302
|
-
@@refs << newref
|
303
|
-
return newref
|
304
|
-
else
|
305
|
-
return ref
|
306
|
-
end
|
307
|
-
end
|
308
|
-
}
|
309
|
-
|
310
|
-
}
|
311
|
-
|
312
|
-
# if we get here, there was no match
|
313
|
-
newref = MU::Config::Ref.new(cfg)
|
314
|
-
@@ref_semaphore.synchronize {
|
315
|
-
@@refs << newref
|
316
|
-
return newref
|
317
|
-
}
|
318
|
-
end
|
319
|
-
|
320
|
-
# A way of dynamically defining +attr_reader+ without leaking memory
|
321
|
-
def self.define_reader(name)
|
322
|
-
define_method(name) {
|
323
|
-
instance_variable_get("@#{name.to_s}")
|
324
|
-
}
|
325
|
-
end
|
326
|
-
|
327
|
-
# @param cfg [Hash]: A Basket of Kittens configuration hash containing
|
328
|
-
# lookup information for a cloud object
|
329
|
-
def initialize(cfg)
|
330
|
-
cfg.keys.each { |field|
|
331
|
-
next if field == "tag"
|
332
|
-
if !cfg[field].nil?
|
333
|
-
self.instance_variable_set("@#{field}".to_sym, cfg[field])
|
334
|
-
elsif !cfg[field.to_sym].nil?
|
335
|
-
self.instance_variable_set("@#{field.to_s}".to_sym, cfg[field.to_sym])
|
336
|
-
end
|
337
|
-
MU::Config::Ref.define_reader(field)
|
338
|
-
}
|
339
|
-
if cfg['tag'] and cfg['tag']['key'] and
|
340
|
-
!cfg['tag']['key'].empty? and cfg['tag']['value']
|
341
|
-
@tag_key = cfg['tag']['key']
|
342
|
-
@tag_value = cfg['tag']['value']
|
343
|
-
end
|
344
|
-
|
345
|
-
if @deploy_id and !@mommacat
|
346
|
-
@mommacat = MU::MommaCat.getLitter(@deploy_id, set_context_to_me: false)
|
347
|
-
elsif @mommacat and !@deploy_id
|
348
|
-
@deploy_id = @mommacat.deploy_id
|
349
|
-
end
|
350
|
-
|
351
|
-
kitten(shallow: true) if @mommacat # try to populate the actual cloud object for this
|
352
|
-
end
|
353
|
-
|
354
|
-
# Comparison operator
|
355
|
-
def <=>(other)
|
356
|
-
return 1 if other.nil?
|
357
|
-
self.to_s <=> other.to_s
|
358
|
-
end
|
359
|
-
|
360
|
-
# Base configuration schema for declared kittens referencing other cloud objects. This is essentially a set of filters that we're going to pass to {MU::MommaCat.findStray}.
|
361
|
-
# @param aliases [Array<Hash>]: Key => value mappings to set backwards-compatibility aliases for attributes, such as the ubiquitous +vpc_id+ (+vpc_id+ => +id+).
|
362
|
-
# @return [Hash]
|
363
|
-
def self.schema(aliases = [], type: nil, parent_obj: nil, desc: nil, omit_fields: [])
|
364
|
-
parent_obj ||= caller[1].gsub(/.*?\/([^\.\/]+)\.rb:.*/, '\1')
|
365
|
-
desc ||= "Reference a #{type ? "'#{type}' resource" : "resource" } from this #{parent_obj ? "'#{parent_obj}'" : "" } resource"
|
366
|
-
schema = {
|
367
|
-
"type" => "object",
|
368
|
-
"#MU_REFERENCE" => true,
|
369
|
-
"minProperties" => 1,
|
370
|
-
"description" => desc,
|
371
|
-
"properties" => {
|
372
|
-
"id" => {
|
373
|
-
"type" => "string",
|
374
|
-
"description" => "Cloud identifier of a resource we want to reference, typically used when leveraging resources not managed by MU"
|
375
|
-
},
|
376
|
-
"name" => {
|
377
|
-
"type" => "string",
|
378
|
-
"description" => "The short (internal Mu) name of a resource we're attempting to reference. Typically used when referring to a sibling resource elsewhere in the same deploy, or in another known Mu deploy in conjunction with +deploy_id+."
|
379
|
-
},
|
380
|
-
"type" => {
|
381
|
-
"type" => "string",
|
382
|
-
"description" => "The resource type we're attempting to reference.",
|
383
|
-
"enum" => MU::Cloud.resource_types.values.map { |t| t[:cfg_plural] }
|
384
|
-
},
|
385
|
-
"deploy_id" => {
|
386
|
-
"type" => "string",
|
387
|
-
"description" => "Our target resource should be found in this Mu deploy."
|
388
|
-
},
|
389
|
-
"credentials" => MU::Config.credentials_primitive,
|
390
|
-
"region" => MU::Config.region_primitive,
|
391
|
-
"cloud" => MU::Config.cloud_primitive,
|
392
|
-
"tag" => {
|
393
|
-
"type" => "object",
|
394
|
-
"description" => "If the target resource supports tagging and our resource implementations +find+ method supports it, we can attempt to locate it by tag.",
|
395
|
-
"properties" => {
|
396
|
-
"key" => {
|
397
|
-
"type" => "string",
|
398
|
-
"description" => "The tag or label key to search against"
|
399
|
-
},
|
400
|
-
"value" => {
|
401
|
-
"type" => "string",
|
402
|
-
"description" => "The tag or label value to match"
|
403
|
-
}
|
404
|
-
}
|
405
|
-
}
|
406
|
-
}
|
407
|
-
}
|
408
|
-
if !["folders", "habitats"].include?(type)
|
409
|
-
schema["properties"]["habitat"] = MU::Config::Habitat.reference
|
410
|
-
end
|
411
|
-
|
412
|
-
if omit_fields
|
413
|
-
omit_fields.each { |f|
|
414
|
-
schema["properties"].delete(f)
|
415
|
-
}
|
416
|
-
end
|
417
|
-
|
418
|
-
if !type.nil?
|
419
|
-
schema["required"] = ["type"]
|
420
|
-
schema["properties"]["type"]["default"] = type
|
421
|
-
schema["properties"]["type"]["enum"] = [type]
|
422
|
-
end
|
423
|
-
|
424
|
-
aliases.each { |a|
|
425
|
-
a.each_pair { |k, v|
|
426
|
-
if schema["properties"][v]
|
427
|
-
schema["properties"][k] = schema["properties"][v].dup
|
428
|
-
schema["properties"][k]["description"] = "Alias for <tt>#{v}</tt>"
|
429
|
-
else
|
430
|
-
MU.log "Reference schema alias #{k} wants to alias #{v}, but no such attribute exists", MU::WARN, details: caller[4]
|
431
|
-
end
|
432
|
-
}
|
433
|
-
}
|
434
|
-
|
435
|
-
schema
|
436
|
-
end
|
437
|
-
|
438
|
-
# Decompose into a plain-jane {MU::Config::BasketOfKittens} hash fragment,
|
439
|
-
# of the sort that would have been used to declare this reference in the
|
440
|
-
# first place.
|
441
|
-
def to_h
|
442
|
-
me = { }
|
443
|
-
|
444
|
-
self.instance_variables.each { |var|
|
445
|
-
next if [:@obj, :@mommacat, :@tag_key, :@tag_value].include?(var)
|
446
|
-
val = self.instance_variable_get(var)
|
447
|
-
next if val.nil?
|
448
|
-
val = val.to_h if val.is_a?(MU::Config::Ref)
|
449
|
-
me[var.to_s.sub(/^@/, '')] = val
|
450
|
-
}
|
451
|
-
if @tag_key and !@tag_key.empty?
|
452
|
-
me['tag'] = {
|
453
|
-
'key' => @tag_key,
|
454
|
-
'value' => @tag_value
|
455
|
-
}
|
456
|
-
end
|
457
|
-
me
|
458
|
-
end
|
459
|
-
|
460
|
-
# Getter for the #{id} instance variable that attempts to populate it if
|
461
|
-
# it's not set.
|
462
|
-
# @return [String,nil]
|
463
|
-
def id
|
464
|
-
return @id if @id
|
465
|
-
kitten # if it's not defined, attempt to define it
|
466
|
-
@id
|
467
|
-
end
|
468
|
-
|
469
|
-
# Alias for {id}
|
470
|
-
# @return [String,nil]
|
471
|
-
def cloud_id
|
472
|
-
id
|
473
|
-
end
|
474
|
-
|
475
|
-
# Return a {MU::Cloud} object for this reference. This is only meant to be
|
476
|
-
# called in a live deploy, which is to say that if called during initial
|
477
|
-
# configuration parsing, results may be incorrect.
|
478
|
-
# @param mommacat [MU::MommaCat]: A deploy object which will be searched for the referenced resource if provided, before restoring to broader, less efficient searches.
|
479
|
-
def kitten(mommacat = @mommacat, shallow: false)
|
480
|
-
return nil if !@cloud or !@type
|
481
|
-
|
482
|
-
if @obj
|
483
|
-
@deploy_id ||= @obj.deploy_id
|
484
|
-
@id ||= @obj.cloud_id
|
485
|
-
@name ||= @obj.config['name']
|
486
|
-
return @obj
|
487
|
-
end
|
488
|
-
|
489
|
-
if mommacat
|
490
|
-
@obj = mommacat.findLitterMate(type: @type, name: @name, cloud_id: @id, credentials: @credentials, debug: false)
|
491
|
-
if @obj # initialize missing attributes, if we can
|
492
|
-
@id ||= @obj.cloud_id
|
493
|
-
@mommacat ||= mommacat
|
494
|
-
@obj.intoDeploy(@mommacat) # make real sure these are set
|
495
|
-
@deploy_id ||= mommacat.deploy_id
|
496
|
-
if !@name
|
497
|
-
if @obj.config and @obj.config['name']
|
498
|
-
@name = @obj.config['name']
|
499
|
-
elsif @obj.mu_name
|
500
|
-
if @type == "folders"
|
501
|
-
MU.log "would assign name '#{@obj.mu_name}' in ref to this folder if I were feeling aggressive", MU::WARN, details: self.to_h
|
502
|
-
end
|
503
|
-
# @name = @obj.mu_name
|
504
|
-
end
|
505
|
-
end
|
506
|
-
return @obj
|
507
|
-
else
|
508
|
-
# MU.log "Failed to find a live '#{@type.to_s}' object named #{@name}#{@id ? " (#{@id})" : "" }#{ @habitat ? " in habitat #{@habitat}" : "" }", MU::WARN, details: self
|
509
|
-
end
|
510
|
-
end
|
511
|
-
|
512
|
-
if !@obj and !(@cloud == "Google" and @id and @type == "users" and MU::Cloud::Google::User.cannedServiceAcctName?(@id)) and !shallow
|
513
|
-
|
514
|
-
begin
|
515
|
-
hab_arg = if @habitat.nil?
|
516
|
-
[nil]
|
517
|
-
elsif @habitat.is_a?(MU::Config::Ref)
|
518
|
-
[@habitat.id]
|
519
|
-
elsif @habitat.is_a?(Hash)
|
520
|
-
[@habitat["id"]]
|
521
|
-
else
|
522
|
-
[@habitat.to_s]
|
523
|
-
end
|
524
|
-
|
525
|
-
found = MU::MommaCat.findStray(
|
526
|
-
@cloud,
|
527
|
-
@type,
|
528
|
-
name: @name,
|
529
|
-
cloud_id: @id,
|
530
|
-
deploy_id: @deploy_id,
|
531
|
-
region: @region,
|
532
|
-
habitats: hab_arg,
|
533
|
-
credentials: @credentials,
|
534
|
-
dummy_ok: (["habitats", "folders", "users", "groups", "vpcs"].include?(@type))
|
535
|
-
)
|
536
|
-
@obj ||= found.first if found
|
537
|
-
rescue ThreadError => e
|
538
|
-
# Sometimes MommaCat calls us in a potential deadlock situation;
|
539
|
-
# don't be the cause of a fatal error if so, we don't need this
|
540
|
-
# object that badly.
|
541
|
-
raise e if !e.message.match(/recursive locking/)
|
542
|
-
rescue SystemExit => e
|
543
|
-
# XXX this is temporary, to cope with some debug stuff that's in findStray
|
544
|
-
# for the nonce
|
545
|
-
return
|
546
|
-
end
|
547
|
-
end
|
548
|
-
|
549
|
-
if @obj
|
550
|
-
@deploy_id ||= @obj.deploy_id
|
551
|
-
@id ||= @obj.cloud_id
|
552
|
-
@name ||= @obj.config['name']
|
553
|
-
end
|
554
|
-
|
555
|
-
@obj
|
556
|
-
end
|
557
|
-
|
558
|
-
end
|
559
|
-
|
560
|
-
# A wrapper for config leaves that came from ERB parameters instead of raw
|
561
|
-
# YAML or JSON. Will behave like a string for things that expect that
|
562
|
-
# sort of thing. Code that needs to know that this leaf was the result of
|
563
|
-
# a parameter will be able to tell by the object class being something
|
564
|
-
# other than a plain string, array, or hash.
|
565
|
-
class Tail
|
566
|
-
@value = nil
|
567
|
-
@name = nil
|
568
|
-
@prettyname = nil
|
569
|
-
@description = nil
|
570
|
-
@prefix = ""
|
571
|
-
@suffix = ""
|
572
|
-
@is_list_element = false
|
573
|
-
@pseudo = false
|
574
|
-
@runtimecode = nil
|
575
|
-
@valid_values = []
|
576
|
-
@index = 0
|
577
|
-
attr_reader :description
|
578
|
-
attr_reader :pseudo
|
579
|
-
attr_reader :index
|
580
|
-
attr_reader :value
|
581
|
-
attr_reader :runtimecode
|
582
|
-
attr_reader :valid_values
|
583
|
-
attr_reader :is_list_element
|
584
|
-
|
585
|
-
def initialize(name, value, prettyname = nil, cloudtype = "String", valid_values = [], description = "", is_list_element = false, prefix: "", suffix: "", pseudo: false, runtimecode: nil, index: 0)
|
586
|
-
@name = name
|
587
|
-
@bindings = {}
|
588
|
-
@value = value
|
589
|
-
@valid_values = valid_values
|
590
|
-
@pseudo = pseudo
|
591
|
-
@index = index
|
592
|
-
@runtimecode = runtimecode
|
593
|
-
@cloudtype = cloudtype
|
594
|
-
@is_list_element = is_list_element
|
595
|
-
@description ||=
|
596
|
-
if !description.nil?
|
597
|
-
description
|
598
|
-
else
|
599
|
-
""
|
600
|
-
end
|
601
|
-
@prettyname ||=
|
602
|
-
if !prettyname.nil?
|
603
|
-
prettyname
|
604
|
-
else
|
605
|
-
@name.capitalize.gsub(/[^a-z0-9]/i, "")
|
606
|
-
end
|
607
|
-
@prefix = prefix if !prefix.nil?
|
608
|
-
@suffix = suffix if !suffix.nil?
|
609
|
-
end
|
610
|
-
|
611
|
-
# Return the parameter name of this Tail
|
612
|
-
def getName
|
613
|
-
@name
|
614
|
-
end
|
615
|
-
# Return the platform-specific cloud type of this Tail
|
616
|
-
def getCloudType
|
617
|
-
@cloudtype
|
618
|
-
end
|
619
|
-
# Return the human-friendly name of this Tail
|
620
|
-
def getPrettyName
|
621
|
-
@prettyname
|
622
|
-
end
|
623
|
-
# Walk like a String
|
624
|
-
def to_s
|
625
|
-
@prefix.to_s+@value.to_s+@suffix.to_s
|
626
|
-
end
|
627
|
-
# Quack like a String
|
628
|
-
def to_str
|
629
|
-
to_s
|
630
|
-
end
|
631
|
-
# Upcase like a String
|
632
|
-
def upcase
|
633
|
-
to_s.upcase
|
634
|
-
end
|
635
|
-
# Downcase like a String
|
636
|
-
def downcase
|
637
|
-
to_s.downcase
|
638
|
-
end
|
639
|
-
# Check for emptiness like a String
|
640
|
-
def empty?
|
641
|
-
to_s.empty?
|
642
|
-
end
|
643
|
-
# Match like a String
|
644
|
-
def match(*args)
|
645
|
-
to_s.match(*args)
|
646
|
-
end
|
647
|
-
# Check for equality like a String
|
648
|
-
def ==(o)
|
649
|
-
(o.class == self.class or o.class == "String") && o.to_s == to_s
|
650
|
-
end
|
651
|
-
# Concatenate like a string
|
652
|
-
def +(o)
|
653
|
-
return to_s if o.nil?
|
654
|
-
to_s + o.to_s
|
655
|
-
end
|
656
|
-
# Perform global substitutions like a String
|
657
|
-
def gsub(*args)
|
658
|
-
to_s.gsub(*args)
|
659
|
-
end
|
660
|
-
end
|
661
|
-
|
662
|
-
# Wrapper method for creating a {MU::Config::Tail} object as a reference to
|
663
|
-
# a parameter that's valid in the loaded configuration.
|
664
|
-
# @param param [<String>]: The name of the parameter to which this should be tied.
|
665
|
-
# @param value [<String>]: The value of the parameter to return when asked
|
666
|
-
# @param prettyname [<String>]: A human-friendly parameter name to be used when generating CloudFormation templates and the like
|
667
|
-
# @param cloudtype [<String>]: A platform-specific identifier used by cloud layers to identify a parameter's type, e.g. AWS::EC2::VPC::Id
|
668
|
-
# @param valid_values [Array<String>]: A list of acceptable String values for the given parameter.
|
669
|
-
# @param description [<String>]: A long-form description of what the parameter does.
|
670
|
-
# @param list_of [<String>]: Indicates that the value should be treated as a member of a list (array) by the cloud layer.
|
671
|
-
# @param prefix [<String>]: A static String that should be prefixed to the stored value when queried
|
672
|
-
# @param suffix [<String>]: A static String that should be appended to the stored value when queried
|
673
|
-
# @param pseudo [<Boolean>]: This is a pseudo-parameter, automatically provided, and not available as user input.
|
674
|
-
# @param runtimecode [<String>]: Actual code to allow the cloud layer to interpret literally in its own idiom, e.g. '"Ref" : "AWS::StackName"' for CloudFormation
|
675
|
-
def getTail(param, value: nil, prettyname: nil, cloudtype: "String", valid_values: [], description: nil, list_of: nil, prefix: "", suffix: "", pseudo: false, runtimecode: nil)
|
676
|
-
if value.nil?
|
677
|
-
if @@parameters.nil? or !@@parameters.has_key?(param)
|
678
|
-
MU.log "Parameter '#{param}' (#{param.class.name}) referenced in config but not provided (#{caller[0]})", MU::DEBUG, details: @@parameters
|
679
|
-
return nil
|
680
|
-
# raise DeployParamError
|
681
|
-
else
|
682
|
-
value = @@parameters[param]
|
683
|
-
end
|
684
|
-
end
|
685
|
-
if !prettyname.nil?
|
686
|
-
prettyname.gsub!(/[^a-z0-9]/i, "") # comply with CloudFormation restrictions
|
687
|
-
end
|
688
|
-
if value.is_a?(MU::Config::Tail)
|
689
|
-
MU.log "Parameter #{param} is using a nested parameter as a value. This rarely works, depending on the target cloud. YMMV.", MU::WARN
|
690
|
-
tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
|
691
|
-
elsif !list_of.nil? or (@@tails.has_key?(param) and @@tails[param].is_a?(Array))
|
692
|
-
tail = []
|
693
|
-
count = 0
|
694
|
-
value.split(/\s*,\s*/).each { |subval|
|
695
|
-
if @@tails.has_key?(param) and !@@tails[param][count].nil?
|
696
|
-
subval = @@tails[param][count].values.first.to_s if subval.nil?
|
697
|
-
list_of = @@tails[param][count].values.first.getName if list_of.nil?
|
698
|
-
prettyname = @@tails[param][count].values.first.getPrettyName if prettyname.nil?
|
699
|
-
description = @@tails[param][count].values.first.description if description.nil?
|
700
|
-
valid_values = @@tails[param][count].values.first.valid_values if valid_values.nil? or valid_values.empty?
|
701
|
-
cloudtype = @@tails[param][count].values.first.getCloudType if @@tails[param][count].values.first.getCloudType != "String"
|
702
|
-
end
|
703
|
-
prettyname = param.capitalize if prettyname.nil?
|
704
|
-
tail << { list_of => MU::Config::Tail.new(list_of, subval, prettyname, cloudtype, valid_values, description, true, pseudo: pseudo, index: count) }
|
705
|
-
count = count + 1
|
706
|
-
}
|
707
|
-
else
|
708
|
-
if @@tails.has_key?(param)
|
709
|
-
pseudo = @@tails[param].pseudo
|
710
|
-
value = @@tails[param].to_s if value.nil?
|
711
|
-
prettyname = @@tails[param].getPrettyName if prettyname.nil?
|
712
|
-
description = @@tails[param].description if description.nil?
|
713
|
-
valid_values = @@tails[param].valid_values if valid_values.nil? or valid_values.empty?
|
714
|
-
cloudtype = @@tails[param].getCloudType if @@tails[param].getCloudType != "String"
|
715
|
-
end
|
716
|
-
tail = MU::Config::Tail.new(param, value, prettyname, cloudtype, valid_values, description, prefix: prefix, suffix: suffix, pseudo: pseudo, runtimecode: runtimecode)
|
717
|
-
end
|
718
|
-
|
719
|
-
if valid_values and valid_values.size > 0 and value
|
720
|
-
if !valid_values.include?(value)
|
721
|
-
raise DeployParamError, "Invalid parameter value '#{value}' supplied for '#{param}'"
|
722
|
-
end
|
723
|
-
end
|
724
|
-
@@tails[param] = tail
|
725
|
-
|
726
|
-
tail
|
727
|
-
end
|
728
|
-
|
729
107
|
# Load up our YAML or JSON and parse it through ERB, optionally substituting
|
730
108
|
# externally-supplied parameters.
|
731
109
|
def resolveConfig(path: @@config_path, param_pass: false, cloud: nil)
|
732
110
|
config = nil
|
733
111
|
@param_pass = param_pass
|
734
112
|
|
113
|
+
if cloud
|
114
|
+
MU.log "Exposing cloud variable to ERB with value of #{cloud}", MU::DEBUG
|
115
|
+
end
|
116
|
+
|
735
117
|
# Catch calls to missing variables in Basket of Kittens files when being
|
736
118
|
# parsed by ERB, and replace with placeholders for parameters. This
|
737
119
|
# method_missing is only defined innside {MU::Config.resolveConfig}
|
@@ -862,17 +244,15 @@ return
|
|
862
244
|
# @param cloud [String]: Sets a parameter named 'cloud', and insert it as the default cloud platform if not already declared
|
863
245
|
# @return [Hash]: The complete validated configuration for a deployment.
|
864
246
|
def initialize(path, skipinitialupdates = false, params: {}, updating: nil, default_credentials: nil, cloud: nil)
|
865
|
-
$myPublicIp
|
866
|
-
$myRoot
|
247
|
+
$myPublicIp ||= MU.mu_public_ip
|
248
|
+
$myRoot ||= MU.myRoot
|
867
249
|
$myRoot.freeze
|
868
250
|
|
869
|
-
$myAZ
|
251
|
+
$myAZ ||= MU.myAZ.freeze
|
870
252
|
$myAZ.freeze
|
871
|
-
$myRegion
|
253
|
+
$myRegion ||= MU.curRegion.freeze
|
872
254
|
$myRegion.freeze
|
873
255
|
|
874
|
-
$myAppName = nil
|
875
|
-
|
876
256
|
@kittens = {}
|
877
257
|
@kittencfg_semaphore = Mutex.new
|
878
258
|
@@config_path = path
|
@@ -920,7 +300,7 @@ return
|
|
920
300
|
# you can't specify parameters in an included file, because ERB is what's
|
921
301
|
# doing the including, and parameters need to already be resolved so that
|
922
302
|
# ERB can use them.
|
923
|
-
param_cfg,
|
303
|
+
param_cfg, _raw_erb_params_only = resolveConfig(path: @@config_path, param_pass: true, cloud: cloud)
|
924
304
|
if param_cfg.has_key?("parameters")
|
925
305
|
param_cfg["parameters"].each { |param|
|
926
306
|
if param.has_key?("default") and param["default"].nil?
|
@@ -976,7 +356,7 @@ return
|
|
976
356
|
$parameters = @@parameters.dup
|
977
357
|
$parameters.freeze
|
978
358
|
|
979
|
-
tmp_cfg,
|
359
|
+
tmp_cfg, _raw_erb = resolveConfig(path: @@config_path, cloud: cloud)
|
980
360
|
|
981
361
|
# Convert parameter entries that constitute whole config keys into
|
982
362
|
# {MU::Config::Tail} objects.
|
@@ -1022,8 +402,6 @@ return
|
|
1022
402
|
exit 1
|
1023
403
|
end
|
1024
404
|
|
1025
|
-
types = MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }
|
1026
|
-
|
1027
405
|
MU::Cloud.resource_types.values.map { |v| v[:cfg_plural] }.each { |type|
|
1028
406
|
if @config[type]
|
1029
407
|
@config[type].each { |k|
|
@@ -1041,164 +419,6 @@ return
|
|
1041
419
|
@config.freeze
|
1042
420
|
end
|
1043
421
|
|
1044
|
-
# Output the dependencies of this BoK stack as a directed acyclic graph.
|
1045
|
-
# Very useful for debugging.
|
1046
|
-
def visualizeDependencies
|
1047
|
-
# GraphViz won't like MU::Config::Tail, pare down to plain Strings
|
1048
|
-
config = MU::Config.stripConfig(@config)
|
1049
|
-
begin
|
1050
|
-
g = GraphViz.new(:G, :type => :digraph)
|
1051
|
-
# Generate a GraphViz node for each resource in this stack
|
1052
|
-
nodes = {}
|
1053
|
-
MU::Cloud.resource_types.each_pair { |classname, attrs|
|
1054
|
-
nodes[attrs[:cfg_name]] = {}
|
1055
|
-
if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
|
1056
|
-
config[attrs[:cfg_plural]].each { |resource|
|
1057
|
-
nodes[attrs[:cfg_name]][resource['name']] = g.add_nodes("#{classname}: #{resource['name']}")
|
1058
|
-
}
|
1059
|
-
end
|
1060
|
-
}
|
1061
|
-
# Now add edges corresponding to the dependencies they list
|
1062
|
-
MU::Cloud.resource_types.each_pair { |classname, attrs|
|
1063
|
-
if config.has_key?(attrs[:cfg_plural]) and config[attrs[:cfg_plural]]
|
1064
|
-
config[attrs[:cfg_plural]].each { |resource|
|
1065
|
-
if resource.has_key?("dependencies")
|
1066
|
-
me = nodes[attrs[:cfg_name]][resource['name']]
|
1067
|
-
resource["dependencies"].each { |dep|
|
1068
|
-
parent = nodes[dep['type']][dep['name']]
|
1069
|
-
g.add_edges(me, parent)
|
1070
|
-
}
|
1071
|
-
end
|
1072
|
-
}
|
1073
|
-
end
|
1074
|
-
}
|
1075
|
-
# Spew some output?
|
1076
|
-
MU.log "Emitting dependency graph as /tmp/#{config['appname']}.jpg", MU::NOTICE
|
1077
|
-
g.output(:jpg => "/tmp/#{config['appname']}.jpg")
|
1078
|
-
rescue Exception => e
|
1079
|
-
MU.log "Failed to generate GraphViz dependency tree: #{e.inspect}. This should only matter to developers.", MU::WARN, details: e.backtrace
|
1080
|
-
end
|
1081
|
-
end
|
1082
|
-
|
1083
|
-
# Generate a documentation-friendly dummy Ruby class for our mu.yaml main
|
1084
|
-
# config.
|
1085
|
-
def self.emitConfigAsRuby
|
1086
|
-
example = %Q{---
|
1087
|
-
public_address: 1.2.3.4
|
1088
|
-
mu_admin_email: egtlabs@eglobaltech.com
|
1089
|
-
mu_admin_name: Joe Schmoe
|
1090
|
-
mommacat_port: 2260
|
1091
|
-
banner: My Example Mu Master
|
1092
|
-
mu_repository: git://github.com/cloudamatic/mu.git
|
1093
|
-
repos:
|
1094
|
-
- https://github.com/cloudamatic/mu_demo_platform
|
1095
|
-
allow_invade_foreign_vpcs: true
|
1096
|
-
ansible_dir:
|
1097
|
-
aws:
|
1098
|
-
egtdev:
|
1099
|
-
region: us-east-1
|
1100
|
-
log_bucket_name: egt-mu-log-bucket
|
1101
|
-
default: true
|
1102
|
-
name: egtdev
|
1103
|
-
personal:
|
1104
|
-
region: us-east-2
|
1105
|
-
log_bucket_name: my-mu-log-bucket
|
1106
|
-
name: personal
|
1107
|
-
google:
|
1108
|
-
egtlabs:
|
1109
|
-
project: egt-labs-admin
|
1110
|
-
credentials_file: /opt/mu/etc/google.json
|
1111
|
-
region: us-east4
|
1112
|
-
log_bucket_name: hexabucket-761234
|
1113
|
-
default: true
|
1114
|
-
}
|
1115
|
-
mu_yaml_schema = eval(%Q{
|
1116
|
-
$NOOP = true
|
1117
|
-
load "#{MU.myRoot}/bin/mu-configure"
|
1118
|
-
$CONFIGURABLES
|
1119
|
-
})
|
1120
|
-
return if mu_yaml_schema.nil? or !mu_yaml_schema.is_a?(Hash)
|
1121
|
-
muyamlpath = "#{MU.myRoot}/modules/mu/mu.yaml.rb"
|
1122
|
-
MU.log "Converting mu.yaml schema to Ruby objects in #{muyamlpath}"
|
1123
|
-
muyaml_rb = File.new(muyamlpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
|
1124
|
-
muyaml_rb.puts "# Configuration schema for mu.yaml. See also {https://github.com/cloudamatic/mu/wiki/Configuration the Mu wiki}."
|
1125
|
-
muyaml_rb.puts "#"
|
1126
|
-
muyaml_rb.puts "# Example:"
|
1127
|
-
muyaml_rb.puts "#"
|
1128
|
-
muyaml_rb.puts "# <pre>"
|
1129
|
-
example.split(/\n/).each { |line|
|
1130
|
-
muyaml_rb.puts "# "+line+" " # markdooooown
|
1131
|
-
}
|
1132
|
-
muyaml_rb.puts "# </pre>"
|
1133
|
-
muyaml_rb.puts "module MuYAML"
|
1134
|
-
muyaml_rb.puts "\t# The configuration file format for Mu's main config file."
|
1135
|
-
self.printMuYamlSchema(muyaml_rb, [], { "subtree" => mu_yaml_schema })
|
1136
|
-
muyaml_rb.puts "end"
|
1137
|
-
muyaml_rb.close
|
1138
|
-
end
|
1139
|
-
|
1140
|
-
# Take the schema we've defined and create a dummy Ruby class tree out of
|
1141
|
-
# it, basically so we can leverage Yard to document it.
|
1142
|
-
def self.emitSchemaAsRuby
|
1143
|
-
kittenpath = "#{MU.myRoot}/modules/mu/kittens.rb"
|
1144
|
-
MU.log "Converting Basket of Kittens schema to Ruby objects in #{kittenpath}"
|
1145
|
-
kitten_rb = File.new(kittenpath, File::CREAT|File::TRUNC|File::RDWR, 0644)
|
1146
|
-
kitten_rb.puts "### THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT ###"
|
1147
|
-
kitten_rb.puts "#"
|
1148
|
-
kitten_rb.puts "#"
|
1149
|
-
kitten_rb.puts "#"
|
1150
|
-
kitten_rb.puts "module MU"
|
1151
|
-
kitten_rb.puts "class Config"
|
1152
|
-
kitten_rb.puts "\t# The configuration file format for Mu application stacks."
|
1153
|
-
self.printSchema(kitten_rb, ["BasketofKittens"], MU::Config.docSchema)
|
1154
|
-
kitten_rb.puts "end"
|
1155
|
-
kitten_rb.puts "end"
|
1156
|
-
kitten_rb.close
|
1157
|
-
|
1158
|
-
end
|
1159
|
-
|
1160
|
-
# Take an IP block and split it into a more-or-less arbitrary number of
|
1161
|
-
# subnets.
|
1162
|
-
# @param ip_block [String]: CIDR of the network to subdivide
|
1163
|
-
# @param subnets_desired [Integer]: Number of subnets we want back
|
1164
|
-
# @param max_mask [Integer]: The highest netmask we're allowed to use for a subnet (various by cloud provider)
|
1165
|
-
# @return [MU::Config::Tail]: Resulting subnet tails, or nil if an error occurred.
|
1166
|
-
def divideNetwork(ip_block, subnets_desired, max_mask = 28)
|
1167
|
-
cidr = NetAddr::IPv4Net.parse(ip_block.to_s)
|
1168
|
-
|
1169
|
-
# Ugly but reliable method of landing on the right subnet size
|
1170
|
-
subnet_bits = cidr.netmask.prefix_len
|
1171
|
-
begin
|
1172
|
-
subnet_bits += 1
|
1173
|
-
if subnet_bits > max_mask
|
1174
|
-
MU.log "Can't subdivide #{cidr.to_s} into #{subnets_desired.to_s}", MU::ERR
|
1175
|
-
raise MuError, "Subnets smaller than /#{max_mask} not permitted"
|
1176
|
-
end
|
1177
|
-
end while cidr.subnet_count(subnet_bits) < subnets_desired
|
1178
|
-
|
1179
|
-
if cidr.subnet_count(subnet_bits) > subnets_desired
|
1180
|
-
MU.log "Requested #{subnets_desired.to_s} subnets from #{cidr.to_s}, leaving #{(cidr.subnet_count(subnet_bits)-subnets_desired).to_s} unused /#{subnet_bits.to_s}s available", MU::NOTICE
|
1181
|
-
end
|
1182
|
-
|
1183
|
-
begin
|
1184
|
-
subnets = []
|
1185
|
-
(0..subnets_desired).each { |x|
|
1186
|
-
subnets << cidr.nth_subnet(subnet_bits, x).to_s
|
1187
|
-
}
|
1188
|
-
rescue RuntimeError => e
|
1189
|
-
if e.message.match(/exceeds subnets available for allocation/)
|
1190
|
-
MU.log e.message, MU::ERR
|
1191
|
-
MU.log "I'm attempting to create #{subnets_desired} subnets (one public and one private for each Availability Zone), of #{subnet_size} addresses each, but that's too many for a /#{cidr.netmask.prefix_len} network. Either declare a larger network, or explicitly declare a list of subnets with few enough entries to fit.", MU::ERR
|
1192
|
-
return nil
|
1193
|
-
else
|
1194
|
-
raise e
|
1195
|
-
end
|
1196
|
-
end
|
1197
|
-
|
1198
|
-
subnets = getTail("subnetblocks", value: subnets.join(","), cloudtype: "CommaDelimitedList", description: "IP Address ranges to be used for VPC subnets", prettyname: "SubnetIpBlocks", list_of: "ip_block").map { |tail| tail["ip_block"] }
|
1199
|
-
subnets
|
1200
|
-
end
|
1201
|
-
|
1202
422
|
# See if a given resource is configured in the current stack
|
1203
423
|
# @param name [String]: The name of the resource being checked
|
1204
424
|
# @param type [String]: The type of resource being checked
|
@@ -1206,7 +426,7 @@ $CONFIGURABLES
|
|
1206
426
|
def haveLitterMate?(name, type, has_multiple: false)
|
1207
427
|
@kittencfg_semaphore.synchronize {
|
1208
428
|
matches = []
|
1209
|
-
|
429
|
+
_shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
|
1210
430
|
if @kittens[cfg_plural]
|
1211
431
|
@kittens[cfg_plural].each { |kitten|
|
1212
432
|
if kitten['name'].to_s == name.to_s or
|
@@ -1233,7 +453,7 @@ $CONFIGURABLES
|
|
1233
453
|
# @param type [String]: The type of resource being removed
|
1234
454
|
def removeKitten(name, type)
|
1235
455
|
@kittencfg_semaphore.synchronize {
|
1236
|
-
|
456
|
+
_shortclass, _cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
|
1237
457
|
deletia = nil
|
1238
458
|
if @kittens[cfg_plural]
|
1239
459
|
@kittens[cfg_plural].each { |kitten|
|
@@ -1247,42 +467,6 @@ $CONFIGURABLES
|
|
1247
467
|
}
|
1248
468
|
end
|
1249
469
|
|
1250
|
-
# FirewallRules can reference other FirewallRules, which means we need to do
|
1251
|
-
# an extra pass to make sure we get all intra-stack dependencies correct.
|
1252
|
-
# @param acl [Hash]: The configuration hash for the FirewallRule to check
|
1253
|
-
# @return [Hash]
|
1254
|
-
def resolveIntraStackFirewallRefs(acl, delay_validation = false)
|
1255
|
-
acl["rules"].each { |acl_include|
|
1256
|
-
if acl_include['sgs']
|
1257
|
-
acl_include['sgs'].each { |sg_ref|
|
1258
|
-
if haveLitterMate?(sg_ref, "firewall_rules")
|
1259
|
-
acl["dependencies"] ||= []
|
1260
|
-
found = false
|
1261
|
-
acl["dependencies"].each { |dep|
|
1262
|
-
if dep["type"] == "firewall_rule" and dep["name"] == sg_ref
|
1263
|
-
dep["no_create_wait"] = true
|
1264
|
-
found = true
|
1265
|
-
end
|
1266
|
-
}
|
1267
|
-
if !found
|
1268
|
-
acl["dependencies"] << {
|
1269
|
-
"type" => "firewall_rule",
|
1270
|
-
"name" => sg_ref,
|
1271
|
-
"no_create_wait" => true
|
1272
|
-
}
|
1273
|
-
end
|
1274
|
-
siblingfw = haveLitterMate?(sg_ref, "firewall_rules")
|
1275
|
-
if !siblingfw["#MU_VALIDATED"]
|
1276
|
-
# XXX raise failure somehow
|
1277
|
-
insertKitten(siblingfw, "firewall_rules", delay_validation: delay_validation)
|
1278
|
-
end
|
1279
|
-
end
|
1280
|
-
}
|
1281
|
-
end
|
1282
|
-
}
|
1283
|
-
acl
|
1284
|
-
end
|
1285
|
-
|
1286
470
|
# Insert a resource into the current stack
|
1287
471
|
# @param descriptor [Hash]: The configuration description, as from a Basket of Kittens
|
1288
472
|
# @param type [String]: The type of resource being added
|
@@ -1290,7 +474,7 @@ $CONFIGURABLES
|
|
1290
474
|
# @param ignore_duplicates [Boolean]: Do not raise an exception if we attempt to insert a resource with a +name+ field that's already in use
|
1291
475
|
def insertKitten(descriptor, type, delay_validation = false, ignore_duplicates: false, overwrite: false)
|
1292
476
|
append = false
|
1293
|
-
start = Time.now
|
477
|
+
# start = Time.now
|
1294
478
|
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
|
1295
479
|
MU.log "insertKitten on #{cfg_name} #{descriptor['name']} (delay_validation: #{delay_validation.to_s})", MU::DEBUG, details: caller[0]
|
1296
480
|
|
@@ -1617,7 +801,7 @@ $CONFIGURABLES
|
|
1617
801
|
plain_cfg.delete("parent_block") if cfg_plural == "vpcs"
|
1618
802
|
begin
|
1619
803
|
JSON::Validator.validate!(myschema, plain_cfg)
|
1620
|
-
rescue JSON::Schema::ValidationError
|
804
|
+
rescue JSON::Schema::ValidationError
|
1621
805
|
pp plain_cfg
|
1622
806
|
# Use fully_validate to get the complete error list, save some time
|
1623
807
|
errors = JSON::Validator.fully_validate(myschema, plain_cfg)
|
@@ -1665,165 +849,63 @@ $CONFIGURABLES
|
|
1665
849
|
ok
|
1666
850
|
end
|
1667
851
|
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
}
|
852
|
+
# For our resources which specify intra-stack dependencies, make sure those
|
853
|
+
# dependencies are actually declared.
|
854
|
+
# TODO check for loops
|
855
|
+
def self.check_dependencies(config)
|
856
|
+
ok = true
|
1674
857
|
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1680
|
-
|
1681
|
-
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
|
1688
|
-
|
858
|
+
config.each_pair { |type, values|
|
859
|
+
if values.instance_of?(Array)
|
860
|
+
values.each { |resource|
|
861
|
+
if resource.kind_of?(Hash) and !resource["dependencies"].nil?
|
862
|
+
append = []
|
863
|
+
delete = []
|
864
|
+
resource["dependencies"].each { |dependency|
|
865
|
+
_shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(dependency["type"])
|
866
|
+
found = false
|
867
|
+
names_seen = []
|
868
|
+
if !config[cfg_plural].nil?
|
869
|
+
config[cfg_plural].each { |service|
|
870
|
+
names_seen << service["name"].to_s
|
871
|
+
found = true if service["name"].to_s == dependency["name"].to_s
|
872
|
+
if service["virtual_name"]
|
873
|
+
names_seen << service["virtual_name"].to_s
|
874
|
+
if service["virtual_name"].to_s == dependency["name"].to_s
|
875
|
+
found = true
|
876
|
+
append_me = dependency.dup
|
877
|
+
append_me['name'] = service['name']
|
878
|
+
append << append_me
|
879
|
+
delete << dependency
|
880
|
+
end
|
881
|
+
end
|
882
|
+
}
|
883
|
+
end
|
884
|
+
if !found
|
885
|
+
MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen
|
886
|
+
ok = false
|
887
|
+
end
|
888
|
+
}
|
889
|
+
if append.size > 0
|
890
|
+
append.uniq!
|
891
|
+
resource["dependencies"].concat(append)
|
892
|
+
end
|
893
|
+
if delete.size > 0
|
894
|
+
delete.each { |delete_me|
|
895
|
+
resource["dependencies"].delete(delete_me)
|
896
|
+
}
|
897
|
+
end
|
898
|
+
end
|
899
|
+
}
|
900
|
+
end
|
1689
901
|
}
|
902
|
+
return ok
|
1690
903
|
end
|
1691
904
|
|
1692
|
-
#
|
1693
|
-
#
|
1694
|
-
|
1695
|
-
|
1696
|
-
"type" => "string",
|
1697
|
-
"description" => "Specify a non-default set of credentials to use when authenticating to cloud provider APIs, as listed in `mu.yaml` under each provider's subsection. If "
|
1698
|
-
}
|
1699
|
-
end
|
1700
|
-
|
1701
|
-
# Configuration chunk for creating resource tags as an array of key/value
|
1702
|
-
# pairs.
|
1703
|
-
# @return [Hash]
|
1704
|
-
def self.optional_tags_primitive
|
1705
|
-
{
|
1706
|
-
"type" => "boolean",
|
1707
|
-
"description" => "Tag the resource with our optional tags (+MU-HANDLE+, +MU-MASTER-NAME+, +MU-OWNER+).",
|
1708
|
-
"default" => true
|
1709
|
-
}
|
1710
|
-
end
|
1711
|
-
|
1712
|
-
# Configuration chunk for creating resource tags as an array of key/value
|
1713
|
-
# pairs.
|
1714
|
-
# @return [Hash]
|
1715
|
-
def self.tags_primitive
|
1716
|
-
{
|
1717
|
-
"type" => "array",
|
1718
|
-
"minItems" => 1,
|
1719
|
-
"items" => {
|
1720
|
-
"description" => "Tags to apply to this resource. Will apply at the cloud provider level and in node groomers, where applicable.",
|
1721
|
-
"type" => "object",
|
1722
|
-
"title" => "tags",
|
1723
|
-
"required" => ["key", "value"],
|
1724
|
-
"additionalProperties" => false,
|
1725
|
-
"properties" => {
|
1726
|
-
"key" => {
|
1727
|
-
"type" => "string",
|
1728
|
-
},
|
1729
|
-
"value" => {
|
1730
|
-
"type" => "string",
|
1731
|
-
}
|
1732
|
-
}
|
1733
|
-
}
|
1734
|
-
}
|
1735
|
-
end
|
1736
|
-
|
1737
|
-
# Configuration chunk for choosing a cloud provider
|
1738
|
-
# @return [Hash]
|
1739
|
-
def self.cloud_primitive
|
1740
|
-
{
|
1741
|
-
"type" => "string",
|
1742
|
-
# "default" => MU::Config.defaultCloud, # applyInheritedDefaults does this better
|
1743
|
-
"enum" => MU::Cloud.supportedClouds
|
1744
|
-
}
|
1745
|
-
end
|
1746
|
-
|
1747
|
-
# Generate configuration for the general-pursose ADMIN firewall rulesets
|
1748
|
-
# (security groups in AWS). Note that these are unique to regions and
|
1749
|
-
# individual VPCs (as well as Classic, which is just a degenerate case of
|
1750
|
-
# a VPC for our purposes.
|
1751
|
-
# @param vpc [Hash]: A VPC reference as defined in our config schema. This originates with the calling resource, so we'll peel out just what we need (a name or cloud id of a VPC).
|
1752
|
-
# @param admin_ip [String]: Optional string of an extra IP address to allow blanket access to the calling resource.
|
1753
|
-
# @param cloud [String]: The parent resource's cloud plugin identifier
|
1754
|
-
# @param region [String]: Cloud provider region, if applicable.
|
1755
|
-
# @return [Hash<String>]: A dependency description that the calling resource can then add to itself.
|
1756
|
-
def adminFirewallRuleset(vpc: nil, admin_ip: nil, region: nil, cloud: nil, credentials: nil, rules_only: false)
|
1757
|
-
if !cloud or (cloud == "AWS" and !region)
|
1758
|
-
raise MuError, "Cannot call adminFirewallRuleset without specifying the parent's region and cloud provider"
|
1759
|
-
end
|
1760
|
-
hosts = Array.new
|
1761
|
-
hosts << "#{MU.my_public_ip}/32" if MU.my_public_ip
|
1762
|
-
hosts << "#{MU.my_private_ip}/32" if MU.my_private_ip
|
1763
|
-
hosts << "#{MU.mu_public_ip}/32" if MU.mu_public_ip
|
1764
|
-
hosts << "#{admin_ip}/32" if admin_ip
|
1765
|
-
hosts.uniq!
|
1766
|
-
|
1767
|
-
rules = []
|
1768
|
-
if cloud == "Google"
|
1769
|
-
rules = [
|
1770
|
-
{ "ingress" => true, "proto" => "all", "hosts" => hosts },
|
1771
|
-
{ "egress" => true, "proto" => "all", "hosts" => hosts }
|
1772
|
-
]
|
1773
|
-
else
|
1774
|
-
rules = [
|
1775
|
-
{ "proto" => "tcp", "port_range" => "0-65535", "hosts" => hosts },
|
1776
|
-
{ "proto" => "udp", "port_range" => "0-65535", "hosts" => hosts },
|
1777
|
-
{ "proto" => "icmp", "port_range" => "-1", "hosts" => hosts }
|
1778
|
-
]
|
1779
|
-
end
|
1780
|
-
|
1781
|
-
resclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("FirewallRule")
|
1782
|
-
|
1783
|
-
if rules_only
|
1784
|
-
return rules
|
1785
|
-
end
|
1786
|
-
|
1787
|
-
name = "admin"
|
1788
|
-
name += credentials.to_s if credentials
|
1789
|
-
realvpc = nil
|
1790
|
-
if vpc
|
1791
|
-
realvpc = {}
|
1792
|
-
['vpc_name', 'vpc_id'].each { |p|
|
1793
|
-
if vpc[p]
|
1794
|
-
vpc[p.sub(/^vpc_/, '')] = vpc[p]
|
1795
|
-
vpc.delete(p)
|
1796
|
-
end
|
1797
|
-
}
|
1798
|
-
['cloud', 'id', 'name', 'deploy_id', 'habitat', 'credentials'].each { |field|
|
1799
|
-
realvpc[field] = vpc[field] if !vpc[field].nil?
|
1800
|
-
}
|
1801
|
-
if !realvpc['id'].nil? and !realvpc['id'].empty?
|
1802
|
-
# Stupid kludge for Google cloud_ids which are sometimes URLs and
|
1803
|
-
# sometimes not. Requirements are inconsistent from scenario to
|
1804
|
-
# scenario.
|
1805
|
-
name = name + "-" + realvpc['id'].gsub(/.*\//, "")
|
1806
|
-
realvpc['id'] = getTail("id", value: realvpc['id'], prettyname: "Admin Firewall Ruleset #{name} Target VPC", cloudtype: "AWS::EC2::VPC::Id") if realvpc["id"].is_a?(String)
|
1807
|
-
elsif !realvpc['name'].nil?
|
1808
|
-
name = name + "-" + realvpc['name']
|
1809
|
-
end
|
1810
|
-
end
|
1811
|
-
|
1812
|
-
|
1813
|
-
acl = {"name" => name, "rules" => rules, "vpc" => realvpc, "cloud" => cloud, "admin" => true, "credentials" => credentials }
|
1814
|
-
if cloud == "Google" and acl["vpc"] and acl["vpc"]["habitat"]
|
1815
|
-
acl['project'] = acl["vpc"]["habitat"]["id"] || acl["vpc"]["habitat"]["name"]
|
1816
|
-
end
|
1817
|
-
acl.delete("vpc") if !acl["vpc"]
|
1818
|
-
if !resclass.isGlobal? and !region.nil? and !region.empty?
|
1819
|
-
acl["region"] = region
|
1820
|
-
end
|
1821
|
-
@admin_firewall_rules << acl if !@admin_firewall_rules.include?(acl)
|
1822
|
-
return {"type" => "firewall_rule", "name" => name}
|
1823
|
-
end
|
1824
|
-
|
1825
|
-
private
|
1826
|
-
|
905
|
+
# Ugly text-manipulation to recursively resolve some placeholder strings
|
906
|
+
# we put in for ERB include() directives.
|
907
|
+
# @param lines [String]
|
908
|
+
# @return [String]
|
1827
909
|
def self.resolveYAMLAnchors(lines)
|
1828
910
|
new_text = ""
|
1829
911
|
lines.each_line { |line|
|
@@ -1843,7 +925,6 @@ $CONFIGURABLES
|
|
1843
925
|
return new_text
|
1844
926
|
end
|
1845
927
|
|
1846
|
-
|
1847
928
|
# Given a path to a config file, try to guess whether it's YAML or JSON.
|
1848
929
|
# @param path [String]: The path to the file to check.
|
1849
930
|
def self.guessFormat(path)
|
@@ -1852,10 +933,10 @@ $CONFIGURABLES
|
|
1852
933
|
stripped = raw.gsub(/<%.*?%>,?/, "").gsub(/,[\n\s]*([\]\}])/, '\1')
|
1853
934
|
begin
|
1854
935
|
JSON.parse(stripped)
|
1855
|
-
rescue JSON::ParserError
|
936
|
+
rescue JSON::ParserError
|
1856
937
|
begin
|
1857
938
|
YAML.load(raw.gsub(/<%.*?%>/, ""))
|
1858
|
-
rescue Psych::SyntaxError
|
939
|
+
rescue Psych::SyntaxError
|
1859
940
|
# Ok, well neither of those worked, let's assume that filenames are
|
1860
941
|
# meaningful.
|
1861
942
|
if path.match(/\.(yaml|yml)$/i)
|
@@ -1935,7 +1016,7 @@ $CONFIGURABLES
|
|
1935
1016
|
end
|
1936
1017
|
begin
|
1937
1018
|
erb = ERB.new(File.read(file), nil, "<>")
|
1938
|
-
rescue Errno::ENOENT
|
1019
|
+
rescue Errno::ENOENT
|
1939
1020
|
retries = retries + 1
|
1940
1021
|
if retries == 1
|
1941
1022
|
file = File.dirname(MU::Config.config_path)+"/"+orig_filename
|
@@ -1960,12 +1041,12 @@ $CONFIGURABLES
|
|
1960
1041
|
parsed_cfg = nil
|
1961
1042
|
begin
|
1962
1043
|
parsed_cfg = JSON.parse(erb.result(binding))
|
1963
|
-
parsed_as = :json
|
1044
|
+
# parsed_as = :json
|
1964
1045
|
rescue JSON::ParserError => e
|
1965
1046
|
MU.log e.inspect, MU::DEBUG
|
1966
1047
|
begin
|
1967
1048
|
parsed_cfg = YAML.load(MU::Config.resolveYAMLAnchors(erb.result(binding)))
|
1968
|
-
parsed_as = :yaml
|
1049
|
+
# parsed_as = :yaml
|
1969
1050
|
rescue Psych::SyntaxError => e
|
1970
1051
|
MU.log e.inspect, MU::DEBUG
|
1971
1052
|
MU.log "#{file} parsed neither as JSON nor as YAML, including as raw text", MU::WARN if @param_pass
|
@@ -1980,16 +1061,11 @@ $CONFIGURABLES
|
|
1980
1061
|
$yaml_refs[file] = ""+YAML.dump(parsed_cfg).sub(/^---\n/, "")
|
1981
1062
|
return "# MU::Config.include PLACEHOLDER #{file} REDLOHECALP"
|
1982
1063
|
end
|
1983
|
-
rescue SyntaxError
|
1064
|
+
rescue SyntaxError
|
1984
1065
|
raise ValidationError, "ERB in #{file} threw a syntax error"
|
1985
1066
|
end
|
1986
1067
|
end
|
1987
1068
|
|
1988
|
-
# (see #include)
|
1989
|
-
def include(file)
|
1990
|
-
MU::Config.include(file, get_binding(@@tails.keys.sort), param_pass = @param_pass)
|
1991
|
-
end
|
1992
|
-
|
1993
1069
|
@@bindings = {}
|
1994
1070
|
# Keep a cache of bindings we've created as sandbox contexts for ERB
|
1995
1071
|
# processing, so we don't keep reloading the entire Mu library inside new
|
@@ -1998,6 +1074,13 @@ $CONFIGURABLES
|
|
1998
1074
|
@@bindings
|
1999
1075
|
end
|
2000
1076
|
|
1077
|
+
private
|
1078
|
+
|
1079
|
+
# (see #include)
|
1080
|
+
def include(file)
|
1081
|
+
MU::Config.include(file, get_binding(@@tails.keys.sort), @param_pass)
|
1082
|
+
end
|
1083
|
+
|
2001
1084
|
# Namespace magic to pass to ERB's result method.
|
2002
1085
|
def get_binding(keyset)
|
2003
1086
|
environment = $environment
|
@@ -2012,242 +1095,6 @@ $CONFIGURABLES
|
|
2012
1095
|
MU::Config.global_bindings[keyset]
|
2013
1096
|
end
|
2014
1097
|
|
2015
|
-
def applySchemaDefaults(conf_chunk = config, schema_chunk = schema, depth = 0, siblings = nil, type: nil)
|
2016
|
-
return if schema_chunk.nil?
|
2017
|
-
|
2018
|
-
if conf_chunk != nil and schema_chunk["properties"].kind_of?(Hash) and conf_chunk.is_a?(Hash)
|
2019
|
-
|
2020
|
-
if schema_chunk["properties"]["creation_style"].nil? or
|
2021
|
-
schema_chunk["properties"]["creation_style"] != "existing"
|
2022
|
-
schema_chunk["properties"].each_pair { |key, subschema|
|
2023
|
-
shortclass = if conf_chunk[key]
|
2024
|
-
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(key)
|
2025
|
-
shortclass
|
2026
|
-
else
|
2027
|
-
nil
|
2028
|
-
end
|
2029
|
-
|
2030
|
-
new_val = applySchemaDefaults(conf_chunk[key], subschema, depth+1, conf_chunk, type: shortclass).dup
|
2031
|
-
|
2032
|
-
conf_chunk[key] = Marshal.load(Marshal.dump(new_val)) if !new_val.nil?
|
2033
|
-
}
|
2034
|
-
end
|
2035
|
-
elsif schema_chunk["type"] == "array" and conf_chunk.kind_of?(Array)
|
2036
|
-
conf_chunk.map! { |item|
|
2037
|
-
# If we're working on a resource type, go get implementation-specific
|
2038
|
-
# schema information so that we set those defaults correctly.
|
2039
|
-
realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
|
2040
|
-
|
2041
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(item["cloud"]).const_get(type)
|
2042
|
-
toplevel_required, cloudschema = cloudclass.schema(self)
|
2043
|
-
|
2044
|
-
newschema = schema_chunk["items"].dup
|
2045
|
-
newschema["properties"].merge!(cloudschema)
|
2046
|
-
newschema
|
2047
|
-
else
|
2048
|
-
schema_chunk["items"].dup
|
2049
|
-
end
|
2050
|
-
|
2051
|
-
applySchemaDefaults(item, realschema, depth+1, conf_chunk, type: type).dup
|
2052
|
-
}
|
2053
|
-
else
|
2054
|
-
if conf_chunk.nil? and !schema_chunk["default_if"].nil? and !siblings.nil?
|
2055
|
-
schema_chunk["default_if"].each { |cond|
|
2056
|
-
if siblings[cond["key_is"]] == cond["value_is"]
|
2057
|
-
return Marshal.load(Marshal.dump(cond["set"]))
|
2058
|
-
end
|
2059
|
-
}
|
2060
|
-
end
|
2061
|
-
if conf_chunk.nil? and schema_chunk["default"] != nil
|
2062
|
-
return Marshal.load(Marshal.dump(schema_chunk["default"]))
|
2063
|
-
end
|
2064
|
-
end
|
2065
|
-
|
2066
|
-
return conf_chunk
|
2067
|
-
end
|
2068
|
-
|
2069
|
-
# For our resources which specify intra-stack dependencies, make sure those
|
2070
|
-
# dependencies are actually declared.
|
2071
|
-
# TODO check for loops
|
2072
|
-
def self.check_dependencies(config)
|
2073
|
-
ok = true
|
2074
|
-
|
2075
|
-
config.each_pair { |type, values|
|
2076
|
-
if values.instance_of?(Array)
|
2077
|
-
values.each { |resource|
|
2078
|
-
if resource.kind_of?(Hash) and !resource["dependencies"].nil?
|
2079
|
-
append = []
|
2080
|
-
delete = []
|
2081
|
-
resource["dependencies"].each { |dependency|
|
2082
|
-
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(dependency["type"])
|
2083
|
-
found = false
|
2084
|
-
names_seen = []
|
2085
|
-
if !config[cfg_plural].nil?
|
2086
|
-
config[cfg_plural].each { |service|
|
2087
|
-
names_seen << service["name"].to_s
|
2088
|
-
found = true if service["name"].to_s == dependency["name"].to_s
|
2089
|
-
if service["virtual_name"]
|
2090
|
-
names_seen << service["virtual_name"].to_s
|
2091
|
-
if service["virtual_name"].to_s == dependency["name"].to_s
|
2092
|
-
found = true
|
2093
|
-
append_me = dependency.dup
|
2094
|
-
append_me['name'] = service['name']
|
2095
|
-
append << append_me
|
2096
|
-
delete << dependency
|
2097
|
-
end
|
2098
|
-
end
|
2099
|
-
}
|
2100
|
-
end
|
2101
|
-
if !found
|
2102
|
-
MU.log "Missing dependency: #{type}{#{resource['name']}} needs #{cfg_name}{#{dependency['name']}}", MU::ERR, details: names_seen
|
2103
|
-
ok = false
|
2104
|
-
end
|
2105
|
-
}
|
2106
|
-
if append.size > 0
|
2107
|
-
append.uniq!
|
2108
|
-
resource["dependencies"].concat(append)
|
2109
|
-
end
|
2110
|
-
if delete.size > 0
|
2111
|
-
delete.each { |delete_me|
|
2112
|
-
resource["dependencies"].delete(delete_me)
|
2113
|
-
}
|
2114
|
-
end
|
2115
|
-
end
|
2116
|
-
}
|
2117
|
-
end
|
2118
|
-
}
|
2119
|
-
return ok
|
2120
|
-
end
|
2121
|
-
|
2122
|
-
|
2123
|
-
# Verify that a server or server_pool has a valid AD config referencing
|
2124
|
-
# valid Vaults for credentials.
|
2125
|
-
def self.check_vault_refs(server)
|
2126
|
-
ok = true
|
2127
|
-
server['vault_access'] = [] if server['vault_access'].nil?
|
2128
|
-
server['groomer'] ||= self.defaultGroomer
|
2129
|
-
groomclass = MU::Groomer.loadGroomer(server['groomer'])
|
2130
|
-
|
2131
|
-
begin
|
2132
|
-
if !server['active_directory'].nil?
|
2133
|
-
["domain_admin_vault", "domain_join_vault"].each { |vault_class|
|
2134
|
-
server['vault_access'] << {
|
2135
|
-
"vault" => server['active_directory'][vault_class]['vault'],
|
2136
|
-
"item" => server['active_directory'][vault_class]['item']
|
2137
|
-
}
|
2138
|
-
item = groomclass.getSecret(
|
2139
|
-
vault: server['active_directory'][vault_class]['vault'],
|
2140
|
-
item: server['active_directory'][vault_class]['item'],
|
2141
|
-
)
|
2142
|
-
["username_field", "password_field"].each { |field|
|
2143
|
-
if !item.has_key?(server['active_directory'][vault_class][field])
|
2144
|
-
ok = false
|
2145
|
-
MU.log "I don't see a value named #{field} in Chef Vault #{server['active_directory'][vault_class]['vault']}:#{server['active_directory'][vault_class]['item']}", MU::ERR
|
2146
|
-
end
|
2147
|
-
}
|
2148
|
-
}
|
2149
|
-
end
|
2150
|
-
|
2151
|
-
if !server['windows_auth_vault'].nil?
|
2152
|
-
server['use_cloud_provider_windows_password'] = false
|
2153
|
-
|
2154
|
-
server['vault_access'] << {
|
2155
|
-
"vault" => server['windows_auth_vault']['vault'],
|
2156
|
-
"item" => server['windows_auth_vault']['item']
|
2157
|
-
}
|
2158
|
-
item = groomclass.getSecret(
|
2159
|
-
vault: server['windows_auth_vault']['vault'],
|
2160
|
-
item: server['windows_auth_vault']['item']
|
2161
|
-
)
|
2162
|
-
["password_field", "ec2config_password_field", "sshd_password_field"].each { |field|
|
2163
|
-
if !item.has_key?(server['windows_auth_vault'][field])
|
2164
|
-
MU.log "No value named #{field} in Chef Vault #{server['windows_auth_vault']['vault']}:#{server['windows_auth_vault']['item']}, will use a generated password.", MU::NOTICE
|
2165
|
-
server['windows_auth_vault'].delete(field)
|
2166
|
-
end
|
2167
|
-
}
|
2168
|
-
end
|
2169
|
-
# Check all of the non-special ones while we're at it
|
2170
|
-
server['vault_access'].each { |v|
|
2171
|
-
next if v['vault'] == "splunk" and v['item'] == "admin_user"
|
2172
|
-
item = groomclass.getSecret(vault: v['vault'], item: v['item'])
|
2173
|
-
}
|
2174
|
-
rescue MuError
|
2175
|
-
MU.log "Can't load a Chef Vault I was configured to use. Does it exist?", MU::ERR
|
2176
|
-
ok = false
|
2177
|
-
end
|
2178
|
-
return ok
|
2179
|
-
end
|
2180
|
-
|
2181
|
-
|
2182
|
-
# Given a bare hash describing a resource, insert default values which can
|
2183
|
-
# be inherited from its parent or from the root of the BoK.
|
2184
|
-
# @param kitten [Hash]: A resource descriptor
|
2185
|
-
# @param type [String]: The type of resource this is ("servers" etc)
|
2186
|
-
def applyInheritedDefaults(kitten, type)
|
2187
|
-
return if !kitten.is_a?(Hash)
|
2188
|
-
kitten['cloud'] ||= @config['cloud']
|
2189
|
-
kitten['cloud'] ||= MU::Config.defaultCloud
|
2190
|
-
|
2191
|
-
if !MU::Cloud.supportedClouds.include?(kitten['cloud'])
|
2192
|
-
return
|
2193
|
-
end
|
2194
|
-
|
2195
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud'])
|
2196
|
-
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
|
2197
|
-
resclass = Object.const_get("MU").const_get("Cloud").const_get(kitten['cloud']).const_get(shortclass)
|
2198
|
-
|
2199
|
-
schema_fields = ["us_only", "scrub_mu_isms", "credentials", "billing_acct"]
|
2200
|
-
if !resclass.isGlobal?
|
2201
|
-
kitten['region'] ||= @config['region']
|
2202
|
-
kitten['region'] ||= cloudclass.myRegion(kitten['credentials'])
|
2203
|
-
schema_fields << "region"
|
2204
|
-
end
|
2205
|
-
|
2206
|
-
kitten['credentials'] ||= @config['credentials']
|
2207
|
-
kitten['credentials'] ||= cloudclass.credConfig(name_only: true)
|
2208
|
-
|
2209
|
-
kitten['us_only'] ||= @config['us_only']
|
2210
|
-
kitten['us_only'] ||= false
|
2211
|
-
|
2212
|
-
kitten['scrub_mu_isms'] ||= @config['scrub_mu_isms']
|
2213
|
-
kitten['scrub_mu_isms'] ||= false
|
2214
|
-
|
2215
|
-
if kitten['cloud'] == "Google"
|
2216
|
-
# TODO this should be cloud-generic (handle AWS accounts, Azure subscriptions)
|
2217
|
-
if resclass.canLiveIn.include?(:Habitat)
|
2218
|
-
kitten["project"] ||= MU::Cloud::Google.defaultProject(kitten['credentials'])
|
2219
|
-
schema_fields << "project"
|
2220
|
-
end
|
2221
|
-
if kitten['region'].nil? and !kitten['#MU_CLOUDCLASS'].nil? and
|
2222
|
-
!resclass.isGlobal? and
|
2223
|
-
![MU::Cloud::VPC, MU::Cloud::FirewallRule].include?(kitten['#MU_CLOUDCLASS'])
|
2224
|
-
if MU::Cloud::Google.myRegion((kitten['credentials'])).nil?
|
2225
|
-
raise ValidationError, "Google '#{type}' resource '#{kitten['name']}' declared without a region, but no default Google region declared in mu.yaml under #{kitten['credentials'].nil? ? "default" : kitten['credentials']} credential set"
|
2226
|
-
end
|
2227
|
-
kitten['region'] ||= MU::Cloud::Google.myRegion
|
2228
|
-
end
|
2229
|
-
elsif kitten["cloud"] == "AWS" and !resclass.isGlobal? and !kitten['region']
|
2230
|
-
if MU::Cloud::AWS.myRegion.nil?
|
2231
|
-
raise ValidationError, "AWS resource declared without a region, but no default AWS region found"
|
2232
|
-
end
|
2233
|
-
kitten['region'] ||= MU::Cloud::AWS.myRegion
|
2234
|
-
end
|
2235
|
-
|
2236
|
-
|
2237
|
-
kitten['billing_acct'] ||= @config['billing_acct'] if @config['billing_acct']
|
2238
|
-
|
2239
|
-
kitten["dependencies"] ||= []
|
2240
|
-
|
2241
|
-
# Make sure the schema knows about these "new" fields, so that validation
|
2242
|
-
# doesn't trip over them.
|
2243
|
-
schema_fields.each { |field|
|
2244
|
-
if @@schema["properties"][field]
|
2245
|
-
MU.log "Adding #{field} to schema for #{type} #{kitten['cloud']}", MU::DEBUG, details: @@schema["properties"][field]
|
2246
|
-
@@schema["properties"][type]["items"]["properties"][field] ||= @@schema["properties"][field]
|
2247
|
-
end
|
2248
|
-
}
|
2249
|
-
end
|
2250
|
-
|
2251
1098
|
def validate(config = @config)
|
2252
1099
|
ok = true
|
2253
1100
|
|
@@ -2277,9 +1124,11 @@ $CONFIGURABLES
|
|
2277
1124
|
}
|
2278
1125
|
}
|
2279
1126
|
|
1127
|
+
newrules = []
|
2280
1128
|
@kittens["firewall_rules"].each { |acl|
|
2281
|
-
|
1129
|
+
newrules << resolveIntraStackFirewallRefs(acl)
|
2282
1130
|
}
|
1131
|
+
@kittens["firewall_rules"] = newrules
|
2283
1132
|
|
2284
1133
|
# VPCs do complex things in their cloud-layer validation that other
|
2285
1134
|
# resources tend to need, like subnet allocation, so hit them early.
|
@@ -2321,7 +1170,7 @@ $CONFIGURABLES
|
|
2321
1170
|
ruleset = haveLitterMate?("database"+db['name'], "firewall_rules")
|
2322
1171
|
if ruleset
|
2323
1172
|
["server_pools", "servers"].each { |type|
|
2324
|
-
|
1173
|
+
_shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(type)
|
2325
1174
|
@kittens[cfg_plural].each { |server|
|
2326
1175
|
server["dependencies"].each { |dep|
|
2327
1176
|
if dep["type"] == "database" and dep["name"] == db["name"]
|
@@ -2378,510 +1227,13 @@ $CONFIGURABLES
|
|
2378
1227
|
# end
|
2379
1228
|
end
|
2380
1229
|
|
2381
|
-
# Emit our mu.yaml schema in a format that YARD can comprehend and turn into
|
2382
|
-
# documentation.
|
2383
|
-
def self.printMuYamlSchema(muyaml_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
|
2384
|
-
return if schema.nil?
|
2385
|
-
if schema["subtree"]
|
2386
|
-
printme = Array.new
|
2387
|
-
# order sub-elements by whether they're required, so we can use YARD's
|
2388
|
-
# grouping tags on them
|
2389
|
-
have_required = schema["subtree"].keys.any? { |k| schema["subtree"][k]["required"] }
|
2390
|
-
prop_list = schema["subtree"].keys.sort { |a, b|
|
2391
|
-
if schema["subtree"][a]["required"] and !schema["subtree"][b]["required"]
|
2392
|
-
-1
|
2393
|
-
elsif !schema["subtree"][a]["required"] and schema["subtree"][b]["required"]
|
2394
|
-
1
|
2395
|
-
else
|
2396
|
-
a <=> b
|
2397
|
-
end
|
2398
|
-
}
|
2399
|
-
|
2400
|
-
req = false
|
2401
|
-
printme << "# @!group Optional parameters" if !have_required
|
2402
|
-
prop_list.each { |name|
|
2403
|
-
prop = schema["subtree"][name]
|
2404
|
-
if prop["required"]
|
2405
|
-
printme << "# @!group Required parameters" if !req
|
2406
|
-
req = true
|
2407
|
-
else
|
2408
|
-
if req
|
2409
|
-
printme << "# @!endgroup"
|
2410
|
-
printme << "# @!group Optional parameters"
|
2411
|
-
end
|
2412
|
-
req = false
|
2413
|
-
end
|
2414
|
-
|
2415
|
-
printme << self.printMuYamlSchema(muyaml_rb, class_hierarchy+ [name], prop, false, req)
|
2416
|
-
}
|
2417
|
-
printme << "# @!endgroup"
|
2418
|
-
|
2419
|
-
desc = (schema['desc'] || schema['title'])
|
2420
|
-
|
2421
|
-
tabs = 1
|
2422
|
-
class_hierarchy.each { |classname|
|
2423
|
-
if classname == class_hierarchy.last and desc
|
2424
|
-
muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{desc}\n"
|
2425
|
-
end
|
2426
|
-
muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
|
2427
|
-
tabs = tabs + 1
|
2428
|
-
}
|
2429
|
-
printme.each { |lines|
|
2430
|
-
if !lines.nil? and lines.is_a?(String)
|
2431
|
-
lines.lines.each { |line|
|
2432
|
-
muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
|
2433
|
-
}
|
2434
|
-
end
|
2435
|
-
}
|
2436
|
-
|
2437
|
-
class_hierarchy.each { |classname|
|
2438
|
-
tabs = tabs - 1
|
2439
|
-
muyaml_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
|
2440
|
-
}
|
2441
|
-
|
2442
|
-
# And now that we've dealt with our children, pass our own rendered
|
2443
|
-
# commentary back up to our caller.
|
2444
|
-
name = class_hierarchy.last
|
2445
|
-
if in_array
|
2446
|
-
type = "Array<#{class_hierarchy.join("::")}>"
|
2447
|
-
else
|
2448
|
-
type = class_hierarchy.join("::")
|
2449
|
-
end
|
2450
|
-
|
2451
|
-
docstring = "\n"
|
2452
|
-
docstring = docstring + "# **REQUIRED**\n" if required
|
2453
|
-
# docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
|
2454
|
-
docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if desc
|
2455
|
-
docstring = docstring + "#\n"
|
2456
|
-
docstring = docstring + "# @return [#{type}]\n"
|
2457
|
-
docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
|
2458
|
-
docstring = docstring + "attr_accessor :#{name}"
|
2459
|
-
return docstring
|
2460
|
-
|
2461
|
-
else
|
2462
|
-
in_array = schema["array"]
|
2463
|
-
name = class_hierarchy.last
|
2464
|
-
type = if schema['boolean']
|
2465
|
-
"Boolean"
|
2466
|
-
else
|
2467
|
-
"String"
|
2468
|
-
end
|
2469
|
-
if in_array
|
2470
|
-
type = "Array<#{type}>"
|
2471
|
-
end
|
2472
|
-
docstring = "\n"
|
2473
|
-
|
2474
|
-
prefixes = []
|
2475
|
-
prefixes << "# **REQUIRED**" if schema["required"] and schema['default'].nil?
|
2476
|
-
# prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
|
2477
|
-
prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
|
2478
|
-
if !schema['pattern'].nil?
|
2479
|
-
# XXX unquoted regex chars confuse the hell out of YARD. How do we
|
2480
|
-
# quote {}[] etc in YARD-speak?
|
2481
|
-
prefixes << "# **Must match pattern `#{schema['pattern'].to_s.gsub(/\n/, "\n#")}`**"
|
2482
|
-
end
|
2483
|
-
|
2484
|
-
desc = (schema['desc'] || schema['title'])
|
2485
|
-
if prefixes.size > 0
|
2486
|
-
docstring += prefixes.join(",\n")
|
2487
|
-
if desc and desc.size > 1
|
2488
|
-
docstring += " - "
|
2489
|
-
end
|
2490
|
-
docstring += "\n"
|
2491
|
-
end
|
2492
|
-
|
2493
|
-
docstring = docstring + "# #{desc.gsub(/\n/, "\n#")}\n" if !desc.nil?
|
2494
|
-
docstring = docstring + "#\n"
|
2495
|
-
docstring = docstring + "# @return [#{type}]\n"
|
2496
|
-
docstring = docstring + "attr_accessor :#{name}"
|
2497
|
-
|
2498
|
-
return docstring
|
2499
|
-
end
|
2500
|
-
|
2501
|
-
end
|
2502
|
-
|
2503
|
-
# Emit our Basket of Kittens schema in a format that YARD can comprehend
|
2504
|
-
# and turn into documentation.
|
2505
|
-
def self.printSchema(kitten_rb, class_hierarchy, schema, in_array = false, required = false, prefix: nil)
|
2506
|
-
return if schema.nil?
|
2507
|
-
|
2508
|
-
if schema["type"] == "object"
|
2509
|
-
printme = []
|
2510
|
-
|
2511
|
-
if !schema["properties"].nil?
|
2512
|
-
# order sub-elements by whether they're required, so we can use YARD's
|
2513
|
-
# grouping tags on them
|
2514
|
-
if !schema["required"].nil? and schema["required"].size > 0
|
2515
|
-
prop_list = schema["properties"].keys.sort_by { |name|
|
2516
|
-
schema["required"].include?(name) ? 0 : 1
|
2517
|
-
}
|
2518
|
-
else
|
2519
|
-
prop_list = schema["properties"].keys
|
2520
|
-
end
|
2521
|
-
req = false
|
2522
|
-
printme << "# @!group Optional parameters" if schema["required"].nil? or schema["required"].size == 0
|
2523
|
-
prop_list.each { |name|
|
2524
|
-
prop = schema["properties"][name]
|
2525
|
-
|
2526
|
-
if class_hierarchy.size == 1
|
2527
|
-
|
2528
|
-
_shortclass, cfg_name, cfg_plural, _classname = MU::Cloud.getResourceNames(name)
|
2529
|
-
if cfg_name
|
2530
|
-
example_path = MU.myRoot+"/modules/mu/config/"+cfg_name+".yml"
|
2531
|
-
if File.exist?(example_path)
|
2532
|
-
example = "#\n# Examples:\n#\n"
|
2533
|
-
# XXX these variables are all parameters from the BoKs in
|
2534
|
-
# modules/tests. A really clever implementation would read
|
2535
|
-
# and parse them to get default values, perhaps, instead of
|
2536
|
-
# hard-coding them here.
|
2537
|
-
instance_type = "t2.medium"
|
2538
|
-
db_size = "db.t2.medium"
|
2539
|
-
vpc_name = "some_vpc"
|
2540
|
-
logs_name = "some_loggroup"
|
2541
|
-
queues_name = "some_queue"
|
2542
|
-
server_pools_name = "some_server_pool"
|
2543
|
-
["simple", "complex"].each { |complexity|
|
2544
|
-
erb = ERB.new(File.read(example_path), nil, "<>")
|
2545
|
-
example += "# !!!yaml\n"
|
2546
|
-
example += "# ---\n"
|
2547
|
-
example += "# appname: #{complexity}\n"
|
2548
|
-
example += "# #{cfg_plural}:\n"
|
2549
|
-
firstline = true
|
2550
|
-
erb.result(binding).split(/\n/).each { |l|
|
2551
|
-
l.chomp!
|
2552
|
-
l.sub!(/#.*/, "") if !l.match(/#(?:INTERNET|NAT|DENY)/)
|
2553
|
-
next if l.empty? or l.match(/^\s+$/)
|
2554
|
-
if firstline
|
2555
|
-
l = "- "+l
|
2556
|
-
firstline = false
|
2557
|
-
else
|
2558
|
-
l = " "+l
|
2559
|
-
end
|
2560
|
-
example += "# "+l+" "+"\n"
|
2561
|
-
}
|
2562
|
-
example += "# \n#\n" if complexity == "simple"
|
2563
|
-
}
|
2564
|
-
schema["properties"][name]["items"]["description"] ||= ""
|
2565
|
-
if !schema["properties"][name]["items"]["description"].empty?
|
2566
|
-
schema["properties"][name]["items"]["description"] += "\n"
|
2567
|
-
end
|
2568
|
-
schema["properties"][name]["items"]["description"] += example
|
2569
|
-
end
|
2570
|
-
end
|
2571
|
-
end
|
2572
|
-
|
2573
|
-
if !schema["required"].nil? and schema["required"].include?(name)
|
2574
|
-
printme << "# @!group Required parameters" if !req
|
2575
|
-
req = true
|
2576
|
-
else
|
2577
|
-
if req
|
2578
|
-
printme << "# @!endgroup"
|
2579
|
-
printme << "# @!group Optional parameters"
|
2580
|
-
end
|
2581
|
-
req = false
|
2582
|
-
end
|
2583
|
-
|
2584
|
-
printme << self.printSchema(kitten_rb, class_hierarchy+ [name], prop, false, req, prefix: schema["prefix"])
|
2585
|
-
}
|
2586
|
-
printme << "# @!endgroup"
|
2587
|
-
end
|
2588
|
-
|
2589
|
-
tabs = 1
|
2590
|
-
class_hierarchy.each { |classname|
|
2591
|
-
if classname == class_hierarchy.last and !schema['description'].nil?
|
2592
|
-
kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "# #{schema['description']}\n"
|
2593
|
-
end
|
2594
|
-
kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "class #{classname}"
|
2595
|
-
tabs = tabs + 1
|
2596
|
-
}
|
2597
|
-
printme.each { |lines|
|
2598
|
-
if !lines.nil? and lines.is_a?(String)
|
2599
|
-
lines.lines.each { |line|
|
2600
|
-
kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + line
|
2601
|
-
}
|
2602
|
-
end
|
2603
|
-
}
|
2604
|
-
|
2605
|
-
class_hierarchy.each { |classname|
|
2606
|
-
tabs = tabs - 1
|
2607
|
-
kitten_rb.puts ["\t"].cycle(tabs).to_a.join('') + "end"
|
2608
|
-
}
|
2609
|
-
|
2610
|
-
# And now that we've dealt with our children, pass our own rendered
|
2611
|
-
# commentary back up to our caller.
|
2612
|
-
name = class_hierarchy.last
|
2613
|
-
if in_array
|
2614
|
-
type = "Array<#{class_hierarchy.join("::")}>"
|
2615
|
-
else
|
2616
|
-
type = class_hierarchy.join("::")
|
2617
|
-
end
|
2618
|
-
|
2619
|
-
docstring = "\n"
|
2620
|
-
docstring = docstring + "# **REQUIRED**\n" if required
|
2621
|
-
docstring = docstring + "# **"+schema["prefix"]+"**\n" if schema["prefix"]
|
2622
|
-
docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
|
2623
|
-
docstring = docstring + "#\n"
|
2624
|
-
docstring = docstring + "# @return [#{type}]\n"
|
2625
|
-
docstring = docstring + "# @see #{class_hierarchy.join("::")}\n"
|
2626
|
-
docstring = docstring + "attr_accessor :#{name}"
|
2627
|
-
return docstring
|
2628
|
-
|
2629
|
-
elsif schema["type"] == "array"
|
2630
|
-
return self.printSchema(kitten_rb, class_hierarchy, schema['items'], true, required, prefix: prefix)
|
2631
|
-
else
|
2632
|
-
name = class_hierarchy.last
|
2633
|
-
if schema['type'].nil?
|
2634
|
-
MU.log "Couldn't determine schema type in #{class_hierarchy.join(" => ")}", MU::WARN, details: schema
|
2635
|
-
return nil
|
2636
|
-
end
|
2637
|
-
if in_array
|
2638
|
-
type = "Array<#{schema['type'].capitalize}>"
|
2639
|
-
else
|
2640
|
-
type = schema['type'].capitalize
|
2641
|
-
end
|
2642
|
-
docstring = "\n"
|
2643
|
-
|
2644
|
-
prefixes = []
|
2645
|
-
prefixes << "# **REQUIRED**" if required and schema['default'].nil?
|
2646
|
-
prefixes << "# **"+schema["prefix"]+"**" if schema["prefix"]
|
2647
|
-
prefixes << "# **Default: `#{schema['default']}`**" if !schema['default'].nil?
|
2648
|
-
if !schema['enum'].nil? and !schema["enum"].empty?
|
2649
|
-
prefixes << "# **Must be one of: `#{schema['enum'].join(', ')}`**"
|
2650
|
-
elsif !schema['pattern'].nil?
|
2651
|
-
# XXX unquoted regex chars confuse the hell out of YARD. How do we
|
2652
|
-
# quote {}[] etc in YARD-speak?
|
2653
|
-
prefixes << "# **Must match pattern `#{schema['pattern'].gsub(/\n/, "\n#")}`**"
|
2654
|
-
end
|
2655
|
-
|
2656
|
-
if prefixes.size > 0
|
2657
|
-
docstring += prefixes.join(",\n")
|
2658
|
-
if schema['description'] and schema['description'].size > 1
|
2659
|
-
docstring += " - "
|
2660
|
-
end
|
2661
|
-
docstring += "\n"
|
2662
|
-
end
|
2663
|
-
|
2664
|
-
docstring = docstring + "# #{schema['description'].gsub(/\n/, "\n#")}\n" if !schema['description'].nil?
|
2665
|
-
docstring = docstring + "#\n"
|
2666
|
-
docstring = docstring + "# @return [#{type}]\n"
|
2667
|
-
docstring = docstring + "attr_accessor :#{name}"
|
2668
|
-
|
2669
|
-
return docstring
|
2670
|
-
end
|
2671
|
-
|
2672
|
-
end
|
2673
|
-
|
2674
|
-
def self.dependencies_primitive
|
2675
|
-
{
|
2676
|
-
"type" => "array",
|
2677
|
-
"items" => {
|
2678
|
-
"type" => "object",
|
2679
|
-
"description" => "Declare other objects which this resource requires. This resource will wait until the others are available to create itself.",
|
2680
|
-
"required" => ["name", "type"],
|
2681
|
-
"additionalProperties" => false,
|
2682
|
-
"properties" => {
|
2683
|
-
"name" => {"type" => "string"},
|
2684
|
-
"type" => {
|
2685
|
-
"type" => "string",
|
2686
|
-
"enum" => MU::Cloud.resource_types.values.map { |v| v[:cfg_name] }
|
2687
|
-
},
|
2688
|
-
"phase" => {
|
2689
|
-
"type" => "string",
|
2690
|
-
"description" => "Which part of the creation process of the resource we depend on should we wait for before starting our own creation? Defaults are usually sensible, but sometimes you want, say, a Server to wait on another Server to be completely ready (through its groom phase) before starting up.",
|
2691
|
-
"enum" => ["create", "groom"]
|
2692
|
-
},
|
2693
|
-
"no_create_wait" => {
|
2694
|
-
"type" => "boolean",
|
2695
|
-
"default" => false,
|
2696
|
-
"description" => "By default, it's assumed that we want to wait on our parents' creation phase, in addition to whatever is declared in this stanza. Setting this flag will bypass waiting on our parent resource's creation, so that our create or groom phase can instead depend only on the parent's groom phase. "
|
2697
|
-
}
|
2698
|
-
}
|
2699
|
-
}
|
2700
|
-
}
|
2701
|
-
end
|
2702
|
-
|
2703
|
-
CIDR_PATTERN = "^\\d+\\.\\d+\\.\\d+\\.\\d+\/[0-9]{1,2}$"
|
2704
|
-
CIDR_DESCRIPTION = "CIDR-formatted IP block, e.g. 1.2.3.4/32"
|
2705
|
-
CIDR_PRIMITIVE = {
|
2706
|
-
"type" => "string",
|
2707
|
-
"pattern" => CIDR_PATTERN,
|
2708
|
-
"description" => CIDR_DESCRIPTION
|
2709
|
-
}
|
2710
|
-
|
2711
|
-
# Have a default value available for config schema elements that take an
|
2712
|
-
# email address.
|
2713
|
-
# @return [String]
|
2714
|
-
def self.notification_email
|
2715
|
-
if MU.chef_user == "mu"
|
2716
|
-
ENV['MU_ADMIN_EMAIL']
|
2717
|
-
else
|
2718
|
-
MU.userEmail
|
2719
|
-
end
|
2720
|
-
end
|
2721
|
-
|
2722
|
-
# Load and validate the schema for an individual resource class, optionally
|
2723
|
-
# merging cloud-specific schema components.
|
2724
|
-
# @param type [String]: The resource type to load
|
2725
|
-
# @param cloud [String]: A specific cloud, whose implementation's schema of this resource we will merge
|
2726
|
-
# @return [Hash]
|
2727
|
-
def self.loadResourceSchema(type, cloud: nil)
|
2728
|
-
valid = true
|
2729
|
-
shortclass, cfg_name, cfg_plural, classname = MU::Cloud.getResourceNames(type)
|
2730
|
-
schemaclass = Object.const_get("MU").const_get("Config").const_get(shortclass)
|
2731
|
-
|
2732
|
-
[:schema, :validate].each { |method|
|
2733
|
-
if !schemaclass.respond_to?(method)
|
2734
|
-
MU.log "MU::Config::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
|
2735
|
-
return [nil, false] if method == :schema
|
2736
|
-
valid = false
|
2737
|
-
end
|
2738
|
-
}
|
2739
|
-
|
2740
|
-
schema = schemaclass.schema.dup
|
2741
|
-
|
2742
|
-
schema["properties"]["virtual_name"] = {
|
2743
|
-
"description" => "Internal use.",
|
2744
|
-
"type" => "string"
|
2745
|
-
}
|
2746
|
-
schema["properties"]["dependencies"] = MU::Config.dependencies_primitive
|
2747
|
-
schema["properties"]["cloud"] = MU::Config.cloud_primitive
|
2748
|
-
schema["properties"]["credentials"] = MU::Config.credentials_primitive
|
2749
|
-
schema["title"] = type.to_s
|
2750
|
-
|
2751
|
-
if cloud
|
2752
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(shortclass)
|
2753
|
-
|
2754
|
-
if cloudclass.respond_to?(:schema)
|
2755
|
-
reqd, cloudschema = cloudclass.schema
|
2756
|
-
cloudschema.each { |key, cfg|
|
2757
|
-
if schema["properties"][key]
|
2758
|
-
schemaMerge(schema["properties"][key], cfg, cloud)
|
2759
|
-
else
|
2760
|
-
schema["properties"][key] = cfg.dup
|
2761
|
-
end
|
2762
|
-
}
|
2763
|
-
else
|
2764
|
-
MU.log "MU::Cloud::#{cloud}::#{type}.#{method.to_s} doesn't seem to be implemented", MU::ERR
|
2765
|
-
valid = false
|
2766
|
-
end
|
2767
|
-
|
2768
|
-
end
|
2769
|
-
|
2770
|
-
return [schema, valid]
|
2771
|
-
end
|
2772
|
-
|
2773
|
-
@@schema = {
|
2774
|
-
"$schema" => "http://json-schema.org/draft-04/schema#",
|
2775
|
-
"title" => "MU Application",
|
2776
|
-
"type" => "object",
|
2777
|
-
"description" => "A MU application stack, consisting of at least one resource.",
|
2778
|
-
"required" => ["admins", "appname"],
|
2779
|
-
"properties" => {
|
2780
|
-
"appname" => {
|
2781
|
-
"type" => "string",
|
2782
|
-
"description" => "A name for your application stack. Should be short, but easy to differentiate from other applications.",
|
2783
|
-
},
|
2784
|
-
"scrub_mu_isms" => {
|
2785
|
-
"type" => "boolean",
|
2786
|
-
"description" => "When 'cloud' is set to 'CloudFormation,' use this flag to strip out Mu-specific artifacts (tags, standard userdata, naming conventions, etc) to yield a clean, source-agnostic template. Setting this flag here will override declarations in individual resources."
|
2787
|
-
},
|
2788
|
-
"project" => {
|
2789
|
-
"type" => "string",
|
2790
|
-
"description" => "**GOOGLE ONLY**: The project into which to deploy resources"
|
2791
|
-
},
|
2792
|
-
"billing_acct" => {
|
2793
|
-
"type" => "string",
|
2794
|
-
"description" => "**GOOGLE ONLY**: Billing account ID to associate with a newly-created Google Project. If not specified, will attempt to locate a billing account associated with the default project for our credentials.",
|
2795
|
-
},
|
2796
|
-
"region" => MU::Config.region_primitive,
|
2797
|
-
"credentials" => MU::Config.credentials_primitive,
|
2798
|
-
"us_only" => {
|
2799
|
-
"type" => "boolean",
|
2800
|
-
"description" => "For resources which span regions, restrict to regions inside the United States",
|
2801
|
-
"default" => false
|
2802
|
-
},
|
2803
|
-
"conditions" => {
|
2804
|
-
"type" => "array",
|
2805
|
-
"items" => {
|
2806
|
-
"type" => "object",
|
2807
|
-
"required" => ["name", "cloudcode"],
|
2808
|
-
"description" => "CloudFormation-specific. Define Conditions as in http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/conditions-section-structure.html. Arguments must use the cloudCode() macro.",
|
2809
|
-
"properties" => {
|
2810
|
-
"name" => { "required" => true, "type" => "string" },
|
2811
|
-
"cloudcode" => { "required" => true, "type" => "string" },
|
2812
|
-
}
|
2813
|
-
}
|
2814
|
-
},
|
2815
|
-
"parameters" => {
|
2816
|
-
"type" => "array",
|
2817
|
-
"items" => {
|
2818
|
-
"type" => "object",
|
2819
|
-
"title" => "parameter",
|
2820
|
-
"description" => "Parameters to be substituted elsewhere in this Basket of Kittens as ERB variables (<%= varname %>)",
|
2821
|
-
"additionalProperties" => false,
|
2822
|
-
"properties" => {
|
2823
|
-
"name" => { "required" => true, "type" => "string" },
|
2824
|
-
"default" => { "type" => "string" },
|
2825
|
-
"list_of" => {
|
2826
|
-
"type" => "string",
|
2827
|
-
"description" => "Treat the value as a comma-separated list of values with this key name, equivalent to CloudFormation's various List<> types. For example, set to 'subnet_id' to pass values as an array of subnet identifiers as the 'subnets' argument of a VPC stanza."
|
2828
|
-
},
|
2829
|
-
"prettyname" => {
|
2830
|
-
"type" => "string",
|
2831
|
-
"description" => "An alternative name to use when generating parameter fields in, for example, CloudFormation templates"
|
2832
|
-
},
|
2833
|
-
"description" => {"type" => "string"},
|
2834
|
-
"cloudtype" => {
|
2835
|
-
"type" => "string",
|
2836
|
-
"description" => "A platform-specific string describing the type of validation to use for this parameter. E.g. when generating a CloudFormation template, set to AWS::EC2::Image::Id to validate input as an AMI identifier."
|
2837
|
-
},
|
2838
|
-
"required" => {
|
2839
|
-
"type" => "boolean",
|
2840
|
-
"default" => true
|
2841
|
-
},
|
2842
|
-
"valid_values" => {
|
2843
|
-
"type" => "array",
|
2844
|
-
"description" => "List of valid values for this parameter. Can only be a list of static strings, for now.",
|
2845
|
-
"items" => {
|
2846
|
-
"type" => "string"
|
2847
|
-
}
|
2848
|
-
}
|
2849
|
-
}
|
2850
|
-
}
|
2851
|
-
},
|
2852
|
-
# TODO availability zones (or an array thereof)
|
2853
|
-
|
2854
|
-
"admins" => {
|
2855
|
-
"type" => "array",
|
2856
|
-
"items" => {
|
2857
|
-
"type" => "object",
|
2858
|
-
"title" => "admin",
|
2859
|
-
"description" => "Administrative contacts for this application stack. Will be automatically set to invoking Mu user, if not specified.",
|
2860
|
-
"required" => ["name", "email"],
|
2861
|
-
"additionalProperties" => false,
|
2862
|
-
"properties" => {
|
2863
|
-
"name" => {"type" => "string"},
|
2864
|
-
"email" => {"type" => "string"},
|
2865
|
-
"public_key" => {
|
2866
|
-
"type" => "string",
|
2867
|
-
"description" => "An OpenSSH-style public key string. This will be installed on all instances created in this deployment."
|
2868
|
-
}
|
2869
|
-
}
|
2870
|
-
},
|
2871
|
-
"minItems" => 1,
|
2872
|
-
"uniqueItems" => true
|
2873
|
-
}
|
2874
|
-
},
|
2875
|
-
"additionalProperties" => false
|
2876
|
-
}
|
2877
|
-
|
2878
1230
|
failed = []
|
2879
1231
|
|
2880
1232
|
# Load all of the config stub files at the Ruby level
|
2881
1233
|
MU::Cloud.resource_types.each_pair { |type, cfg|
|
2882
1234
|
begin
|
2883
1235
|
require "mu/config/#{cfg[:cfg_name]}"
|
2884
|
-
rescue LoadError
|
1236
|
+
rescue LoadError
|
2885
1237
|
# raise MuError, "MU::Config implemention of #{type} missing from modules/mu/config/#{cfg[:cfg_name]}.rb"
|
2886
1238
|
MU.log "MU::Config::#{type} stub class is missing", MU::ERR
|
2887
1239
|
failed << type
|
@@ -2889,7 +1241,6 @@ $CONFIGURABLES
|
|
2889
1241
|
end
|
2890
1242
|
}
|
2891
1243
|
|
2892
|
-
|
2893
1244
|
MU::Cloud.resource_types.each_pair { |type, cfg|
|
2894
1245
|
begin
|
2895
1246
|
schema, valid = loadResourceSchema(type)
|