cloud-mu 3.1.5 → 3.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/tasks/main.yml +16 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +3 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +2 -0
- data/extras/image-generators/AWS/win2k16.yaml +2 -0
- data/extras/image-generators/AWS/win2k19.yaml +2 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +86 -98
- data/modules/mu/adoption.rb +373 -58
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1733
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +123 -81
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +90 -90
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +34 -20
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +69 -4
- data/modules/mu/groomers/chef.rb +51 -4
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/mommacat.rb +160 -874
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +497 -0
- data/modules/mu/mommacat/storage.rb +252 -194
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +258 -57
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +6 -6
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +41 -24
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +122 -92
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -163,7 +163,7 @@ module MU
|
|
163
163
|
dnsthread = Thread.new {
|
164
164
|
if !MU::Cloud::AWS.isGovCloud?
|
165
165
|
MU.dupGlobals(parent_thread_id)
|
166
|
-
generic_mu_dns = MU::Cloud
|
166
|
+
generic_mu_dns = MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: @mu_name, target: "#{lb.dns_name}.", cloudclass: MU::Cloud::LoadBalancer, sync_wait: @config['dns_sync_wait'])
|
167
167
|
end
|
168
168
|
}
|
169
169
|
|
@@ -239,16 +239,35 @@ module MU
|
|
239
239
|
end
|
240
240
|
end
|
241
241
|
|
242
|
+
redirect_block = Proc.new { |r|
|
243
|
+
{
|
244
|
+
:protocol => r['protocol'],
|
245
|
+
:port => r['port'].to_s,
|
246
|
+
:host => r['host'],
|
247
|
+
:path => r['path'],
|
248
|
+
:query => r['query'],
|
249
|
+
:status_code => "HTTP_"+r['status_code'].to_s
|
250
|
+
}
|
251
|
+
}
|
252
|
+
|
242
253
|
if !@config['classic']
|
243
254
|
@config["listeners"].each { |l|
|
244
|
-
if
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
255
|
+
action = if l['redirect']
|
256
|
+
{
|
257
|
+
:type => "redirect",
|
258
|
+
:redirect_config => redirect_block.call(l['redirect'])
|
259
|
+
}
|
260
|
+
else
|
261
|
+
if !@targetgroups.has_key?(l['targetgroup'])
|
262
|
+
raise MuError, "Listener in #{@mu_name} configured for target group #{l['targetgroup']}, but I don't have data on a targetgroup by that name"
|
263
|
+
end
|
264
|
+
{
|
249
265
|
:target_group_arn => @targetgroups[l['targetgroup']].target_group_arn,
|
250
266
|
:type => "forward"
|
251
|
-
}
|
267
|
+
}
|
268
|
+
end
|
269
|
+
listen_descriptor = {
|
270
|
+
:default_actions => [ action ],
|
252
271
|
:load_balancer_arn => lb.load_balancer_arn,
|
253
272
|
:port => l['lb_port'],
|
254
273
|
:protocol => l['lb_protocol']
|
@@ -276,10 +295,17 @@ module MU
|
|
276
295
|
:actions => []
|
277
296
|
}
|
278
297
|
rule['actions'].each { |a|
|
279
|
-
rule_descriptor[:actions] <<
|
280
|
-
|
281
|
-
|
282
|
-
|
298
|
+
rule_descriptor[:actions] << if a['action'] == "forward"
|
299
|
+
{
|
300
|
+
:target_group_arn => @targetgroups[a['targetgroup']].target_group_arn,
|
301
|
+
:type => a['action']
|
302
|
+
}
|
303
|
+
elsif a['action'] == "redirect"
|
304
|
+
{
|
305
|
+
:redirect_config => redirect_block.call(rule['redirect']),
|
306
|
+
:type => a['action']
|
307
|
+
}
|
308
|
+
end
|
283
309
|
}
|
284
310
|
MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).create_rule(rule_descriptor)
|
285
311
|
}
|
@@ -536,7 +562,7 @@ module MU
|
|
536
562
|
}
|
537
563
|
end
|
538
564
|
if !MU::Cloud::AWS.isGovCloud?
|
539
|
-
MU::Cloud
|
565
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: cloud_desc.dns_name)
|
540
566
|
end
|
541
567
|
end
|
542
568
|
|
@@ -557,6 +583,7 @@ module MU
|
|
557
583
|
# Wrapper for cloud_desc method that deals with elb vs. elb2 resources.
|
558
584
|
def cloud_desc(use_cache: true)
|
559
585
|
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
586
|
+
return nil if !@cloud_id
|
560
587
|
if @config['classic']
|
561
588
|
@cloud_desc_cache = MU::Cloud::AWS.elb(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
|
562
589
|
load_balancer_names: [@cloud_id]
|
@@ -566,13 +593,34 @@ module MU
|
|
566
593
|
@cloud_desc_cache = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_load_balancers(
|
567
594
|
names: [@cloud_id]
|
568
595
|
).load_balancers.first
|
569
|
-
if @targetgroups.nil?
|
570
|
-
@deploy.deployment['loadbalancers'].has_key?(@config['name']) and
|
571
|
-
@deploy.deployment['loadbalancers'][@config['name']].has_key?("targetgroups")
|
596
|
+
if @targetgroups.nil?
|
572
597
|
@targetgroups = {}
|
573
|
-
|
574
|
-
|
575
|
-
|
598
|
+
if !@deploy.nil? and
|
599
|
+
@deploy.deployment['loadbalancers'] and
|
600
|
+
@deploy.deployment['loadbalancers'][@config['name']] and
|
601
|
+
@deploy.deployment['loadbalancers'][@config['name']]["targetgroups"]
|
602
|
+
@deploy.deployment['loadbalancers'][@config['name']]["targetgroups"].each_pair { |tg_name, tg_arn|
|
603
|
+
@targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
|
604
|
+
}
|
605
|
+
else
|
606
|
+
pp @config['targetgroups']
|
607
|
+
MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(load_balancer_arn: @cloud_desc_cache.load_balancer_arn).target_groups.each { |tg|
|
608
|
+
tg_name = tg.target_group_name
|
609
|
+
if @config['targetgroups']
|
610
|
+
@config['targetgroups'].each { |tg_cfg|
|
611
|
+
if tg_name = @deploy.getResourceName(tg_cfg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
|
612
|
+
tg_name = tg_cfg['name']
|
613
|
+
break
|
614
|
+
end
|
615
|
+
}
|
616
|
+
end
|
617
|
+
@targetgroups[tg_name] = tg
|
618
|
+
}
|
619
|
+
# @config['targetgroups'].each { |tg|
|
620
|
+
# tg_name = @deploy.getResourceName(tg["name"], max_length: 32, disallowed_chars: /[^A-Za-z0-9-]/)
|
621
|
+
# @targetgroups[tg_name] = MU::Cloud::AWS.elb2(region: @config['region'], credentials: @config['credentials']).describe_target_groups(target_group_arns: [tg_arn]).target_groups.first
|
622
|
+
# }
|
623
|
+
end
|
576
624
|
end
|
577
625
|
|
578
626
|
return @cloud_desc_cache
|
@@ -645,8 +693,8 @@ module MU
|
|
645
693
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
646
694
|
# @param region [String]: The cloud provider region
|
647
695
|
# @return [void]
|
648
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
649
|
-
if (
|
696
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
697
|
+
if (deploy_id.nil? or deploy_id.empty?) and (!flags or !flags["vpc_id"])
|
650
698
|
raise MuError, "Can't touch ELBs without MU-ID or vpc_id flag"
|
651
699
|
end
|
652
700
|
|
@@ -656,7 +704,7 @@ module MU
|
|
656
704
|
# @param region [String]: The cloud provider region
|
657
705
|
# @param ignoremaster [Boolean]: Whether to ignore the MU-MASTER-IP tag
|
658
706
|
# @param classic [Boolean]: Whether to look for a classic ELB instead of an ALB (ELB2)
|
659
|
-
def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false)
|
707
|
+
def self.checkForTagMatch(arn, region, ignoremaster, credentials, classic = false, deploy_id: MU.deploy_id)
|
660
708
|
tags = []
|
661
709
|
if classic
|
662
710
|
tags = MU::Cloud::AWS.elb(credentials: credentials, region: region).describe_tags(
|
@@ -673,7 +721,7 @@ module MU
|
|
673
721
|
if !tags.nil?
|
674
722
|
tags.each { |tag|
|
675
723
|
saw_tags << tag.key
|
676
|
-
muid_match = true if tag.key == "MU-ID" and tag.value ==
|
724
|
+
muid_match = true if tag.key == "MU-ID" and tag.value == deploy_id
|
677
725
|
mumaster_match = true if tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
678
726
|
}
|
679
727
|
end
|
@@ -699,14 +747,14 @@ module MU
|
|
699
747
|
matched = true if lb.vpc_id == flags['vpc_id']
|
700
748
|
else
|
701
749
|
if classic
|
702
|
-
matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic)
|
750
|
+
matched = self.checkForTagMatch(lb.load_balancer_name, region, ignoremaster, credentials, classic, deploy_id: deploy_id)
|
703
751
|
else
|
704
|
-
matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic)
|
752
|
+
matched = self.checkForTagMatch(lb.load_balancer_arn, region, ignoremaster, credentials, classic, deploy_id: deploy_id)
|
705
753
|
end
|
706
754
|
end
|
707
755
|
if matched
|
708
756
|
if !MU::Cloud::AWS.isGovCloud?
|
709
|
-
MU::Cloud
|
757
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: lb.load_balancer_name, target: lb.dns_name, cloudclass: MU::Cloud::LoadBalancer, delete: true) if !noop
|
710
758
|
end
|
711
759
|
if classic
|
712
760
|
MU.log "Removing Elastic Load Balancer #{lb.load_balancer_name}"
|
@@ -747,7 +795,7 @@ module MU
|
|
747
795
|
|
748
796
|
|
749
797
|
tgs.each { |tg|
|
750
|
-
if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials)
|
798
|
+
if self.checkForTagMatch(tg.target_group_arn, region, ignoremaster, credentials, deploy_id: deploy_id)
|
751
799
|
MU.log "Removing Load Balancer Target Group #{tg.target_group_name}"
|
752
800
|
retries = 0
|
753
801
|
begin
|
@@ -793,26 +841,7 @@ module MU
|
|
793
841
|
}
|
794
842
|
}
|
795
843
|
},
|
796
|
-
"ingress_rules" =>
|
797
|
-
"items" => {
|
798
|
-
"properties" => {
|
799
|
-
"sgs" => {
|
800
|
-
"type" => "array",
|
801
|
-
"items" => {
|
802
|
-
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
803
|
-
"type" => "string"
|
804
|
-
}
|
805
|
-
},
|
806
|
-
"lbs" => {
|
807
|
-
"type" => "array",
|
808
|
-
"items" => {
|
809
|
-
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
810
|
-
"type" => "string"
|
811
|
-
}
|
812
|
-
}
|
813
|
-
}
|
814
|
-
}
|
815
|
-
}
|
844
|
+
"ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
|
816
845
|
}
|
817
846
|
[toplevel_required, schema]
|
818
847
|
end
|
@@ -830,7 +859,7 @@ module MU
|
|
830
859
|
(!listener["ssl_certificate_id"].nil? and !listener["ssl_certificate_id"].empty?)
|
831
860
|
if lb['cloud'] != "CloudFormation" # XXX or maybe do this anyway?
|
832
861
|
begin
|
833
|
-
listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region'])
|
862
|
+
listener["ssl_certificate_id"] = MU::Cloud::AWS.findSSLCertificate(name: listener["ssl_certificate_name"].to_s, id: listener["ssl_certificate_id"].to_s, region: lb['region']).first
|
834
863
|
rescue MuError
|
835
864
|
ok = false
|
836
865
|
next
|
@@ -923,6 +952,7 @@ module MU
|
|
923
952
|
return matches
|
924
953
|
|
925
954
|
end
|
955
|
+
|
926
956
|
end
|
927
957
|
end
|
928
958
|
end
|
@@ -202,14 +202,14 @@ module MU
|
|
202
202
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
203
203
|
# @param region [String]: The cloud provider region
|
204
204
|
# @return [void]
|
205
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
205
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
206
206
|
MU.log "AWS::Log.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
207
207
|
MU.log "Placeholder: AWS Log artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
208
208
|
|
209
209
|
log_groups = self.find(credentials: credentials, region: region).values
|
210
210
|
if !log_groups.empty?
|
211
211
|
log_groups.each{ |lg|
|
212
|
-
if lg.log_group_name.match(
|
212
|
+
if lg.log_group_name.match(deploy_id)
|
213
213
|
log_streams = MU::Cloud::AWS.cloudwatchlogs(credentials: credentials, region: region).describe_log_streams(log_group_name: lg.log_group_name).log_streams
|
214
214
|
if !log_streams.empty?
|
215
215
|
log_streams.each{ |ls|
|
@@ -232,9 +232,9 @@ module MU
|
|
232
232
|
|
233
233
|
# unless noop
|
234
234
|
# MU::Cloud::AWS.iam(credentials: credentials).list_roles.roles.each{ |role|
|
235
|
-
# match_string = "#{
|
236
|
-
# Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud
|
237
|
-
# MU::Cloud
|
235
|
+
# match_string = "#{deploy_id}.*CloudTrail"
|
236
|
+
# Maybe we should have a more generic way to delete IAM profiles and policies. The call itself should be moved from MU::Cloud.resourceClass("AWS", "Server").
|
237
|
+
# MU::Cloud.resourceClass("AWS", "Server").removeIAMProfile(role.role_name) if role.role_name.match(match_string)
|
238
238
|
# }
|
239
239
|
# end
|
240
240
|
end
|
@@ -80,6 +80,7 @@ module MU
|
|
80
80
|
# @return [Hash]: AWS doesn't return anything but the SQS URL, so supplement with attributes
|
81
81
|
def cloud_desc(use_cache: true)
|
82
82
|
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
83
|
+
return nil if !@cloud_id
|
83
84
|
|
84
85
|
if !@cloud_id
|
85
86
|
resp = MU::Cloud::AWS.sqs(region: @config['region'], credentials: @config['credentials']).list_queues(
|
@@ -133,12 +134,12 @@ module MU
|
|
133
134
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
134
135
|
# @param region [String]: The cloud provider region
|
135
136
|
# @return [void]
|
136
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
137
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
137
138
|
MU.log "AWS::MsgQueue.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
138
139
|
MU.log "Placeholder: AWS MsgQueue artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
139
140
|
|
140
141
|
resp = MU::Cloud::AWS.sqs(credentials: credentials, region: region).list_queues(
|
141
|
-
queue_name_prefix:
|
142
|
+
queue_name_prefix: deploy_id
|
142
143
|
)
|
143
144
|
if resp and resp.queue_urls
|
144
145
|
threads = []
|
@@ -194,7 +195,15 @@ module MU
|
|
194
195
|
|
195
196
|
# Go fetch its attributes
|
196
197
|
fetch = if args[:cloud_id]
|
197
|
-
|
198
|
+
if args[:cloud_id] !~ /^https?:\/\//
|
199
|
+
[begin
|
200
|
+
MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).get_queue_url(queue_name: args[:cloud_id]).queue_url
|
201
|
+
rescue Aws::SQS::Errors::NonExistentQueue
|
202
|
+
return found
|
203
|
+
end]
|
204
|
+
else
|
205
|
+
[args[:cloud_id]]
|
206
|
+
end
|
198
207
|
else
|
199
208
|
resp = MU::Cloud::AWS.sqs(region: args[:region], credentials: args[:credentials]).list_queues
|
200
209
|
resp.queue_urls
|
@@ -327,16 +336,10 @@ module MU
|
|
327
336
|
failq.delete("failqueue")
|
328
337
|
ok = false if !configurator.insertKitten(failq, "msg_queues")
|
329
338
|
queue['failqueue']['name'] = failq['name']
|
330
|
-
queue[
|
331
|
-
"name" => failq['name'],
|
332
|
-
"type" => "msg_queue"
|
333
|
-
}
|
339
|
+
MU::Config.addDependency(queue, failq["name"], "msg_queue")
|
334
340
|
else
|
335
341
|
if configurator.haveLitterMate?(queue['failqueue']['name'], "msg_queue")
|
336
|
-
queue['
|
337
|
-
"name" => queue['failqueue']['name'],
|
338
|
-
"type" => "msg_queue"
|
339
|
-
}
|
342
|
+
MU::Config.addDependency(queue, queue['failqueue']['name'], "msg_queue")
|
340
343
|
else
|
341
344
|
failq = MU::Cloud::AWS::MsgQueue.find(cloud_id: queue['failqueue']['name'])
|
342
345
|
if !failq
|
@@ -47,7 +47,11 @@ module MU
|
|
47
47
|
}
|
48
48
|
end
|
49
49
|
|
50
|
+
type_map = {}
|
51
|
+
|
50
52
|
@config['attributes'].each { |attr|
|
53
|
+
type_map[attr['name']] = attr['type']
|
54
|
+
|
51
55
|
params[:attribute_definitions] << {
|
52
56
|
:attribute_name => attr['name'],
|
53
57
|
:attribute_type => attr['type']
|
@@ -67,6 +71,11 @@ module MU
|
|
67
71
|
}
|
68
72
|
end
|
69
73
|
}
|
74
|
+
# apparently the HASH key always has to be before RANGE, so sort it
|
75
|
+
# lexically by that field and call it a day
|
76
|
+
params[:key_schema].sort! { |a, b|
|
77
|
+
a[:key_type] <=> b[:key_type]
|
78
|
+
}
|
70
79
|
|
71
80
|
if @config['secondary_indexes']
|
72
81
|
@config['secondary_indexes'].each { |idx|
|
@@ -99,7 +108,11 @@ module MU
|
|
99
108
|
}
|
100
109
|
end
|
101
110
|
|
102
|
-
|
111
|
+
if @tags
|
112
|
+
params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
|
113
|
+
end
|
114
|
+
|
115
|
+
MU.log "Creating DynamoDB table #{@mu_name}", MU::NOTICE, details: params
|
103
116
|
|
104
117
|
resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
|
105
118
|
@cloud_id = @mu_name
|
@@ -109,8 +122,24 @@ module MU
|
|
109
122
|
sleep 5 if resp.table.table_status == "CREATING"
|
110
123
|
end while resp.table.table_status == "CREATING"
|
111
124
|
|
112
|
-
|
113
125
|
tagTable if !@config['scrub_mu_isms']
|
126
|
+
|
127
|
+
if @config['populate'] and !@config['populate'].empty?
|
128
|
+
MU.log "Preloading #{@mu_name} with #{@config['populate'].size.to_s} items"
|
129
|
+
items_to_write = @config['populate'].dup
|
130
|
+
begin
|
131
|
+
batch = items_to_write.slice!(0, (items_to_write.length >= 25 ? 25 : items_to_write.length))
|
132
|
+
begin
|
133
|
+
MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).batch_write_item(
|
134
|
+
request_items: {
|
135
|
+
@cloud_id => batch.map { |i| { put_request: { item: i } } }
|
136
|
+
}
|
137
|
+
)
|
138
|
+
rescue ::Aws::DynamoDB::Errors::ValidationException => e
|
139
|
+
MU.log e.message, MU::ERR, details: item
|
140
|
+
end
|
141
|
+
end while !items_to_write.empty?
|
142
|
+
end
|
114
143
|
end
|
115
144
|
|
116
145
|
# Apply tags to this DynamoDB table
|
@@ -143,6 +172,7 @@ module MU
|
|
143
172
|
# Called automatically by {MU::Deploy#createResources}
|
144
173
|
def groom
|
145
174
|
tagTable if !@config['scrub_mu_isms']
|
175
|
+
MU.log "NoSQL Table #{@config['name']}: #{@cloud_id}", MU::SUMMARY
|
146
176
|
end
|
147
177
|
|
148
178
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
@@ -163,7 +193,7 @@ module MU
|
|
163
193
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
164
194
|
# @param region [String]: The cloud provider region
|
165
195
|
# @return [void]
|
166
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
196
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
167
197
|
MU.log "AWS::NoSQLDb.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
168
198
|
|
169
199
|
resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables
|
@@ -183,7 +213,7 @@ module MU
|
|
183
213
|
deploy_match = false
|
184
214
|
master_match = false
|
185
215
|
tags.tags.each { |tag|
|
186
|
-
if tag.key == "MU-ID" and tag.value ==
|
216
|
+
if tag.key == "MU-ID" and tag.value == deploy_id
|
187
217
|
deploy_match = true
|
188
218
|
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
189
219
|
master_match = true
|
@@ -214,7 +244,8 @@ module MU
|
|
214
244
|
# Return the metadata for this user cofiguration
|
215
245
|
# @return [Hash]
|
216
246
|
def notify
|
217
|
-
|
247
|
+
return nil if !@cloud_id or !cloud_desc(use_cache: false)
|
248
|
+
MU.structToHash(cloud_desc, stringify_keys: true)
|
218
249
|
end
|
219
250
|
|
220
251
|
# Locate an existing DynamoDB table
|
@@ -244,6 +275,59 @@ module MU
|
|
244
275
|
found
|
245
276
|
end
|
246
277
|
|
278
|
+
# Reverse-map our cloud description into a runnable config hash.
|
279
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
280
|
+
# calculate our own accordingly based on what's live in the cloud.
|
281
|
+
def toKitten(**_args)
|
282
|
+
bok = {
|
283
|
+
"cloud" => "AWS",
|
284
|
+
"credentials" => @config['credentials'],
|
285
|
+
"cloud_id" => @cloud_id,
|
286
|
+
"region" => @config['region']
|
287
|
+
}
|
288
|
+
|
289
|
+
if !cloud_desc
|
290
|
+
MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
|
291
|
+
return nil
|
292
|
+
end
|
293
|
+
bok['name'] = cloud_desc.table_name
|
294
|
+
bok['read_capacity'] = cloud_desc.provisioned_throughput.read_capacity_units
|
295
|
+
bok['write_capacity'] = cloud_desc.provisioned_throughput.write_capacity_units
|
296
|
+
|
297
|
+
|
298
|
+
cloud_desc.attribute_definitions.each { |attr|
|
299
|
+
bok['attributes'] ||= []
|
300
|
+
newattr = {
|
301
|
+
"name" => attr.attribute_name,
|
302
|
+
"type" => attr.attribute_type
|
303
|
+
}
|
304
|
+
if cloud_desc.key_schema
|
305
|
+
cloud_desc.key_schema.each { |key|
|
306
|
+
next if key.attribute_name == attr.attribute_name
|
307
|
+
if key.key_type == "RANGE"
|
308
|
+
newattr["primary_partition"] = true
|
309
|
+
elsif key.key_type == "HASH"
|
310
|
+
newattr["primary_sort"] = true
|
311
|
+
end
|
312
|
+
}
|
313
|
+
end
|
314
|
+
bok['attributes'] << newattr
|
315
|
+
}
|
316
|
+
|
317
|
+
if cloud_desc.stream_specification and cloud_desc.stream_specification.stream_enabled
|
318
|
+
|
319
|
+
bok['stream'] = cloud_desc.stream_specification.stream_view_type
|
320
|
+
# cloud_desc.latest_stream_arn
|
321
|
+
# MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams
|
322
|
+
end
|
323
|
+
|
324
|
+
bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @config['region']).scan(
|
325
|
+
table_name: @cloud_id
|
326
|
+
).items
|
327
|
+
|
328
|
+
bok
|
329
|
+
end
|
330
|
+
|
247
331
|
# Cloud-specific configuration properties.
|
248
332
|
# @param _config [MU::Config]: The calling MU::Config object
|
249
333
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
@@ -252,6 +336,13 @@ module MU
|
|
252
336
|
|
253
337
|
|
254
338
|
schema = {
|
339
|
+
"populate" => {
|
340
|
+
"type" => "array",
|
341
|
+
"items" => {
|
342
|
+
"type" => "object",
|
343
|
+
"description" => "Key-value pairs, compatible with the +attributes+ schema, with which to populate this +table+ during its initial creation."
|
344
|
+
}
|
345
|
+
},
|
255
346
|
"attributes" => {
|
256
347
|
"type" => "array",
|
257
348
|
"minItems" => 1,
|