cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +16 -54
  3. data/Berksfile.lock +14 -62
  4. data/bin/mu-aws-setup +131 -108
  5. data/bin/mu-configure +311 -74
  6. data/bin/mu-gcp-setup +84 -62
  7. data/bin/mu-load-config.rb +46 -2
  8. data/bin/mu-self-update +11 -9
  9. data/bin/mu-upload-chef-artifacts +4 -4
  10. data/{mu.gemspec → cloud-mu.gemspec} +2 -2
  11. data/cookbooks/awscli/Berksfile +8 -0
  12. data/cookbooks/mu-activedirectory/Berksfile +11 -0
  13. data/cookbooks/mu-firewall/Berksfile +9 -0
  14. data/cookbooks/mu-firewall/metadata.rb +1 -1
  15. data/cookbooks/mu-glusterfs/Berksfile +10 -0
  16. data/cookbooks/mu-jenkins/Berksfile +14 -0
  17. data/cookbooks/mu-master/Berksfile +23 -0
  18. data/cookbooks/mu-master/attributes/default.rb +1 -1
  19. data/cookbooks/mu-master/metadata.rb +2 -2
  20. data/cookbooks/mu-master/recipes/default.rb +1 -1
  21. data/cookbooks/mu-master/recipes/init.rb +7 -3
  22. data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
  23. data/cookbooks/mu-mongo/Berksfile +10 -0
  24. data/cookbooks/mu-openvpn/Berksfile +11 -0
  25. data/cookbooks/mu-php54/Berksfile +13 -0
  26. data/cookbooks/mu-splunk/Berksfile +10 -0
  27. data/cookbooks/mu-tools/Berksfile +21 -0
  28. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  29. data/cookbooks/mu-utility/Berksfile +9 -0
  30. data/cookbooks/mu-utility/metadata.rb +2 -1
  31. data/cookbooks/nagios/Berksfile +7 -4
  32. data/cookbooks/s3fs/Berksfile +9 -0
  33. data/environments/dev.json +6 -6
  34. data/environments/prod.json +6 -6
  35. data/modules/mu.rb +20 -42
  36. data/modules/mu/cleanup.rb +102 -100
  37. data/modules/mu/cloud.rb +90 -28
  38. data/modules/mu/clouds/aws.rb +449 -218
  39. data/modules/mu/clouds/aws/alarm.rb +29 -17
  40. data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
  41. data/modules/mu/clouds/aws/collection.rb +25 -18
  42. data/modules/mu/clouds/aws/container_cluster.rb +73 -66
  43. data/modules/mu/clouds/aws/database.rb +124 -116
  44. data/modules/mu/clouds/aws/dnszone.rb +27 -20
  45. data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
  46. data/modules/mu/clouds/aws/folder.rb +18 -3
  47. data/modules/mu/clouds/aws/function.rb +77 -23
  48. data/modules/mu/clouds/aws/group.rb +19 -12
  49. data/modules/mu/clouds/aws/habitat.rb +153 -0
  50. data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
  51. data/modules/mu/clouds/aws/log.rb +30 -23
  52. data/modules/mu/clouds/aws/msg_queue.rb +29 -20
  53. data/modules/mu/clouds/aws/notifier.rb +222 -0
  54. data/modules/mu/clouds/aws/role.rb +178 -90
  55. data/modules/mu/clouds/aws/search_domain.rb +40 -24
  56. data/modules/mu/clouds/aws/server.rb +169 -137
  57. data/modules/mu/clouds/aws/server_pool.rb +60 -83
  58. data/modules/mu/clouds/aws/storage_pool.rb +59 -31
  59. data/modules/mu/clouds/aws/user.rb +36 -27
  60. data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
  61. data/modules/mu/clouds/aws/vpc.rb +250 -189
  62. data/modules/mu/clouds/azure.rb +132 -0
  63. data/modules/mu/clouds/cloudformation.rb +65 -1
  64. data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
  65. data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
  66. data/modules/mu/clouds/cloudformation/collection.rb +7 -0
  67. data/modules/mu/clouds/cloudformation/database.rb +7 -0
  68. data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
  69. data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
  70. data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
  71. data/modules/mu/clouds/cloudformation/log.rb +7 -0
  72. data/modules/mu/clouds/cloudformation/server.rb +7 -0
  73. data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
  74. data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
  75. data/modules/mu/clouds/google.rb +214 -110
  76. data/modules/mu/clouds/google/container_cluster.rb +42 -24
  77. data/modules/mu/clouds/google/database.rb +15 -6
  78. data/modules/mu/clouds/google/firewall_rule.rb +17 -25
  79. data/modules/mu/clouds/google/group.rb +13 -5
  80. data/modules/mu/clouds/google/habitat.rb +105 -0
  81. data/modules/mu/clouds/google/loadbalancer.rb +28 -20
  82. data/modules/mu/clouds/google/server.rb +93 -354
  83. data/modules/mu/clouds/google/server_pool.rb +18 -10
  84. data/modules/mu/clouds/google/user.rb +22 -14
  85. data/modules/mu/clouds/google/vpc.rb +97 -69
  86. data/modules/mu/config.rb +133 -38
  87. data/modules/mu/config/alarm.rb +25 -0
  88. data/modules/mu/config/cache_cluster.rb +5 -3
  89. data/modules/mu/config/cache_cluster.yml +23 -0
  90. data/modules/mu/config/database.rb +25 -16
  91. data/modules/mu/config/database.yml +3 -3
  92. data/modules/mu/config/function.rb +1 -2
  93. data/modules/mu/config/{project.rb → habitat.rb} +10 -10
  94. data/modules/mu/config/notifier.rb +85 -0
  95. data/modules/mu/config/notifier.yml +9 -0
  96. data/modules/mu/config/role.rb +1 -1
  97. data/modules/mu/config/search_domain.yml +2 -2
  98. data/modules/mu/config/server.rb +13 -1
  99. data/modules/mu/config/server.yml +3 -3
  100. data/modules/mu/config/server_pool.rb +3 -1
  101. data/modules/mu/config/storage_pool.rb +3 -1
  102. data/modules/mu/config/storage_pool.yml +19 -0
  103. data/modules/mu/config/vpc.rb +70 -8
  104. data/modules/mu/groomers/chef.rb +2 -3
  105. data/modules/mu/kittens.rb +500 -122
  106. data/modules/mu/master.rb +5 -5
  107. data/modules/mu/mommacat.rb +151 -91
  108. data/modules/tests/super_complex_bok.yml +12 -0
  109. data/modules/tests/super_simple_bok.yml +12 -0
  110. data/spec/mu/clouds/azure_spec.rb +82 -0
  111. data/spec/spec_helper.rb +105 -0
  112. metadata +26 -5
  113. data/modules/mu/clouds/aws/notification.rb +0 -139
  114. data/modules/mu/config/notification.rb +0 -44
@@ -64,7 +64,7 @@ module MU
64
64
  if @config['all_account_vpcs']
65
65
  # If we've been told to make this domain available account-wide, do so
66
66
  MU::Cloud::AWS.listRegions(@config['us_only']).each { |region|
67
- known_vpcs = MU::Cloud::AWS.ec2(region).describe_vpcs.vpcs
67
+ known_vpcs = MU::Cloud::AWS.ec2(region: region).describe_vpcs.vpcs
68
68
 
69
69
  MU.log "Enumerating VPCs in #{region}", MU::DEBUG, details: known_vpcs
70
70
 
@@ -330,11 +330,11 @@ module MU
330
330
  # @param vpc_id [String]: The cloud identifier of the VPC
331
331
  # @param region [String]: The cloud provider's region
332
332
  # @param remove [Boolean]: Whether to remove access (default: grant access)
333
- def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false)
333
+ def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false, credentials: nil)
334
334
 
335
335
  if !remove
336
336
  MU.log "Granting VPC #{vpc_id} access to zone #{id}"
337
- MU::Cloud::AWS.route53(region).associate_vpc_with_hosted_zone(
337
+ MU::Cloud::AWS.route53(credentials: credentials).associate_vpc_with_hosted_zone(
338
338
  hosted_zone_id: id,
339
339
  vpc: {
340
340
  :vpc_id => vpc_id,
@@ -345,7 +345,7 @@ module MU
345
345
  else
346
346
  MU.log "Revoking VPC #{vpc_id} access to zone #{id}"
347
347
  begin
348
- MU::Cloud::AWS.route53(region).disassociate_vpc_from_hosted_zone(
348
+ MU::Cloud::AWS.route53(credentials: credentials).disassociate_vpc_from_hosted_zone(
349
349
  hosted_zone_id: id,
350
350
  vpc: {
351
351
  :vpc_id => vpc_id,
@@ -400,7 +400,7 @@ module MU
400
400
  target_zone = "/hostedzone/"+alias_zone if !alias_zone.match(/^\/hostedzone\//)
401
401
  else
402
402
  MU::Cloud::AWS.listRegions.each { |region|
403
- MU::Cloud::AWS.elb(region).describe_load_balancers.load_balancer_descriptions.each { |elb|
403
+ MU::Cloud::AWS.elb(region: region).describe_load_balancers.load_balancer_descriptions.each { |elb|
404
404
  elb_dns = elb.dns_name.downcase
405
405
  elb_dns.chomp!(".")
406
406
  if target_name == elb_dns
@@ -657,14 +657,21 @@ module MU
657
657
  end
658
658
  end
659
659
 
660
+ # Does this resource type exist as a global (cloud-wide) artifact, or
661
+ # is it localized to a region/zone?
662
+ # @return [Boolean]
663
+ def self.isGlobal?
664
+ true
665
+ end
666
+
660
667
  # Called by {MU::Cleanup}. Locates resources that were created by the
661
668
  # currently-loaded deployment, and purges them.
662
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
669
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
663
670
  checks_to_clean = []
664
671
  threads = []
665
- MU::Cloud::AWS.route53(region).list_health_checks.health_checks.each { |check|
672
+ MU::Cloud::AWS.route53(credentials: credentials).list_health_checks.health_checks.each { |check|
666
673
  begin
667
- tags = MU::Cloud::AWS.route53(region).list_tags_for_resource(
674
+ tags = MU::Cloud::AWS.route53(credentials: credentials).list_tags_for_resource(
668
675
  resource_type: "healthcheck",
669
676
  resource_id: check.id
670
677
  ).resource_tag_set.tags
@@ -692,7 +699,7 @@ module MU
692
699
  MU.log "Removing health check #{check.id}"
693
700
  retries = 5
694
701
  begin
695
- MU::Cloud::AWS.route53(region).delete_health_check(health_check_id: check.id) if !noop
702
+ MU::Cloud::AWS.route53(credentials: credentials).delete_health_check(health_check_id: check.id) if !noop
696
703
  rescue Aws::Route53::Errors::NoSuchHealthCheck => e
697
704
  MU.log "Health Check '#{check.id}' disappeared before I could remove it", MU::WARN, details: e.inspect
698
705
  rescue Aws::Route53::Errors::InvalidInput => e
@@ -721,11 +728,11 @@ module MU
721
728
  if !noop
722
729
  begin
723
730
  # Clean up resource records first
724
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id)
731
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
725
732
  rrsets.resource_record_sets.each { |rrset|
726
733
  next if zone.name == rrset.name and (rrset.type == "NS" or rrset.type == "SOA")
727
734
  records = []
728
- MU::Cloud::AWS.route53(region).change_resource_record_sets(
735
+ MU::Cloud::AWS.route53(credentials: credentials).change_resource_record_sets(
729
736
  hosted_zone_id: zone.id,
730
737
  change_batch: {
731
738
  changes: [
@@ -738,7 +745,7 @@ module MU
738
745
  )
739
746
  }
740
747
 
741
- MU::Cloud::AWS.route53(region).delete_hosted_zone(id: zone.id)
748
+ MU::Cloud::AWS.route53(credentials: credentials).delete_hosted_zone(id: zone.id)
742
749
  rescue Aws::Route53::Errors::PriorRequestNotComplete
743
750
  MU.log "Still waiting for all records in DNS Zone '#{zone.name}' (#{zone.id}) to delete", MU::WARN
744
751
  sleep 20
@@ -754,17 +761,17 @@ module MU
754
761
  }
755
762
 
756
763
  # Lets try cleaning MU DNS records in all zones.
757
- MU::Cloud::AWS.route53(region).list_hosted_zones.hosted_zones.each { |zone|
764
+ MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones.hosted_zones.each { |zone|
758
765
  begin
759
766
  zone_rrsets = []
760
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id)
767
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
761
768
  rrsets.resource_record_sets.each { |record|
762
769
  zone_rrsets << record
763
770
  }
764
771
 
765
772
  # AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
766
773
  while rrsets.next_record_name && rrsets.next_record_type
767
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
774
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
768
775
  rrsets.resource_record_sets.each { |record|
769
776
  zone_rrsets << record
770
777
  }
@@ -871,10 +878,10 @@ module MU
871
878
  # @param region [String]: The cloud provider region
872
879
  # @param flags [Hash]: Optional flags
873
880
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching DNSZones
874
- def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, flags: {})
881
+ def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, credentials: nil, flags: {})
875
882
  matches = {}
876
883
 
877
- resp = MU::Cloud::AWS.route53(region).list_hosted_zones(
884
+ resp = MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones(
878
885
  max_items: 100
879
886
  )
880
887
 
@@ -882,13 +889,13 @@ module MU
882
889
  if !cloud_id.nil? and !cloud_id.empty?
883
890
  if zone.id == cloud_id
884
891
  begin
885
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
892
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
886
893
  rescue Aws::Route53::Errors::NoSuchHostedZone
887
894
  MU.log "Hosted zone #{zone.id} doesn't exist"
888
895
  end
889
896
  elsif zone.name == cloud_id or zone.name == cloud_id+"."
890
897
  begin
891
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
898
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
892
899
  rescue Aws::Route53::Errors::NoSuchHostedZone
893
900
  MU.log "Hosted zone #{zone.id} doesn't exist"
894
901
  end
@@ -896,7 +903,7 @@ module MU
896
903
  end
897
904
  if !deploy_id.nil? and !deploy_id.empty? and zone.config.comment == deploy_id
898
905
  begin
899
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
906
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
900
907
  rescue Aws::Route53::Errors::NoSuchHostedZone
901
908
  MU.log "Hosted zone #{zone.id} doesn't exist"
902
909
  end
@@ -51,7 +51,6 @@ module MU
51
51
  vpc_id = @vpc.cloud_id if !@vpc.nil?
52
52
  groupname = @mu_name
53
53
  description = groupname
54
- MU.log "Creating EC2 Security Group #{groupname}"
55
54
 
56
55
  sg_struct = {
57
56
  :group_name => groupname,
@@ -62,14 +61,16 @@ module MU
62
61
  end
63
62
 
64
63
  begin
65
- secgroup = MU::Cloud::AWS.ec2(@config['region']).create_security_group(sg_struct)
64
+ MU.log "Creating EC2 Security Group #{groupname}", details: sg_struct
65
+
66
+ secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
66
67
  @cloud_id = secgroup.group_id
67
68
  rescue Aws::EC2::Errors::InvalidGroupDuplicate => e
68
69
  MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
69
70
  filters = [{name: "group-name", values: [groupname]}]
70
71
  filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
71
72
 
72
- secgroup = MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(filters: filters).security_groups.first
73
+ secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
73
74
  deploy_id = @deploy.deploy_id if !@deploy_id.nil?
74
75
  if secgroup.nil?
75
76
  raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
@@ -78,25 +79,25 @@ module MU
78
79
  end
79
80
 
80
81
  begin
81
- MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(group_ids: [secgroup.group_id])
82
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
82
83
  rescue Aws::EC2::Errors::InvalidGroupNotFound => e
83
84
  MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
84
85
  sleep 10
85
86
  retry
86
87
  end
87
88
 
88
- MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'])
89
- MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'])
89
+ MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
90
+ MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
90
91
 
91
92
  if @config['optional_tags']
92
93
  MU::MommaCat.listOptionalTags.each { |key, value|
93
- MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'])
94
+ MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
94
95
  }
95
96
  end
96
97
 
97
98
  if @config['tags']
98
99
  @config['tags'].each { |tag|
99
- MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'])
100
+ MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
100
101
  }
101
102
  end
102
103
 
@@ -167,12 +168,12 @@ module MU
167
168
 
168
169
  begin
169
170
  if egress
170
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
171
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
171
172
  group_id: @cloud_id,
172
173
  ip_permissions: ec2_rule
173
174
  )
174
175
  else
175
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
176
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
176
177
  group_id: @cloud_id,
177
178
  ip_permissions: ec2_rule
178
179
  )
@@ -185,7 +186,7 @@ module MU
185
186
  # Canonical Amazon Resource Number for this resource
186
187
  # @return [String]
187
188
  def arn
188
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.account_number+":security-group/"+@cloud_id
189
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":security-group/"+@cloud_id
189
190
  end
190
191
 
191
192
  # Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
@@ -195,11 +196,11 @@ module MU
195
196
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
196
197
  # @param flags [Hash]: Optional flags
197
198
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching FirewallRules
198
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
199
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
199
200
 
200
201
  if !cloud_id.nil? and !cloud_id.empty?
201
202
  begin
202
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(group_ids: [cloud_id])
203
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(group_ids: [cloud_id])
203
204
  return {cloud_id => resp.data.security_groups.first}
204
205
  rescue ArgumentError => e
205
206
  MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::WARN, details: caller
@@ -212,7 +213,7 @@ module MU
212
213
 
213
214
  map = {}
214
215
  if !tag_key.nil? and !tag_value.nil?
215
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
216
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
216
217
  filters: [
217
218
  {name: "tag:#{tag_key}", values: [tag_value]}
218
219
  ]
@@ -227,12 +228,19 @@ module MU
227
228
  map
228
229
  end
229
230
 
231
+ # Does this resource type exist as a global (cloud-wide) artifact, or
232
+ # is it localized to a region/zone?
233
+ # @return [Boolean]
234
+ def self.isGlobal?
235
+ false
236
+ end
237
+
230
238
  # Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
231
239
  # @param noop [Boolean]: If true, will only print what would be done
232
240
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
233
241
  # @param region [String]: The cloud provider region
234
242
  # @return [void]
235
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
243
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
236
244
  tagfilters = [
237
245
  {name: "tag:MU-ID", values: [MU.deploy_id]}
238
246
  ]
@@ -240,7 +248,7 @@ module MU
240
248
  tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
241
249
  end
242
250
 
243
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
251
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
244
252
  filters: tagfilters
245
253
  )
246
254
 
@@ -303,13 +311,13 @@ module MU
303
311
  begin
304
312
 
305
313
  if ingress_to_revoke.size > 0
306
- MU::Cloud::AWS.ec2(region).revoke_security_group_ingress(
314
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
307
315
  group_id: sg.group_id,
308
316
  ip_permissions: ingress_to_revoke
309
317
  )
310
318
  end
311
319
  if egress_to_revoke.size > 0
312
- MU::Cloud::AWS.ec2(region).revoke_security_group_egress(
320
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
313
321
  group_id: sg.group_id,
314
322
  ip_permissions: egress_to_revoke
315
323
  )
@@ -325,7 +333,7 @@ module MU
325
333
 
326
334
  retries = 0
327
335
  begin
328
- MU::Cloud::AWS.ec2(region).delete_security_group(group_id: sg.group_id) if !noop
336
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
329
337
  rescue Aws::EC2::Errors::InvalidGroupNotFound
330
338
  MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
331
339
  rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
@@ -459,13 +467,13 @@ module MU
459
467
  MU.log "Rules for EC2 Security Group #{@mu_name} (#{@cloud_id}): #{ec2_rules}", MU::DEBUG
460
468
  begin
461
469
  if ingress
462
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
470
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
463
471
  group_id: @cloud_id,
464
472
  ip_permissions: ec2_rules
465
473
  )
466
474
  end
467
475
  if egress
468
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
476
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
469
477
  group_id: @cloud_id,
470
478
  ip_permissions: ec2_rules
471
479
  )
@@ -571,7 +579,7 @@ module MU
571
579
  if !lb.nil? and !lb.cloud_desc.nil?
572
580
  lb.cloud_desc.security_groups.each { |lb_sg|
573
581
  ec2_rule[:user_id_group_pairs] << {
574
- user_id: MU.account_number,
582
+ user_id: MU::Cloud::AWS.credToAcct(@config['credentials']),
575
583
  group_id: lb_sg
576
584
  }
577
585
  }
@@ -48,21 +48,36 @@ module MU
48
48
  }
49
49
  end
50
50
 
51
+ # Does this resource type exist as a global (cloud-wide) artifact, or
52
+ # is it localized to a region/zone?
53
+ # @return [Boolean]
54
+ def self.isGlobal?
55
+ true
56
+ end
57
+
51
58
  # Remove all logs associated with the currently loaded deployment.
52
59
  # @param noop [Boolean]: If true, will only print what would be done
53
60
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
54
61
  # @param region [String]: The cloud provider region
55
62
  # @return [void]
56
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
63
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
57
64
  end
58
65
 
59
- # Locate an existing log group.
66
+ # Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
60
67
  # @param cloud_id [String]: The cloud provider's identifier for this resource.
61
68
  # @param region [String]: The cloud provider region.
62
69
  # @param flags [Hash]: Optional flags
63
70
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
64
- def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
71
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
65
72
  found = nil
73
+
74
+ if cloud_id
75
+ else
76
+ resp = MU::Cloud::AWS.orgs(credentials: credentials).describe_organization
77
+ found ||= {}
78
+ found[resp.organization.id] = resp.organization
79
+ end
80
+
66
81
  found
67
82
  end
68
83
 
@@ -35,16 +35,23 @@ module MU
35
35
  @mu_name ||= @deploy.getResourceName(@config["name"])
36
36
  end
37
37
 
38
-
38
+ # Given an IAM role name, resolve to ARN. Will attempt to identify any
39
+ # sibling Mu role resources by this name first, and failing that, will
40
+ # do a plain get_role() to the IAM API for the provided name.
41
+ # @param name [String]
39
42
  def get_role_arn(name)
43
+ sib_role = @deploy.findLitterMate(name: name, type: "roles")
44
+ return sib_role.cloudobj.arn if sib_role
45
+
40
46
  begin
41
- role = MU::Cloud::AWS.iam(@config['region']).get_role({
47
+ role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
42
48
  role_name: name.to_s
43
49
  })
44
50
  return role['role']['arn']
45
51
  rescue Exception => e
46
- Mu.log "#{e}", MU::ERR
52
+ MU.log "#{e}", MU::ERR
47
53
  end
54
+ nil
48
55
  end
49
56
 
50
57
  def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
@@ -52,7 +59,7 @@ module MU
52
59
  ## get vpc_id
53
60
  ## get sub_id and verify its in the same vpc
54
61
  ## get sg_id and verify its in the same vpc
55
- ec2_client = MU::Cloud::AWS.ec2(region)
62
+ ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
56
63
 
57
64
  vpc_filter = ec2_client.describe_vpcs({
58
65
  filters: [{ name: 'tag-value', values: [vpc_name] }]
@@ -94,7 +101,7 @@ module MU
94
101
  def assign_tag(resource_arn, tag_list, region=@config['region'])
95
102
  begin
96
103
  tag_list.each do |each_pair|
97
- tag_resp = MU::Cloud::AWS.lambda(region).tag_resource({
104
+ tag_resp = MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
98
105
  resource: resource_arn,
99
106
  tags: each_pair
100
107
  })
@@ -164,11 +171,22 @@ module MU
164
171
  lambda_properties[:vpc_config] = vpc_conf
165
172
  end
166
173
 
167
- MU::Cloud::AWS.lambda(@config['region']).create_function(lambda_properties)
174
+ retries = 0
175
+ begin
176
+ MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
177
+ rescue Aws::Lambda::Errors::InvalidParameterValueException => e
178
+ # Freshly-made IAM roles sometimes aren't really ready
179
+ if retries < 5
180
+ sleep 10
181
+ retries += 1
182
+ retry
183
+ end
184
+ raise e
185
+ end
168
186
  end
169
187
 
170
188
  def groom
171
- desc = MU::Cloud::AWS.lambda(@config['region']).get_function(
189
+ desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
172
190
  function_name: @mu_name
173
191
  )
174
192
  func_arn = desc.configuration.function_arn if !desc.empty?
@@ -199,7 +217,7 @@ module MU
199
217
 
200
218
  MU.log trigger_properties, MU::DEBUG
201
219
  begin
202
- add_trigger = MU::Cloud::AWS.lambda(@config['region']).add_permission(trigger_properties)
220
+ add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
203
221
  rescue Aws::Lambda::Errors::ResourceConflictException
204
222
  # XXX check properly for existence
205
223
  end
@@ -216,11 +234,11 @@ module MU
216
234
  arn = nil
217
235
  case svc.downcase
218
236
  when 'sns'
219
- arn = "arn:aws:sns:#{@config['region']}:#{MU.account_number}:#{name}"
237
+ arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
220
238
  when 'alarm','events', 'event', 'cloudwatch_event'
221
- arn = "arn:aws:events:#{@config['region']}:#{MU.account_number}:rule/#{name}"
239
+ arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
222
240
  when 'apigateway'
223
- arn = "arn:aws:apigateway:#{@config['region']}:#{MU.account_number}:#{name}"
241
+ arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
224
242
  when 's3'
225
243
  arn = ''
226
244
  end
@@ -237,15 +255,16 @@ module MU
237
255
  case trig_type
238
256
 
239
257
  when 'sns'
240
-
241
- sns_client = MU::Cloud::AWS.sns(@config['region'])
258
+ # XXX don't do this, use MU::Cloud::AWS::Notification
259
+ sns_client = MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials'])
242
260
  sub_to_what = sns_client.subscribe({
243
261
  topic_arn: trig_arn,
244
262
  protocol: protocol,
245
263
  endpoint: func_arn
246
264
  })
247
265
  when 'event','cloudwatch_event', 'events'
248
- client = MU::Cloud::AWS.cloudwatch_events(@config['region']).put_targets({
266
+ # XXX don't do this, use MU::Cloud::AWS::Log
267
+ client = MU::Cloud::AWS.cloudwatch_events(region: @config['region'], credentials: @config['credentials']).put_targets({
249
268
  rule: @config['trigger']['name'],
250
269
  targets: [
251
270
  {
@@ -268,23 +287,27 @@ module MU
268
287
  return deploy_struct
269
288
  end
270
289
 
271
-
272
-
290
+ # Does this resource type exist as a global (cloud-wide) artifact, or
291
+ # is it localized to a region/zone?
292
+ # @return [Boolean]
293
+ def self.isGlobal?
294
+ false
295
+ end
273
296
 
274
297
  # Remove all functions associated with the currently loaded deployment.
275
298
  # @param noop [Boolean]: If true, will only print what would be done
276
299
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
277
300
  # @param region [String]: The cloud provider region
278
301
  # @return [void]
279
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
280
- MU::Cloud::AWS.lambda(region).list_functions.functions.each { |f|
281
- desc = MU::Cloud::AWS.lambda(region).get_function(
302
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
303
+ MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
304
+ desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
282
305
  function_name: f.function_name
283
306
  )
284
307
  if desc.tags and desc.tags["MU-ID"] == MU.deploy_id
285
308
  MU.log "Deleting Lambda function #{f.function_name}"
286
309
  if !noop
287
- MU::Cloud::AWS.lambda(region).delete_function(
310
+ MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
288
311
  function_name: f.function_name
289
312
  )
290
313
  end
@@ -304,10 +327,10 @@ module MU
304
327
  # @param region [String]: The cloud provider region.
305
328
  # @param flags [Hash]: Optional flags
306
329
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
307
- def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, flags: {})
330
+ def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, credentials: nil, flags: {})
308
331
  func = nil
309
332
  if !func_name.nil?
310
- all_functions = MU::Cloud::AWS.lambda(region).list_functions
333
+ all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
311
334
  if all_functions.include?(func_name)
312
335
  all_functions.functions.each do |x|
313
336
  if x.function_name == func_name
@@ -329,7 +352,13 @@ module MU
329
352
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
330
353
  def self.schema(config)
331
354
  toplevel_required = []
332
- schema = {}
355
+ schema = {
356
+ "iam_role" => {
357
+ "type" => "string",
358
+ "description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with the AWSLambdaBasicExecutionRole policy attached. To grant other permissions for your function, create a Mu 'role' resource and use the 'import' and 'policies' parameters to add permissions. See also: https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html"
359
+ }
360
+ # XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
361
+ }
333
362
  [toplevel_required, schema]
334
363
  end
335
364
 
@@ -340,6 +369,31 @@ module MU
340
369
  def self.validateConfig(function, configurator)
341
370
  ok = true
342
371
 
372
+ if !function['iam_role']
373
+ roledesc = {
374
+ "name" => function['name']+"execrole",
375
+ "credentials" => function['credentials'],
376
+ "can_assume" => [
377
+ {
378
+ "entity_id" => "lambda.amazonaws.com",
379
+ "entity_type" => "service"
380
+ }
381
+ ],
382
+ "import" => [
383
+ "AWSLambdaBasicExecutionRole"
384
+ ]
385
+ }
386
+ configurator.insertKitten(roledesc, "roles")
387
+
388
+ function['dependencies'] ||= []
389
+ function['iam_role'] = function['name']+"execrole"
390
+
391
+ function['dependencies'] << {
392
+ "type" => "role",
393
+ "name" => function['name']+"execrole"
394
+ }
395
+ end
396
+
343
397
  ok
344
398
  end
345
399