cloud-mu 3.1.6 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +1 -1
- data/bin/mu-adopt +15 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +37 -1
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +4 -4
- data/cookbooks/mu-tools/attributes/default.rb +7 -0
- data/cookbooks/mu-tools/libraries/helper.rb +87 -3
- data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
- data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
- data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
- data/cookbooks/mu-tools/resources/disk.rb +33 -12
- data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
- data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
- data/extras/clean-stock-amis +10 -2
- data/extras/generate-stock-images +7 -3
- data/extras/image-generators/AWS/centos7.yaml +19 -16
- data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
- data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
- data/modules/mommacat.ru +2 -2
- data/modules/mu.rb +84 -97
- data/modules/mu/adoption.rb +359 -59
- data/modules/mu/cleanup.rb +67 -44
- data/modules/mu/cloud.rb +108 -1754
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +178 -0
- data/modules/mu/config.rb +122 -80
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +4 -5
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +81 -9
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +30 -34
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +5 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +10 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +13 -17
- data/modules/mu/defaults/AWS.yaml +106 -106
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +15 -0
- data/modules/mu/groomers/chef.rb +3 -0
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +22 -1
- data/modules/mu/mommacat.rb +71 -26
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +82 -3
- data/modules/mu/mommacat/search.rb +59 -16
- data/modules/mu/mommacat/storage.rb +119 -48
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +248 -62
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
- data/modules/mu/providers/aws/database.rb +1747 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +469 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +4 -1
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +15 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
- data/modules/mu/{clouds → providers}/google/database.rb +2 -9
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +4 -4
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
- data/modules/mu/{clouds → providers}/google/role.rb +46 -35
- data/modules/mu/{clouds → providers}/google/server.rb +26 -11
- data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
- data/modules/mu/{clouds → providers}/google/user.rb +32 -22
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +2 -2
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +126 -98
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -89,7 +89,7 @@ Looking elsewhere in `cloud.rb` let's see what all we have to do:
|
|
89
89
|
generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config]
|
90
90
|
```
|
91
91
|
|
92
|
-
Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/
|
92
|
+
Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/providers/aws/function.rb`:
|
93
93
|
|
94
94
|
```
|
95
95
|
module MU
|
@@ -39,7 +39,7 @@ module MU
|
|
39
39
|
end
|
40
40
|
|
41
41
|
# List all AWS projects available to our credentials
|
42
|
-
def self.listHabitats(credentials = nil)
|
42
|
+
def self.listHabitats(credentials = nil, use_cache: true)
|
43
43
|
cfg = credConfig(credentials)
|
44
44
|
return [] if !cfg or !cfg['account_number']
|
45
45
|
[cfg['account_number']]
|
@@ -550,7 +550,9 @@ end
|
|
550
550
|
def self.credToAcct(name = nil)
|
551
551
|
creds = credConfig(name)
|
552
552
|
|
553
|
-
|
553
|
+
if creds['account_number'] and !creds['account_number'].empty?
|
554
|
+
return creds['account_number']
|
555
|
+
end
|
554
556
|
|
555
557
|
acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
|
556
558
|
acct_num.to_s
|
@@ -672,8 +674,8 @@ end
|
|
672
674
|
next
|
673
675
|
end
|
674
676
|
acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4]
|
675
|
-
|
676
|
-
|
677
|
+
cfg['account_number'] ||= acct_num.to_s
|
678
|
+
if acct_num.to_s == name.to_s
|
677
679
|
@@acct_to_profile_map[name.to_s] = cfg
|
678
680
|
return name_only ? name.to_s : cfg
|
679
681
|
end
|
@@ -805,50 +807,47 @@ end
|
|
805
807
|
|
806
808
|
@@instance_types ||= {}
|
807
809
|
@@instance_types[region] ||= {}
|
808
|
-
next_token = nil
|
809
810
|
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
|
815
|
-
|
816
|
-
|
817
|
-
|
818
|
-
|
819
|
-
|
820
|
-
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
type: "TERM_MATCH"
|
830
|
-
}
|
831
|
-
],
|
832
|
-
next_token: next_token
|
833
|
-
)
|
834
|
-
resp.price_list.each { |pricing|
|
835
|
-
data = JSON.parse(pricing)
|
836
|
-
type = data["product"]["attributes"]["instanceType"]
|
837
|
-
next if @@instance_types[region].has_key?(type)
|
838
|
-
@@instance_types[region][type] = {}
|
839
|
-
["ecu", "vcpu", "memory", "storage"].each { |a|
|
840
|
-
@@instance_types[region][type][a] = data["product"]["attributes"][a]
|
811
|
+
# Pricing API isn't widely available, so ask a region we know supports
|
812
|
+
# it
|
813
|
+
resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
|
814
|
+
service_code: "AmazonEC2",
|
815
|
+
filters: [
|
816
|
+
{
|
817
|
+
field: "productFamily",
|
818
|
+
value: "Compute Instance",
|
819
|
+
type: "TERM_MATCH"
|
820
|
+
},
|
821
|
+
{
|
822
|
+
field: "tenancy",
|
823
|
+
value: "Shared",
|
824
|
+
type: "TERM_MATCH"
|
825
|
+
},
|
826
|
+
{
|
827
|
+
field: "location",
|
828
|
+
value: human_region,
|
829
|
+
type: "TERM_MATCH"
|
841
830
|
}
|
842
|
-
|
843
|
-
|
844
|
-
|
831
|
+
]
|
832
|
+
)
|
833
|
+
resp.price_list.each { |pricing|
|
834
|
+
data = JSON.parse(pricing)
|
835
|
+
type = data["product"]["attributes"]["instanceType"]
|
836
|
+
next if @@instance_types[region].has_key?(type)
|
837
|
+
@@instance_types[region][type] = {}
|
838
|
+
["ecu", "vcpu", "memory", "storage"].each { |a|
|
839
|
+
@@instance_types[region][type][a] = data["product"]["attributes"][a]
|
845
840
|
}
|
846
|
-
|
847
|
-
|
841
|
+
@@instance_types[region][type]["memory"].sub!(/ GiB/, "")
|
842
|
+
@@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f
|
843
|
+
@@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f
|
844
|
+
}
|
848
845
|
|
849
846
|
@@instance_types
|
850
847
|
end
|
851
848
|
|
849
|
+
@@certificates = {}
|
850
|
+
|
852
851
|
# AWS can stash API-available certificates in Amazon Certificate Manager
|
853
852
|
# or in IAM. Rather than make people crazy trying to get the syntax
|
854
853
|
# correct in our Baskets of Kittens, let's have a helper that tries to do
|
@@ -857,21 +856,24 @@ end
|
|
857
856
|
# @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised.
|
858
857
|
# @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied.
|
859
858
|
# @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
|
860
|
-
def self.findSSLCertificate(name: nil, id: nil, region: myRegion)
|
861
|
-
if name.nil?
|
859
|
+
def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true)
|
860
|
+
if (name.nil? or name.empty?) and (id.nil? or id.empty?)
|
862
861
|
raise MuError, "Can't call findSSLCertificate without specifying either a name or an id"
|
863
862
|
end
|
863
|
+
if id and @@certificates[id]
|
864
|
+
return [id, @@certificates[id]]
|
865
|
+
end
|
864
866
|
|
865
867
|
if !name.nil? and !name.empty?
|
866
868
|
matches = []
|
867
|
-
acmcerts = MU::Cloud::AWS.acm(region: region).list_certificates(
|
869
|
+
acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates(
|
868
870
|
certificate_statuses: ["ISSUED"]
|
869
871
|
)
|
870
872
|
acmcerts.certificate_summary_list.each { |cert|
|
871
873
|
matches << cert.certificate_arn if cert.domain_name == name
|
872
874
|
}
|
873
875
|
begin
|
874
|
-
iamcert = MU::Cloud::AWS.iam.get_server_certificate(
|
876
|
+
iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate(
|
875
877
|
server_certificate_name: name
|
876
878
|
)
|
877
879
|
rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity
|
@@ -881,32 +883,45 @@ end
|
|
881
883
|
matches << iamcert.server_certificate.server_certificate_metadata.arn
|
882
884
|
end
|
883
885
|
if matches.size == 1
|
884
|
-
|
886
|
+
id = matches.first
|
885
887
|
elsif matches.size == 0
|
886
|
-
|
888
|
+
if raise_on_missing
|
889
|
+
raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}"
|
890
|
+
else
|
891
|
+
return nil
|
892
|
+
end
|
887
893
|
elsif matches.size > 1
|
888
894
|
raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use."
|
889
895
|
end
|
890
896
|
end
|
891
897
|
|
898
|
+
domains = []
|
899
|
+
|
892
900
|
if id.match(/^arn:aws(?:-us-gov)?:acm/)
|
893
|
-
resp = MU::Cloud::AWS.acm(region: region).
|
901
|
+
resp = MU::Cloud::AWS.acm(region: region).describe_certificate(
|
894
902
|
certificate_arn: id
|
895
903
|
)
|
896
|
-
|
904
|
+
|
905
|
+
if resp.nil? or resp.certificate.nil?
|
897
906
|
raise MuError, "No such ACM certificate '#{id}'"
|
898
907
|
end
|
908
|
+
domains << resp.certificate.domain_name
|
909
|
+
if resp.certificate.subject_alternative_names
|
910
|
+
domains.concat(resp.certificate.subject_alternative_names)
|
911
|
+
end
|
899
912
|
elsif id.match(/^arn:aws(?:-us-gov)?:iam/)
|
900
913
|
resp = MU::Cloud::AWS.iam.list_server_certificates
|
901
914
|
if resp.nil?
|
902
915
|
raise MuError, "No such IAM certificate '#{id}'"
|
903
916
|
end
|
904
917
|
resp.server_certificate_metadata_list.each { |cert|
|
918
|
+
|
905
919
|
if cert.arn == id
|
906
920
|
if cert.expiration < Time.now
|
907
921
|
MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN
|
908
922
|
end
|
909
|
-
|
923
|
+
@@certificates[id] = [cert.server_certificate_name]
|
924
|
+
return [id, [cert.server_certificate_name]]
|
910
925
|
end
|
911
926
|
}
|
912
927
|
raise MuError, "No such IAM certificate '#{id}'"
|
@@ -914,7 +929,56 @@ end
|
|
914
929
|
raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM"
|
915
930
|
end
|
916
931
|
|
917
|
-
id
|
932
|
+
@@certificates[id] = domains.uniq
|
933
|
+
[id, domains.uniq]
|
934
|
+
end
|
935
|
+
|
936
|
+
# Given a domain name and an ACM or IAM certificate identifier, sort out
|
937
|
+
# whether the domain name is "covered" by the certificate
|
938
|
+
# @param name [String]
|
939
|
+
# @param cert_id [String]
|
940
|
+
# @return [Boolean]
|
941
|
+
def self.nameMatchesCertificate(name, cert_id)
|
942
|
+
_id, domains = findSSLCertificate(id: cert_id)
|
943
|
+
return false if !domains
|
944
|
+
domains.each { |dom|
|
945
|
+
if dom == name or
|
946
|
+
(dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/)
|
947
|
+
return true
|
948
|
+
end
|
949
|
+
}
|
950
|
+
false
|
951
|
+
end
|
952
|
+
|
953
|
+
# Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate,
|
954
|
+
# look up and validate the specified certificate. This is intended to be
|
955
|
+
# invoked from resource implementations' +validateConfig+ methods.
|
956
|
+
# @param certblock [Hash,MU::Config::Ref]:
|
957
|
+
# @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any
|
958
|
+
# @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any
|
959
|
+
# @return [Boolean]
|
960
|
+
def self.resolveSSLCertificate(certblock, region: nil, credentials: nil)
|
961
|
+
return false if !certblock
|
962
|
+
ok = true
|
963
|
+
|
964
|
+
certblock['region'] ||= region if !certblock['id']
|
965
|
+
certblock['credentials'] ||= credentials
|
966
|
+
cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate(
|
967
|
+
name: certblock["name"],
|
968
|
+
id: certblock["id"],
|
969
|
+
region: certblock['region'],
|
970
|
+
credentials: certblock['credentials']
|
971
|
+
)
|
972
|
+
|
973
|
+
if cert_arn
|
974
|
+
certblock['id'] ||= cert_arn
|
975
|
+
end
|
976
|
+
|
977
|
+
['region', 'credentials'].each { |field|
|
978
|
+
certblock.delete(field) if certblock[field].nil?
|
979
|
+
}
|
980
|
+
|
981
|
+
[cert_arn, cert_domains]
|
918
982
|
end
|
919
983
|
|
920
984
|
# Amazon Certificate Manager API
|
@@ -1034,6 +1098,14 @@ end
|
|
1034
1098
|
@@cloudwatchlogs_api[credentials][region]
|
1035
1099
|
end
|
1036
1100
|
|
1101
|
+
# Amazon's CloudWatchEvents API
|
1102
|
+
def self.cloudwatchevents(region: MU.curRegion, credentials: nil)
|
1103
|
+
region ||= myRegion
|
1104
|
+
@@cloudwatchevents_api[credentials] ||= {}
|
1105
|
+
@@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials)
|
1106
|
+
@@cloudwatchevents_api[credentials][region]
|
1107
|
+
end
|
1108
|
+
|
1037
1109
|
# Amazon's CloudFront API
|
1038
1110
|
def self.cloudfront(region: MU.curRegion, credentials: nil)
|
1039
1111
|
region ||= myRegion
|
@@ -1122,6 +1194,14 @@ end
|
|
1122
1194
|
@@dynamo_api[credentials][region]
|
1123
1195
|
end
|
1124
1196
|
|
1197
|
+
# Amazon's DynamoStream API
|
1198
|
+
def self.dynamostream(region: MU.curRegion, credentials: nil)
|
1199
|
+
region ||= myRegion
|
1200
|
+
@@dynamostream_api[credentials] ||= {}
|
1201
|
+
@@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials)
|
1202
|
+
@@dynamostream_api[credentials][region]
|
1203
|
+
end
|
1204
|
+
|
1125
1205
|
# Amazon's Pricing API
|
1126
1206
|
def self.pricing(region: MU.curRegion, credentials: nil)
|
1127
1207
|
region ||= myRegion
|
@@ -1170,6 +1250,14 @@ end
|
|
1170
1250
|
@@kms_api[credentials][region]
|
1171
1251
|
end
|
1172
1252
|
|
1253
|
+
# Amazon's CloudFront API
|
1254
|
+
def self.cloudfront(region: MU.curRegion, credentials: nil)
|
1255
|
+
region ||= myRegion
|
1256
|
+
@@cloudfront_api[credentials] ||= {}
|
1257
|
+
@@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials)
|
1258
|
+
@@cloudfront_api[credentials][region]
|
1259
|
+
end
|
1260
|
+
|
1173
1261
|
# Amazon's Organizations API
|
1174
1262
|
def self.orgs(credentials: nil)
|
1175
1263
|
@@organizations_api ||= {}
|
@@ -1251,7 +1339,7 @@ end
|
|
1251
1339
|
# Mu Master, if we're in AWS.
|
1252
1340
|
# @return [void]
|
1253
1341
|
def self.openFirewallForClients
|
1254
|
-
MU::Cloud.
|
1342
|
+
MU::Cloud.resourceClass("AWS", :FirewallRule)
|
1255
1343
|
begin
|
1256
1344
|
if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|
1257
1345
|
::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|
@@ -1430,6 +1518,7 @@ end
|
|
1430
1518
|
def initialize(region: nil, api: "EC2", credentials: nil)
|
1431
1519
|
@cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
|
1432
1520
|
@credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
|
1521
|
+
@api_name = api
|
1433
1522
|
|
1434
1523
|
if !@cred_obj
|
1435
1524
|
raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
|
@@ -1447,6 +1536,8 @@ end
|
|
1447
1536
|
params[:credentials] = @cred_obj
|
1448
1537
|
|
1449
1538
|
MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
|
1539
|
+
require "aws-sdk-#{api.downcase}"
|
1540
|
+
|
1450
1541
|
@api = Object.const_get("Aws::#{api}::Client").new(params)
|
1451
1542
|
end
|
1452
1543
|
|
@@ -1454,21 +1545,113 @@ end
|
|
1454
1545
|
# Catch-all for AWS client methods. Essentially a pass-through with some
|
1455
1546
|
# rescues for known silly endpoint behavior.
|
1456
1547
|
def method_missing(method_sym, *arguments)
|
1457
|
-
|
1548
|
+
# make sure error symbols are loaded for our exception handling later
|
1549
|
+
require "aws-sdk-lambda"
|
1550
|
+
require "aws-sdk-rds"
|
1551
|
+
require "aws-sdk-ec2"
|
1552
|
+
require "aws-sdk-route53"
|
1553
|
+
require "aws-sdk-iam"
|
1554
|
+
require "aws-sdk-efs"
|
1555
|
+
require "aws-sdk-pricing"
|
1556
|
+
require "aws-sdk-apigateway"
|
1557
|
+
require "aws-sdk-ecs"
|
1558
|
+
require "aws-sdk-eks"
|
1559
|
+
require "aws-sdk-cloudwatchlogs"
|
1560
|
+
require "aws-sdk-cloudwatchevents"
|
1561
|
+
require "aws-sdk-elasticloadbalancing"
|
1562
|
+
require "aws-sdk-elasticloadbalancingv2"
|
1563
|
+
require "aws-sdk-autoscaling"
|
1564
|
+
|
1565
|
+
known_concats = {
|
1566
|
+
"Pricing" => {
|
1567
|
+
:get_products => :price_list
|
1568
|
+
}
|
1569
|
+
}
|
1458
1570
|
|
1459
1571
|
retries = 0
|
1460
1572
|
begin
|
1461
|
-
MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
|
1462
|
-
|
1463
|
-
|
1464
|
-
|
1465
|
-
|
1466
|
-
|
1467
|
-
|
1468
|
-
|
1573
|
+
MU.log "Calling #{@api_name}.#{method_sym} in #{@region}", MU::DEBUG, details: arguments
|
1574
|
+
|
1575
|
+
retval = if !arguments.nil? and arguments.size == 1
|
1576
|
+
@api.method(method_sym).call(arguments[0])
|
1577
|
+
elsif !arguments.nil? and arguments.size > 0
|
1578
|
+
@api.method(method_sym).call(*arguments)
|
1579
|
+
else
|
1580
|
+
@api.method(method_sym).call
|
1581
|
+
end
|
1582
|
+
|
1583
|
+
if !retval.nil?
|
1584
|
+
begin
|
1585
|
+
page_markers = {
|
1586
|
+
:marker => :marker,
|
1587
|
+
:next_token => :next_token,
|
1588
|
+
:next_marker => :marker
|
1589
|
+
}
|
1590
|
+
paginator = nil
|
1591
|
+
new_page = nil
|
1592
|
+
page_markers.each_key { |m|
|
1593
|
+
if !retval.nil? and retval.respond_to?(m)
|
1594
|
+
paginator = m
|
1595
|
+
new_page = retval.send(m)
|
1596
|
+
break
|
1597
|
+
end
|
1598
|
+
}
|
1599
|
+
|
1600
|
+
if paginator and new_page and !new_page.empty?
|
1601
|
+
resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
|
1602
|
+
concat_to = MU.structToHash(resp).keys.reject { |m|
|
1603
|
+
m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
|
1604
|
+
}
|
1605
|
+
|
1606
|
+
if concat_to.empty? and known_concats[@api_name] and
|
1607
|
+
known_concats[@api_name][method_sym]
|
1608
|
+
concat_to << known_concats[@api_name][method_sym]
|
1609
|
+
end
|
1610
|
+
|
1611
|
+
if concat_to.empty? and method_sym.to_s.match(/^(?:describe|list)_(.*)/)
|
1612
|
+
my_attr = Regexp.last_match[1].to_sym
|
1613
|
+
concat_to << my_attr if resp.respond_to?(my_attr)
|
1614
|
+
end
|
1615
|
+
|
1616
|
+
if concat_to.size != 1
|
1617
|
+
raise MuError.new "Tried to figure out where I might append paginated results for a #{@api_name}.#{method_sym}, but failed", details: MU.structToHash(resp).keys
|
1618
|
+
else
|
1619
|
+
concat_to = concat_to.first
|
1620
|
+
new_args = arguments ? arguments.dup : [{}]
|
1621
|
+
begin
|
1622
|
+
if new_args.is_a?(Array)
|
1623
|
+
new_args << {} if new_args.empty?
|
1624
|
+
if new_args.size == 1 and new_args.first.is_a?(Hash)
|
1625
|
+
new_args[0][page_markers[paginator]] = new_page
|
1626
|
+
else
|
1627
|
+
MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
|
1628
|
+
end
|
1629
|
+
elsif new_args.is_a?(Hash)
|
1630
|
+
new_args[page_markers[paginator]] = new_page
|
1631
|
+
end
|
1632
|
+
|
1633
|
+
MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args
|
1634
|
+
|
1635
|
+
# resp = if !arguments.nil? and arguments.size == 1
|
1636
|
+
# @api.method(method_sym).call(new_args[0])
|
1637
|
+
# elsif !arguments.nil? and arguments.size > 0
|
1638
|
+
resp = @api.method(method_sym).call(*new_args)
|
1639
|
+
# end
|
1640
|
+
break if resp.nil?
|
1641
|
+
resp = resp.__getobj__ if resp.respond_to?(:__getobj__)
|
1642
|
+
retval.send(concat_to).concat(resp.send(concat_to))
|
1643
|
+
new_page = resp.send(paginator) if !resp.nil?
|
1644
|
+
end while !resp.nil? and !new_page.nil? and !new_page.empty?
|
1645
|
+
end
|
1646
|
+
end
|
1647
|
+
rescue StandardError => e
|
1648
|
+
MU.log "Made a good-faith effort to auto-paginate API call to #{method_sym} and failed with #{e.message}", MU::DEBUG, details: arguments
|
1649
|
+
raise e
|
1650
|
+
end
|
1469
1651
|
end
|
1652
|
+
|
1470
1653
|
return retval
|
1471
|
-
rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
1654
|
+
rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
1472
1655
|
if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
|
1473
1656
|
MU.log e.inspect, MU::ERR
|
1474
1657
|
raise e
|
@@ -1510,6 +1693,7 @@ end
|
|
1510
1693
|
@@wafglobal = {}
|
1511
1694
|
@@waf = {}
|
1512
1695
|
@@cloudwatchlogs_api = {}
|
1696
|
+
@@cloudwatchevents_api = {}
|
1513
1697
|
@@cloudfront_api = {}
|
1514
1698
|
@@elasticache_api = {}
|
1515
1699
|
@@sns_api = {}
|
@@ -1528,6 +1712,8 @@ end
|
|
1528
1712
|
@@kms_api ={}
|
1529
1713
|
@@organization_api ={}
|
1530
1714
|
@@dynamo_api ={}
|
1715
|
+
@@dynamostream_api ={}
|
1716
|
+
@@cloudfront_api ={}
|
1531
1717
|
end
|
1532
1718
|
end
|
1533
1719
|
end
|