cloud-mu 3.1.5 → 3.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/tasks/main.yml +16 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +3 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +2 -0
- data/extras/image-generators/AWS/win2k16.yaml +2 -0
- data/extras/image-generators/AWS/win2k19.yaml +2 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +86 -98
- data/modules/mu/adoption.rb +373 -58
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1733
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +123 -81
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +90 -90
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +34 -20
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +69 -4
- data/modules/mu/groomers/chef.rb +51 -4
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/mommacat.rb +160 -874
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +497 -0
- data/modules/mu/mommacat/storage.rb +252 -194
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +258 -57
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +6 -6
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +41 -24
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +122 -92
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
|
@@ -27,8 +27,8 @@ module MU
|
|
|
27
27
|
|
|
28
28
|
# Called automatically by {MU::Deploy#createResources}
|
|
29
29
|
def create
|
|
30
|
-
MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name)
|
|
31
30
|
@cloud_id = @mu_name
|
|
31
|
+
MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
|
|
32
32
|
MU.log "Created SNS topic #{@mu_name}"
|
|
33
33
|
end
|
|
34
34
|
|
|
@@ -36,17 +36,48 @@ module MU
|
|
|
36
36
|
def groom
|
|
37
37
|
if @config['subscriptions']
|
|
38
38
|
@config['subscriptions'].each { |sub|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
if sub['resource'] and !sub['endpoint']
|
|
40
|
+
endpoint_obj = nil
|
|
41
|
+
MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) {
|
|
42
|
+
endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy)
|
|
43
|
+
}
|
|
44
|
+
sub['endpoint'] = endpoint_obj.arn
|
|
45
|
+
end
|
|
46
|
+
subscribe(sub['endpoint'], sub['type'])
|
|
46
47
|
}
|
|
47
48
|
end
|
|
48
49
|
end
|
|
49
50
|
|
|
51
|
+
# Subscribe something to this SNS topic
|
|
52
|
+
# @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
|
|
53
|
+
# @param protocol [String]: The protocol being subscribed
|
|
54
|
+
def subscribe(endpoint, protocol)
|
|
55
|
+
self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Subscribe something to an SNS topic
|
|
59
|
+
# @param cloud_id [String]: The short name or ARN of an existing SNS topic
|
|
60
|
+
# @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
|
|
61
|
+
# @param protocol [String]: The protocol being subscribed
|
|
62
|
+
# @param region [String]: The region of the target SNS topic
|
|
63
|
+
# @param credentials [String]:
|
|
64
|
+
def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil)
|
|
65
|
+
topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first
|
|
66
|
+
if !topic
|
|
67
|
+
raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}"
|
|
68
|
+
end
|
|
69
|
+
arn = topic["TopicArn"]
|
|
70
|
+
|
|
71
|
+
resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
|
|
72
|
+
|
|
73
|
+
resp.each { |subscription|
|
|
74
|
+
return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE
|
|
78
|
+
MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
|
|
79
|
+
end
|
|
80
|
+
|
|
50
81
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
|
51
82
|
# is it localized to a region/zone?
|
|
52
83
|
# @return [Boolean]
|
|
@@ -65,12 +96,12 @@ module MU
|
|
|
65
96
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
66
97
|
# @param region [String]: The cloud provider region
|
|
67
98
|
# @return [void]
|
|
68
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
99
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
69
100
|
MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
70
101
|
MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
|
71
102
|
|
|
72
103
|
MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
|
|
73
|
-
if topic.topic_arn.match(
|
|
104
|
+
if topic.topic_arn.match(deploy_id)
|
|
74
105
|
# We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN.
|
|
75
106
|
# This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
|
|
76
107
|
MU.log "Deleting SNS topic: #{topic.topic_arn}"
|
|
@@ -91,6 +122,7 @@ module MU
|
|
|
91
122
|
# Return the metadata for this user cofiguration
|
|
92
123
|
# @return [Hash]
|
|
93
124
|
def notify
|
|
125
|
+
return nil if !@cloud_id or !cloud_desc(use_cache: false)
|
|
94
126
|
desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
|
|
95
127
|
MU.structToHash(desc)
|
|
96
128
|
end
|
|
@@ -101,9 +133,16 @@ module MU
|
|
|
101
133
|
found = {}
|
|
102
134
|
|
|
103
135
|
if args[:cloud_id]
|
|
104
|
-
arn =
|
|
105
|
-
|
|
106
|
-
|
|
136
|
+
arn = if args[:cloud_id].match(/^arn:/)
|
|
137
|
+
args[:cloud_id]
|
|
138
|
+
else
|
|
139
|
+
"arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
|
|
140
|
+
end
|
|
141
|
+
begin
|
|
142
|
+
desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
|
|
143
|
+
found[args[:cloud_id]] = desc if desc
|
|
144
|
+
rescue ::Aws::SNS::Errors::NotFound
|
|
145
|
+
end
|
|
107
146
|
else
|
|
108
147
|
next_token = nil
|
|
109
148
|
begin
|
|
@@ -120,6 +159,58 @@ module MU
|
|
|
120
159
|
found
|
|
121
160
|
end
|
|
122
161
|
|
|
162
|
+
# Reverse-map our cloud description into a runnable config hash.
|
|
163
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
|
164
|
+
# calculate our own accordingly based on what's live in the cloud.
|
|
165
|
+
def toKitten(**_args)
|
|
166
|
+
bok = {
|
|
167
|
+
"cloud" => "AWS",
|
|
168
|
+
"credentials" => @config['credentials'],
|
|
169
|
+
"cloud_id" => @cloud_id,
|
|
170
|
+
"region" => @config['region']
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if !cloud_desc
|
|
174
|
+
MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
|
|
175
|
+
return nil
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"]
|
|
179
|
+
svcmap = {
|
|
180
|
+
"lambda" => "functions",
|
|
181
|
+
"sqs" => "msg_queues"
|
|
182
|
+
}
|
|
183
|
+
MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
|
|
184
|
+
bok['subscriptions'] ||= []
|
|
185
|
+
|
|
186
|
+
bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
|
|
187
|
+
_wholestring, service, region, account, id = Regexp.last_match.to_a
|
|
188
|
+
{
|
|
189
|
+
"type" => sub.protocol,
|
|
190
|
+
"resource" => MU::Config::Ref.get(
|
|
191
|
+
type: svcmap[service],
|
|
192
|
+
region: region,
|
|
193
|
+
credentials: @credentials,
|
|
194
|
+
id: id,
|
|
195
|
+
cloud: "AWS",
|
|
196
|
+
habitat: MU::Config::Ref.get(
|
|
197
|
+
id: account,
|
|
198
|
+
cloud: "AWS",
|
|
199
|
+
credentials: @credentials
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
else
|
|
204
|
+
{
|
|
205
|
+
"type" => sub.protocol,
|
|
206
|
+
"endpoint" => sub.endpoint
|
|
207
|
+
}
|
|
208
|
+
end
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
bok
|
|
212
|
+
end
|
|
213
|
+
|
|
123
214
|
# Cloud-specific configuration properties.
|
|
124
215
|
# @param _config [MU::Config]: The calling MU::Config object
|
|
125
216
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
|
@@ -130,11 +221,10 @@ module MU
|
|
|
130
221
|
"type" => "array",
|
|
131
222
|
"items" => {
|
|
132
223
|
"type" => "object",
|
|
133
|
-
"required" => ["endpoint"],
|
|
134
224
|
"properties" => {
|
|
135
225
|
"type" => {
|
|
136
226
|
"type" => "string",
|
|
137
|
-
"description" => "",
|
|
227
|
+
"description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.",
|
|
138
228
|
"enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"]
|
|
139
229
|
}
|
|
140
230
|
}
|
|
@@ -148,26 +238,42 @@ module MU
|
|
|
148
238
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::notifier}, bare and unvalidated.
|
|
149
239
|
|
|
150
240
|
# @param notifier [Hash]: The resource to process and validate
|
|
151
|
-
# @param
|
|
241
|
+
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
|
152
242
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
|
153
|
-
def self.validateConfig(notifier,
|
|
243
|
+
def self.validateConfig(notifier, configurator)
|
|
154
244
|
ok = true
|
|
155
245
|
|
|
156
246
|
if notifier['subscriptions']
|
|
157
247
|
notifier['subscriptions'].each { |sub|
|
|
248
|
+
if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type'])
|
|
249
|
+
sub['resource']['cloud'] = "AWS"
|
|
250
|
+
MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type'])
|
|
251
|
+
end
|
|
158
252
|
if !sub["type"]
|
|
159
|
-
if sub[
|
|
160
|
-
sub[
|
|
161
|
-
|
|
162
|
-
sub[
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
elsif sub[
|
|
166
|
-
sub["
|
|
167
|
-
|
|
168
|
-
sub["
|
|
169
|
-
|
|
170
|
-
|
|
253
|
+
sub['type'] = if sub['resource']
|
|
254
|
+
if sub['resource']['type'] == "functions"
|
|
255
|
+
"lambda"
|
|
256
|
+
elsif sub['resource']['type'] == "msg_queues"
|
|
257
|
+
"sqs"
|
|
258
|
+
end
|
|
259
|
+
elsif sub['endpoint']
|
|
260
|
+
if sub["endpoint"].match(/^http:/i)
|
|
261
|
+
"http"
|
|
262
|
+
elsif sub["endpoint"].match(/^https:/i)
|
|
263
|
+
"https"
|
|
264
|
+
elsif sub["endpoint"].match(/:sqs:/i)
|
|
265
|
+
"sqs"
|
|
266
|
+
elsif sub["endpoint"].match(/:lambda:/i)
|
|
267
|
+
"lambda"
|
|
268
|
+
elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
|
|
269
|
+
"sms"
|
|
270
|
+
elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
|
|
271
|
+
"email"
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
if !sub['type']
|
|
276
|
+
MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub
|
|
171
277
|
ok = false
|
|
172
278
|
end
|
|
173
279
|
end
|
|
@@ -178,40 +284,6 @@ module MU
|
|
|
178
284
|
end
|
|
179
285
|
|
|
180
286
|
|
|
181
|
-
# Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc...
|
|
182
|
-
# Will create the subscription only if it doesn't already exist.
|
|
183
|
-
# @param arn [String]: The cloud provider's identifier of the notifier group.
|
|
184
|
-
# @param protocol [String]: The type of the subscription (eg. email,https, etc..).
|
|
185
|
-
# @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) ..
|
|
186
|
-
# @param region [String]: The cloud provider region.
|
|
187
|
-
def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil)
|
|
188
|
-
retries = 0
|
|
189
|
-
begin
|
|
190
|
-
resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
|
|
191
|
-
rescue Aws::SNS::Errors::NotFound
|
|
192
|
-
if retries < 5
|
|
193
|
-
MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource"
|
|
194
|
-
retries += 1
|
|
195
|
-
sleep 30
|
|
196
|
-
retry
|
|
197
|
-
else
|
|
198
|
-
raise MuError, "Couldn't find topic #{arn}, giving up"
|
|
199
|
-
end
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
already_subscribed = false
|
|
203
|
-
if resp && !resp.empty?
|
|
204
|
-
resp.each { |subscription|
|
|
205
|
-
already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint
|
|
206
|
-
}
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
unless already_subscribed
|
|
210
|
-
MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
|
|
211
|
-
MU.log "Subscribed #{endpoint} to SNS topic #{arn}"
|
|
212
|
-
end
|
|
213
|
-
end
|
|
214
|
-
|
|
215
287
|
# Test if a notifier group exists
|
|
216
288
|
# Create a new notifier group. Will check if the group exists before creating it.
|
|
217
289
|
# @param topic_name [String]: The cloud provider's name for the notifier group.
|
|
@@ -30,7 +30,7 @@ module MU
|
|
|
30
30
|
end
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
@mu_name ||= @deploy.getResourceName(@config["name"])
|
|
33
|
+
@mu_name ||= @deploy.getResourceName(@config["name"], max_length: 64)
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# Called automatically by {MU::Deploy#createResources}
|
|
@@ -92,13 +92,14 @@ module MU
|
|
|
92
92
|
configured_policies = []
|
|
93
93
|
|
|
94
94
|
if @config['raw_policies']
|
|
95
|
+
MU.log "Attaching #{@config['raw_policies'].size.to_s} raw #{@config['raw_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
|
|
95
96
|
configured_policies = @config['raw_policies'].map { |p|
|
|
96
97
|
@mu_name+"-"+p.keys.first.upcase
|
|
97
98
|
}
|
|
98
99
|
end
|
|
99
100
|
|
|
100
101
|
if @config['attachable_policies']
|
|
101
|
-
MU.log "Attaching #{@config['attachable_policies'].size.to_s} #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
|
|
102
|
+
MU.log "Attaching #{@config['attachable_policies'].size.to_s} external #{@config['attachable_policies'].size > 1 ? "policies" : "policy"} to role #{@mu_name}", MU::NOTICE
|
|
102
103
|
configured_policies.concat(@config['attachable_policies'].map { |p|
|
|
103
104
|
id = if p.is_a?(MU::Config::Ref)
|
|
104
105
|
p.cloud_id
|
|
@@ -109,17 +110,16 @@ module MU
|
|
|
109
110
|
end
|
|
110
111
|
id.gsub(/.*?\/([^:\/]+)$/, '\1')
|
|
111
112
|
})
|
|
112
|
-
configured_policies.each { |pol|
|
|
113
|
-
}
|
|
114
113
|
end
|
|
115
114
|
|
|
115
|
+
# Purge anything that doesn't belong
|
|
116
116
|
if !@config['bare_policies']
|
|
117
117
|
attached_policies = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_role_policies(
|
|
118
118
|
role_name: @mu_name
|
|
119
119
|
).attached_policies
|
|
120
120
|
attached_policies.each { |a|
|
|
121
121
|
if !configured_policies.include?(a.policy_name)
|
|
122
|
-
MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE
|
|
122
|
+
MU.log "Removing IAM policy #{a.policy_name} from role #{@mu_name}", MU::NOTICE, details: configured_policies
|
|
123
123
|
MU::Cloud::AWS::Role.purgePolicy(a.policy_arn, @config['credentials'])
|
|
124
124
|
end
|
|
125
125
|
}
|
|
@@ -153,6 +153,7 @@ module MU
|
|
|
153
153
|
policy.values.each { |p|
|
|
154
154
|
p["Version"] ||= "2012-10-17"
|
|
155
155
|
}
|
|
156
|
+
|
|
156
157
|
policy_name = basename+"-"+policy.keys.first.upcase
|
|
157
158
|
|
|
158
159
|
arn = "arn:"+(MU::Cloud::AWS.isGovCloud? ? "aws-us-gov" : "aws")+":iam::"+MU::Cloud::AWS.credToAcct(credentials)+":policy#{path}/#{policy_name}"
|
|
@@ -216,7 +217,22 @@ module MU
|
|
|
216
217
|
# populated with one or both depending on what this resource has
|
|
217
218
|
# defined.
|
|
218
219
|
def cloud_desc(use_cache: true)
|
|
219
|
-
|
|
220
|
+
|
|
221
|
+
# we might inherit a naive cached description from the base cloud
|
|
222
|
+
# layer; rearrange it to our tastes
|
|
223
|
+
if @cloud_desc_cache.is_a?(::Aws::IAM::Types::Role)
|
|
224
|
+
new_desc = {
|
|
225
|
+
"role" => @cloud_desc_cache
|
|
226
|
+
}
|
|
227
|
+
@cloud_desc_cache = new_desc
|
|
228
|
+
elsif @cloud_desc_cache.is_a?(::Aws::IAM::Types::Policy)
|
|
229
|
+
new_desc = {
|
|
230
|
+
"policies" => [@cloud_desc_cache]
|
|
231
|
+
}
|
|
232
|
+
@cloud_desc_cache = new_desc
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
return @cloud_desc_cache if @cloud_desc_cache and !@cloud_desc_cache.empty? and use_cache
|
|
220
236
|
|
|
221
237
|
@cloud_desc_cache = {}
|
|
222
238
|
if @config['bare_policies']
|
|
@@ -419,14 +435,14 @@ end
|
|
|
419
435
|
# @param noop [Boolean]: If true, will only print what would be done
|
|
420
436
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
421
437
|
# @return [void]
|
|
422
|
-
def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
|
|
438
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, credentials: nil, flags: {})
|
|
423
439
|
|
|
424
440
|
resp = MU::Cloud::AWS.iam(credentials: credentials).list_policies(
|
|
425
|
-
path_prefix: "/"+
|
|
441
|
+
path_prefix: "/"+deploy_id+"/"
|
|
426
442
|
)
|
|
427
443
|
if resp and resp.policies
|
|
428
444
|
resp.policies.each { |policy|
|
|
429
|
-
MU.log "Deleting IAM policy /#{
|
|
445
|
+
MU.log "Deleting IAM policy /#{deploy_id}/#{policy.policy_name}"
|
|
430
446
|
if !noop
|
|
431
447
|
purgePolicy(policy.arn, credentials)
|
|
432
448
|
end
|
|
@@ -437,19 +453,23 @@ end
|
|
|
437
453
|
roles = MU::Cloud::AWS::Role.find(credentials: credentials).values
|
|
438
454
|
roles.each { |r|
|
|
439
455
|
next if !r.respond_to?(:role_name)
|
|
440
|
-
if r.path.match(/^\/#{Regexp.quote(
|
|
456
|
+
if r.path.match(/^\/#{Regexp.quote(deploy_id)}/)
|
|
441
457
|
deleteme << r
|
|
442
458
|
next
|
|
443
459
|
end
|
|
444
460
|
# For some dumb reason, the list output that .find gets doesn't
|
|
445
461
|
# include the tags, so we need to fetch each role individually to
|
|
446
462
|
# check tags. Hardly seems efficient.
|
|
447
|
-
desc =
|
|
463
|
+
desc = begin
|
|
464
|
+
MU::Cloud::AWS.iam(credentials: credentials).get_role(role_name: r.role_name)
|
|
465
|
+
rescue Aws::IAM::Errors::NoSuchEntity
|
|
466
|
+
next
|
|
467
|
+
end
|
|
448
468
|
if desc.role and desc.role.tags and desc.role.tags
|
|
449
469
|
master_match = false
|
|
450
470
|
deploy_match = false
|
|
451
471
|
desc.role.tags.each { |t|
|
|
452
|
-
if t.key == "MU-ID" and t.value ==
|
|
472
|
+
if t.key == "MU-ID" and t.value == deploy_id
|
|
453
473
|
deploy_match = true
|
|
454
474
|
elsif t.key == "MU-MASTER-IP" and t.value == MU.mu_public_ip
|
|
455
475
|
master_match = true
|
|
@@ -516,7 +536,7 @@ end
|
|
|
516
536
|
|
|
517
537
|
begin
|
|
518
538
|
# managed policies get fetched by ARN, roles by plain name. Ok!
|
|
519
|
-
if args[:cloud_id].match(/^arn
|
|
539
|
+
if args[:cloud_id].match(/^arn:.*?:policy\//)
|
|
520
540
|
resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_policy(
|
|
521
541
|
policy_arn: args[:cloud_id]
|
|
522
542
|
)
|
|
@@ -525,39 +545,26 @@ end
|
|
|
525
545
|
end
|
|
526
546
|
else
|
|
527
547
|
resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).get_role(
|
|
528
|
-
role_name: args[:cloud_id]
|
|
548
|
+
role_name: args[:cloud_id].sub(/^arn:.*?\/([^:\/]+)$/, '\1') # XXX if it's an ARN, actually parse it and look in the correct account when applicable
|
|
529
549
|
)
|
|
550
|
+
|
|
530
551
|
if resp and resp.role
|
|
531
|
-
found[
|
|
552
|
+
found[resp.role.role_name] = resp.role
|
|
532
553
|
end
|
|
533
554
|
end
|
|
534
555
|
rescue ::Aws::IAM::Errors::NoSuchEntity
|
|
535
556
|
end
|
|
536
557
|
|
|
537
558
|
else
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
)
|
|
543
|
-
break if !resp or !resp.roles
|
|
544
|
-
resp.roles.each { |role|
|
|
545
|
-
found[role.role_name] = role
|
|
546
|
-
}
|
|
547
|
-
marker = resp.marker
|
|
548
|
-
end while marker
|
|
559
|
+
resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_roles
|
|
560
|
+
resp.roles.each { |role|
|
|
561
|
+
found[role.role_name] = role
|
|
562
|
+
}
|
|
549
563
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
)
|
|
555
|
-
break if !resp or !resp.policies
|
|
556
|
-
resp.policies.each { |pol|
|
|
557
|
-
found[pol.arn] = pol
|
|
558
|
-
}
|
|
559
|
-
marker = resp.marker
|
|
560
|
-
end while marker
|
|
564
|
+
resp = MU::Cloud::AWS.iam(credentials: args[:credentials]).list_policies(scope: "Local")
|
|
565
|
+
resp.policies.each { |pol|
|
|
566
|
+
found[pol.arn] = pol
|
|
567
|
+
}
|
|
561
568
|
end
|
|
562
569
|
|
|
563
570
|
found
|
|
@@ -615,7 +622,6 @@ end
|
|
|
615
622
|
)
|
|
616
623
|
JSON.parse(URI.decode(version.policy_version.document))
|
|
617
624
|
end
|
|
618
|
-
|
|
619
625
|
bok["policies"] = MU::Cloud::AWS::Role.doc2MuPolicies(pol.policy_name, doc, bok["policies"])
|
|
620
626
|
end
|
|
621
627
|
}
|
|
@@ -695,6 +701,7 @@ end
|
|
|
695
701
|
end
|
|
696
702
|
|
|
697
703
|
bok["attachable_policies"].uniq! if bok["attachable_policies"]
|
|
704
|
+
bok["name"].gsub!(/[^a-zA-Z0-9_\-]/, "_")
|
|
698
705
|
|
|
699
706
|
bok
|
|
700
707
|
end
|
|
@@ -707,6 +714,10 @@ end
|
|
|
707
714
|
def self.doc2MuPolicies(basename, doc, policies = [])
|
|
708
715
|
policies ||= []
|
|
709
716
|
|
|
717
|
+
if !doc["Statement"].is_a?(Array)
|
|
718
|
+
doc["Statement"] = [doc["Statement"]]
|
|
719
|
+
end
|
|
720
|
+
|
|
710
721
|
doc["Statement"].each { |s|
|
|
711
722
|
if !s["Action"]
|
|
712
723
|
MU.log "Statement in policy document for #{basename} didn't have an Action field", MU::WARN, details: doc
|
|
@@ -804,6 +815,19 @@ end
|
|
|
804
815
|
}
|
|
805
816
|
end
|
|
806
817
|
|
|
818
|
+
if @config['raw_policies']
|
|
819
|
+
raw_arns = MU::Cloud::AWS::Role.manageRawPolicies(
|
|
820
|
+
@config['raw_policies'],
|
|
821
|
+
basename: @deploy.getResourceName(@config['name']),
|
|
822
|
+
credentials: @credentials
|
|
823
|
+
)
|
|
824
|
+
raw_arns.each { |p_arn|
|
|
825
|
+
mypolicies << MU::Cloud::AWS.iam(credentials: @config['credentials']).get_policy(
|
|
826
|
+
policy_arn: p_arn
|
|
827
|
+
).policy
|
|
828
|
+
}
|
|
829
|
+
end
|
|
830
|
+
|
|
807
831
|
mypolicies.each { |p|
|
|
808
832
|
if entitytype == "user"
|
|
809
833
|
resp = MU::Cloud::AWS.iam(credentials: @config['credentials']).list_attached_user_policies(
|
|
@@ -925,7 +949,7 @@ end
|
|
|
925
949
|
toplevel_required = []
|
|
926
950
|
aws_resource_types = MU::Cloud.resource_types.keys.reject { |t|
|
|
927
951
|
begin
|
|
928
|
-
MU::Cloud.
|
|
952
|
+
MU::Cloud.resourceClass("AWS", t)
|
|
929
953
|
false
|
|
930
954
|
rescue MuCloudResourceNotImplemented
|
|
931
955
|
true
|
|
@@ -1087,11 +1111,7 @@ end
|
|
|
1087
1111
|
role['policies'].each { |policy|
|
|
1088
1112
|
policy['targets'].each { |target|
|
|
1089
1113
|
if target['type']
|
|
1090
|
-
role['
|
|
1091
|
-
role['dependencies'] << {
|
|
1092
|
-
"name" => target['identifier'],
|
|
1093
|
-
"type" => target['type']
|
|
1094
|
-
}
|
|
1114
|
+
MU::Config.addDependency(role, target['identifier'], target['type'], no_create_wait: true)
|
|
1095
1115
|
end
|
|
1096
1116
|
}
|
|
1097
1117
|
}
|
|
@@ -1107,13 +1127,14 @@ end
|
|
|
1107
1127
|
# @param policies [Array<Hash>]: One or more policy chunks
|
|
1108
1128
|
# @param deploy_obj [MU::MommaCat]: Deployment object to use when looking up sibling Mu resources
|
|
1109
1129
|
# @return [Array<Hash>]
|
|
1110
|
-
def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false)
|
|
1130
|
+
def self.genPolicyDocument(policies, deploy_obj: nil, bucket_style: false, version: "2012-10-17", doc_id: nil)
|
|
1111
1131
|
if policies
|
|
1112
1132
|
name = nil
|
|
1113
1133
|
doc = {
|
|
1114
|
-
"Version" =>
|
|
1134
|
+
"Version" => version,
|
|
1115
1135
|
"Statement" => []
|
|
1116
1136
|
}
|
|
1137
|
+
doc["Id"] = doc_id if doc_id
|
|
1117
1138
|
policies.each { |policy|
|
|
1118
1139
|
policy["flag"] ||= "Allow"
|
|
1119
1140
|
statement = {
|
|
@@ -1154,7 +1175,14 @@ end
|
|
|
1154
1175
|
raise MuError, "Couldn't find a #{grantee["type"]} named #{grantee["identifier"]} when generating IAM policy"
|
|
1155
1176
|
end
|
|
1156
1177
|
else
|
|
1157
|
-
bucket_prefix = grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
|
|
1178
|
+
bucket_prefix = if grantee["identifier"].match(/^[^\.]+\.amazonaws\.com$/)
|
|
1179
|
+
"Service"
|
|
1180
|
+
elsif grantee["identifier"] =~ /^[a-f0-9]+$/
|
|
1181
|
+
"CanonicalUser"
|
|
1182
|
+
else
|
|
1183
|
+
"AWS"
|
|
1184
|
+
end
|
|
1185
|
+
|
|
1158
1186
|
if bucket_style
|
|
1159
1187
|
statement["Principal"] << { bucket_prefix => grantee["identifier"] }
|
|
1160
1188
|
else
|
|
@@ -1186,7 +1214,7 @@ end
|
|
|
1186
1214
|
statement["Resource"] << id+"/*"
|
|
1187
1215
|
end
|
|
1188
1216
|
else
|
|
1189
|
-
raise MuError, "Couldn't find a #{target["
|
|
1217
|
+
raise MuError, "Couldn't find a #{target["type"]} named #{target["identifier"]} when generating IAM policy"
|
|
1190
1218
|
end
|
|
1191
1219
|
else
|
|
1192
1220
|
target["identifier"] += target["path"] if target["path"]
|