cloud-mu 3.1.6 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +1 -1
- data/bin/mu-adopt +15 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +37 -1
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +4 -4
- data/cookbooks/mu-tools/attributes/default.rb +7 -0
- data/cookbooks/mu-tools/libraries/helper.rb +87 -3
- data/cookbooks/mu-tools/recipes/apply_security.rb +39 -23
- data/cookbooks/mu-tools/recipes/aws_api.rb +13 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +4 -0
- data/cookbooks/mu-tools/recipes/rsyslog.rb +8 -1
- data/cookbooks/mu-tools/resources/disk.rb +33 -12
- data/cookbooks/mu-tools/resources/mommacat_request.rb +1 -2
- data/cookbooks/mu-tools/templates/centos-8/sshd_config.erb +215 -0
- data/extras/clean-stock-amis +10 -2
- data/extras/generate-stock-images +7 -3
- data/extras/image-generators/AWS/centos7.yaml +19 -16
- data/extras/image-generators/AWS/{rhel7.yaml → rhel71.yaml} +0 -0
- data/extras/image-generators/AWS/{win2k12.yaml → win2k12r2.yaml} +0 -0
- data/modules/mommacat.ru +2 -2
- data/modules/mu.rb +84 -97
- data/modules/mu/adoption.rb +359 -59
- data/modules/mu/cleanup.rb +67 -44
- data/modules/mu/cloud.rb +108 -1754
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +178 -0
- data/modules/mu/config.rb +122 -80
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +4 -5
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +81 -9
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +30 -34
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +5 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +10 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +13 -17
- data/modules/mu/defaults/AWS.yaml +106 -106
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +15 -0
- data/modules/mu/groomers/chef.rb +3 -0
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +22 -1
- data/modules/mu/mommacat.rb +71 -26
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +82 -3
- data/modules/mu/mommacat/search.rb +59 -16
- data/modules/mu/mommacat/storage.rb +119 -48
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +248 -62
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +65 -63
- data/modules/mu/providers/aws/database.rb +1747 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +291 -133
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +469 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -78
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +120 -145
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +141 -73
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +4 -1
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +1 -1
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +15 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +2 -2
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +29 -14
- data/modules/mu/{clouds → providers}/google/database.rb +2 -9
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +4 -4
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +2 -2
- data/modules/mu/{clouds → providers}/google/role.rb +46 -35
- data/modules/mu/{clouds → providers}/google/server.rb +26 -11
- data/modules/mu/{clouds → providers}/google/server_pool.rb +11 -11
- data/modules/mu/{clouds → providers}/google/user.rb +32 -22
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +38 -3
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +2 -2
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +126 -98
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -124,7 +124,7 @@ module MU
|
|
124
124
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
125
125
|
# @param region [String]: The cloud provider region
|
126
126
|
# @return [void]
|
127
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
127
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
128
128
|
MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
129
129
|
MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
130
130
|
alarms = []
|
@@ -132,7 +132,7 @@ module MU
|
|
132
132
|
# This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well.
|
133
133
|
MU::Cloud::AWS.cloudwatch(credentials: credentials, region: region).describe_alarms.each { |page|
|
134
134
|
page.metric_alarms.map(&:alarm_name).each { |alarm_name|
|
135
|
-
alarms << alarm_name if alarm_name.match(
|
135
|
+
alarms << alarm_name if alarm_name.match(deploy_id)
|
136
136
|
}
|
137
137
|
}
|
138
138
|
|
@@ -321,7 +321,7 @@ module MU
|
|
321
321
|
if !depclass.nil?
|
322
322
|
dimension["depclass"] = depclass
|
323
323
|
if !dimension["name"].nil? and !dimension["name"].empty?
|
324
|
-
alarm
|
324
|
+
MU::Config.addDependency(alarm, dimension["name"], depclass)
|
325
325
|
end
|
326
326
|
end
|
327
327
|
}
|
@@ -21,6 +21,12 @@ module MU
|
|
21
21
|
@@region_cache = {}
|
22
22
|
@@region_cache_semaphore = Mutex.new
|
23
23
|
|
24
|
+
# Map some filename extensions to mime types. S3 does most of this on
|
25
|
+
# its own, add to this for cases it doesn't cover.
|
26
|
+
MIME_MAP = {
|
27
|
+
".svg" => "image/svg+xml"
|
28
|
+
}
|
29
|
+
|
24
30
|
# Initialize this cloud resource object. Calling +super+ will invoke the initializer defined under {MU::Cloud}, which should set the attribtues listed in {MU::Cloud::PUBLIC_ATTRS} as well as applicable dependency shortcuts, like +@vpc+, for us.
|
25
31
|
# @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
|
26
32
|
def initialize(**args)
|
@@ -81,6 +87,35 @@ module MU
|
|
81
87
|
|
82
88
|
end
|
83
89
|
|
90
|
+
# @return [String]
|
91
|
+
def url
|
92
|
+
"https://#{@cloud_id}.s3.amazonaws.com"
|
93
|
+
end
|
94
|
+
|
95
|
+
# Grant access via our bucket policy to the specified resource
|
96
|
+
# @param principal [String]
|
97
|
+
# @param permissions [Array<String>]
|
98
|
+
# @param paths [Array<String>]
|
99
|
+
def allowPrincipal(principal, permissions: ["GetObject", "ListBucket"], paths: [""], doc_id: nil, name: nil)
|
100
|
+
@config['policies'] ||= []
|
101
|
+
name ||= principal.sub(/.*?([0-9a-z\-_]+)$/i, '\1')
|
102
|
+
@config['policies'] << {
|
103
|
+
"name" => name,
|
104
|
+
"grant_to" => [ { "identifier" => principal } ],
|
105
|
+
"permissions" => permissions.map { |p| "s3:"+p },
|
106
|
+
"flag" => "allow",
|
107
|
+
"targets" => paths.map { |p|
|
108
|
+
{
|
109
|
+
"path" => p,
|
110
|
+
"type" => "bucket",
|
111
|
+
"identifier" => @config['name']
|
112
|
+
}
|
113
|
+
}
|
114
|
+
}
|
115
|
+
|
116
|
+
applyPolicies(doc_id: doc_id)
|
117
|
+
end
|
118
|
+
|
84
119
|
# Called automatically by {MU::Deploy#createResources}
|
85
120
|
def groom
|
86
121
|
|
@@ -90,20 +125,46 @@ module MU
|
|
90
125
|
tagBucket if !@config['scrub_mu_isms']
|
91
126
|
|
92
127
|
current = cloud_desc
|
93
|
-
if @config['policies']
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
128
|
+
applyPolicies if @config['policies']
|
129
|
+
|
130
|
+
if @config['versioning'] and current["versioning"].status != "Enabled"
|
131
|
+
MU.log "Enabling versioning on S3 bucket #{@cloud_id}", MU::NOTICE
|
132
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_versioning(
|
133
|
+
bucket: @cloud_id,
|
134
|
+
versioning_configuration: {
|
135
|
+
mfa_delete: "Disabled",
|
136
|
+
status: "Enabled"
|
137
|
+
}
|
138
|
+
)
|
139
|
+
elsif !@config['versioning'] and current["versioning"].status == "Enabled"
|
140
|
+
MU.log "Suspending versioning on S3 bucket #{@cloud_id}", MU::NOTICE
|
141
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_versioning(
|
142
|
+
bucket: @cloud_id,
|
143
|
+
versioning_configuration: {
|
144
|
+
mfa_delete: "Disabled",
|
145
|
+
status: "Suspended"
|
146
|
+
}
|
147
|
+
)
|
148
|
+
end
|
99
149
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
150
|
+
if @config['upload']
|
151
|
+
@config['upload'].each { |batch|
|
152
|
+
urlbase = "s3://"+@cloud_id+batch['destination']
|
153
|
+
urlbase += "/" if urlbase !~ /\/$/
|
154
|
+
upload_me = if File.directory?(batch['source'])
|
155
|
+
Dir[batch['source']+'/**/*'].reject {|d|
|
156
|
+
File.directory?(d)
|
157
|
+
}.map { |f|
|
158
|
+
[ f, urlbase+f.sub(/^#{Regexp.quote(batch['source'])}\/?/, '') ]
|
159
|
+
}
|
160
|
+
else
|
161
|
+
batch['source'].match(/([^\/]+)$/)
|
162
|
+
[ [batch['source'], urlbase+Regexp.last_match[1]] ]
|
163
|
+
end
|
164
|
+
|
165
|
+
Hash[upload_me].each_pair { |file, url|
|
166
|
+
self.class.upload(url, file: file, credentials: @credentials, region: @config['region'], acl: batch['acl'])
|
167
|
+
}
|
107
168
|
}
|
108
169
|
end
|
109
170
|
|
@@ -120,6 +181,23 @@ module MU
|
|
120
181
|
}
|
121
182
|
}
|
122
183
|
)
|
184
|
+
['web_error_object', 'web_index_object'].each { |key|
|
185
|
+
begin
|
186
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).head_object(
|
187
|
+
bucket: @cloud_id,
|
188
|
+
key: @config[key]
|
189
|
+
)
|
190
|
+
rescue Aws::S3::Errors::NotFound
|
191
|
+
MU.log "Uploading placeholder #{@config[key]} to bucket #{@cloud_id}"
|
192
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_object(
|
193
|
+
acl: "public-read",
|
194
|
+
bucket: @cloud_id,
|
195
|
+
key: @config[key],
|
196
|
+
body: ""
|
197
|
+
)
|
198
|
+
end
|
199
|
+
}
|
200
|
+
# XXX check if error and index objs exist, and if not provide placeholders
|
123
201
|
elsif !@config['web'] and !current["website"].nil?
|
124
202
|
MU.log "Disabling web service on S3 bucket #{@cloud_id}", MU::NOTICE
|
125
203
|
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).delete_bucket_website(
|
@@ -127,25 +205,38 @@ module MU
|
|
127
205
|
)
|
128
206
|
end
|
129
207
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
mfa_delete: "Disabled",
|
136
|
-
status: "Enabled"
|
208
|
+
symbolify_keys = Proc.new { |parent|
|
209
|
+
if parent.is_a?(Hash)
|
210
|
+
newhash = {}
|
211
|
+
parent.each_pair { |k, v|
|
212
|
+
newhash[k.to_sym] = symbolify_keys.call(v)
|
137
213
|
}
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
214
|
+
newhash
|
215
|
+
elsif parent.is_a?(Array)
|
216
|
+
newarr = []
|
217
|
+
parent.each { |child|
|
218
|
+
newarr << symbolify_keys.call(child)
|
219
|
+
}
|
220
|
+
newarr
|
221
|
+
else
|
222
|
+
parent
|
223
|
+
end
|
224
|
+
}
|
225
|
+
|
226
|
+
if @config['cors']
|
227
|
+
MU.log "Setting CORS rules on #{@cloud_id}", details: @config['cors']
|
228
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_cors(
|
142
229
|
bucket: @cloud_id,
|
143
|
-
|
144
|
-
|
145
|
-
status: "Suspended"
|
230
|
+
cors_configuration: {
|
231
|
+
cors_rules: symbolify_keys.call(@config['cors'])
|
146
232
|
}
|
147
233
|
)
|
148
234
|
end
|
235
|
+
|
236
|
+
MU.log "Bucket #{@config['name']}: s3://#{@cloud_id}", MU::SUMMARY
|
237
|
+
if @config['web']
|
238
|
+
MU.log "Bucket #{@config['name']} web access: http://#{@cloud_id}.s3-website-#{@config['region']}.amazonaws.com/", MU::SUMMARY
|
239
|
+
end
|
149
240
|
end
|
150
241
|
|
151
242
|
# Upload a file to a bucket.
|
@@ -160,7 +251,7 @@ module MU
|
|
160
251
|
|
161
252
|
if file and !file.empty?
|
162
253
|
if !File.exist?(file) or !File.readable?(file)
|
163
|
-
raise MuError, "Unable to read #{file} for upload to #{url}"
|
254
|
+
raise MuError, "Unable to read #{file} for upload to #{url} (I'm at #{Dir.pwd}"
|
164
255
|
else
|
165
256
|
data = File.read(file)
|
166
257
|
end
|
@@ -177,12 +268,19 @@ module MU
|
|
177
268
|
|
178
269
|
begin
|
179
270
|
MU.log "Writing #{path} to S3 bucket #{bucket}"
|
180
|
-
|
271
|
+
params = {
|
181
272
|
acl: acl,
|
182
273
|
bucket: bucket,
|
183
274
|
key: path,
|
184
275
|
body: data
|
185
|
-
|
276
|
+
}
|
277
|
+
|
278
|
+
MIME_MAP.each_pair { |extension, content_type|
|
279
|
+
if path =~ /#{Regexp.quote(extension)}$/i
|
280
|
+
params[:content_type] = content_type
|
281
|
+
end
|
282
|
+
}
|
283
|
+
MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(params)
|
186
284
|
rescue Aws::S3::Errors => e
|
187
285
|
raise MuError, "Got #{e.inspect} trying to write #{path} to #{bucket} (region: #{region}, credentials: #{credentials})"
|
188
286
|
end
|
@@ -207,7 +305,7 @@ module MU
|
|
207
305
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
208
306
|
# @param region [String]: The cloud provider region
|
209
307
|
# @return [void]
|
210
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
308
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
211
309
|
MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
212
310
|
|
213
311
|
resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets
|
@@ -242,7 +340,7 @@ module MU
|
|
242
340
|
deploy_match = false
|
243
341
|
master_match = false
|
244
342
|
tags.each { |tag|
|
245
|
-
if tag.key == "MU-ID" and tag.value ==
|
343
|
+
if tag.key == "MU-ID" and tag.value == deploy_id
|
246
344
|
deploy_match = true
|
247
345
|
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
248
346
|
master_match = true
|
@@ -254,6 +352,21 @@ module MU
|
|
254
352
|
MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
|
255
353
|
end
|
256
354
|
end
|
355
|
+
rescue Aws::S3::Errors::BucketNotEmpty => e
|
356
|
+
if flags["skipsnapshots"]
|
357
|
+
del = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_objects(bucket: bucket.name).contents.map { |o| { key: o.key } }
|
358
|
+
del.concat(MU::Cloud::AWS.s3(credentials: credentials, region: region).list_object_versions(bucket: bucket.name).versions.map { |o| { key: o.key, version_id: o.version_id } })
|
359
|
+
|
360
|
+
MU.log "Purging #{del.size.to_s} objects and versions from #{bucket.name}"
|
361
|
+
begin
|
362
|
+
batch = del.slice!(0, (del.length >= 1000 ? 1000 : del.length))
|
363
|
+
MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_objects(bucket: bucket.name, delete: { objects: batch } ) if !noop
|
364
|
+
end while del.size > 0
|
365
|
+
|
366
|
+
retry if !noop
|
367
|
+
else
|
368
|
+
MU.log "Bucket #{bucket.name} is non-empty, will preserve it and its contents. Use --skipsnapshots to forcibly remove.", MU::WARN
|
369
|
+
end
|
257
370
|
rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect
|
258
371
|
next
|
259
372
|
end
|
@@ -279,9 +392,32 @@ module MU
|
|
279
392
|
def self.find(**args)
|
280
393
|
found = {}
|
281
394
|
|
395
|
+
args[:region] ||= MU::Cloud::AWS.myRegion(args[:credentials])
|
396
|
+
if args[:flags] and args[:flags][:allregions]
|
397
|
+
args[:allregions] = args[:flags][:allregions]
|
398
|
+
end
|
399
|
+
minimal = args[:full] ? false : true
|
400
|
+
|
401
|
+
location = Proc.new { |name|
|
402
|
+
begin
|
403
|
+
loc_resp = MU::Cloud::AWS.s3(credentials: args[:credentials], region: args[:region]).get_bucket_location(bucket: name)
|
404
|
+
|
405
|
+
if loc_resp.location_constraint and !loc_resp.location_constraint.empty?
|
406
|
+
loc_resp.location_constraint
|
407
|
+
else
|
408
|
+
nil
|
409
|
+
end
|
410
|
+
rescue Aws::S3::Errors::AccessDenied
|
411
|
+
nil
|
412
|
+
end
|
413
|
+
}
|
414
|
+
|
282
415
|
if args[:cloud_id]
|
283
416
|
begin
|
284
|
-
found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal:
|
417
|
+
found[args[:cloud_id]] = describe_bucket(args[:cloud_id], minimal: minimal, credentials: args[:credentials], region: args[:region])
|
418
|
+
found[args[:cloud_id]]['region'] ||= location.call(args[:cloud_id])
|
419
|
+
found[args[:cloud_id]]['region'] ||= args[:region]
|
420
|
+
found[args[:cloud_id]]['name'] ||= args[:cloud_id]
|
285
421
|
rescue ::Aws::S3::Errors::NoSuchBucket
|
286
422
|
end
|
287
423
|
else
|
@@ -289,11 +425,14 @@ module MU
|
|
289
425
|
if resp and resp.buckets
|
290
426
|
resp.buckets.each { |b|
|
291
427
|
begin
|
292
|
-
|
293
|
-
if !
|
428
|
+
bucket_region = location.call(b.name)
|
429
|
+
if !args[:allregions] and bucket_region != args[:region]
|
294
430
|
next
|
295
431
|
end
|
296
|
-
|
432
|
+
bucket_region ||= args[:region]
|
433
|
+
found[b.name] = describe_bucket(b.name, minimal: minimal, credentials: args[:credentials], region: bucket_region)
|
434
|
+
found[b.name]["region"] ||= bucket_region
|
435
|
+
found[b.name]['name'] ||= b.name
|
297
436
|
rescue Aws::S3::Errors::AccessDenied
|
298
437
|
end
|
299
438
|
}
|
@@ -303,22 +442,96 @@ module MU
|
|
303
442
|
found
|
304
443
|
end
|
305
444
|
|
445
|
+
# Reverse-map our cloud description into a runnable config hash.
|
446
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
447
|
+
# calculate our own accordingly based on what's live in the cloud.
|
448
|
+
def toKitten(**_args)
|
449
|
+
bok = {
|
450
|
+
"cloud" => "AWS",
|
451
|
+
"credentials" => @config['credentials'],
|
452
|
+
"cloud_id" => @cloud_id
|
453
|
+
}
|
454
|
+
|
455
|
+
if @cloud_id =~ /espier/i
|
456
|
+
MU.log @cloud_id, MU::WARN, details: cloud_desc
|
457
|
+
end
|
458
|
+
|
459
|
+
if !cloud_desc
|
460
|
+
MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
|
461
|
+
return nil
|
462
|
+
end
|
463
|
+
|
464
|
+
nil
|
465
|
+
end
|
466
|
+
|
306
467
|
# Cloud-specific configuration properties.
|
307
468
|
# @param _config [MU::Config]: The calling MU::Config object
|
308
469
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
309
470
|
def self.schema(_config)
|
310
471
|
toplevel_required = []
|
311
472
|
schema = {
|
312
|
-
"policies" => MU::Cloud
|
313
|
-
"
|
314
|
-
"
|
315
|
-
|
316
|
-
|
473
|
+
"policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema,
|
474
|
+
"upload" => {
|
475
|
+
"items" => {
|
476
|
+
"properties" => {
|
477
|
+
"acl" => {
|
478
|
+
"type" => "string",
|
479
|
+
"enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
|
480
|
+
"default" => "private"
|
481
|
+
}
|
482
|
+
}
|
483
|
+
}
|
317
484
|
},
|
318
485
|
"storage_class" => {
|
319
486
|
"type" => "string",
|
320
487
|
"enum" => ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA", "INTELLIGENT_TIERING", "GLACIER"],
|
321
488
|
"default" => "STANDARD"
|
489
|
+
},
|
490
|
+
"cors" => {
|
491
|
+
"type" => "array",
|
492
|
+
"items" => {
|
493
|
+
"type" => "object",
|
494
|
+
"description" => "AWS S3 Cross-origin resource sharing policy",
|
495
|
+
"required" => ["allowed_origins"],
|
496
|
+
"properties" => {
|
497
|
+
"allowed_headers" => {
|
498
|
+
"type" => "array",
|
499
|
+
"default" => ["*"],
|
500
|
+
"items" => {
|
501
|
+
"type" => "string",
|
502
|
+
"description" => "Specifies which headers are allowed in a preflight request through the +Access-Control-Request-Headers+ header."
|
503
|
+
}
|
504
|
+
},
|
505
|
+
"allowed_methods" => {
|
506
|
+
"type" => "array",
|
507
|
+
"default" => ["GET"],
|
508
|
+
"items" => {
|
509
|
+
"type" => "string",
|
510
|
+
"enum" => %w{GET PUT POST DELETE HEAD},
|
511
|
+
"description" => "Specifies which HTTP methods for which cross-domain request are permitted"
|
512
|
+
}
|
513
|
+
},
|
514
|
+
"allowed_origins" => {
|
515
|
+
"type" => "array",
|
516
|
+
"items" => {
|
517
|
+
"type" => "string",
|
518
|
+
"description" => "Origins (in URL form) for which cross-domain request are permitted"
|
519
|
+
}
|
520
|
+
},
|
521
|
+
"expose_headers" => {
|
522
|
+
"type" => "array",
|
523
|
+
"items" => {
|
524
|
+
"type" => "string",
|
525
|
+
"description" => "Headers in the response which should be visible to the requesting application"
|
526
|
+
}
|
527
|
+
},
|
528
|
+
"max_age_seconds" => {
|
529
|
+
"type" => "integer",
|
530
|
+
"default" => 3600,
|
531
|
+
"description" => "Maximum cache time for preflight requests"
|
532
|
+
}
|
533
|
+
}
|
534
|
+
}
|
322
535
|
}
|
323
536
|
}
|
324
537
|
[toplevel_required, schema]
|
@@ -384,6 +597,27 @@ module MU
|
|
384
597
|
desc
|
385
598
|
end
|
386
599
|
|
600
|
+
private
|
601
|
+
|
602
|
+
def applyPolicies(doc_id: nil)
|
603
|
+
return if !@config['policies']
|
604
|
+
|
605
|
+
@config['policies'].each { |pol|
|
606
|
+
pol['grant_to'] ||= [
|
607
|
+
{ "id" => "*" }
|
608
|
+
]
|
609
|
+
}
|
610
|
+
|
611
|
+
policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true, version: "2008-10-17", doc_id: doc_id)
|
612
|
+
policy_docs.each { |doc|
|
613
|
+
MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first)
|
614
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
|
615
|
+
bucket: @cloud_id,
|
616
|
+
policy: JSON.generate(doc.values.first)
|
617
|
+
)
|
618
|
+
}
|
619
|
+
end
|
620
|
+
|
387
621
|
end
|
388
622
|
end
|
389
623
|
end
|